allows init spark in cluster mode

enriquea · May 14, 2024 · 16002e7 · 16002e7
1 parent e7dfb84
commit 16002e7
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/fsspark/config/context.py b/fsspark/config/context.py
@@ -6,12 +6,14 @@
                                             PYARROW_SETTINGS,
                                             PANDAS_ON_SPARK_API_SETTINGS)
 
-
 os.environ['PYARROW_IGNORE_TIMEZONE'] = "1"
+
+
 # os.environ['JAVA_HOME'] = "/Library/Java/JavaVirtualMachines/jdk1.8.0_162.jdk/Contents/Home"
 # os.environ['SPARK_HOME'] = "/usr/local/spark-3.3.0-bin-hadoop3"
 
-def init_spark(apply_pyarrow_settings: bool = True,
+def init_spark(master: str = "local[8]",
+               apply_pyarrow_settings: bool = True,
                apply_extra_spark_settings: bool = True,
                apply_pandas_settings: bool = True) -> SparkSession:
     """
@@ -24,7 +26,7 @@ def init_spark(apply_pyarrow_settings: bool = True,
 
     # init or get spark session.
     spark = (SparkSession.builder
-             .master("local[8]")
+             .master(master)
              .appName("fs-spark")
              )