Apache Spark
Check version
$java --version
openjdk 17To build the code.
./gradlew clean build
./gradlew shadowJar
cp build/libs/project08-fat-1.0.0-all.jar /tmp/data/
cp src/main/resources/word-file.txt /tmp/data
cp src/main/resources/customers.json /tmp/datacd docker
docker-compose up -d
docker-compose down
ipconfig getifaddr en0Scala version check on spark cluster
spark-shell
scala.util.Properties.versionString
res0: String = version 2.12.18cd /opt/bitnami/spark
cd bin
spark-submit --class com.demo.project08.Main --master spark://10.177.182.61:7077 /tmp/data/project08-fat-1.0.0-all.jar