{"id":1158724,"date":"2025-01-13T18:44:14","date_gmt":"2025-01-13T10:44:14","guid":{"rendered":"https:\/\/docs.pingcode.com\/ask\/ask-ask\/1158724.html"},"modified":"2025-01-13T18:44:16","modified_gmt":"2025-01-13T10:44:16","slug":"%e5%a6%82%e4%bd%95%e4%bd%bf%e7%94%a8python%e7%9a%84spark-2","status":"publish","type":"post","link":"https:\/\/docs.pingcode.com\/ask\/1158724.html","title":{"rendered":"\u5982\u4f55\u4f7f\u7528python\u7684spark"},"content":{"rendered":"

\"\u5982\u4f55\u4f7f\u7528python\u7684spark\"<\/p>\n

\u8981\u4f7f\u7528Python\u4e2d\u7684Spark\uff0c\u9996\u5148\u9700\u8981\u5b89\u88c5Spark\u548c\u76f8\u5173\u7684Python\u5e93\uff0c\u5176\u6b21\u9700\u8981\u7406\u89e3Spark\u7684\u57fa\u672c\u6982\u5ff5\u548c\u64cd\u4f5c\uff0c\u6700\u540e\u901a\u8fc7\u5b9e\u9645\u793a\u4f8b\u6765\u5c55\u793a\u5982\u4f55\u4f7f\u7528Spark\u8fdb\u884c\u6570\u636e\u5904\u7406\u548c\u5206\u6790\u3002<\/strong> \u5b89\u88c5Spark\u3001\u5b89\u88c5PySpark\u5e93\u3001\u7406\u89e3Spark\u7684\u57fa\u672c\u6982\u5ff5\u3001\u8fdb\u884c\u6570\u636e\u5904\u7406\u548c\u5206\u6790\u3002\u4e0b\u9762\u5c06\u8be6\u7ec6\u5c55\u5f00\u5982\u4f55\u8fdb\u884c\u8fd9\u4e9b\u64cd\u4f5c\u3002<\/p>\n<\/p>\n

\u4e00\u3001\u5b89\u88c5Spark<\/h3>\n<\/p>\n

1\u3001\u4e0b\u8f7d\u5e76\u5b89\u88c5Spark<\/h4>\n<\/p>\n

\u8981\u4f7f\u7528Spark\uff0c\u9996\u5148\u9700\u8981\u4e0b\u8f7d\u548c\u5b89\u88c5Spark\u3002\u53ef\u4ee5\u4eceApache Spark\u7684\u5b98\u65b9\u7f51\u7ad9\u4e0b\u8f7d\u6700\u65b0\u7248\u672c\u7684Spark\u3002\u4e0b\u8f7d\u5b8c\u6210\u540e\uff0c\u89e3\u538b\u7f29\u6587\u4ef6\u5e76\u5c06\u5176\u653e\u7f6e\u5728\u5408\u9002\u7684\u4f4d\u7f6e\u3002<\/p>\n<\/p>\n

\u5b98\u65b9\u7f51\u7ad9: https:\/\/spark.apache.org\/downloads.html<\/a><\/p>\n<\/p>\n

2\u3001\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf<\/h4>\n<\/p>\n

\u4e3a\u4e86\u8ba9\u7cfb\u7edf\u8bc6\u522bSpark\uff0c\u9700\u8981\u8bbe\u7f6e\u73af\u5883\u53d8\u91cf\u3002\u53ef\u4ee5\u5728~\/.bashrc<\/code>\u6216~\/.zshrc<\/code>\u6587\u4ef6\u4e2d\u6dfb\u52a0\u4ee5\u4e0b\u51e0\u884c\uff1a<\/p>\n<\/p>\n

export SPARK_HOME=\/path\/to\/spark<\/p>\n
export PATH=$SPARK_HOME\/bin:$PATH<\/p>\n
<\/code><\/pre>\n<\/p>\n
\u7136\u540e\u5728\u7ec8\u7aef\u4e2d\u6267\u884csource ~\/.bashrc<\/code>\u6216source ~\/.zshrc<\/code>\u4f7f\u73af\u5883\u53d8\u91cf\u751f\u6548\u3002<\/p>\n<\/p>\n
3\u3001\u5b89\u88c5Java<\/h4>\n<\/p>\n
Spark\u8fd0\u884c\u5728Java\u865a\u62df\u673a\uff08JVM\uff09\u4e0a\uff0c\u56e0\u6b64\u9700\u8981\u5b89\u88c5Java\u3002\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u5b89\u88c5OpenJDK\uff1a<\/p>\n<\/p>\n
sudo apt-get update<\/p>\n
sudo apt-get install openjdk-8-jdk<\/p>\n
<\/code><\/pre>\n<\/p>\n
\u4e8c\u3001\u5b89\u88c5PySpark\u5e93<\/h3>\n<\/p>\n
1\u3001\u5b89\u88c5PySpark<\/h4>\n<\/p>\n
PySpark\u662fSpark\u7684Python API\u3002\u53ef\u4ee5\u4f7f\u7528pip\u5b89\u88c5PySpark\uff1a<\/p>\n<\/p>\n
pip install pyspark<\/p>\n
<\/code><\/pre>\n<\/p>\n
2\u3001\u9a8c\u8bc1\u5b89\u88c5<\/h4>\n<\/p>\n
\u5b89\u88c5\u5b8c\u6210\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u547d\u4ee4\u9a8c\u8bc1\u5b89\u88c5\u662f\u5426\u6210\u529f\uff1a<\/p>\n<\/p>\n
import pyspark<\/p>\n
from pyspark.sql import SparkSession<\/p>\n
spark = SparkSession.builder.appName("example").getOrCreate()<\/p>\n
print(spark.version)<\/p>\n
<\/code><\/pre>\n<\/p>\n
\u4e09\u3001\u7406\u89e3Spark\u7684\u57fa\u672c\u6982\u5ff5<\/h3>\n<\/p>\n
1\u3001SparkContext<\/h4>\n<\/p>\n
SparkContext\u662fSpark\u7684\u6838\u5fc3\u5bf9\u8c61\uff0c\u5b83\u8d1f\u8d23\u4e0e\u96c6\u7fa4\u7ba1\u7406\u5668\uff08\u5982YARN\u3001Mesos\u6216Standalone\uff09\u901a\u4fe1\uff0c\u5e76\u7ba1\u7406Spark\u5e94\u7528\u7a0b\u5e8f\u7684\u751f\u547d\u5468\u671f\u3002\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u4ee3\u7801\u521b\u5efaSparkContext\uff1a<\/p>\n<\/p>\n
from pyspark import SparkContext<\/p>\n
sc = SparkContext(master="local", appName="example")<\/p>\n
<\/code><\/pre>\n<\/p>\n
2\u3001RDD\uff08\u5f39\u6027\u5206\u5e03\u5f0f\u6570\u636e\u96c6\uff09<\/h4>\n<\/p>\n
RDD\u662fSpark\u7684\u6838\u5fc3\u62bd\u8c61\uff0c\u5b83\u8868\u793a\u4e00\u4e2a\u4e0d\u53ef\u53d8\u7684\u5206\u5e03\u5f0f\u6570\u636e\u96c6\u3002RDD\u53ef\u4ee5\u901a\u8fc7\u5e76\u884c\u64cd\u4f5c\u8fdb\u884c\u5904\u7406\uff0c\u4f8b\u5982map\u3001filter\u548creduce\u3002\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u4ee3\u7801\u521b\u5efaRDD\uff1a<\/p>\n<\/p>\n
rdd = sc.parallelize([1, 2, 3, 4, 5])<\/p>\n
<\/code><\/pre>\n<\/p>\n
3\u3001DataFrame<\/h4>\n<\/p>\n
DataFrame\u662fSpark SQL\u4e2d\u7684\u6838\u5fc3\u6570\u636e\u7ed3\u6784\uff0c\u5b83\u7c7b\u4f3c\u4e8ePandas\u7684DataFrame\u3002DataFrame\u63d0\u4f9b\u4e86\u66f4\u9ad8\u7ea7\u522b\u7684API\uff0c\u53ef\u4ee5\u7528\u4e8e\u590d\u6742\u7684\u6570\u636e\u5206\u6790\u548c\u64cd\u4f5c\u3002\u53ef\u4ee5\u901a\u8fc7\u4ee5\u4e0b\u4ee3\u7801\u521b\u5efaDataFrame\uff1a<\/p>\n<\/p>\n
from pyspark.sql import SparkSession<\/p>\n
spark = SparkSession.builder.appName("example").getOrCreate()<\/p>\n
data = [("Alice", 1), ("Bob", 2), ("Cathy", 3)]<\/p>\n
df = spark.createDataFrame(data, ["name", "age"])<\/p>\n
<\/code><\/pre>\n<\/p>\n
\u56db\u3001\u8fdb\u884c\u6570\u636e\u5904\u7406\u548c\u5206\u6790<\/h3>\n<\/p>\n
1\u3001\u8bfb\u53d6\u6570\u636e<\/h4>\n<\/p>\n
\u53ef\u4ee5\u901a\u8fc7Spark\u8bfb\u53d6\u5404\u79cd\u683c\u5f0f\u7684\u6570\u636e\uff0c\u5305\u62ecCSV\u3001JSON\u3001Parquet\u7b49\u3002\u4ee5\u4e0b\u793a\u4f8b\u5c55\u793a\u4e86\u5982\u4f55\u8bfb\u53d6CSV\u6587\u4ef6\uff1a<\/p>\n<\/p>\n
df = spark.read.csv("path\/to\/file.csv", header=True, inferSchema=True)<\/p>\n
<\/code><\/pre>\n<\/p>\n
2\u3001\u6570\u636e\u8f6c\u6362<\/h4>\n<\/p>\n
\u53ef\u4ee5\u901a\u8fc7DataFrame\u7684API\u8fdb\u884c\u6570\u636e\u8f6c\u6362\uff0c\u4f8b\u5982\u9009\u62e9\u5217\u3001\u8fc7\u6ee4\u884c\u548c\u805a\u5408\u6570\u636e\u3002\u4ee5\u4e0b\u793a\u4f8b\u5c55\u793a\u4e86\u5982\u4f55\u8fdb\u884c\u57fa\u672c\u7684\u6570\u636e\u8f6c\u6362\uff1a<\/p>\n<\/p>\n
# \u9009\u62e9\u5217<\/p>\n
df.select("name", "age").show()<\/p>\n
\u8fc7\u6ee4\u884c<\/strong><\/h2>\n
df.filter(df["age"] > 2).show()<\/p>\n
\u805a\u5408\u6570\u636e<\/strong><\/h2>\n
df.groupBy("age").count().show()<\/p>\n
<\/code><\/pre>\n<\/p>\n
3\u3001\u6570\u636e\u5199\u5165<\/h4>\n<\/p>\n
\u53ef\u4ee5\u5c06\u5904\u7406\u540e\u7684\u6570\u636e\u5199\u5165\u5404\u79cd\u683c\u5f0f\u7684\u6587\u4ef6\u3002\u4ee5\u4e0b\u793a\u4f8b\u5c55\u793a\u4e86\u5982\u4f55\u5c06DataFrame\u5199\u5165Parquet\u6587\u4ef6\uff1a<\/p>\n<\/p>\n
df.write.parquet("path\/to\/output.parquet")<\/p>\n
<\/code><\/pre>\n<\/p>\n
\u4e94\u3001\u793a\u4f8b\u9879\u76ee<\/h3>\n<\/p>\n
\u4e3a\u4e86\u66f4\u597d\u5730\u7406\u89e3\u5982\u4f55\u4f7f\u7528Python\u7684Spark\uff0c\u4e0b\u9762\u901a\u8fc7\u4e00\u4e2a\u793a\u4f8b\u9879\u76ee\u5c55\u793a\u5982\u4f55\u8fdb\u884c\u6570\u636e\u5904\u7406\u548c\u5206\u6790\u3002<\/p>\n<\/p>\n
1\u3001\u9879\u76ee\u80cc\u666f<\/h4>\n<\/p>\n
\u5047\u8bbe\u6211\u4eec\u6709\u4e00\u4e2a\u5305\u542b\u7528\u6237\u4fe1\u606f\u7684CSV\u6587\u4ef6\uff0c\u6587\u4ef6\u5305\u542b\u7528\u6237\u7684\u59d3\u540d\u3001\u5e74\u9f84\u548c\u57ce\u5e02\u3002\u6211\u4eec\u9700\u8981\u5bf9\u8fd9\u4e9b\u6570\u636e\u8fdb\u884c\u5904\u7406\uff0c\u5305\u62ec\u8bfb\u53d6\u6570\u636e\u3001\u8fc7\u6ee4\u6570\u636e\u3001\u805a\u5408\u6570\u636e\u548c\u5199\u5165\u7ed3\u679c\u3002<\/p>\n<\/p>\n
2\u3001\u8bfb\u53d6\u6570\u636e<\/h4>\n<\/p>\n
\u9996\u5148\uff0c\u6211\u4eec\u9700\u8981\u8bfb\u53d6CSV\u6587\u4ef6\uff1a<\/p>\n<\/p>\n
from pyspark.sql import SparkSession<\/p>\n
spark = SparkSession.builder.appName("UserAnalysis").getOrCreate()<\/p>\n
df = spark.read.csv("path\/to\/users.csv", header=True, inferSchema=True)<\/p>\n
<\/code><\/pre>\n<\/p>\n
3\u3001\u6570\u636e\u8f6c\u6362<\/h4>\n<\/p>\n
\u63a5\u4e0b\u6765\uff0c\u6211\u4eec\u9700\u8981\u5bf9\u6570\u636e\u8fdb\u884c\u8f6c\u6362\uff0c\u5305\u62ec\u9009\u62e9\u5217\u3001\u8fc7\u6ee4\u884c\u548c\u805a\u5408\u6570\u636e\uff1a<\/p>\n<\/p>\n
# \u9009\u62e9\u5217<\/p>\n
df = df.select("name", "age", "city")<\/p>\n
\u8fc7\u6ee4\u5e74\u9f84\u5927\u4e8e30\u7684\u7528\u6237<\/strong><\/h2>\n
df = df.filter(df["age"] > 30)<\/p>\n
\u6309\u57ce\u5e02\u805a\u5408\u7528\u6237\u6570\u91cf<\/strong><\/h2>\n
df = df.groupBy("city").count()<\/p>\n
<\/code><\/pre>\n<\/p>\n
4\u3001\u5199\u5165\u7ed3\u679c<\/h4>\n<\/p>\n
\u6700\u540e\uff0c\u6211\u4eec\u5c06\u5904\u7406\u540e\u7684\u6570\u636e\u5199\u5165Parquet\u6587\u4ef6\uff1a<\/p>\n<\/p>\n
df.write.parquet("path\/to\/output.parquet")<\/p>\n
<\/code><\/pre>\n<\/p>\n
\u516d\u3001\u603b\u7ed3<\/h3>\n<\/p>\n
\u901a\u8fc7\u4e0a\u8ff0\u6b65\u9aa4\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528Python\u4e2d\u7684Spark\u8fdb\u884c\u6570\u636e\u5904\u7406\u548c\u5206\u6790\u3002\u9996\u5148\u9700\u8981\u5b89\u88c5Spark\u548cPySpark\u5e93\uff0c\u7136\u540e\u7406\u89e3Spark\u7684\u57fa\u672c\u6982\u5ff5\uff0c\u5305\u62ecSparkContext\u3001RDD\u548cDataFrame\u3002\u63a5\u7740\uff0c\u901a\u8fc7\u5b9e\u9645\u793a\u4f8b\u5c55\u793a\u4e86\u5982\u4f55\u8fdb\u884c\u6570\u636e\u8bfb\u53d6\u3001\u8f6c\u6362\u548c\u5199\u5165\u64cd\u4f5c\u3002Spark\u5f3a\u5927\u7684\u5e76\u884c\u5904\u7406\u80fd\u529b\u548c\u9ad8\u6548\u7684\u6570\u636e\u5904\u7406API\u4f7f\u5f97\u5b83\u5728\u5927\u6570\u636e\u5904\u7406\u548c\u5206\u6790\u4e2d\u5177\u6709\u5e7f\u6cdb\u7684\u5e94\u7528\u524d\u666f\u3002<\/p>\n<\/p>\n
\u76f8\u5173\u95ee\u7b54FAQs\uff1a<\/strong><\/h2>\n
 \u5982\u4f55\u5f00\u59cb\u4f7f\u7528Python\u7684Spark\u8fdb\u884c\u6570\u636e\u5904\u7406\uff1f<\/strong>
\u8981\u5f00\u59cb\u4f7f\u7528Python\u7684Spark\u8fdb\u884c\u6570\u636e\u5904\u7406\uff0c\u9996\u5148\u9700\u8981\u5b89\u88c5Apache Spark\u548cPySpark\u3002\u53ef\u4ee5\u901a\u8fc7pip\u547d\u4ee4\u5b89\u88c5PySpark\u3002\u5b89\u88c5\u5b8c\u6210\u540e\uff0c\u60a8\u53ef\u4ee5\u4f7f\u7528SparkSession\u6765\u521b\u5efa\u4e00\u4e2aSpark\u5e94\u7528\u7a0b\u5e8f\uff0c\u901a\u8fc7\u8be5\u4f1a\u8bdd\u53ef\u4ee5\u52a0\u8f7d\u6570\u636e\u3001\u6267\u884c\u8f6c\u6362\u548c\u64cd\u4f5c\u3002\u5728\u5b9e\u9645\u64cd\u4f5c\u4e2d\uff0c\u901a\u5e38\u4f1a\u4f7f\u7528RDD\uff08\u5f39\u6027\u5206\u5e03\u5f0f\u6570\u636e\u96c6\uff09\u6216DataFrame\u6765\u8fdb\u884c\u6570\u636e\u5904\u7406\u3002<\/p>\n
Python\u7684Spark\u5728\u5927\u6570\u636e\u5206\u6790\u4e2d\u7684\u5e94\u7528\u6709\u54ea\u4e9b\uff1f<\/strong>
Python\u7684Spark\u5e7f\u6cdb\u5e94\u7528\u4e8e\u5927\u6570\u636e\u5206\u6790\u7684\u591a\u4e2a\u9886\u57df\uff0c\u5305\u62ec\u6570\u636e\u6e05\u6d17\u3001\u6570\u636e\u8f6c\u6362\u548c\u673a\u5668\u5b66\u4e60<\/a>\u3002\u501f\u52a9Spark\u7684\u5206\u5e03\u5f0f\u8ba1\u7b97\u80fd\u529b\uff0c\u7528\u6237\u80fd\u591f\u9ad8\u6548\u5730\u5904\u7406\u6d77\u91cf\u6570\u636e\u3002\u6b64\u5916\uff0cSpark\u63d0\u4f9b\u4e86\u591a\u79cd\u5185\u7f6e\u5e93\uff0c\u5982Spark SQL\u7528\u4e8e\u7ed3\u6784\u5316\u6570\u636e\u5904\u7406\uff0cMLlib\u7528\u4e8e\u673a\u5668\u5b66\u4e60\uff0cGraphX\u7528\u4e8e\u56fe\u5f62\u5904\u7406\u7b49\uff0c\u4f7f\u5f97\u6570\u636e\u5206\u6790\u8fc7\u7a0b\u66f4\u52a0\u7075\u6d3b\u548c\u9ad8\u6548\u3002<\/p>\n
\u5728\u4f7f\u7528Python\u7684Spark\u65f6\uff0c\u5982\u4f55\u4f18\u5316\u6027\u80fd\uff1f<\/strong>
\u5728\u4f7f\u7528Python\u7684Spark\u65f6\uff0c\u53ef\u4ee5\u901a\u8fc7\u591a\u79cd\u65b9\u6cd5\u6765\u4f18\u5316\u6027\u80fd\u3002\u4f8b\u5982\uff0c\u5408\u7406\u4f7f\u7528\u6301\u4e45\u5316\u673a\u5236\uff08\u5982cache\u548cpersist\uff09\u53ef\u4ee5\u51cf\u5c11\u91cd\u590d\u8ba1\u7b97\u7684\u5f00\u9500\uff1b\u4f18\u5316\u6570\u636e\u5206\u533a\uff0c\u901a\u8fc7\u9002\u5f53\u7684\u5206\u533a\u6570\u6765\u5e73\u8861\u8d1f\u8f7d\uff1b\u4f7f\u7528\u5e7f\u64ad\u53d8\u91cf\u6765\u51cf\u5c11\u6570\u636e\u4f20\u8f93\u7684\u5f00\u9500\uff1b\u4ee5\u53ca\u907f\u514d\u5728Driver\u7a0b\u5e8f\u4e2d\u6267\u884c\u5927\u6570\u636e\u96c6\u7684\u64cd\u4f5c\uff0c\u5c3d\u91cf\u5c06\u8ba1\u7b97\u4efb\u52a1\u5206\u6563\u5230Worker\u8282\u70b9\u4e0a\uff0c\u63d0\u5347\u6574\u4f53\u8fd0\u884c\u6548\u7387\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"\u8981\u4f7f\u7528Python\u4e2d\u7684Spark\uff0c\u9996\u5148\u9700\u8981\u5b89\u88c5Spark\u548c\u76f8\u5173\u7684Python\u5e93\uff0c\u5176\u6b21\u9700\u8981\u7406\u89e3Spark\u7684\u57fa\u672c\u6982 […]","protected":false},"author":3,"featured_media":1158728,"comment_status":"closed","ping_status":"","sticky":false,"template":"","format":"standard","meta":{"_acf_changed":false,"footnotes":""},"categories":[37],"tags":[],"acf":[],"_links":{"self":[{"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/posts\/1158724"}],"collection":[{"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/users\/3"}],"replies":[{"embeddable":true,"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/comments?post=1158724"}],"version-history":[{"count":"1","href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/posts\/1158724\/revisions"}],"predecessor-version":[{"id":1158731,"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/posts\/1158724\/revisions\/1158731"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/media\/1158728"}],"wp:attachment":[{"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/media?parent=1158724"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/categories?post=1158724"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/docs.pingcode.com\/wp-json\/wp\/v2\/tags?post=1158724"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}