构建东莞地区数据中台的技术实践

// 安装Hadoop环境

sudo apt-get install default-jdk

wget https://downloads.apache.org/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz

tar -xzf hadoop-3.3.1.tar.gz

]]>

# 配置hadoop-env.sh

export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64

# 配置core-site.xml

fs.defaultFS

hdfs://localhost:9000

# 配置hdfs-site.xml

dfs.replication

1

]]>

// 使用Spark读取HDFS上的数据

val data = sc.textFile("hdfs://localhost:9000/data")

val counts = data.flatMap(line => line.split(" "))

.map(word => (word, 1))

.reduceByKey(_ + _)

counts.collect().foreach(println)

]]>

// 安装Zookeeper

sudo apt-get install zookeeperd

]]>