这是我的docker compose文件:
version: "3.4"
services:
spark-master:
image: bitnami/spark:latest
container_name: master
ports:
- "4040:4040"
- "7077:7077"
networks:
- spark-network
environment:
- SPARK_MODE=master
volumes:
- jupyter-notebooks:/jupyter-notebooks
spark-worker-1:
image: bitnami/spark:latest
container_name: worker-1
ports:
- "8081:8081"
networks:
- spark-network
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark-master:7077
volumes:
- jupyter-notebooks:/jupyter-notebooks
depends_on:
- spark-master
jupyter:
image: jupyter/pyspark-notebook:latest
container_name: jupyter
ports:
- "8888:8888"
volumes:
- jupyter-notebooks:/home/jovyan/work
- ./JDBC_Driver:/Drivers/SQL_Sever/jdbc
networks:
- spark-network
depends_on:
- spark-worker-1
- hive
- postgres
sql-server:
image: mcr.microsoft.com/mssql/server:2022-latest
container_name: sql-server-container
environment:
MSSQL_SA_PASSWORD: Mo*012105
ACCEPT_EULA: Y
ports:
- "1433:1433"
volumes:
- sql-server-data:/var/opt/mssql
networks:
- spark-network
postgres:
image: postgres:latest
container_name: postgres
environment:
POSTGRES_DB: hive
POSTGRES_USER: hiveuser
POSTGRES_PASSWORD: hivepassword
ports:
- "5432:5432"
volumes:
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
networks:
postgres_net:
hive:
image: apache/hive:3.1.3
container_name: hive
environment:
- HIVE_METASTORE_URI=thrift://postgres:9083
- HADOOP_HOME=/opt/hadoop
ports:
- "10000:10000" # Hive server2 port
- "9083:9083" # Hive Metastore port
depends_on:
- postgres
volumes:
- ./emails.csv:/data/emails.csv
networks:
spark-network:
postgres_net:
volumes:
jupyter-notebooks:
sql-server-data:
postgres_data:
networks:
spark-network:
postgres_net:
字符串
这是我从hive服务日志中得到的错误:
Initialization script completed
2023-09-11 22:02:34 schemaTool completed
2023-09-11 22:02:34 [WARN] Failed to create directory: /home/hive/.beeline
2023-09-11 22:02:34 No such file or directory
2023-09-11 22:02:34 + '[' 0 -eq 0 ']'
2023-09-11 22:02:34 + echo 'Initialized schema successfully..'
2023-09-11 22:02:34 Initialized schema successfully..
2023-09-11 22:02:34 + '[' '' == hiveserver2 ']'
2023-09-11 22:02:34 + '[' '' == metastore ']'
2023-09-11 22:02:34 + exec /opt/hive/bin/hive --skiphadoopversion --skiphbasecp --service
2023-09-11 22:02:37 SLF4J: Class path contains multiple SLF4J bindings.
2023-09-11 22:02:37 SLF4J: Found binding in [jar:file:/opt/hive/lib/log4j-slf4j-impl-2.17.1.jar!/org/slf4j/impl/StaticLoggerBinder.
型
1条答案
按热度按时间avkwfej41#
你可以尝试this repo,元存储区正在连接到PostgreSQL数据库运行