dldbwls1123

<pre><code class="language-dockercompose"> airflow-dag-processor:
 <<: *airflow-common
 command: dag-processor
 healthcheck:
 test: ["CMD-SHELL", 'airflow jobs check --job-type DagProcessorJob --hostname "$${HOSTNAME}"']
 interval: 30s
 timeout: 10s
 retries: 5
 start_period: 30s
 restart: always
 depends_on:
 <<: *airflow-common-depends-on
 airflow-init:
 condition: service_completed_successfully

 airflow-worker:
 <<: *airflow-common
 command: celery worker
 healthcheck:
 # yamllint disable rule:line-length
 test:
 - "CMD-SHELL"
 - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
 interval: 30s
 timeout: 10s
 retries: 5
 start_period: 30s
 environment:
 <<: *airflow-common-env
 # Required to handle warm shutdown of the celery workers properly
 # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
 DUMB_INIT_SETSID: "0"
 restart: always
 depends_on:
 <<: *airflow-common-depends-on
 airflow-apiserver:
 condition: service_healthy
 airflow-init:
 condition: service_completed_successfully
 networks:
 network_custom:
 ipv4_address: 172.28.0.8

 airflow-triggerer:
 <<: *airflow-common
 command: triggerer
 healthcheck:
 test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
 interval: 30s
 timeout: 10s
 retries: 5
 start_period: 30s
 restart: always
 depends_on:
 <<: *airflow-common-depends-on
 airflow-init:
 condition: service_completed_successfully
 networks:
 network_custom:
 ipv4_address: 172.28.0.9

 airflow-init:
 <<: *airflow-common
 entrypoint: /bin/bash
 # yamllint disable rule:line-length
 command:
 - -c
 - |
 if [[ -z "${AIRFLOW_UID}" ]]; then
 echo
 echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
 echo "If you are on Linux, you SHOULD follow the instructions below to set "
 echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
 echo "For other operating systems you can get rid of the warning with manually created .env file:"
 echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user"
 echo
 export AIRFLOW_UID=$(id -u)
 fi
 one_meg=1048576
 mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
 cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
 disk_available=$$(df / | tail -1 | awk '{print $$4}')
 warning_resources="false"
 if (( mem_available < 4000 )) ; then
 echo
 echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
 echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
 echo
 warning_resources="true"
 fi
 if (( cpus_available < 2 )); then
 echo
 echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
 echo "At least 2 CPUs recommended. You have $${cpus_available}"
 echo
 warning_resources="true"
 fi
 if (( disk_available < one_meg * 10 )); then
 echo
 echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
 echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
 echo
 warning_resources="true"
 fi
 if [[ $${warning_resources} == "true" ]]; then
 echo
 echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
 echo "Please follow the instructions to increase amount of resources available:"
 echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin"
 echo
 fi
 echo
 echo "Creating missing opt dirs if missing:"
 echo
 mkdir -v -p /opt/airflow/{logs,dags,plugins,config}
 echo
 echo "Airflow version:"
 /entrypoint airflow version
 echo
 echo "Files in shared volumes:"
 echo
 ls -la /opt/airflow/{logs,dags,plugins,config}
 echo
 echo "Running airflow config list to create default config file if missing."
 echo
 /entrypoint airflow config list >/dev/null
 echo
 echo "Files in shared volumes:"
 echo
 ls -la /opt/airflow/{logs,dags,plugins,config}
 echo
 echo "Change ownership of files in /opt/airflow to ${AIRFLOW_UID}:0"
 echo
 chown -R "${AIRFLOW_UID}:0" /opt/airflow/
 echo
 echo "Change ownership of files in shared volumes to ${AIRFLOW_UID}:0"
 echo
 chown -v -R "${AIRFLOW_UID}:0" /opt/airflow/{logs,dags,plugins,config}
 echo
 echo "Files in shared volumes:"
 echo
 ls -la /opt/airflow/{logs,dags,plugins,config}

 # yamllint enable rule:line-length
 environment:
 <<: *airflow-common-env
 _AIRFLOW_DB_MIGRATE: 'true'
 _AIRFLOW_WWW_USER_CREATE: 'true'
 _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
 _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
 _PIP_ADDITIONAL_REQUIREMENTS: ''
 user: "0:0"
 networks:
 network_custom:
 ipv4_address: 172.28.0.10 

 airflow-cli:
 <<: *airflow-common
 profiles:
 - debug
 environment:
 <<: *airflow-common-env
 CONNECTION_CHECK_MAX_COUNT: "0"
 # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
 command:
 - bash
 - -c
 - airflow
 depends_on:
 <<: *airflow-common-depends-on

 # You can enable flower by adding "--profile flower" option e.g. docker-compose --profile flower up
 # or by explicitly targeted on the command line e.g. docker-compose up flower.
 # See: https://docs.docker.com/compose/profiles/
 flower:
 <<: *airflow-common
 command: celery flower
 profiles:
 - flower
 ports:
 - "5555:5555"
 healthcheck:
 test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
 interval: 30s
 timeout: 10s
 retries: 5
 start_period: 30s
 restart: always
 depends_on:
 <<: *airflow-common-depends-on
 airflow-init:
 condition: service_completed_successfully

volumes:
 postgres-db-volume:
 postgres-custom-db-volume:

networks:
 network_custom:
 driver: bridge
 ipam:
 driver: default
 config:
 - subnet: 172.28.0.0/16
 gateway: 172.28.0.1</code></pre>글자 수 제한으로 나눠서 올립니다

더이상 airflow web에 dag 파일이 안 올라갑니다ㅜㅜ

hue 클러스터 삭제 후 다시 설치하고 해봐도 같은 에러가 뜹니다 결과물도 안 나오고 INFO : Kill Command = /opt/cloudera/parcels/CDH-6.3.2-1.cdh6.3.2.p0.1605554/lib/hadoop/bin/hadoop job -kill job_1751019956646_0003
INFO : Hadoop job information for Stage-1: number of mappers: 1; number of reducers: 4
INFO : 2025-06-27 20:22:46,041 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:23:46,394 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:24:46,543 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:25:46,701 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:26:46,778 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:27:47,101 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:28:47,116 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:29:47,215 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:30:47,413 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:31:48,417 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:32:49,362 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:33:50,343 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:34:51,288 Stage-1 map = 0%, reduce = 0%
INFO : 2025-06-27 20:35:52,241 Stage-1 map = 0%, reduce = 0% 이 상태만 지속됩니다ㅜ

dldbwls1123

Posts