![]() context import Context 51 52 53 class EmrAddStepsOperator (BaseOperator ): 54 """ 55 An operator that adds steps to an existing EMR job_flow. types import NOTSET, ArgNotSet 48 49 if TYPE_CHECKING : 50 from airflow. helpers import exactly_one, prune_dict 47 from airflow. waiter_with_logging import wait 46 from airflow. emr import ( 33 EmrAddStepsTrigger, 34 EmrContainerTrigger, 35 EmrCreateJobFlowTrigger, 36 EmrServerlessCancelJobsTrigger, 37 EmrServerlessCreateApplicationTrigger, 38 EmrServerlessDeleteApplicationTrigger, 39 EmrServerlessStartApplicationTrigger, 40 EmrServerlessStartJobTrigger, 41 EmrServerlessStopApplicationTrigger, 42 EmrTerminateJobFlowTrigger, 43 ) 44 from airflow. emr import EmrClusterLink, EmrLogsLink, get_log_uri 32 from airflow. emr import EmrContainerHook, EmrHook, EmrServerlessHook 31 from airflow. models import BaseOperator 30 from airflow. exceptions import AirflowException, AirflowProviderDeprecationWarning 29 from airflow. configuration import conf 28 from airflow. 18 from _future_ import annotations 19 20 import ast 21 import warnings 22 from datetime import timedelta 23 from functools import cached_property 24 from typing import TYPE_CHECKING, Any, Sequence 25 from uuid import uuid4 26 27 from airflow. See the License for the 16 # specific language governing permissions and limitations 17 # under the License. ![]() You may obtain a copy of the License at 9 # 10 # 11 # 12 # Unless required by applicable law or agreed to in writing, 13 # software distributed under the License is distributed on an 14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 # KIND, either express or implied. The ASF licenses this file 6 # to you under the Apache License, Version 2.0 (the 7 # "License") you may not use this file except in compliance 8 # with the License. See the NOTICE file 4 # distributed with this work for additional information 5 # regarding copyright ownership. If the above usecase is possible, please let us know how can I add the binaries in MWAA.As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers.Īlternatively you can here view or download the uninterpreted source code file.įor more information about "emr.py" see the Fossies "Dox" file reference documentation and the last Fossies "Diffs" side-by-side code changes report: 2.6.3_vs_2.7.0.ġ # 2 # Licensed to the Apache Software Foundation (ASF) under one 3 # or more contributor license agreements. Can I Achieve the above use case with AWS MWAA? Till now I have explored everything and I am unable to find a way to add the binaries on MWAA. ![]() The same code works fine, whenever queried from Airflow to EMR using HiveOperator.ĭoes MWAA support integrations to AWS Services only? Like EMR ClusterLaunch, EMR AddStep, AthenaOperators etc. Talking about my on premises airflow setup on EKS, We added all the binaries (hive, hadoop, hdfs) by keeping the base image - Apache Airflow. Similar to this, If I want to use HDFSSensor, it will require hadoop/hdfs client. ![]() Raise child_exception_type(errno_num, err_msg, err_filename) FileNotFoundError: No such file or directory: 'beeline': 'beeline' _cli(hql=self.hql, schema=self.schema, hive_conf=self.hiveconfs)įile "/usr/local/lib/python3.7/site-packages/airflow/hooks/hive_hooks.py", line 258, in run_cliįile "/usr/lib64/python3.7/subprocess.py", line 800, in initįile "/usr/lib64/python3.7/subprocess.py", line 1551, in _execute_child Result = task_copy.execute(context=context)įile "/usr/local/lib/python3.7/site-packages/airflow/operators/hive_operator.py", line 137, in execute ERROR - No such file or directory: 'beeline': 'beeline'įile "/usr/local/lib/python3.7/site-packages/airflow/models/taskinstance.py", line 984, in _run_raw_task Hql='CREATE TABLE XXXX.XXXX STORED AS ORC AS SELECT DISTINCT * from XXXX.XXXX limit 2' Hive_cli_conn_id='hive_emr_dag_connection', Sample Code ( The below code works fine with our airflow installed on EKS) Please see the below dag code and stack trace. But due to missing binaries on MWAA, it says no such beeline directory. Rather than adding a step on EMR, I want to hit the Hive Query directly using HiveOperator from AWS MWAA. I am using HiveOperator to execute a query on EMR cluster using beeline client from AWS MWAA.
0 Comments
Leave a Reply. |
AuthorWrite something about yourself. No need to be fancy, just an overview. ArchivesCategories |