Airtunnel
1.0.1
Tutorial: loading the university data model
Configuration and Setup
Design Principles
Physical data store setup
Architecture & classes
Extending Airtunnel
Known Limitations
Development & testing
Data Asset Declaration Files
API reference
Airtunnel
Docs
»
Index
Index
A
|
B
|
C
|
D
|
E
|
F
|
G
|
I
|
K
|
L
|
M
|
N
|
O
|
P
|
R
|
S
|
T
|
U
|
W
A
airtunnel (module)
airtunnel.data_asset (module)
airtunnel.data_store (module)
airtunnel.declaration_store (module)
airtunnel.metadata (module)
airtunnel.metadata.adapter (module)
airtunnel.metadata.entities (module)
airtunnel.operators (module)
airtunnel.operators.archival (module)
airtunnel.operators.ingestion (module)
airtunnel.operators.loading (module)
airtunnel.operators.sql (module)
airtunnel.operators.sql.sql_helpers (module)
airtunnel.operators.sql.sqloperator (module)
airtunnel.operators.transformation (module)
airtunnel.paths (module)
airtunnel.sensors (module)
airtunnel.sensors.ingestion (module)
airtunnel.sensors.metadata (module)
all() (airtunnel.declaration_store.DataAssetDeclaration property)
archival (airtunnel.operators.Colours attribute)
archive_ingest() (airtunnel.declaration_store.DataAssetDeclaration property)
archive_ready() (airtunnel.declaration_store.DataAssetDeclaration property)
asset_name() (airtunnel.declaration_store.DataAssetDeclaration property)
AwaitAssetAncestorsUpdatedSensor() (in module airtunnel.sensors.metadata)
AwaitLoadStatusSensor() (in module airtunnel.sensors.metadata)
B
BaseDataAsset (class in airtunnel)
(class in airtunnel.data_asset)
BaseDataAssetIO (class in airtunnel)
(class in airtunnel.data_asset)
BaseDataStoreAdapter (class in airtunnel.data_store)
BaseHookFactory (class in airtunnel.metadata.adapter)
BaseMetaAdapter (class in airtunnel.metadata.adapter)
C
Colours (class in airtunnel.operators)
copy() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
D
dag_exec_date() (airtunnel.metadata.entities.IngestedFileMetadata property)
(airtunnel.metadata.entities.Lineage property)
(airtunnel.metadata.entities.LoadStatus property)
dag_id() (airtunnel.metadata.entities.IngestedFileMetadata property)
(airtunnel.metadata.entities.Lineage property)
(airtunnel.metadata.entities.LoadStatus property)
data_sources() (airtunnel.metadata.entities.Lineage property)
data_target() (airtunnel.metadata.entities.Lineage property)
DataAssetArchiveOperator() (in module airtunnel.operators.archival)
DataAssetDeclaration (class in airtunnel.declaration_store)
declarations (airtunnel.BaseDataAsset attribute)
(airtunnel.data_asset.BaseDataAsset attribute)
(airtunnel.data_asset.PandasDataAsset attribute)
(airtunnel.data_asset.PySparkDataAsset attribute)
(airtunnel.data_asset.ShellDataAsset attribute)
(airtunnel.data_asset.SQLDataAsset attribute)
(airtunnel.PandasDataAsset attribute)
(airtunnel.PySparkDataAsset attribute)
(airtunnel.SQLDataAsset attribute)
DeclarationSchemas (class in airtunnel.declaration_store)
DefaultSQLHookFactory (class in airtunnel.metadata.adapter)
delete() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
DERIVED_DATA_ASSET (airtunnel.declaration_store.DeclarationSchemas attribute)
discovered_files_xcom_key() (airtunnel.BaseDataAsset property)
(airtunnel.data_asset.BaseDataAsset property)
E
execute() (airtunnel.operators.sql.sqloperator.SQLOperator method)
(airtunnel.operators.transformation.PandasTransformationOperator method)
(airtunnel.operators.transformation.PySparkTransformationOperator method)
(airtunnel.operators.transformation.SQLTransformationOperator method)
execute_script() (in module airtunnel.operators.sql.sql_helpers)
execute_statement() (in module airtunnel.operators.sql.sql_helpers)
exists() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
extra_declarations() (airtunnel.declaration_store.DataAssetDeclaration property)
F
fetch_all_declarations() (in module airtunnel.declaration_store)
file_create_time() (airtunnel.metadata.entities.IngestedFileMetadata property)
file_mod_time() (airtunnel.metadata.entities.IngestedFileMetadata property)
filepath() (airtunnel.metadata.entities.IngestedFileMetadata property)
filesize() (airtunnel.metadata.entities.IngestedFileMetadata property)
FN_DAG_ID (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_DATA_ASSET (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_DATA_ASSET_SRC (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_DATA_ASSET_TRG (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_EXEC_DATE (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_FILE_CREATE_TIME (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_FILE_MOD_TIME (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_FILE_PATH (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_FILE_SIZE (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_LOAD_TIME (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_METADATA_JSON (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_METADATA_TIME (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_METADATA_TYPE (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
FN_TASK_ID (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
for_asset() (airtunnel.metadata.entities.IngestedFileMetadata property)
format_sql_script() (in module airtunnel.operators.sql.sql_helpers)
formatted_sql_script() (airtunnel.data_asset.SQLDataAsset method)
(airtunnel.SQLDataAsset method)
from_names() (airtunnel.data_asset.ShellDataAsset static method)
G
get_configured_data_store_adapter() (in module airtunnel.data_store)
get_configured_meta_adapter() (in module airtunnel.metadata.adapter)
get_configured_meta_adapter_hook() (in module airtunnel.metadata.adapter)
get_raw_sql_script() (airtunnel.data_asset.SQLDataAsset method)
(airtunnel.SQLDataAsset method)
glob() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
I
in_storage_format() (airtunnel.declaration_store.DataAssetDeclaration property)
ingest_archive_path() (airtunnel.BaseDataAsset property)
(airtunnel.data_asset.BaseDataAsset property)
ingest_file_glob() (airtunnel.declaration_store.DataAssetDeclaration property)
IngestArchiveOperator() (in module airtunnel.operators.archival)
INGESTED_DATA_ASSET (airtunnel.declaration_store.DeclarationSchemas attribute)
IngestedFileMetadata (class in airtunnel.metadata.entities)
ingestion (airtunnel.operators.Colours attribute)
IngestOperator() (in module airtunnel.operators.ingestion)
inspect() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
is_csv_input() (airtunnel.declaration_store.DataAssetDeclaration property)
is_csv_output() (airtunnel.declaration_store.DataAssetDeclaration property)
is_parquet_input() (airtunnel.declaration_store.DataAssetDeclaration property)
is_parquet_output() (airtunnel.declaration_store.DataAssetDeclaration property)
is_within() (airtunnel.metadata.entities.LoadStatus method)
is_xls_input() (airtunnel.declaration_store.DataAssetDeclaration property)
K
key_columns() (airtunnel.declaration_store.DataAssetDeclaration property)
L
landing_path() (airtunnel.BaseDataAsset property)
(airtunnel.data_asset.BaseDataAsset property)
Lineage (class in airtunnel.metadata.entities)
lineage_from_sql_script() (airtunnel.metadata.entities.Lineage static method)
lineage_from_sql_statement() (airtunnel.metadata.entities.Lineage static method)
listdir() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
LOAD_SCHEMA (airtunnel.declaration_store.DeclarationSchemas attribute)
load_sql_script() (in module airtunnel.operators.sql.sql_helpers)
load_time() (airtunnel.metadata.entities.LoadStatus property)
loading (airtunnel.operators.Colours attribute)
LoadStatus (class in airtunnel.metadata.entities)
LocalDataStoreAdapter (class in airtunnel.data_store)
M
make_hook() (airtunnel.metadata.adapter.BaseHookFactory static method)
(airtunnel.metadata.adapter.DefaultSQLHookFactory static method)
make_ready_temp_path() (airtunnel.BaseDataAsset method)
(airtunnel.data_asset.BaseDataAsset method)
makedirs() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
modification_times() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
move() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
N
name (airtunnel.BaseDataAsset attribute)
(airtunnel.data_asset.BaseDataAsset attribute)
(airtunnel.data_asset.PandasDataAsset attribute)
(airtunnel.data_asset.PySparkDataAsset attribute)
(airtunnel.data_asset.ShellDataAsset attribute)
(airtunnel.data_asset.SQLDataAsset attribute)
(airtunnel.PandasDataAsset attribute)
(airtunnel.PySparkDataAsset attribute)
(airtunnel.SQLDataAsset attribute)
O
open() (airtunnel.data_store.BaseDataStoreAdapter static method)
(airtunnel.data_store.LocalDataStoreAdapter static method)
out_comp_codec() (airtunnel.declaration_store.DataAssetDeclaration property)
out_storage_format() (airtunnel.declaration_store.DataAssetDeclaration property)
output_filename() (airtunnel.BaseDataAsset property)
(airtunnel.data_asset.BaseDataAsset property)
P
PandasDataAsset (class in airtunnel)
(class in airtunnel.data_asset)
PandasDataAssetIO (class in airtunnel)
(class in airtunnel.data_asset)
PandasTransformationOperator (class in airtunnel.operators.transformation)
pickedup_files() (airtunnel.BaseDataAsset method)
(airtunnel.data_asset.BaseDataAsset method)
prepare_sql_params() (in module airtunnel.operators.sql.sql_helpers)
prepare_staging_ready_path() (airtunnel.BaseDataAssetIO static method)
(airtunnel.data_asset.BaseDataAssetIO static method)
PySparkDataAsset (class in airtunnel)
(class in airtunnel.data_asset)
PySparkDataAssetIO (class in airtunnel)
(class in airtunnel.data_asset)
PySparkTransformationOperator (class in airtunnel.operators.transformation)
R
read_data_asset() (airtunnel.BaseDataAssetIO static method)
(airtunnel.data_asset.BaseDataAssetIO static method)
(airtunnel.data_asset.PandasDataAssetIO static method)
(airtunnel.data_asset.PySparkDataAssetIO static method)
(airtunnel.PandasDataAssetIO static method)
(airtunnel.PySparkDataAssetIO static method)
read_inspected_files() (airtunnel.metadata.adapter.BaseMetaAdapter method)
(airtunnel.metadata.adapter.SQLMetaAdapter method)
read_lineage() (airtunnel.metadata.adapter.BaseMetaAdapter method)
(airtunnel.metadata.adapter.SQLMetaAdapter method)
read_load_status() (airtunnel.metadata.adapter.BaseMetaAdapter method)
(airtunnel.metadata.adapter.SQLMetaAdapter method)
ready_archive_path() (airtunnel.BaseDataAsset method)
(airtunnel.data_asset.BaseDataAsset method)
ready_path() (airtunnel.BaseDataAsset property)
(airtunnel.data_asset.BaseDataAsset property)
rebuild_for_store() (airtunnel.BaseDataAsset method)
(airtunnel.data_asset.BaseDataAsset method)
(airtunnel.data_asset.PandasDataAsset method)
(airtunnel.data_asset.PySparkDataAsset method)
(airtunnel.data_asset.ShellDataAsset method)
(airtunnel.data_asset.SQLDataAsset method)
(airtunnel.PandasDataAsset method)
(airtunnel.PySparkDataAsset method)
(airtunnel.SQLDataAsset method)
rename_fields_as_declared() (airtunnel.data_asset.PandasDataAsset method)
(airtunnel.data_asset.PySparkDataAsset method)
(airtunnel.PandasDataAsset method)
(airtunnel.PySparkDataAsset method)
retrieve_data_asset() (airtunnel.BaseDataAssetIO static method)
(airtunnel.data_asset.BaseDataAssetIO static method)
(airtunnel.data_asset.PandasDataAssetIO static method)
(airtunnel.data_asset.PySparkDataAssetIO static method)
(airtunnel.PandasDataAssetIO static method)
(airtunnel.PySparkDataAssetIO static method)
retrieve_from_store() (airtunnel.BaseDataAsset method)
(airtunnel.data_asset.BaseDataAsset method)
(airtunnel.data_asset.PandasDataAsset method)
(airtunnel.data_asset.PySparkDataAsset method)
(airtunnel.data_asset.ShellDataAsset method)
(airtunnel.data_asset.SQLDataAsset method)
(airtunnel.PandasDataAsset method)
(airtunnel.PySparkDataAsset method)
(airtunnel.SQLDataAsset method)
run_ddl() (airtunnel.declaration_store.DataAssetDeclaration property)
S
ShellDataAsset (class in airtunnel.data_asset)
SourceFileIsReadySensor() (in module airtunnel.sensors.ingestion)
split_sql_script() (in module airtunnel.operators.sql.sql_helpers)
SQLDataAsset (class in airtunnel)
(class in airtunnel.data_asset)
SQLMetaAdapter (class in airtunnel.metadata.adapter)
SQLOperator (class in airtunnel.operators.sql.sqloperator)
SQLTransformationOperator (class in airtunnel.operators.transformation)
STAGING_ASSETS (airtunnel.declaration_store.DeclarationSchemas attribute)
staging_assets() (airtunnel.declaration_store.DataAssetDeclaration property)
staging_pickedup_path() (airtunnel.BaseDataAsset method)
(airtunnel.data_asset.BaseDataAsset method)
staging_ready_path() (airtunnel.BaseDataAsset property)
(airtunnel.data_asset.BaseDataAsset property)
StagingToReadyOperator() (in module airtunnel.operators.loading)
T
TABLES (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
task_id() (airtunnel.metadata.entities.IngestedFileMetadata property)
(airtunnel.metadata.entities.Lineage property)
(airtunnel.metadata.entities.LoadStatus property)
TN_GENERIC_METADATA (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
TN_INFILE_METADATA (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
TN_LINEAGE (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
TN_LOAD_STATUS (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
TN_LOAD_STATUS_HIST (airtunnel.metadata.adapter.SQLMetaAdapter attribute)
to_full_data_asset() (airtunnel.data_asset.ShellDataAsset method)
transform_date_formats() (airtunnel.declaration_store.DataAssetDeclaration property)
transform_renames() (airtunnel.declaration_store.DataAssetDeclaration property)
transformation (airtunnel.operators.Colours attribute)
U
ui_color (airtunnel.operators.transformation.PandasTransformationOperator attribute)
(airtunnel.operators.transformation.PySparkTransformationOperator attribute)
(airtunnel.operators.transformation.SQLTransformationOperator attribute)
W
write_data_asset() (airtunnel.BaseDataAssetIO static method)
(airtunnel.data_asset.BaseDataAssetIO static method)
(airtunnel.data_asset.PandasDataAssetIO static method)
(airtunnel.data_asset.PySparkDataAssetIO static method)
(airtunnel.PandasDataAssetIO static method)
(airtunnel.PySparkDataAssetIO static method)
write_generic_metadata() (airtunnel.metadata.adapter.BaseMetaAdapter method)
(airtunnel.metadata.adapter.SQLMetaAdapter method)
write_inspected_files() (airtunnel.metadata.adapter.BaseMetaAdapter method)
(airtunnel.metadata.adapter.SQLMetaAdapter method)
write_lineage() (airtunnel.metadata.adapter.BaseMetaAdapter method)
(airtunnel.metadata.adapter.SQLMetaAdapter method)
write_load_status() (airtunnel.metadata.adapter.BaseMetaAdapter method)
(airtunnel.metadata.adapter.SQLMetaAdapter method)