|
new_cluster: &new_cluster
|
|
new_cluster:
|
|
num_workers: 3
|
|
spark_version: 13.3.x-cpu-ml-scala2.12
|
|
node_type_id: Standard_D3_v2
|
|
custom_tags:
|
|
clusterSource: mlops-stacks_0.4
|
|
|
|
resources:
|
|
jobs:
|
|
model_training_job:
|
|
name: ${bundle.target}-training-job
|
|
job_clusters:
|
|
- job_cluster_key: model_training_job_cluster
|
|
<<: *new_cluster
|
|
tasks:
|
|
- task_key: Train
|
|
job_cluster_key: model_training_job_cluster
|
|
notebook_task:
|
|
notebook_path: ../training/notebooks/TrainWithFeatureStore.py
|
|
base_parameters:
|
|
env: ${bundle.target}
|
|
|
|
training_data_path: /databricks-datasets/nyctaxi-with-zipcodes/subsampled
|
|
experiment_name: ${var.experiment_name}
|
|
model_name: ${bundle.target}.[schema].${var.model_name}
|
|
pickup_features_table: ${bundle.target}.[schema].trip_pickup_features
|
|
dropoff_features_table: ${bundle.target}.[schema].trip_dropoff_features
|
|
git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}
|
|
|