new_cluster: &new_cluster
  new_cluster:
    num_workers: 3
    spark_version: 13.3.x-cpu-ml-scala2.12
    node_type_id: Standard_D3_v2
    custom_tags:
      clusterSource: mlops-stacks_0.4

resources:
  jobs:
    model_training_job:
      name: ${bundle.target}-training-job
      job_clusters:
        - job_cluster_key: model_training_job_cluster
          <<: *new_cluster
      tasks:
        - task_key: Train
          job_cluster_key: model_training_job_cluster
          notebook_task:
            notebook_path: ../training/notebooks/TrainWithFeatureStore.py
            base_parameters:
              env: ${bundle.target}
  
              training_data_path: /databricks-datasets/nyctaxi-with-zipcodes/subsampled
              experiment_name: ${var.experiment_name}
              model_name: ${bundle.target}.[schema].${var.model_name}
              pickup_features_table: ${bundle.target}.[schema].trip_pickup_features
              dropoff_features_table: ${bundle.target}.[schema].trip_dropoff_features
              git_source_info: url:${bundle.git.origin_url}; branch:${bundle.git.branch}; commit:${bundle.git.commit}