Examples

Create a pipeline

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Python_Script_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

pipeline_step = (
    PipelineStep("Python_Script_Step")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step])
  )

pipeline.create()

Run a job as a step

  • YAML
  • Python
rom ads.jobs import Job, DataScienceJob, ScriptRuntime
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    DataScienceJob()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

job = (
    Job()
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)
job.create() # create a job

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: dataScienceJob
    spec:
      description: A step running a job
      jobId: {job_id}
      name: Job_Step
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id, job_id=job.id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()
from ads.jobs import Job, DataScienceJob, ScriptRuntime
from ads.pipeline import Pipeline, PipelineStep
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    DataScienceJob()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

job = (
    Job()
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)
job.create() # create a job

pipeline_step = (
    PipelineStep("Job_Step")
    .with_description("A step running a job")
    .with_job_id(job.id)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step])
  )

pipeline.create()

pipeline_run = pipeline.run()

Run a python script as a step

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Python_Script_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

pipeline_step = (
    PipelineStep("Python_Script_Step")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step])
  )

pipeline.create()

pipeline_run = pipeline.run()

Run a notebook as a step

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a notebook
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Notebook_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: tensorflow26_p37_cpu_v2
            type: service
          env:
          - name: GREETINGS
            value: Welcome to OCI Data Science
          notebookEncoding: utf-8
          notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb
          outputURI: oci://<bucket_name>@<namespace>/<prefix>
        type: notebook
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime
import os

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    NotebookRuntime()
    .with_notebook(
        path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb",
        encoding='utf-8'
    )
    .with_service_conda("tensorflow26_p37_cpu_v2")
    .with_environment_variable(GREETINGS="Welcome to OCI Data Science")
    .with_output("oci://<bucket_name>@<namespace>/<prefix>")
)

pipeline_step = (
    PipelineStep("Notebook_Step")
    .with_description("A step running a notebook")
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
    Pipeline("A single step pipeline")
    .with_compartment_id(compartment_id)
    .with_project_id(project_id)
    .with_step_details([pipeline_step])
)

pipeline.create()

pipeline_run = pipeline.run()

Run two steps with the same infrastructure

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Python_Script_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
  - kind: customScript
    spec:
      description: A step running a notebook
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Notebook_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: tensorflow26_p37_cpu_v2
            type: service
          env:
          - name: GREETINGS
            value: Welcome to OCI Data Science
          notebookEncoding: utf-8
          notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb
          outputURI: oci://<bucket_name>@<namespace>/<prefix>
        type: notebook
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

step_one_runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

pipeline_step_one = (
    PipelineStep("Python_Script_Step")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(step_one_runtime)
)

step_two_runtime = (
    NotebookRuntime()
    .with_notebook(
        path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb",
        encoding='utf-8'
    )
    .with_service_conda("tensorflow26_p37_cpu_v2")
    .with_environment_variable(GREETINGS="Welcome to OCI Data Science")
    .with_output("oci://<bucket_name>@<namespace>/<prefix>")
)

pipeline_step_two = (
    PipelineStep("Notebook_Step")
    .with_description("A step running a notebook")
    .with_infrastructure(infrastructure)
    .with_runtime(step_two_runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step_one, pipeline_step_two])
  )

pipeline.create()

pipeline_run = pipeline.run()

Run two steps in parallel

In the example below, when DAG is not specified, the steps in the pipeline run in parallel.

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Python_Script_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
  - kind: customScript
    spec:
      description: A step running a notebook
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Notebook_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: tensorflow26_p37_cpu_v2
            type: service
          env:
          - name: GREETINGS
            value: Welcome to OCI Data Science
          notebookEncoding: utf-8
          notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb
          outputURI: oci://<bucket_name>@<namespace>/<prefix>
        type: notebook
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

step_one_runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

pipeline_step_one = (
    PipelineStep("Python_Script_Step")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(step_one_runtime)
)

step_two_runtime = (
    NotebookRuntime()
    .with_notebook(
        path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb",
        encoding='utf-8'
    )
    .with_service_conda("tensorflow26_p37_cpu_v2")
    .with_environment_variable(GREETINGS="Welcome to OCI Data Science")
    .with_output("oci://<bucket_name>@<namespace>/<prefix>")
)

pipeline_step_two = (
    PipelineStep("Notebook_Step")
    .with_description("A step running a notebook")
    .with_infrastructure(infrastructure)
    .with_runtime(step_two_runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step_one, pipeline_step_two])
  )

pipeline.create()

pipeline_run = pipeline.run()

Run two steps sequentially

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  dag:
  - Python_Script_Step >> Notebook_Step
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Python_Script_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
  - kind: customScript
    spec:
      description: A step running a notebook
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Notebook_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: tensorflow26_p37_cpu_v2
            type: service
          env:
          - name: GREETINGS
            value: Welcome to OCI Data Science
          notebookEncoding: utf-8
          notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb
          outputURI: oci://<bucket_name>@<namespace>/<prefix>
        type: notebook
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

step_one_runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

pipeline_step_one = (
    PipelineStep("Python_Script_Step")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(step_one_runtime)
)

step_two_runtime = (
    NotebookRuntime()
    .with_notebook(
        path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb",
        encoding='utf-8'
    )
    .with_service_conda("tensorflow26_p37_cpu_v2")
    .with_environment_variable(GREETINGS="Welcome to OCI Data Science")
    .with_output("oci://<bucket_name>@<namespace>/<prefix>")
)

pipeline_step_two = (
    PipelineStep("Notebook_Step")
    .with_description("A step running a notebook")
    .with_infrastructure(infrastructure)
    .with_runtime(step_two_runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step_one, pipeline_step_two])
      .with_dag(["Python_Script_Step >> Notebook_Step"])
  )

pipeline.create()

pipeline_run = pipeline.run()

Run multiple steps with dependencies specified in DAG

In this example, step_1 and step_2 run in parallel and step_3 runs after step_1 and step_2 are complete.

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: An example pipeline
  projectId: {project_id}
  dag:
  - (step_1, step_2) >> step_3
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: step_1
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
  - kind: customScript
    spec:
      description: A step running a notebook
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: step_2
      runtime:
        kind: runtime
        spec:
          conda:
            slug: tensorflow26_p37_cpu_v2
            type: service
          notebookEncoding: utf-8
          notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb
        type: notebook
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: step_3
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()      # create the pipeline
pipeline.show()        # visualize the pipeline

pipeline_run = pipeline.run()   # run the pipeline

pipeline_run.show(wait=True)    # watch the pipeline run status
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime, NotebookRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

script_runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

notebook_runtime = (
    NotebookRuntime()
    .with_notebook(
        path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb",
        encoding='utf-8'
    )
    .with_service_conda("tensorflow26_p37_cpu_v2")
)

pipeline_step_1 = (
    PipelineStep("step_1")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(script_runtime)
)

pipeline_step_2 = (
    PipelineStep("step_2")
    .with_description("A step running a notebook")
    .with_infrastructure(infrastructure)
    .with_runtime(notebook_runtime)
)

pipeline_step_3 = (
    PipelineStep("step_3")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(script_runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("An example pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step_1, pipeline_step_2, pipeline_step_3])
      .with_dag(["(step_1, step_2) >> step_3"])
  )

pipeline.create()      # create the pipeline
pipeline.show()        # visualize the pipeline

pipeline_run = pipeline.run()   # run the pipeline

pipeline_run.show(wait=True)    # watch the pipeline run status

Set environment variables in a step

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a notebook
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Notebook_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: tensorflow26_p37_cpu_v2
            type: service
          env:
          - name: GREETINGS
            value: Welcome to OCI Data Science
          notebookEncoding: utf-8
          notebookPathURI: https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb
          outputURI: oci://<bucket_name>@<namespace>/<prefix>
        type: notebook
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

pipeline_run = pipeline.run()
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, NotebookRuntime
import os

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    NotebookRuntime()
    .with_notebook(
        path="https://raw.githubusercontent.com/tensorflow/docs/master/site/en/tutorials/customization/basics.ipynb",
        encoding='utf-8'
    )
    .with_service_conda("tensorflow26_p37_cpu_v2")
    .with_environment_variable(GREETINGS="Welcome to OCI Data Science")
    .with_output("oci://<bucket_name>@<namespace>/<prefix>")
)

pipeline_step = (
    PipelineStep("Notebook_Step")
    .with_description("A step running a notebook")
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
    Pipeline("A single step pipeline")
    .with_compartment_id(compartment_id)
    .with_project_id(project_id)
    .with_step_details([pipeline_step])
)

pipeline.create()

pipeline_run = pipeline.run()

Watch status update on a pipeline run

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Python_Script_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()
pipeline_run = pipeline.run()

# pipeline_run.show(mode="text")   # watch pipeline run status in text
pipeline_run.show(wait=True)       # watch pipeline run status in graph
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

pipeline_step = (
    PipelineStep("Python_Script_Step")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step])
  )

pipeline.create()
pipeline_run = pipeline.run()

# pipeline_run.show(mode="text")   # watch pipeline run status in text
pipeline_run.show(wait=True)       # watch pipeline run status in graph

Monitor logs of a pipeline run

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Python_Script_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()
pipeline_run = pipeline.run()

# pipeline_run.watch()  # stream the consolidated log of the pipeline run
pipeline_run.watch(log_type="service_log")      # stream service log of the pipeline run
pipeline_run.watch("Python_Script_Step", log_type="custom_log") # stream custom log of the step run
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

pipeline_step = (
    PipelineStep("Python_Script_Step")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step])
  )

pipeline.create()
pipeline_run = pipeline.run()

# pipeline_run.watch()  # stream the consolidated log of the pipeline run
pipeline_run.watch(log_type="service_log")      # stream service log of the pipeline run
pipeline_run.watch("Python_Script_Step", log_type="custom_log") # stream custom log of the step run

Override configurations when creating a pipeline run

  • YAML
  • Python
from ads.pipeline import Pipeline
import os

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

yaml_string = """
kind: pipeline
spec:
  commandLineArguments: argument --key value
  environmentVariables:
    env: value
  compartmentId: {compartment_id}
  displayName: A single step pipeline
  projectId: {project_id}
  stepDetails:
  - kind: customScript
    spec:
      description: A step running a python script
      infrastructure:
        kind: infrastructure
        spec:
          blockStorageSize: 200
          shapeConfigDetails:
            memoryInGBs: 32
            ocpus: 4
          shapeName: VM.Standard3.Flex
      name: Python_Script_Step
      runtime:
        kind: runtime
        spec:
          conda:
            slug: generalml_p37_cpu_v1
            type: service
          scriptPathURI: script.py
        type: script
type: pipeline
""".format(compartment_id=compartment_id, project_id=project_id)

pipeline = Pipeline.from_yaml(yaml_string)

pipeline.create()

# Override configurations when creating a pipeline run
display_override_name = "RunOverrideName"
configuration_override_details = {
    "maximum_runtime_in_minutes": 30,
    "type": "DEFAULT",
    "environment_variables": {"a": "b"},
    "command_line_arguments": "ARGUMENT --KEY VALUE",
}

step_override_details = [
{
    "step_name": "Python_Script_Step",
    "step_configuration_details": {
        "maximum_runtime_in_minutes": 200,
        "environment_variables": {"1": "2"},
        "command_line_arguments": "argument --key value",
    },
}
]
pipeline_run = pipeline.run(
    display_name=display_override_name,
    configuration_override_details=configuration_override_details,
    step_override_details=step_override_details,
)
from ads.pipeline import Pipeline, PipelineStep, CustomScriptStep, ScriptRuntime
import os

with open("script.py", "w") as f:
    f.write("print('Hello World!')")

infrastructure = (
    CustomScriptStep()
    .with_block_storage_size(200)
    .with_shape_name("VM.Standard3.Flex")
    .with_shape_config_details(ocpus=4, memory_in_gbs=32)
)

runtime = (
    ScriptRuntime()
    .with_source("script.py")
    .with_service_conda("generalml_p37_cpu_v1")
)

pipeline_step = (
    PipelineStep("Python_Script_Step")
    .with_description("A step running a python script")
    .with_infrastructure(infrastructure)
    .with_runtime(runtime)
)

compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']
project_id = os.environ["PROJECT_OCID"]

pipeline = (
      Pipeline("A single step pipeline")
      .with_compartment_id(compartment_id)
      .with_project_id(project_id)
      .with_step_details([pipeline_step])
      .with_argument("argument", key="value")
      .with_environment_variable(env="value")
  )

pipeline.create()

# Override configurations when creating a pipeline run
display_override_name = "RunOverrideName"
configuration_override_details = {
    "maximum_runtime_in_minutes": 30,
    "type": "DEFAULT",
    "environment_variables": {"a": "b"},
    "command_line_arguments": "ARGUMENT --KEY VALUE",
}

step_override_details = [
{
    "step_name": "Python_Script_Step",
    "step_configuration_details": {
        "maximum_runtime_in_minutes": 200,
        "environment_variables": {"1": "2"},
        "command_line_arguments": "argument --key value",
    },
}
]
pipeline_run = pipeline.run(
    display_name=display_override_name,
    configuration_override_details=configuration_override_details,
    step_override_details=step_override_details,
)