feature: add SageMaker lineage, workflow and pipelines support (#448) (#461) (#479) (#485) (#504) (#508) (#513)

Author: danabens

.dictionary

@@ -0,0 +1,38 @@
+args
+arn
+autoscaling
+aws
+bool
+boolean
+boto
+botocore
+clienterror
+cloudwatch
+cron
+config
+dataset
+datasets
+datetime
+desc
+docstring
+entrypoint
+env
+iam
+hyperparameter
+hyperparameters
+jupyter
+kms
+kwargs
+neo
+noqa
+rc
+runtime
+sagemaker
+stdout
+str
+subdirectories
+subnet
+subnets
+unexpectedstatusexception
+uri
+vpc

.pylintrc

@@ -292,7 +292,7 @@ spelling-dict=
 spelling-ignore-words=
 
 # A path to a file that contains private dictionary; one word per line.
-spelling-private-dict-file=
+spelling-private-dict-file=.dictionary
 
 # Tells whether to store unknown words to indicated private dictionary in
 # --spelling-private-dict-file option instead of raising a message.

doc/api/utility/inputs.rst

@@ -5,3 +5,4 @@ Inputs
     :members:
     :undoc-members:
     :show-inheritance:
+    :noindex:

setup.py

@@ -33,6 +33,7 @@ def read_version():
 
 # Declare minimal set for installation
 required_packages = [
+    "attrs",
     "boto3>=1.14.12",
     "google-pasta",
     "numpy>=1.9.0",
@@ -73,6 +74,7 @@ def read_version():
         "apache-airflow==1.10.11",
         "fabric>=2.0",
         "requests>=2.20.0, <3",
+        "sagemaker-experiments",
     ],
 )
 

src/sagemaker/__init__.py

@@ -49,6 +49,7 @@
 from sagemaker.local.local_session import LocalSession  # noqa: F401
 
 from sagemaker.model import Model, ModelPackage  # noqa: F401
+from sagemaker.model_metrics import ModelMetrics, MetricsSource  # noqa: F401
 from sagemaker.pipeline import PipelineModel  # noqa: F401
 from sagemaker.predictor import Predictor  # noqa: F401
 from sagemaker.processing import Processor, ScriptProcessor  # noqa: F401

src/sagemaker/analytics.py

@@ -10,7 +10,7 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-"""Placeholder docstring"""
+"""Placeholder docstring."""
 from __future__ import print_function, absolute_import
 
 from abc import ABCMeta, abstractmethod
@@ -22,6 +22,7 @@
 
 from sagemaker.session import Session
 from sagemaker.utils import DeferredError
+from sagemaker.lineage import artifact
 
 logger = logging.getLogger(__name__)
 
@@ -36,8 +37,8 @@
 
 
 class AnalyticsMetricsBase(with_metaclass(ABCMeta, object)):
-    """Base class for tuning job or training job analytics classes. Understands
-    common functionality like persistence and caching.
+    """Base class for tuning job or training job analytics classes.
+    Understands common functionality like persistence and caching.
     """
 
     def __init__(self):
@@ -52,8 +53,8 @@ def export_csv(self, filename):
         self.dataframe().to_csv(filename)
 
     def dataframe(self, force_refresh=False):
-        """A pandas dataframe with lots of interesting results about this
-        object. Created by calling SageMaker List and Describe APIs and
+        """A pandas dataframe with lots of interesting results about this object.
+        Created by calling SageMaker List and Describe APIs and
         converting them into a convenient tabular summary.
 
         Args:
@@ -71,17 +72,15 @@ def _fetch_dataframe(self):
         """Sub-class must calculate the dataframe and return it."""
 
     def clear_cache(self):
-        """Clear the object of all local caches of API methods, so that the next
-        time any properties are accessed they will be refreshed from the
+        """Clear the object of all local caches of API methods.
+        So that the next time any properties are accessed they will be refreshed from the
         service.
         """
         self._dataframe = None
 
 
 class HyperparameterTuningJobAnalytics(AnalyticsMetricsBase):
-    """Fetch results about a hyperparameter tuning job and make them accessible
-    for analytics.
-    """
+    """Fetch results about a hyperparameter tuning job and make them accessible for analytics."""
 
     def __init__(self, hyperparameter_tuning_job_name, sagemaker_session=None):
         """Initialize a ``HyperparameterTuningJobAnalytics`` instance.
@@ -104,7 +103,7 @@ def __init__(self, hyperparameter_tuning_job_name, sagemaker_session=None):
 
     @property
     def name(self):
-        """Name of the HyperparameterTuningJob being analyzed"""
+        """Name of the HyperparameterTuningJob being analyzed."""
         return self._tuning_job_name
 
     def __repr__(self):
@@ -156,8 +155,8 @@ def reshape(training_summary):
 
     @property
     def tuning_ranges(self):
-        """A dictionary describing the ranges of all tuned hyperparameters. The
-        keys are the names of the hyperparameter, and the values are the ranges.
+        """A dictionary describing the ranges of all tuned hyperparameters.
+        The keys are the names of the hyperparameter, and the values are the ranges.
 
         The output can take one of two forms:
 
@@ -208,16 +207,15 @@ def tuning_ranges(self):
         }
 
     def _prepare_parameter_ranges(self, parameter_ranges):
-        """Convert parameter ranges a dictionary using the parameter range names as the keys"""
+        """Convert parameter ranges a dictionary using the parameter range names as the keys."""
         out = {}
         for _, ranges in parameter_ranges.items():
             for param in ranges:
                 out[param["Name"]] = param
         return out
 
     def description(self, force_refresh=False):
-        """Call ``DescribeHyperParameterTuningJob`` for the hyperparameter
-        tuning job.
+        """Call ``DescribeHyperParameterTuningJob`` for the hyperparameter tuning job.
 
         Args:
             force_refresh (bool): Set to True to fetch the latest data from
@@ -236,8 +234,7 @@ def description(self, force_refresh=False):
         return self._tuning_job_describe_result
 
     def training_job_summaries(self, force_refresh=False):
-        """A (paginated) list of everything from
-        ``ListTrainingJobsForTuningJob``.
+        """A (paginated) list of everything from ``ListTrainingJobsForTuningJob``.
 
         Args:
             force_refresh (bool): Set to True to fetch the latest data from
@@ -270,9 +267,7 @@ def training_job_summaries(self, force_refresh=False):
 
 
 class TrainingJobAnalytics(AnalyticsMetricsBase):
-    """Fetch training curve data from CloudWatch Metrics for a specific training
-    job.
-    """
+    """Fetch training curve data from CloudWatch Metrics for a specific training job."""
 
     CLOUDWATCH_NAMESPACE = "/aws/sagemaker/TrainingJobs"
 
@@ -318,7 +313,7 @@ def __init__(
 
     @property
     def name(self):
-        """Name of the TrainingJob being analyzed"""
+        """Name of the TrainingJob being analyzed."""
         return self._training_job_name
 
     def __repr__(self):
@@ -365,7 +360,7 @@ def _fetch_dataframe(self):
         return pd.DataFrame(self._data)
 
     def _fetch_metric(self, metric_name):
-        """Fetch all the values of a named metric, and add them to _data
+        """Fetch all the values of a named metric, and add them to _data.
 
         Args:
             metric_name: The metric name to fetch.
@@ -425,6 +420,75 @@ def _metric_names_for_training_job(self):
         return metric_names
 
 
+class ArtifactAnalytics(AnalyticsMetricsBase):
+    """Fetch artifact data and make them accessible for analytics."""
+
+    def __init__(
+        self,
+        sort_by=None,
+        sort_order=None,
+        source_uri=None,
+        artifact_type=None,
+        sagemaker_session=None,
+    ):
+        """Initialize a ``ArtifactAnalytics`` instance.
+
+        Args:
+            sort_by (str, optional): The name of the resource property used to sort
+                the set of artifacts. Currently only support for sort by Name
+            sort_order(str optional): How trial components are ordered, valid values are Ascending
+                and Descending. The default is Descending.
+            source_uri(dict optional): The artifact source uri for filtering.
+            artifact_type(dict optional): The artifact type for filtering.
+            sagemaker_session (obj, optional): Sagemaker session. Defaults to None.
+        """
+        self._sort_by = sort_by if sort_by == "Name" else None
+        self._sort_order = sort_order
+        self._source_uri = source_uri
+        self._artifact_type = artifact_type
+        self._sagemaker_session = sagemaker_session
+        super(ArtifactAnalytics, self).__init__()
+        self.clear_cache()
+
+    def __repr__(self):
+        """Human-readable representation override."""
+        return "<sagemaker.ArtifactAnalytics>"
+
+    def _reshape_source_type(self, artifact_source_types):
+        """Reshape artifact source type."""
+        out = OrderedDict()
+        for artifact_source_type in artifact_source_types:
+            out["ArtifactSourceType"] = artifact_source_type
+        return out
+
+    def _reshape(self, artifact_summary):
+        """Reshape artifact summary."""
+        out = OrderedDict()
+        out["ArtifactName"] = artifact_summary.artifact_name
+        out["ArtifactArn"] = artifact_summary.artifact_arn
+        out["ArtifactType"] = artifact_summary.artifact_type
+        out["ArtifactSourceUri"] = artifact_summary.source.source_uri
+        out["CreationTime"] = artifact_summary.creation_time
+        out["LastModifiedTime"] = artifact_summary.last_modified_time
+        return out
+
+    def _fetch_dataframe(self):
+        """Return a pandas dataframe with all artifacts."""
+        df = pd.DataFrame([self._reshape(artifact) for artifact in self._get_list_artifacts()])
+        return df
+
+    def _get_list_artifacts(self):
+        """List artifacts."""
+        artifacts = artifact.Artifact.list(
+            source_uri=self._source_uri,
+            artifact_type=self._artifact_type,
+            sort_by=self._sort_by,
+            sort_order=self._sort_order,
+            sagemaker_session=self._sagemaker_session,
+        )
+        return artifacts
+
+
 class ExperimentAnalytics(AnalyticsMetricsBase):
     """Fetch trial component data and make them accessible for analytics."""
 
@@ -486,7 +550,7 @@ def __init__(
 
     @property
     def name(self):
-        """Name of the Experiment being analyzed"""
+        """Name of the Experiment being analyzed."""
         return self._experiment_name
 
     def __repr__(self):
@@ -499,7 +563,7 @@ def clear_cache(self):
         self._trial_components = None
 
     def _reshape_parameters(self, parameters):
-        """Reshape trial component parameters to a pandas column
+        """Reshape trial component parameters to a pandas column.
         Args:
             parameters: trial component parameters
         Returns:
@@ -513,7 +577,7 @@ def _reshape_parameters(self, parameters):
         return out
 
     def _reshape_metrics(self, metrics):
-        """Reshape trial component metrics to a pandas column
+        """Reshape trial component metrics to a pandas column.
         Args:
             metrics: trial component metrics
         Returns:
@@ -533,7 +597,7 @@ def _reshape_metrics(self, metrics):
         return out
 
     def _reshape_artifacts(self, artifacts, _artifact_names):
-        """Reshape trial component input/output artifacts to a pandas column
+        """Reshape trial component input/output artifacts to a pandas column.
         Args:
             artifacts: trial component input/output artifacts
         Returns:
@@ -548,7 +612,8 @@ def _reshape_artifacts(self, artifacts, _artifact_names):
         return out
 
     def _reshape_parents(self, parents):
-        """Reshape trial component parents to a pandas column
+        """Reshape trial component parents to a pandas column.
+
         Args:
             parents: trial component parents (trials and experiments)
         Returns:
@@ -565,7 +630,7 @@ def _reshape_parents(self, parents):
         return out
 
     def _reshape(self, trial_component):
-        """Reshape trial component data to pandas columns
+        """Reshape trial component data to pandas columns.
         Args:
             trial_component: dict representing a trial component
         Returns:
@@ -633,8 +698,7 @@ def _get_trial_components(self, force_refresh=False):
         return self._search(self._search_expression, self._sort_by, self._sort_order)
 
     def _search(self, search_expression, sort_by, sort_order):
-        """
-        Perform a search query using SageMaker Search and return the matching trial components
+        """Perform a search query using SageMaker Search and return the matching trial components.
 
         Args:
             search_expression: Search expression to filter trial components.

src/sagemaker/apiutils/.pydocstylerc

@@ -0,0 +1,2 @@
+[pydocstyle]
+inherit = false

src/sagemaker/apiutils/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Utilities for providing a more pythonic style wrapper of boto."""