feat: adds time_format and timestamp_format and associated tests (#2238)

Author: chalmerlowe

google/cloud/bigquery/external_config.py

@@ -879,6 +879,34 @@ def time_zone(self) -> Optional[str]:
     def time_zone(self, value: Optional[str]):
         self._properties["timeZone"] = value
 
+    @property
+    def time_format(self) -> Optional[str]:
+        """Optional[str]: Format used to parse TIME values. Supports C-style and SQL-style values.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.time_format
+        """
+        result = self._properties.get("timeFormat")
+        return typing.cast(str, result)
+
+    @time_format.setter
+    def time_format(self, value: Optional[str]):
+        self._properties["timeFormat"] = value
+
+    @property
+    def timestamp_format(self) -> Optional[str]:
+        """Optional[str]: Format used to parse TIMESTAMP values. Supports C-style and SQL-style values.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.timestamp_format
+        """
+        result = self._properties.get("timestampFormat")
+        return typing.cast(str, result)
+
+    @timestamp_format.setter
+    def timestamp_format(self, value: Optional[str]):
+        self._properties["timestampFormat"] = value
+
     @property
     def connection_id(self):
         """Optional[str]: [Experimental] ID of a BigQuery Connection API

google/cloud/bigquery/job/load.py

@@ -575,6 +575,32 @@ def time_zone(self) -> Optional[str]:
     def time_zone(self, value: Optional[str]):
         self._set_sub_prop("timeZone", value)
 
+    @property
+    def time_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing TIME values.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.time_format
+        """
+        return self._get_sub_prop("timeFormat")
+
+    @time_format.setter
+    def time_format(self, value: Optional[str]):
+        self._set_sub_prop("timeFormat", value)
+
+    @property
+    def timestamp_format(self) -> Optional[str]:
+        """Optional[str]: Date format used for parsing TIMESTAMP values.
+
+        See:
+        https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.timestamp_format
+        """
+        return self._get_sub_prop("timestampFormat")
+
+    @timestamp_format.setter
+    def timestamp_format(self, value: Optional[str]):
+        self._set_sub_prop("timestampFormat", value)
+
     @property
     def time_partitioning(self):
         """Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies time-based
@@ -930,6 +956,20 @@ def time_zone(self):
         """
         return self.configuration.time_zone
 
+    @property
+    def time_format(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.time_format`.
+        """
+        return self.configuration.time_format
+
+    @property
+    def timestamp_format(self):
+        """See
+        :attr:`google.cloud.bigquery.job.LoadJobConfig.timestamp_format`.
+        """
+        return self.configuration.timestamp_format
+
     @property
     def schema_update_options(self):
         """See

tests/unit/job/test_load.py

@@ -39,13 +39,18 @@ def _setUpConstants(self):
         self.REFERENCE_FILE_SCHEMA_URI = "gs://path/to/reference"
         self.DATE_FORMAT = "%Y-%m-%d"
         self.TIME_ZONE = "UTC"
+        self.TIME_FORMAT = "%H:%M:%S"
+        self.TIMESTAMP_FORMAT = "YYYY-MM-DD HH:MM:SS.SSSSSSZ"
 
     def _make_resource(self, started=False, ended=False):
         resource = super(TestLoadJob, self)._make_resource(started, ended)
         config = resource["configuration"]["load"]
         config["sourceUris"] = [self.SOURCE1]
         config["dateFormat"] = self.DATE_FORMAT
         config["timeZone"] = self.TIME_ZONE
+        config["timeFormat"] = self.TIME_FORMAT
+        config["timestampFormat"] = self.TIMESTAMP_FORMAT
+
         config["destinationTable"] = {
             "projectId": self.PROJECT,
             "datasetId": self.DS_ID,
@@ -163,6 +168,14 @@ def _verifyResourceProperties(self, job, resource):
             self.assertEqual(job.time_zone, config["timeZone"])
         else:
             self.assertIsNone(job.time_zone)
+        if "timeFormat" in config:
+            self.assertEqual(job.time_format, config["timeFormat"])
+        else:
+            self.assertIsNone(job.time_format)
+        if "timestampFormat" in config:
+            self.assertEqual(job.timestamp_format, config["timestampFormat"])
+        else:
+            self.assertIsNone(job.timestamp_format)
 
     def test_ctor(self):
         client = _make_client(project=self.PROJECT)
@@ -207,6 +220,8 @@ def test_ctor(self):
         self.assertIsNone(job.reference_file_schema_uri)
         self.assertIsNone(job.date_format)
         self.assertIsNone(job.time_zone)
+        self.assertIsNone(job.time_format)
+        self.assertIsNone(job.timestamp_format)
 
     def test_ctor_w_config(self):
         from google.cloud.bigquery.schema import SchemaField
@@ -604,7 +619,10 @@ def test_begin_w_alternate_client(self):
             "schemaUpdateOptions": [SchemaUpdateOption.ALLOW_FIELD_ADDITION],
             "dateFormat": self.DATE_FORMAT,
             "timeZone": self.TIME_ZONE,
+            "timeFormat": self.TIME_FORMAT,
+            "timestampFormat": self.TIMESTAMP_FORMAT,
         }
+
         RESOURCE["configuration"]["load"] = LOAD_CONFIGURATION
         conn1 = make_connection()
         client1 = _make_client(project=self.PROJECT, connection=conn1)
@@ -634,6 +652,8 @@ def test_begin_w_alternate_client(self):
         config.reference_file_schema_uri = "gs://path/to/reference"
         config.date_format = self.DATE_FORMAT
         config.time_zone = self.TIME_ZONE
+        config.time_format = self.TIME_FORMAT
+        config.timestamp_format = self.TIMESTAMP_FORMAT
 
         with mock.patch(
             "google.cloud.bigquery.opentelemetry_tracing._get_final_span_attributes"

tests/unit/job/test_load_config.py

@@ -860,6 +860,40 @@ def test_time_zone_setter(self):
         config.time_zone = time_zone
         self.assertEqual(config._properties["load"]["timeZone"], time_zone)
 
+    def test_time_format_missing(self):
+        config = self._get_target_class()()
+        self.assertIsNone(config.time_format)
+
+    def test_time_format_hit(self):
+        time_format = "%H:%M:%S"
+        config = self._get_target_class()()
+        config._properties["load"]["timeFormat"] = time_format
+        self.assertEqual(config.time_format, time_format)
+
+    def test_time_format_setter(self):
+        time_format = "HH24:MI:SS"
+        config = self._get_target_class()()
+        config.time_format = time_format
+        self.assertEqual(config._properties["load"]["timeFormat"], time_format)
+
+    def test_timestamp_format_missing(self):
+        config = self._get_target_class()()
+        self.assertIsNone(config.timestamp_format)
+
+    def test_timestamp_format_hit(self):
+        timestamp_format = "%Y-%m-%dT%H:%M:%S.%fZ"
+        config = self._get_target_class()()
+        config._properties["load"]["timestampFormat"] = timestamp_format
+        self.assertEqual(config.timestamp_format, timestamp_format)
+
+    def test_timestamp_format_setter(self):
+        timestamp_format = "YYYY/MM/DD HH24:MI:SS.FF6 TZR"
+        config = self._get_target_class()()
+        config.timestamp_format = timestamp_format
+        self.assertEqual(
+            config._properties["load"]["timestampFormat"], timestamp_format
+        )
+
     def test_parquet_options_missing(self):
         config = self._get_target_class()()
         self.assertIsNone(config.parquet_options)

tests/unit/test_external_config.py

@@ -27,6 +27,8 @@ class TestExternalConfig(unittest.TestCase):
     SOURCE_URIS = ["gs://foo", "gs://bar"]
     DATE_FORMAT = "MM/DD/YYYY"
     TIME_ZONE = "America/Los_Angeles"
+    TIME_FORMAT = "HH24:MI:SS"
+    TIMESTAMP_FORMAT = "MM/DD/YYYY HH24:MI:SS.FF6 TZR"
 
     BASE_RESOURCE = {
         "sourceFormat": "",
@@ -37,6 +39,8 @@ class TestExternalConfig(unittest.TestCase):
         "compression": "compression",
         "dateFormat": DATE_FORMAT,
         "timeZone": TIME_ZONE,
+        "timeFormat": TIME_FORMAT,
+        "timestampFormat": TIMESTAMP_FORMAT,
     }
 
     def test_from_api_repr_base(self):
@@ -85,6 +89,9 @@ def test_to_api_repr_base(self):
 
         ec.date_format = self.DATE_FORMAT
         ec.time_zone = self.TIME_ZONE
+        ec.time_format = self.TIME_FORMAT
+        ec.timestamp_format = self.TIMESTAMP_FORMAT
+
         exp_schema = {
             "fields": [{"name": "full_name", "type": "STRING", "mode": "REQUIRED"}]
         }
@@ -100,6 +107,8 @@ def test_to_api_repr_base(self):
             "schema": exp_schema,
             "dateFormat": self.DATE_FORMAT,
             "timeZone": self.TIME_ZONE,
+            "timeFormat": self.TIME_FORMAT,
+            "timestampFormat": self.TIMESTAMP_FORMAT,
         }
         self.assertEqual(got_resource, exp_resource)
 
@@ -137,6 +146,8 @@ def _verify_base(self, ec):
         self.assertEqual(ec.source_uris, self.SOURCE_URIS)
         self.assertEqual(ec.date_format, self.DATE_FORMAT)
         self.assertEqual(ec.time_zone, self.TIME_ZONE)
+        self.assertEqual(ec.time_format, self.TIME_FORMAT)
+        self.assertEqual(ec.timestamp_format, self.TIMESTAMP_FORMAT)
 
     def test_to_api_repr_source_format(self):
         ec = external_config.ExternalConfig("CSV")