Aggregation of fields referencing external variables

Changelog.rst

@@ -19,6 +19,8 @@ version 3.8.0
 * Fixed bug that caused a failure when writing a dataset that contains
   a scalar domain ancillary construct
   (https://github.com/NCAS-CMS/cf-python/issues/152)
+* Fixed bug that prevented aggregation of fields with external cell measures
+  (https://github.com/NCAS-CMS/cf-python/issues/150#issuecomment-729747867)
 * Fixed bug that caused rows full of zeros to appear in WGDOS packed
   UM data that contain masked points
   (https://github.com/NCAS-CMS/cf-python/issues/161)

cf/aggregate.py

@@ -600,7 +600,28 @@ def __init__(
         # ------------------------------------------------------------
         self.msr = {}
         info_msr = {}
+        copied_field = False
         for key, msr in f.cell_measures.items():
+            # If the measure is an external variable, remove it because
+            # the dimensions are not known so there is no way to tell if the
+            # aggregation should have changed it. (This is sufficiently
+            # sensible behaviour for now, but will be reviewed in future.)
+            # Note: for CF <=1.8 only cell measures can be external variables.
+            if msr.nc_get_external():
+                # Only create one copy of field if there is >1 external measure
+                if not copied_field:
+                    self.field = self.field.copy()  # copy as will delete msr
+                    f = self.field
+                    copied_field = True
+                f.del_construct(msr.identity())
+                logger.info(
+                    "Removed '{}' construct from a copy of input field {!r} "
+                    "pre-aggregation because it is an external variable so it "
+                    "is not possible to determine the influence the "
+                    "aggregation process should have on it.".format(
+                        msr.identity(), f.identity())
+                )
+                continue
 
             if not self.cell_measure_has_data_and_units(msr):
                 return
@@ -1934,7 +1955,7 @@ def aggregate(fields,
 
                         unaggregatable = True
                         break
-                # --- End: while
+                # --- End: for
 
                 m[:] = [m0]
             # --- End: for
@@ -1961,6 +1982,7 @@ def aggregate(fields,
                     output_fields.extend((m.field for m in meta0))
         else:
             output_fields.extend((m.field for m in meta))
+
     # --- End: for
 
     aggregate.status = status

cf/test/test_external.py

@@ -161,6 +161,51 @@ def test_EXTERNAL_WRITE(self):
         for i in range(len(h)):
             self.assertTrue(external[i].equals(h[i], verbose=2))
 
+    def test_EXTERNAL_AGGREGATE(self):
+        if self.test_only and inspect.stack()[0][3] not in self.test_only:
+            return
+
+        # Read parent file without the external file, taking first field to test
+        f = cf.read(self.parent_file, verbose=0)[0]
+        measure_name = 'measure:area'
+
+        # Split f into parts (take longitude with 3 x 3 = 9 points) so can
+        # test the difference between the aggregated result and original f.
+        # Note all parts retain the external variable cell measure.
+        f_lon_thirds = [f[:, :3], f[:, 3:6], f[:, 6:]]
+
+        g = cf.aggregate(f_lon_thirds)
+
+        self.assertEqual(len(g), 1)
+
+        # Check cell measure construct from external variable has been removed
+        self.assertFalse(g[0].cell_measures())
+
+        # Check aggregated field is identical to original with measure removed
+        f0 = f.copy()
+        f0.del_construct(measure_name)
+        self.assertEqual(g[0], f0)
+
+        # Also check aggregation did not remove the measure from the inputs
+        for part in f_lon_thirds:
+            cell_measure = part.constructs.filter_by_identity(
+                'measure:area').value()
+            self.assertTrue(cell_measure.nc_get_external())
+
+        # Now try aggregating when one part doesn't have the cell measure,
+        # expecting all of the parts to still aggregate back to one field
+        # without the external measure (rather than  2->1 => 1 + 1 = 2 fields).
+        f_lon_thirds[1].del_construct(measure_name)
+        g = cf.aggregate(f_lon_thirds)
+        self.assertEqual(len(g), 1)
+        self.assertFalse(g[0].cell_measures())
+
+        # Also check measure was not removed from, or added to, any input
+        for part in [f_lon_thirds[0], f_lon_thirds[2]]:
+            cm = part.constructs.filter_by_identity('measure:area').value()
+            self.assertTrue(cm.nc_get_external())
+        self.assertFalse(f_lon_thirds[1].cell_measures())
+
 # --- End: class