Support pickling of functions with positional-only arguments (#269)

Author: pierreglaser

.travis.yml

@@ -12,6 +12,9 @@ matrix:
     - os: linux
       dist: trusty
       python: "pypy3"
+    - os: linux
+      if: commit_message =~ /(\[ci python-nightly\])/
+      env: PYTHON_NIGHTLY=1
     - os: linux
       python: 3.7
     - os: linux
@@ -69,38 +72,53 @@ before_install:
         export PATH="$PYTHON_ROOT:$PYTHON_ROOT/Scripts:$PATH";
         python -m pip install --upgrade pip;
     fi
+  - if [[ "$PYTHON_NIGHTLY" == 1 ]]; then
+        export VENV_DIR="$HOME/python38";
+        pushd ..;
+        git clone https://github.com/python/cpython.git;
+        pushd cpython;
+        ./configure;
+        make;
+        ./python -m venv "$VENV_DIR";
+        popd;
+        popd;
+        export PYTHON_EXE="$VENV_DIR/bin/python";
+    else
+        export PYTHON_EXE="python";
+    fi
+
 install:
-  - pip install .
-  - pip install --upgrade -r dev-requirements.txt
-  - pip install tornado
-  - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* ]]; then
-        pip install numpy scipy;
+  - $PYTHON_EXE -m pip install .
+  - $PYTHON_EXE -m pip install --upgrade -r dev-requirements.txt
+  - $PYTHON_EXE -m pip install tornado
+  - if [[ $TRAVIS_PYTHON_VERSION != 'pypy'* && "$PYTHON_NIGHTLY" != 1 ]]; then
+        $PYTHON_EXE -m pip install numpy scipy;
     fi
   - if [[ $PROJECT != "" ]]; then
-        pip install $TEST_REQUIREMENTS;
+        $PYTHON_EXE -m pip install $TEST_REQUIREMENTS;
         pushd ..;
         git clone $PROJECT_URL;
         if [[ $PROJECT == "joblib" ]]; then
             pushd joblib/joblib/externals;
             source vendor_cloudpickle.sh ../../../cloudpickle;
             popd;
         fi;
-        pip install ./$PROJECT;
+        $PYTHON_EXE -m pip install ./$PROJECT;
         popd;
     fi
-  - pip list
+  - $PYTHON_EXE -m pip list
 before_script:
   # stop the build if there are Python syntax errors or undefined names
-  - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
+  - $PYTHON_EXE -m flake8 . --count --verbose --select=E901,E999,F821,F822,F823 --show-source --statistics
   # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
-  - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-  - python ci/install_coverage_subprocess_pth.py
+  - $PYTHON_EXE -m flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+  - $PYTHON_EXE ci/install_coverage_subprocess_pth.py
 script:
-  - COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' pytest -r s
+  - COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' $PYTHON_EXE -m pytest -r s
   - |
     if [[ $PROJECT != "" ]]; then
       pushd ../$PROJECT
-      pytest -vl
+      $PYTHON_EXE -m pytest -vl
       TEST_RETURN_CODE=$?
       popd
       if [[ "$TEST_RETURN_CODE" != "0" ]]; then

CHANGES.md

@@ -1,6 +1,9 @@
 1.1.0
 =====
 
+- Support the pickling of interactively-defined functions with positional-only
+  arguments. ([issue #266](https://github.com/cloudpipe/cloudpickle/pull/266))
+
 - Track the provenance of dynamic classes and enums so as to preseve the
   usual `isinstance` relationship between pickled objects and their
   original class defintions.

cloudpickle/cloudpickle.py

@@ -168,24 +168,43 @@ def inner(value):
             (),
         )
     else:
-        return types.CodeType(
-            co.co_argcount,
-            co.co_kwonlyargcount,
-            co.co_nlocals,
-            co.co_stacksize,
-            co.co_flags,
-            co.co_code,
-            co.co_consts,
-            co.co_names,
-            co.co_varnames,
-            co.co_filename,
-            co.co_name,
-            co.co_firstlineno,
-            co.co_lnotab,
-            co.co_cellvars,  # this is the trickery
-            (),
-        )
-
+        if hasattr(types.CodeType, "co_posonlyargcount"):  # pragma: no branch
+            return types.CodeType(
+                co.co_argcount,
+                co.co_posonlyargcount,  # Python3.8 with PEP570
+                co.co_kwonlyargcount,
+                co.co_nlocals,
+                co.co_stacksize,
+                co.co_flags,
+                co.co_code,
+                co.co_consts,
+                co.co_names,
+                co.co_varnames,
+                co.co_filename,
+                co.co_name,
+                co.co_firstlineno,
+                co.co_lnotab,
+                co.co_cellvars,  # this is the trickery
+                (),
+            )
+        else:
+            return types.CodeType(
+                co.co_argcount,
+                co.co_kwonlyargcount,
+                co.co_nlocals,
+                co.co_stacksize,
+                co.co_flags,
+                co.co_code,
+                co.co_consts,
+                co.co_names,
+                co.co_varnames,
+                co.co_filename,
+                co.co_name,
+                co.co_firstlineno,
+                co.co_lnotab,
+                co.co_cellvars,  # this is the trickery
+                (),
+            )
 
 _cell_set_template_code = _make_cell_set_template_code()
 
@@ -371,12 +390,23 @@ def save_codeobject(self, obj):
         Save a code object
         """
         if PY3:  # pragma: no branch
-            args = (
-                obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
-                obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames,
-                obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
-                obj.co_cellvars
-            )
+            if hasattr(obj, "co_posonlyargcount"):  # pragma: no branch
+                args = (
+                    obj.co_argcount, obj.co_posonlyargcount,
+                    obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
+                    obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
+                    obj.co_varnames, obj.co_filename, obj.co_name,
+                    obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
+                    obj.co_cellvars
+                )
+            else:
+                args = (
+                    obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
+                    obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
+                    obj.co_names, obj.co_varnames, obj.co_filename,
+                    obj.co_name, obj.co_firstlineno, obj.co_lnotab,
+                    obj.co_freevars, obj.co_cellvars
+                )
         else:
             args = (
                 obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code,

tests/cloudpickle_test.py

@@ -1639,6 +1639,32 @@ def f(a, *, b=1):
         """.format(protocol=self.protocol)
         assert_run_python_script(textwrap.dedent(code))
 
+    @pytest.mark.skipif(not hasattr(types.CodeType, "co_posonlyargcount"),
+                        reason="Requires positional-only argument syntax")
+    def test_interactively_defined_func_with_positional_only_argument(self):
+        # Fixes https://github.com/cloudpipe/cloudpickle/issues/266
+        # The source code of this test is bundled in a string and is ran from
+        # the __main__ module of a subprocess in order to avoid a SyntaxError
+        # in versions of python that do not support positional-only argument
+        # syntax.
+        code = """
+        import pytest
+        from cloudpickle import loads, dumps
+
+        def f(a, /, b=1):
+            return a + b
+
+        depickled_f = loads(dumps(f, protocol={protocol}))
+
+        for func in (f, depickled_f):
+            assert func(2) == 3
+            assert func.__code__.co_posonlyargcount == 1
+            with pytest.raises(TypeError):
+                func(a=2)
+
+        """.format(protocol=self.protocol)
+        assert_run_python_script(textwrap.dedent(code))
+
 class Protocol2CloudPickleTest(CloudPickleTest):
 
     protocol = 2