test: populate tables for system test using pytest fixtures #51 (#60)

* test: populate tables for system test using pytest fixtures #51

* test: fixed import order lint errror, fixed numeric test error #51

* allow dataset to not exist when trying to delete it

* try adding autouse=True to the dataset fixture

* typo in autouse

* remove ibis v2 from v1 branch tests

* add option to not refresh dataset

* use more specific dataset ID

* save dataset on ibis-gbq project

Co-authored-by: Tim Swast <swast@google.com>

Author: loudinb

.github/workflows/main.yml

@@ -53,14 +53,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.7", "3.8", "3.9"]
-        ibis_version: ["github"]
         include:
           - python_version: "3.7"
             "ibis_version": "1.2.0"
-          - python_version: "3.7"
-            "ibis_version":  "1.3.0"
           - python_version: "3.8"
+            "ibis_version":  "1.3.0"
+          - python_version: "3.9"
+            "ibis_version": "1.4.0"
+          - python_version: "3.10"
             "ibis_version": "1.4.0"
 
     steps:

.github/workflows/system-tests-pr.yml

@@ -17,14 +17,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.7", "3.8", "3.9"]
-        ibis_version: ["github"]
         include:
           - python_version: "3.7"
             "ibis_version": "1.2.0"
-          - python_version: "3.7"
-            "ibis_version":  "1.3.0"
           - python_version: "3.8"
+            "ibis_version":  "1.3.0"
+          - python_version: "3.10"
             "ibis_version": "1.4.0"
 
     steps:
@@ -55,7 +53,7 @@ jobs:
         GCLOUD_KEY_PASSPHRASE: ${{ secrets.GCLOUD_KEY_PASSPHRASE }}
 
     - name: run tests
-      run: GOOGLE_BIGQUERY_PROJECT_ID="ibis-gbq" GOOGLE_APPLICATION_CREDENTIALS="$HOME/secrets/gcloud-service-key.json" python -m pytest tests/system
+      run: GOOGLE_BIGQUERY_PROJECT_ID="ibis-gbq" GOOGLE_APPLICATION_CREDENTIALS="$HOME/secrets/gcloud-service-key.json" python -m pytest tests/system --save-dataset
 
     # Remove the label so that it can be added again to re-trigger a build
     - name: reset label

.github/workflows/system-tests.yml

@@ -15,14 +15,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.7", "3.8", "3.9"]
         ibis_version: ["github"]
-        include:
           - python_version: "3.7"
             "ibis_version": "1.2.0"
-          - python_version: "3.7"
-            "ibis_version":  "1.3.0"
           - python_version: "3.8"
+            "ibis_version":  "1.3.0"
+          - python_version: "3.10"
             "ibis_version": "1.4.0"
 
     steps:
@@ -51,4 +49,4 @@ jobs:
         GCLOUD_KEY_PASSPHRASE: ${{ secrets.GCLOUD_KEY_PASSPHRASE }}
 
     - name: run tests
-      run: GOOGLE_BIGQUERY_PROJECT_ID="ibis-gbq" GOOGLE_APPLICATION_CREDENTIALS="$HOME/secrets/gcloud-service-key.json" python -m pytest tests/system
+      run: GOOGLE_BIGQUERY_PROJECT_ID="ibis-gbq" GOOGLE_APPLICATION_CREDENTIALS="$HOME/secrets/gcloud-service-key.json" python -m pytest tests/system --save-dataset

environment.yml

@@ -1,13 +1,13 @@
-name: ibis-bigquery-dev
+name: ibis-v1-bigquery-dev
 channels:
 - conda-forge
 dependencies:
 
 # core
-- ibis-framework  # TODO: require Ibis 2.0 when it's released
-- cached_property  # New in Ibis 2.0 https://github.com/ibis-project/ibis/pull/2805
+- ibis-framework <=2.0.0dev
 - google-cloud-bigquery-core >=1.12.0,<3.0.0dev
 - pydata-google-auth
+- sqlalchemy=1.3.23
 
 # dev
 - black=19.10b0 # Same as ibis

setup.py

@@ -41,12 +41,14 @@
     packages=setuptools.find_packages(),
     python_requires=">=3.7",
     install_requires=[
-        # TODO require ibis 2.0 when it's released
-        "ibis-framework >= 1.2.0,<3.0.0dev",
+        "ibis-framework >=1.2.0,<2.0.0dev",
         "google-cloud-bigquery >=1.12.0,<3.0.0dev",
         "google-cloud-bigquery-storage >=1.0.0,<3.0.0dev",
         "pyarrow >=1.0.0,<5.0.0dev",
         "pydata-google-auth",
+        # Workaround for ibis 1.x incompatibility with SQLAlchemy 1.4
+        # AttributeError: module 'sqlalchemy' has no attribute 'Binary'   
+        "sqlalchemy <1.4.0dev",
     ],
     classifiers=[
         release_status,