From 9fb9933cb16ad2077adb2fa5f43e54608096ecf3 Mon Sep 17 00:00:00 2001 From: ValentinCordonnier Date: Thu, 8 Jun 2023 16:54:51 +0200 Subject: [PATCH 1/8] feat(export): add `export_to_pubsub` (#114) --- bigfunctions/export_to_pubsub.yaml | 46 ++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 bigfunctions/export_to_pubsub.yaml diff --git a/bigfunctions/export_to_pubsub.yaml b/bigfunctions/export_to_pubsub.yaml new file mode 100644 index 000000000..e28a5f8ab --- /dev/null +++ b/bigfunctions/export_to_pubsub.yaml @@ -0,0 +1,46 @@ +type: function_py +category: export +author: + name: Valentin Cordonnier + url: https://www.linkedin.com/in/valentin-cordonnier-99482b1b5 + avatar_url: "https://storage.googleapis.com/public-bucket-val/photo-output__01.jpg" +description: Exports data and its attributes to a given Pub/Sub topic. +arguments: + - name: project + type: string + - name: topic + type: string + - name: data + type: string + - name: attributes + type: json +output: + name: result + type: string +examples: + - description: "" + arguments: + - "'your-project'" + - "'my_topic'" + - "'Your message data'" + - '{"attribute1": "value1", "attribute2": "value2"}' + output: "'200 Message successfully published.'" + region: ALL +code: | + import json + from google.cloud import pubsub_v1 + + attributes = json.loads(attributes) if attributes else {} + publisher = google.cloud.pubsub_v1.PublisherClient() + future = publisher.publish( + publisher.topic_path(project, topic), + data.encode("utf-8"), + **attributes) + + try: + future.result() + return "200 Message successfully published." + except Exception as e: + return f"{type(e).__name__}: {str(e)}" +requirements: + google-cloud-pubsub From 960ef9149d78effa3b1329631a0e91f38f938795 Mon Sep 17 00:00:00 2001 From: ValentinCordonnier Date: Wed, 21 Jun 2023 11:16:59 +0200 Subject: [PATCH 2/8] feat(export_to_bigtable): add function (#116) --- bigfunctions/export_to_bigtable.yaml | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 bigfunctions/export_to_bigtable.yaml diff --git a/bigfunctions/export_to_bigtable.yaml b/bigfunctions/export_to_bigtable.yaml new file mode 100644 index 000000000..485a9f705 --- /dev/null +++ b/bigfunctions/export_to_bigtable.yaml @@ -0,0 +1,66 @@ +type: function_py +category: export +author: + name: Valentin Cordonnier + url: https://www.linkedin.com/in/valentin-cordonnier-99482b1b5 + avatar_url: "https://storage.googleapis.com/public-bucket-val/photo-output__01.jpg" +description: Exports a value to a given Bigtable table. +arguments: + - name: project_id + type: string + - name: instance_id + type: string + - name: table_id + type: string + - name: column_family + type: string + - name: column_name + type: string + - name: row_key + type: string + - name: value + type: string +output: + name: result + type: string +examples: + - description: "" + arguments: + - "'your-project'" + - "'your-instance'" + - "'your-table'" + - "'your-family'" + - "'your-column-name'" + - "'your-row-key'" + - "'your-value'" + output: "'200 Message successfully published.'" + region: ALL +code: | + import google.api_core.exceptions + from google.cloud import bigtable + + client = bigtable.Client(project=project_id, admin=True) + + try: + instance = client.instance(instance_id) + + try: + table = instance.table(table_id) + + row = table.direct_row(row_key.encode('utf-8')) + row.set_cell(column_family_id=column_family, column=column_name, value=value.encode('utf-8')) + table.mutate_rows(rows=[row]) + + return "200 Value successfully exported." + + except google.api_core.exceptions.NotFound: + assert False, f'Given table does not exist in given instance OR service Account `get_current_service_account()` does have the permission to access it' + except google.api_core.exceptions.PermissionDenied: + assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' + + except google.api_core.exceptions.NotFound: + assert False, f'Given instance does not exist in given project OR service Account `get_current_service_account()` does have the permission to access it' + except google.api_core.exceptions.PermissionDenied: + assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' +requirements: + google-cloud-bigtable From 5f0ae0c5a7884936af2a5c17b6a72f2e2f777b7e Mon Sep 17 00:00:00 2001 From: ValentinCordonnier Date: Wed, 21 Jun 2023 11:16:59 +0200 Subject: [PATCH 3/8] feat(export_to_bigtable): add function (#116) --- bigfunctions/export_to_bigtable.yaml | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 bigfunctions/export_to_bigtable.yaml diff --git a/bigfunctions/export_to_bigtable.yaml b/bigfunctions/export_to_bigtable.yaml new file mode 100644 index 000000000..485a9f705 --- /dev/null +++ b/bigfunctions/export_to_bigtable.yaml @@ -0,0 +1,66 @@ +type: function_py +category: export +author: + name: Valentin Cordonnier + url: https://www.linkedin.com/in/valentin-cordonnier-99482b1b5 + avatar_url: "https://storage.googleapis.com/public-bucket-val/photo-output__01.jpg" +description: Exports a value to a given Bigtable table. +arguments: + - name: project_id + type: string + - name: instance_id + type: string + - name: table_id + type: string + - name: column_family + type: string + - name: column_name + type: string + - name: row_key + type: string + - name: value + type: string +output: + name: result + type: string +examples: + - description: "" + arguments: + - "'your-project'" + - "'your-instance'" + - "'your-table'" + - "'your-family'" + - "'your-column-name'" + - "'your-row-key'" + - "'your-value'" + output: "'200 Message successfully published.'" + region: ALL +code: | + import google.api_core.exceptions + from google.cloud import bigtable + + client = bigtable.Client(project=project_id, admin=True) + + try: + instance = client.instance(instance_id) + + try: + table = instance.table(table_id) + + row = table.direct_row(row_key.encode('utf-8')) + row.set_cell(column_family_id=column_family, column=column_name, value=value.encode('utf-8')) + table.mutate_rows(rows=[row]) + + return "200 Value successfully exported." + + except google.api_core.exceptions.NotFound: + assert False, f'Given table does not exist in given instance OR service Account `get_current_service_account()` does have the permission to access it' + except google.api_core.exceptions.PermissionDenied: + assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' + + except google.api_core.exceptions.NotFound: + assert False, f'Given instance does not exist in given project OR service Account `get_current_service_account()` does have the permission to access it' + except google.api_core.exceptions.PermissionDenied: + assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' +requirements: + google-cloud-bigtable From 0bc576d71344920377e71f6cec72ffe88e4e6b38 Mon Sep 17 00:00:00 2001 From: ValentinCordonnier Date: Wed, 21 Jun 2023 12:01:45 +0200 Subject: [PATCH 4/8] Revert "feat(export_to_bigtable): add function (#116)" This reverts commit 960ef9149d78effa3b1329631a0e91f38f938795. --- bigfunctions/export_to_bigtable.yaml | 66 ---------------------------- 1 file changed, 66 deletions(-) delete mode 100644 bigfunctions/export_to_bigtable.yaml diff --git a/bigfunctions/export_to_bigtable.yaml b/bigfunctions/export_to_bigtable.yaml deleted file mode 100644 index 485a9f705..000000000 --- a/bigfunctions/export_to_bigtable.yaml +++ /dev/null @@ -1,66 +0,0 @@ -type: function_py -category: export -author: - name: Valentin Cordonnier - url: https://www.linkedin.com/in/valentin-cordonnier-99482b1b5 - avatar_url: "https://storage.googleapis.com/public-bucket-val/photo-output__01.jpg" -description: Exports a value to a given Bigtable table. -arguments: - - name: project_id - type: string - - name: instance_id - type: string - - name: table_id - type: string - - name: column_family - type: string - - name: column_name - type: string - - name: row_key - type: string - - name: value - type: string -output: - name: result - type: string -examples: - - description: "" - arguments: - - "'your-project'" - - "'your-instance'" - - "'your-table'" - - "'your-family'" - - "'your-column-name'" - - "'your-row-key'" - - "'your-value'" - output: "'200 Message successfully published.'" - region: ALL -code: | - import google.api_core.exceptions - from google.cloud import bigtable - - client = bigtable.Client(project=project_id, admin=True) - - try: - instance = client.instance(instance_id) - - try: - table = instance.table(table_id) - - row = table.direct_row(row_key.encode('utf-8')) - row.set_cell(column_family_id=column_family, column=column_name, value=value.encode('utf-8')) - table.mutate_rows(rows=[row]) - - return "200 Value successfully exported." - - except google.api_core.exceptions.NotFound: - assert False, f'Given table does not exist in given instance OR service Account `get_current_service_account()` does have the permission to access it' - except google.api_core.exceptions.PermissionDenied: - assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' - - except google.api_core.exceptions.NotFound: - assert False, f'Given instance does not exist in given project OR service Account `get_current_service_account()` does have the permission to access it' - except google.api_core.exceptions.PermissionDenied: - assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' -requirements: - google-cloud-bigtable From db623fe6a164068ca58d59960209009fa39a5424 Mon Sep 17 00:00:00 2001 From: ValentinCordonnier Date: Wed, 21 Jun 2023 12:06:40 +0200 Subject: [PATCH 5/8] "feat(export): add `export_to_bigtable` (#116)" --- bigfunctions/export_to_bigtable.yaml | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 bigfunctions/export_to_bigtable.yaml diff --git a/bigfunctions/export_to_bigtable.yaml b/bigfunctions/export_to_bigtable.yaml new file mode 100644 index 000000000..485a9f705 --- /dev/null +++ b/bigfunctions/export_to_bigtable.yaml @@ -0,0 +1,66 @@ +type: function_py +category: export +author: + name: Valentin Cordonnier + url: https://www.linkedin.com/in/valentin-cordonnier-99482b1b5 + avatar_url: "https://storage.googleapis.com/public-bucket-val/photo-output__01.jpg" +description: Exports a value to a given Bigtable table. +arguments: + - name: project_id + type: string + - name: instance_id + type: string + - name: table_id + type: string + - name: column_family + type: string + - name: column_name + type: string + - name: row_key + type: string + - name: value + type: string +output: + name: result + type: string +examples: + - description: "" + arguments: + - "'your-project'" + - "'your-instance'" + - "'your-table'" + - "'your-family'" + - "'your-column-name'" + - "'your-row-key'" + - "'your-value'" + output: "'200 Message successfully published.'" + region: ALL +code: | + import google.api_core.exceptions + from google.cloud import bigtable + + client = bigtable.Client(project=project_id, admin=True) + + try: + instance = client.instance(instance_id) + + try: + table = instance.table(table_id) + + row = table.direct_row(row_key.encode('utf-8')) + row.set_cell(column_family_id=column_family, column=column_name, value=value.encode('utf-8')) + table.mutate_rows(rows=[row]) + + return "200 Value successfully exported." + + except google.api_core.exceptions.NotFound: + assert False, f'Given table does not exist in given instance OR service Account `get_current_service_account()` does have the permission to access it' + except google.api_core.exceptions.PermissionDenied: + assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' + + except google.api_core.exceptions.NotFound: + assert False, f'Given instance does not exist in given project OR service Account `get_current_service_account()` does have the permission to access it' + except google.api_core.exceptions.PermissionDenied: + assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' +requirements: + google-cloud-bigtable From 195987f1f749077a0a64f51bf4012f62d1294832 Mon Sep 17 00:00:00 2001 From: ValentinCordonnier Date: Mon, 26 Jun 2023 10:16:45 +0200 Subject: [PATCH 6/8] fix(export): fixed export_to_bigtable --- bigfunctions/export_to_bigtable.yaml | 59 +++++++++++++++------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/bigfunctions/export_to_bigtable.yaml b/bigfunctions/export_to_bigtable.yaml index bf9d83fa3..954bfb6dc 100644 --- a/bigfunctions/export_to_bigtable.yaml +++ b/bigfunctions/export_to_bigtable.yaml @@ -37,41 +37,46 @@ examples: region: ALL code_process_rows_as_batch: true code: | - import google.api_core.exceptions import datetime + import google.api_core.exceptions from google.cloud import bigtable - i = 1 - - for row in rows: - project_id, instance_id, table_id, column_family, column_name, row_key, value = row + project_id, instance_id, table_id, *_ = rows[0] + try: client = bigtable.Client(project=project_id, admin=True) + instance = client.instance(instance_id) + except google.api_core.exceptions.NotFound: + assert False, f'Given instance does not exist in given project OR service Account `get_current_service_account()` does have the permission to access it' + except google.api_core.exceptions.PermissionDenied: + assert False, f'Service Account `get_current_service_account()` does not have access permission for given instance of given project' + + + try: + table = instance.table(table_id) + except google.api_core.exceptions.NotFound: + assert False, f'Given table does not exist in given instance OR service Account `get_current_service_account()` does have the permission to access it' + except google.api_core.exceptions.PermissionDenied: + assert False, f'Service Account `get_current_service_account()` does not have access permission for given table of given instance' + + entities = [] + for i, row in enumerate(rows): + *_, column_family, column_name, row_key, value = row + + row_bigtable = table.direct_row(row_key.encode('utf-8')) + + + row_bigtable.set_cell( + column_family_id=column_family, + column=column_name, + value=value.encode('utf-8'), + timestamp=datetime.datetime.now()+datetime.timedelta(0, i * 0.1) + ) - entities = [] - try: - instance = client.instance(instance_id) + entities.append(row_bigtable) - try: - table = instance.table(table_id) - row_bigtable = table.direct_row(row_key.encode('utf-8')) - row_bigtable.set_cell(column_family_id=column_family, column=column_name, value=value.encode('utf-8'), timestamp=datetime.datetime.now()+datetime.timedelta(0,i*0.1)) - - entities.append(row_bigtable) - i+=1 - - except google.api_core.exceptions.NotFound: - assert False, f'Given table does not exist in given instance OR service Account `get_current_service_account()` does have the permission to access it' - except google.api_core.exceptions.PermissionDenied: - assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' - - except google.api_core.exceptions.NotFound: - assert False, f'Given instance does not exist in given project OR service Account `get_current_service_account()` does have the permission to access it' - except google.api_core.exceptions.PermissionDenied: - assert False, f'Service Account `get_current_service_account()` does not have publish permission into given topic of given project' - table.mutate_rows(rows=entities) - return ["200 Value successfully exported." for entity in entities] + return ["200 Value successfully exported."] * len(entities) requirements: google-cloud-bigtable From 9c8d62a9d691246c86f1cc6d8854b4d0e61ce2b7 Mon Sep 17 00:00:00 2001 From: ValentinCordonnier Date: Mon, 26 Jun 2023 14:15:24 +0200 Subject: [PATCH 7/8] fix(export): fixed export_to_bigtable --- bigfunctions/export_to_bigtable.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigfunctions/export_to_bigtable.yaml b/bigfunctions/export_to_bigtable.yaml index 954bfb6dc..685068e1a 100644 --- a/bigfunctions/export_to_bigtable.yaml +++ b/bigfunctions/export_to_bigtable.yaml @@ -70,7 +70,7 @@ code: | column_family_id=column_family, column=column_name, value=value.encode('utf-8'), - timestamp=datetime.datetime.now()+datetime.timedelta(0, i * 0.1) + timestamp=datetime.datetime.now()+datetime.timedelta(milliseconds=i) ) entities.append(row_bigtable) From e4f6190383eea1a29800aa891c9dd5bacf3fede1 Mon Sep 17 00:00:00 2001 From: ValentinCordonnier Date: Wed, 28 Jun 2023 09:27:10 +0200 Subject: [PATCH 8/8] fix(export): fixed export_to_bigtable description --- bigfunctions/export_to_bigtable.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigfunctions/export_to_bigtable.yaml b/bigfunctions/export_to_bigtable.yaml index 685068e1a..3e7d40adc 100644 --- a/bigfunctions/export_to_bigtable.yaml +++ b/bigfunctions/export_to_bigtable.yaml @@ -4,7 +4,7 @@ author: name: Valentin Cordonnier url: https://www.linkedin.com/in/valentin-cordonnier-99482b1b5 avatar_url: "https://storage.googleapis.com/public-bucket-val/photo-output__01.jpg" -description: Exports a value to a given Bigtable table. +description: Exports values to a given Bigtable table. arguments: - name: project_id type: string