diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c39cff..785050e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### Unreleased changes -* bugfix: `validate` and `send` could incorrect number of total records +* bugfix: failed `send` could bury error message coming from ODS +* bugfix: `validate` and `send` could show incorrect number of total records ### v0.1.6
diff --git a/lightbeam/send.py b/lightbeam/send.py index d4dd4ee..e247aec 100644 --- a/lightbeam/send.py +++ b/lightbeam/send.py @@ -147,7 +147,7 @@ async def do_post(self, endpoint, file_name, data, line_number, data_hash): ssl=self.lightbeam.config["connection"]["verify_ssl"], headers=self.lightbeam.api.headers ) as response: - body = await response.text() + text = await response.text() status = response.status if status!=401: # update status_counts (for every-second status update) @@ -156,20 +156,29 @@ async def do_post(self, endpoint, file_name, data, line_number, data_hash): # warn about errors if response.status not in [ 200, 201 ]: - message = str(response.status) + ": " + util.linearize(json.loads(body).get("message")) + try: + body = json.loads(text) + if "errors" in body: + log_message = str(response.status) + ": " + "; ".join(map(util.linearize, body["errors"])) + elif "message" in body: + log_message = str(response.status) + ": " + util.linearize(body["message"]) + else: + log_message = "" + except json.decoder.JSONDecodeError: + log_message = "" # update run metadata... failures = self.lightbeam.metadata["resources"][endpoint].get("failures", []) do_append = True for index, item in enumerate(failures): - if item["status_code"]==response.status and item["message"]==message and item["file"]==file_name: + if item["status_code"]==response.status and item["message"]==log_message and item["file"]==file_name: failures[index]["line_numbers"].append(line_number) failures[index]["count"] += 1 do_append = False if do_append: failure = { 'status_code': response.status, - 'message': message, + 'message': log_message, 'file': file_name, 'line_numbers': [line_number], 'count': 1 @@ -178,9 +187,9 @@ async def do_post(self, endpoint, file_name, data, line_number, data_hash): self.lightbeam.metadata["resources"][endpoint]["failures"] = failures # update output and counters - self.lightbeam.increment_status_reason(message) + self.lightbeam.increment_status_reason(log_message) if response.status==400: - raise Exception(message) + raise ValueError(log_message) else: self.lightbeam.num_errors += 1 @@ -206,7 +215,7 @@ async def do_post(self, endpoint, file_name, data, line_number, data_hash): except RuntimeError as e: await asyncio.sleep(1) - except Exception as e: + except ValueError as e: status = 400 self.lightbeam.num_errors += 1 self.logger.warn("{0} (at line {1} of {2} )".format(str(e), line_number, file_name))