From db3daa09346271edfa61068993bc02c3f43704cc Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Thu, 28 Apr 2016 13:53:53 +0200 Subject: [PATCH 01/24] oauth.py: Add HTTP proxy support This patch adds support for a HTTP proxy to the geeknote oauth authorization code. The proxy is read from system settings, usually the environment variable https_proxy. Basic proxy authorization is supported. Note that for geeknote to work through a proxy, proxy support for thrift needs to be enabled, too. See https://issues.apache.org/jira/browse/THRIFT-3798 --- geeknote/oauth.py | 51 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/geeknote/oauth.py b/geeknote/oauth.py index 3c44a01..dd173fb 100644 --- a/geeknote/oauth.py +++ b/geeknote/oauth.py @@ -5,7 +5,8 @@ import Cookie import uuid import re -from urllib import urlencode, unquote +import base64 +from urllib import urlencode, unquote, getproxies, proxy_bypass from urlparse import urlparse import out @@ -58,6 +59,26 @@ class GeekNoteAuth(object): incorrectCode = 0 code = None + def __init__(self): + try: + proxy = getproxies()['https'] + except KeyError: + proxy = None + if proxy is None: + self._proxy = None + else: + # This assumes that the proxy is given in URL form. + # A little simpler as _parse_proxy in urllib2.py + self._proxy = urlparse(proxy) + + if proxy is None or not self._proxy.username: + self._proxy_auth = None + else: + user_pass = "%s:%s" % (urlparse.unquote(self._proxy.username), + urlparse.unquote(self._proxy.password)) + self._proxy_auth = { "Proxy-Authorization": + "Basic " + base64.b64encode(user_pass).strip() } + def getTokenRequestData(self, **kwargs): params = { 'oauth_consumer_key': self.consumerKey, @@ -77,9 +98,11 @@ def loadPage(self, url, uri=None, method="GET", params=""): logging.error("Request URL undefined") tools.exitErr() + if not url.startswith("http"): + url = "https://" + url + urlData = urlparse(url) if not uri: - urlData = urlparse(url) - url = urlData.netloc + url = "%s://%s" (urlData.scheme, urlData.netloc) uri = urlData.path + '?' + urlData.query # prepare params, append to uri @@ -97,11 +120,27 @@ def loadPage(self, url, uri=None, method="GET", params=""): if method == "POST": headers["Content-type"] = "application/x-www-form-urlencoded" - logging.debug("Request URL: %s:/%s > %s # %s", url, + if self._proxy is None or proxy_bypass(urlData.hostname): + host = urlData.hostname + port = urlData.port + real_host = real_port = None + else: + host = self._proxy.hostname + port = self._proxy.port + real_host = urlData.hostname + real_port = urlData.port + + logging.debug("Request URL: %s%s > %s # %s", url, uri, unquote(params), headers["Cookie"]) - conn = httplib.HTTPSConnection(url) - conn.request(method, uri, params, headers) + conn = httplib.HTTPSConnection(host, port) + + if real_host is not None: + conn.set_tunnel(real_host, real_port, headers=self._proxy_auth) + if config.DEBUG: + conn.set_debuglevel(1) + + conn.request(method, url + uri, params, headers) response = conn.getresponse() data = response.read() conn.close() From b9f2a984dcf752dae8918c66ff882f7b049b87fa Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Thu, 28 Apr 2016 16:18:25 +0200 Subject: [PATCH 02/24] proxy_support.md: Documentation for proxy support --- proxy_support.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 proxy_support.md diff --git a/proxy_support.md b/proxy_support.md new file mode 100644 index 0000000..ba292e4 --- /dev/null +++ b/proxy_support.md @@ -0,0 +1,45 @@ +HTTP proxy support for geeknote +=============================== + +I recommend to make this work with virtualenv, to avoid overwriting system files. +The important part is to install in the order **thrift, then evernote, then geeknote**. This will make sure that path search order is correct for thrift. + +``` +# Download thrift and geeknote +git clone https://github.com/apache/thrift.git +git clone https://github.com/mwilck/geeknote.git + +# create and enter a virtual environment +virtualenv /var/tmp/geeknote +. /var/tmp/geeknote/bin/activate + +# Apply proxy-support patches for thrift +cd thrift + +## If the patches don't apply, you may need to check out the state that I wrote the patches for: +## git checkout -b proxy e363a34e63 +curl https://issues.apache.org/jira/secure/attachment/12801233/0001-python-THttpClient-Add-support-for-system-proxy-sett.patch | git am +curl https://issues.apache.org/jira/secure/attachment/12801234/0002-Python-THttpClient-Support-proxy-authorization.patch | git am + +# Install thrift from the patched tree +(cd lib/py; python setup.py install) +cd .. + +# Install evernote +pip install evernote + +# Install geeknote +cd geeknote +python setup.py install +``` + +Now `geeknote login`, `geeknote find`, etc. should work behind a proxy if the `http_proxy` environment variable is correctly set. You can now generate a script to activate the virtual environment: + +``` +cat >~/bin/geeknote <<\EOF +#! /bin/bash +. /var/tmp/geeknote/bin/activate +exec geeknote "$@" +EOF +chmod a+x ~/bin/geeknote +``` \ No newline at end of file From 4f3e5712af5e95fa8b9cc75e251a13fc5a280e27 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 20:26:09 +0200 Subject: [PATCH 03/24] out/printLine: fix several outTest failures The previous code wouldn't see the changed sys.stdout in printLine(). This way it works better. --- geeknote/out.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/geeknote/out.py b/geeknote/out.py index 6c3e27b..062e2fd 100644 --- a/geeknote/out.py +++ b/geeknote/out.py @@ -286,7 +286,9 @@ def printDate(timestamp): return datetime.date.strftime(datetime.date.fromtimestamp(timestamp / 1000), "%d.%m.%Y") -def printLine(line, endLine="\n", out=sys.stdout): +def printLine(line, endLine="\n", out=None): + if out is None: + out = sys.stdout message = line + endLine message = tools.stdoutEncode(message) try: From 9a100d91ed5e8352ef6e9ada3a31430a963b42f8 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 20:49:25 +0200 Subject: [PATCH 04/24] outTest: fix error / failure in test_show_note_success() --- tests/outTest.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/outTest.py b/tests/outTest.py index 778333f..716a7ef 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -22,6 +22,9 @@ class UserStub(object): accounting = AccountingStub() +class NoteAttributesStub(object): + pass + class NoteStub(object): title = 'testnote' created = 10000 @@ -29,7 +32,7 @@ class NoteStub(object): content = '##note content' tagNames = ['tag1', 'tag2', 'tag3'] guid = 12345 - + attributes = NoteAttributesStub() class outTestsWithHackedStdout(unittest.TestCase): @@ -106,10 +109,11 @@ def test_show_note_success(self): note = '''################## TITLE ################## testnote =================== META ================== -Created: 01.01.1970 Updated:01.01.1970 \n'''\ -'''----------------- CONTENT ----------------- +Created: 01.01.1970 +Updated: 01.01.1970 +----------------- CONTENT ----------------- Tags: tag1, tag2, tag3 -##note content\n\n\n''' +##note content\n\n''' showNote(NoteStub()) sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), note) From c94eac3ce107313867351af9e4810f413ec6ee39 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:01:15 +0200 Subject: [PATCH 05/24] outTest: fix failure in test_failure_message_success --- tests/outTest.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/outTest.py b/tests/outTest.py index 716a7ef..f2e7c46 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -76,9 +76,13 @@ def test_separator_empty_args_success(self): self.assertEquals(sys.stdout.read(), '\n\n') def test_failure_message_success(self): + sav = sys.stderr + buf = StringIO() + sys.stderr = buf failureMessage('fail') - sys.stdout.seek(0) - self.assertEquals(sys.stdout.read(), 'fail\n') + sys.stderr = sav + buf.seek(0) + self.assertEquals(buf.read(), 'fail\n') def test_success_message_success(self): successMessage('success') From 153de4fa0dc1ed09aebfb9d32bed2a51301de273 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:28:41 +0200 Subject: [PATCH 06/24] out: fix wrong date conversion in printDate The conversion has been broken since be00692f0277. That patch divided the timestamp effectively by 1000 twice. --- geeknote/out.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/geeknote/out.py b/geeknote/out.py index 062e2fd..222eb6c 100644 --- a/geeknote/out.py +++ b/geeknote/out.py @@ -275,15 +275,6 @@ def rawInput(message, isPass=False): def printDate(timestamp): - # Author @ash-2000 https://github.com/ash-2000 - # Check for crashing when timestamp is 13 digits on python2.7 - # pull request #260 - - if len(str(timestamp)) == 13: - timestamp = int(str(timestamp)[0:-3]) - - # --- - return datetime.date.strftime(datetime.date.fromtimestamp(timestamp / 1000), "%d.%m.%Y") def printLine(line, endLine="\n", out=None): From e181dc7f54a3f33bacadf7ee2127fadfe47c34ed Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:30:35 +0200 Subject: [PATCH 07/24] testOut: fix failure in test_print_date Take account of the fact that evernote measures time in milliseconds since 1.1.1970. Together with the previous commit, this fixes the test failure in test_print_date(). --- tests/outTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/outTest.py b/tests/outTest.py index f2e7c46..90283c0 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -170,4 +170,4 @@ def test_search_result_success(self): self.assertEquals(sys.stdout.read(), result) def test_print_date(self): - self.assertEquals(printDate(1000000), '12.01.1970') + self.assertEquals(printDate(1000000000L), '12.01.1970') From 25993368bb2b3a550a08462c615dcdf15ac0ecb6 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:35:28 +0200 Subject: [PATCH 08/24] outTest: fix failures caused by date field width printList() and searchResult() use 18-character wide date fields. outTest hasn't been adapted to this until now. --- tests/outTest.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/outTest.py b/tests/outTest.py index 90283c0..a2a4ee4 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -124,8 +124,8 @@ def test_show_note_success(self): def test_print_list_without_title_success(self): notes_list = '''Total found: 2 - 1 : 01.01.1970 testnote - 2 : 01.01.1970 testnote\n''' + 1 : 01.01.1970 testnote + 2 : 01.01.1970 testnote\n''' printList([NoteStub() for _ in xrange(2)]) sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), notes_list) @@ -133,8 +133,8 @@ def test_print_list_without_title_success(self): def test_print_list_with_title_success(self): notes_list = '''=================== test ================== Total found: 2 - 1 : 01.01.1970 testnote - 2 : 01.01.1970 testnote\n''' + 1 : 01.01.1970 testnote + 2 : 01.01.1970 testnote\n''' printList([NoteStub() for _ in xrange(2)], title='test') sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), notes_list) @@ -142,8 +142,8 @@ def test_print_list_with_title_success(self): def test_print_list_with_urls_success(self): notes_list = '''=================== test ================== Total found: 2 - 1 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 - 2 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 + 1 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 + 2 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 ''' printList([NoteStub() for _ in xrange(2)], title='test', showUrl=True) sys.stdout.seek(0) @@ -153,8 +153,8 @@ def test_print_list_with_selector_success(self): out.rawInput = lambda x: 2 notes_list = '''=================== test ================== Total found: 2 - 1 : 01.01.1970 testnote - 2 : 01.01.1970 testnote + 1 : 01.01.1970 testnote + 2 : 01.01.1970 testnote 0 : -Cancel-\n''' out.printList([NoteStub() for _ in xrange(2)], title='test', showSelector=True) sys.stdout.seek(0) @@ -163,8 +163,8 @@ def test_print_list_with_selector_success(self): def test_search_result_success(self): result = '''Search request: test Total found: 2 - 1 : 01.01.1970 testnote - 2 : 01.01.1970 testnote\n''' + 1 : 01.01.1970 testnote + 2 : 01.01.1970 testnote\n''' SearchResult([NoteStub() for _ in xrange(2)], 'test') sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), result) From 4964f31f9e466294da9ae8296597d8e42644da7a Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:54:01 +0200 Subject: [PATCH 09/24] geeknoteTest: fix error in testError_createSearchRequest1 This must be an assertRaises test. --- tests/geeknoteTest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/geeknoteTest.py b/tests/geeknoteTest.py index 209c3a2..4b116b8 100644 --- a/tests/geeknoteTest.py +++ b/tests/geeknoteTest.py @@ -88,6 +88,5 @@ def test_createSearchRequest2(self): self.assertEqual(testRequest, response) def testError_createSearchRequest1(self): - testRequest = self.notes._createSearchRequest(search="test text", - date="12.31.1999") - self.assertEqual(testRequest, 'exit') + self.assertRaises(SystemExit, self.notes._createSearchRequest, + search="test text", date="12.31.1999") From 495d161ac8d091efd235790c58d41f586380da3a Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 2 May 2016 21:54:52 +0200 Subject: [PATCH 10/24] editorTest: Fix failure in test_ENMLToText The output was wrong by one \n. --- tests/editorTest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/editorTest.py b/tests/editorTest.py index 92e54dc..3e0fa8c 100644 --- a/tests/editorTest.py +++ b/tests/editorTest.py @@ -16,7 +16,6 @@ def setUp(self): _Line 2_ **Line 3** - """ self.HTML_TEXT = "

Header 1

Header 2

Line 1

"\ "Line 2

Line 3

" From dab0fc90494832bc0f34fa746dd2e1bd98ae998c Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Tue, 3 May 2016 14:36:38 +0200 Subject: [PATCH 11/24] editor.py: get rid of Beautifulsoup warning --- geeknote/editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geeknote/editor.py b/geeknote/editor.py index d7e2853..c582e8a 100644 --- a/geeknote/editor.py +++ b/geeknote/editor.py @@ -76,7 +76,7 @@ def checklistInENMLtoSoup(soup): @staticmethod def ENMLtoText(contentENML): - soup = BeautifulSoup(contentENML.decode('utf-8')) + soup = BeautifulSoup(contentENML.decode('utf-8'), "html.parser") for section in soup.select('li > p'): section.replace_with( section.contents[0] ) From 6f4fdb73d8465d39a740dc7fddfc5c9cc9e3cb5c Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 09:35:38 +0200 Subject: [PATCH 12/24] _editWithEditorInThread: use thread.join rather than time.sleep thread.join has the advantage to return immediately when the editor quit, while time.sleep will sleep unconditionally. --- geeknote/geeknote.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/geeknote/geeknote.py b/geeknote/geeknote.py index 42ca567..bb9d744 100644 --- a/geeknote/geeknote.py +++ b/geeknote/geeknote.py @@ -624,7 +624,8 @@ def _editWithEditorInThread(self, inputData, note = None): if not thread.isAlive(): # check if thread is alive here before sleep to avoid losing data saved during this 5 secs break - time.sleep(5) + thread.join(timeout=5) + return result def create(self, title, content=None, tags=None, notebook=None, resource=None): From ba47972c7991e1e5479b0803f589f67e2fe4cf57 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 09:37:26 +0200 Subject: [PATCH 13/24] config.py: use different APP_DIR directory for DEV_MODE Avoid spoiing users's APP_DIR in developer mode. DEV_MODE is likely to be used from a source directory, so use just TOPDIR/config. --- geeknote/config.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/geeknote/config.py b/geeknote/config.py index 31fe120..69767dd 100644 --- a/geeknote/config.py +++ b/geeknote/config.py @@ -31,31 +31,34 @@ # Application path APP_DIR = os.path.join(os.getenv("HOME") or os.getenv("USERPROFILE"), ".geeknote") -ERROR_LOG = os.path.join(APP_DIR, "error.log") # Set default system editor DEF_UNIX_EDITOR = "nano" DEF_WIN_EDITOR = "notepad.exe" EDITOR_OPEN = "WRITE" -DEV_MODE = False +DEV_MODE = True DEBUG = False # Url view the note NOTE_URL = "https://%domain%/Home.action?#n=%s" +if DEV_MODE: + USER_STORE_URI = USER_STORE_URI_SANDBOX + CONSUMER_KEY = CONSUMER_KEY_SANDBOX + CONSUMER_SECRET = CONSUMER_SECRET_SANDBOX + USER_BASE_URL = USER_BASE_URL_SANDBOX + APP_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config") + sys.stderr.write("Developer mode: using %s as application directory\n" % APP_DIR) + +ERROR_LOG = os.path.join(APP_DIR, "error.log") + # validate config try: if not os.path.exists(APP_DIR): os.mkdir(APP_DIR) except Exception, e: - sys.stdout.write("Can not create application dirictory : %s" % APP_DIR) + sys.stderr.write("Can not create application directory : %s" % APP_DIR) exit(1) -if DEV_MODE: - USER_STORE_URI = USER_STORE_URI_SANDBOX - CONSUMER_KEY = CONSUMER_KEY_SANDBOX - CONSUMER_SECRET = CONSUMER_SECRET_SANDBOX - USER_BASE_URL = USER_BASE_URL_SANDBOX - NOTE_URL = NOTE_URL.replace('%domain%', USER_BASE_URL) From 94f74942811bac2a945edf75a19ae0a512d99523 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 09:38:47 +0200 Subject: [PATCH 14/24] requirements.txt: add beautifulsoup4 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index fde3d06..c1323b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ html2text sqlalchemy markdown2 thrift +beautifulsoup4 From b2388dd99f84dbd01d493c7aacd3d7b7db32b577 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 11:09:25 +0200 Subject: [PATCH 15/24] out: support taking credentials from file in DEV_MODE The credentials are read from the APP_DIR/credentials, which shoul have the following format (parsed with python's exec() function): credentials="username:password:two-factor-auth-code" --- geeknote/out.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/geeknote/out.py b/geeknote/out.py index 222eb6c..8ac399a 100644 --- a/geeknote/out.py +++ b/geeknote/out.py @@ -6,6 +6,7 @@ import time import datetime import sys +import os.path import tools from editor import Editor @@ -94,10 +95,33 @@ def draw(): except: pass +def _getCredentialsFromFile(): + # Get evernote credentials from file APP_DIR/credentials + # This is used only for sandbox mode (DEV_MODE=True) for security reasons + if config.DEV_MODE: + creds = os.path.join(config.APP_DIR, "credentials") + if os.path.exists(creds): + # execfile doesn't work reliably for assignments, see python docs + with open(creds, "r") as f: + # this sets "credentials" if correctly formatted + exec f.read() + try: + return credentials.split(":") + except: + sys.stderr.write("""Error reading credentials from %s. +Format should be: +credentials="::" + +""" % creds) + return None @preloaderPause def GetUserCredentials(): """Prompts the user for a username and password.""" + creds = _getCredentialsFromFile() + if creds is not None: + return creds[:2] + try: login = None password = None @@ -117,6 +141,10 @@ def GetUserCredentials(): @preloaderPause def GetUserAuthCode(): """Prompts the user for a two factor auth code.""" + creds = _getCredentialsFromFile() + if creds is not None: + return creds[2] + try: code = None if code is None: From 68cc14b23d2ab97c03c023517ff859bde1f1f580 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 12:49:06 +0200 Subject: [PATCH 16/24] tests/pseudoedit.py: trivial batch editor This will be used by the unit tests instead of a real editor. Its current behavior is to delete all lines containing the word "delete", and leave all else untouched. --- tests/pseudoedit.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/pseudoedit.py diff --git a/tests/pseudoedit.py b/tests/pseudoedit.py new file mode 100644 index 0000000..26999ab --- /dev/null +++ b/tests/pseudoedit.py @@ -0,0 +1,21 @@ +import sys +import re +import unittest + +class pseudoTest(unittest.TestCase): + def test_dummy(self): + self.assertTrue(1) + +def do_edit(filename, filter=None): + with open(filename, "r") as f: + lines = f.readlines() + with open(filename, "w") as f: + for line in lines: + if filter is not None and not filter.match(line): + f.write(line) + +if __name__ == "__main__": + # delete all lines containing the word "delete" + filter = re.compile(r".*\bdelete\b") + for name in sys.argv[1:]: + do_edit(name, filter=filter) From 6a32b6ea2c8975dbf209ad88004230b61082b23e Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 12:50:54 +0200 Subject: [PATCH 17/24] outTests: make test_print_list_with_urls_success work in sandbox In sandbox mode (DEV_MODE=True), test_print_list_with_urls_success() makes wrong assumptions about the URLs printed. --- tests/outTest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/outTest.py b/tests/outTest.py index a2a4ee4..f2cf059 100644 --- a/tests/outTest.py +++ b/tests/outTest.py @@ -3,7 +3,7 @@ import sys import unittest from cStringIO import StringIO -from geeknote.config import VERSION +from geeknote.config import VERSION, USER_BASE_URL from geeknote.out import printDate, printLine, printAbout,\ separator, failureMessage, successMessage, showUser, showNote, \ printList, SearchResult @@ -142,9 +142,10 @@ def test_print_list_with_title_success(self): def test_print_list_with_urls_success(self): notes_list = '''=================== test ================== Total found: 2 - 1 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 - 2 : 01.01.1970 testnote >>> https://www.evernote.com/Home.action?#n=12345 -''' + 1 : 01.01.1970 testnote >>> https://{url}/Home.action?#n=12345 + 2 : 01.01.1970 testnote >>> https://{url}/Home.action?#n=12345 +'''.format(url=USER_BASE_URL) + printList([NoteStub() for _ in xrange(2)], title='test', showUrl=True) sys.stdout.seek(0) self.assertEquals(sys.stdout.read(), notes_list) From a725dee66ce9503a1827d2a71532115cf4350d5f Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 12:52:08 +0200 Subject: [PATCH 18/24] storageTest: Replace Storage hack by proper subclassing The storage "hack" would modify the Storage class and thus affect other unit tests relying on working Storage(). Use a derived class instead. --- tests/storageTest.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/storageTest.py b/tests/storageTest.py index 13f6da5..83a443d 100644 --- a/tests/storageTest.py +++ b/tests/storageTest.py @@ -6,20 +6,18 @@ from geeknote import storage import pickle +class TestStorage(storage.Storage): -def hacked_init(self): - '''Hack for testing''' - engine = create_engine('sqlite:///:memory:', echo=False) - storage.Base.metadata.create_all(engine) - Session = sessionmaker(bind=engine) - self.session = Session() - + def __init__(self): + '''Hack for testing''' + engine = create_engine('sqlite:///:memory:', echo=False) + storage.Base.metadata.create_all(engine) + Session = sessionmaker(bind=engine) + self.session = Session() class storageTest(unittest.TestCase): def setUp(self): - stor = storage.Storage - stor.__init__ = hacked_init - self.storage = stor() + self.storage = TestStorage() self.otoken = 'testoauthtoken' self.userinfo = {'email': 'test@mail.com'} self.tags = {u'tag': 1, u'tag2': 2, u'tag3': 'lol'} From 9818c3d5e95e9baad68c539c22bdb5c4e5996d88 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 12:53:38 +0200 Subject: [PATCH 19/24] geeknoteTest: make editor tests do something real Create a real test case with AssertEqual statements for test_editWithEditorInThread(). Also, use the pseudoedit editor to avoid user interaction in the test cases. --- tests/geeknoteTest.py | 52 +++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/tests/geeknoteTest.py b/tests/geeknoteTest.py index 4b116b8..6b315f5 100644 --- a/tests/geeknoteTest.py +++ b/tests/geeknoteTest.py @@ -1,26 +1,45 @@ # -*- coding: utf-8 -*- +import sys import time import unittest from geeknote.geeknote import * from geeknote import tools from geeknote.editor import Editor +from geeknote.storage import Storage class GeekNoteOver(GeekNote): def __init__(self): pass def loadNoteContent(self, note): - note.content = "note content" + if "content" not in note.__dict__: + note.content = "note content" + def updateNote(self, guid=None, **inputData): + # HACK for testing: this assumes that the guid represents a "note" itself + # see do_test_editWithEditorInThread below + guid.content = inputData["content"] class NotesOver(Notes): def connectToEvertone(self): self.evernote = GeekNoteOver() - class testNotes(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Use our trivial "pseudoedit" program as editor to avoid user interaction + cls.storage = Storage() + cls.saved_editor = cls.storage.getUserprop('editor') + cls.storage.setUserprop('editor', sys.executable + " " + + os.path.join(os.path.dirname(os.path.abspath(__file__)), "pseudoedit.py")) + + @classmethod + def tearDownClass(cls): + if cls.saved_editor: + cls.storage.setUserprop('editor', cls.saved_editor) + def setUp(self): self.notes = NotesOver() self.testNote = tools.Struct(title="note title") @@ -49,17 +68,26 @@ def test_parseInput2(self): ) self.assertEqual(testData["tags"], ["tag1", "tag2"]) + def do_test_editWithEditorInThread(self, txt, expected): + testNote = tools.Struct(title="note title", + content=txt) + # hack to make updateNote work - see above + testNote.guid = testNote + testData = self.notes._parseInput("title", + txt, + "tag1, tag2", + None, testNote) + result = self.notes._editWithEditorInThread(testData, testNote) + self.assertEqual(Editor.ENMLtoText(testNote.content), expected) + def test_editWithEditorInThread(self): - testData = self.notes._parseInput("title", "WRITE", "tag1, tag2", - None, self.testNote) - print ('') - print ('') - print (testData) - print ('') - print ('') - - self.notes._editWithEditorInThread(testData) - + txt = "Please do not change this file" + self.do_test_editWithEditorInThread(txt, txt+'\n') + + def test_editWithEditorInThread2(self): + txt = "Please delete this line, save, and quit the editor" + self.do_test_editWithEditorInThread(txt, "\n") + def test_createSearchRequest1(self): testRequest = self.notes._createSearchRequest( search="test text", From 40bd7aac5640cc93362dc0715bbac64e1ca48e10 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Wed, 4 May 2016 16:20:15 +0200 Subject: [PATCH 20/24] sandboxTest: a unit test for evernote sandbox connection A unit test that connects to the evernote sandbox server and performs various tests. These tests will only be done in DEV_MODE (sandbox server). It is recommended to put the sandbox server credentials into $TOPDIR/config/credentials to avoid user interaction during the test. NOTE: the notebook deletion is forbidden by evernote by design. --- tests/sandboxTest.py | 96 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/sandboxTest.py diff --git a/tests/sandboxTest.py b/tests/sandboxTest.py new file mode 100644 index 0000000..05acf96 --- /dev/null +++ b/tests/sandboxTest.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +import unittest +from geeknote import config +from geeknote.geeknote import User, Notebooks, Notes, GeekNote +from geeknote.storage import Storage +from random import SystemRandom +from string import hexdigits + +# see https://docs.python.org/2.7/library/unittest.html §25.3.6 +# http://thecodeship.com/patterns/guide-to-python-function-decorators/ +# (decorator with empty argument list) +def skipUnlessDevMode(): + if config.DEV_MODE: + return lambda x: x + else: + return unittest.skip("Test only active with DEV_MODE=True") + +class TestSandbox(unittest.TestCase): + + @classmethod + def setUpClass(cls): + # start out with empty auth token. Save current token to restore it later. + cls.storage = Storage() + cls.token = cls.storage.getUserToken() + cls.info = cls.storage.getUserInfo() + cls.storage.removeUser() + cls.notes = set() + cls.nbs = set() + cls.notebook = ("Geeknote test %s please delete" % + "".join(SystemRandom().choice(hexdigits) for x in range(12))) + + @classmethod + def tearDownClass(cls): + if cls.token: + cls.storage.createUser(cls.token, cls.info) + + def setUp(self): + self.user = User() + self.tag = "geeknote_unittest_1" + self.Notes = Notes() + self.Notebooks = Notebooks() + self.Geeknote = self.Notebooks.getEvernote() + + @skipUnlessDevMode() + def test01_userLogin(self): + # This is an implicit test. The GeekNote() call in setUp() will perform + # an automatic login. + self.assertTrue(self.Geeknote.checkAuth()) + + @skipUnlessDevMode() + def test10_createNotebook(self): + self.assertTrue(self.Notebooks.create(self.notebook)) + + @skipUnlessDevMode() + def test15_findNotebook(self): + all = self.Geeknote.findNotebooks() + nb = [nb for nb in all if nb.name == self.notebook] + self.assertTrue(len(nb)==1) + self.nbs.add(nb[0].guid) + + @skipUnlessDevMode() + def test30_createNote(self): + self.Notes.create("note title 01", + content = """\ +# Sample note 01 +This is the note text. +""", + notebook = self.notebook, + tags = self.tag) + + @skipUnlessDevMode() + def test31_findNote(self): + self.Notes.find(notebooks=self.notebook, tags=self.tag) + result = self.storage.getSearch() + self.assertTrue(len(result.notes)==1) + self.notes.add(result.notes[0].guid) + + @skipUnlessDevMode() + def test90_removeNotes(self): + while self.notes: + self.assertTrue(self.Geeknote.removeNote(self.notes.pop())) + + # EXPECTED FAILURE + # "This function is generally not available to third party applications" + # https://dev.evernote.com/doc/reference/NoteStore.html#Fn_NoteStore_expungeNotebook + @skipUnlessDevMode() + def test95_removeNotebooks(self): + while self.nbs: + #self.assertTrue(self.Geeknote.removeNotebook(self.nbs.pop())) + self.assertRaises(SystemExit, self.Geeknote.removeNotebook, self.nbs.pop()) + + @skipUnlessDevMode() + def test99_userLogout(self): + self.user.logout(force=True) + self.assertFalse(self.Geeknote.checkAuth()) + From 6949ef67521f7c3dcb68ced27edcc45ea8db43d8 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 9 May 2016 16:52:31 +0200 Subject: [PATCH 21/24] oauth.py: add degug statement about proxy usage. --- geeknote/oauth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geeknote/oauth.py b/geeknote/oauth.py index dd173fb..1314144 100644 --- a/geeknote/oauth.py +++ b/geeknote/oauth.py @@ -8,7 +8,6 @@ import base64 from urllib import urlencode, unquote, getproxies, proxy_bypass from urlparse import urlparse - import out import tools import config @@ -70,6 +69,7 @@ def __init__(self): # This assumes that the proxy is given in URL form. # A little simpler as _parse_proxy in urllib2.py self._proxy = urlparse(proxy) + logging.debug("Using proxy: %s" % self._proxy.geturl()) if proxy is None or not self._proxy.username: self._proxy_auth = None From 191de8170d1e44ae27b34bf4042e38a5aeeabbff Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 9 May 2016 16:53:09 +0200 Subject: [PATCH 22/24] tests/sandboxTest.py: Add unit test for proxy support This patch adds a proxy test. The sandbox test is repeated with a squid proxy created with the proxyenv package (https://pypi.python.org/pypi/proxyenv). This unit test covers both proxy support for thrift ("normal" evernote requests) and for oauth in geeknote (login/logout calls). This change requires some changes in the setUpClass() procedure of sandboxTest: geeknote needs to be forced to forget the credentials for the login/logout test. Note again that proxy support requires patches for both geeknote and thrift. See https://issues.apache.org/jira/browse/THRIFT-3798. Note also that this test, as the sandbox test in general, is only active with config.DEV_MODE=True. Valid Evernote sandbox credentials are expected in APP_DIR/credentials for this test to run without user interaction. --- tests/sandboxTest.py | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/tests/sandboxTest.py b/tests/sandboxTest.py index 05acf96..1b2a66a 100644 --- a/tests/sandboxTest.py +++ b/tests/sandboxTest.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- import unittest from geeknote import config -from geeknote.geeknote import User, Notebooks, Notes, GeekNote +from geeknote.geeknote import User, Notebooks, Notes, GeekNote, GeekNoteConnector from geeknote.storage import Storage +from geeknote.oauth import GeekNoteAuth from random import SystemRandom from string import hexdigits +from proxyenv.proxyenv import ProxyFactory # see https://docs.python.org/2.7/library/unittest.html §25.3.6 # http://thecodeship.com/patterns/guide-to-python-function-decorators/ @@ -19,15 +21,25 @@ class TestSandbox(unittest.TestCase): @classmethod def setUpClass(cls): + storage = Storage() + # start out with empty auth token. Save current token to restore it later. + cls.token = storage.getUserToken() + cls.info = storage.getUserInfo() + storage.removeUser() + + # Force reconnection and re-authorization because it's part of our test suite + GeekNoteAuth.cookies = {} + GeekNoteConnector.evernote = None + GeekNote.skipInitConnection = False cls.storage = Storage() - cls.token = cls.storage.getUserToken() - cls.info = cls.storage.getUserInfo() - cls.storage.removeUser() cls.notes = set() cls.nbs = set() cls.notebook = ("Geeknote test %s please delete" % "".join(SystemRandom().choice(hexdigits) for x in range(12))) + cls.Notes = Notes() + cls.Notebooks = Notebooks() + cls.Geeknote = cls.Notebooks.getEvernote() @classmethod def tearDownClass(cls): @@ -37,9 +49,6 @@ def tearDownClass(cls): def setUp(self): self.user = User() self.tag = "geeknote_unittest_1" - self.Notes = Notes() - self.Notebooks = Notebooks() - self.Geeknote = self.Notebooks.getEvernote() @skipUnlessDevMode() def test01_userLogin(self): @@ -94,3 +103,22 @@ def test99_userLogout(self): self.user.logout(force=True) self.assertFalse(self.Geeknote.checkAuth()) + +class TestSandboxWithProxy(TestSandbox): + + @classmethod + def setUpClass(cls): + cls.proxy = ProxyFactory()() + cls.proxy.start() + cls.proxy.wait() + cls.proxy.enter_environment() + super(TestSandboxWithProxy, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + super(TestSandboxWithProxy, cls).tearDownClass() + cls.proxy.leave_environment() + try: + cls.proxy.stop() + except: + pass From 4d89b2bb2e6417a468f512c85440634a4ac47a6c Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Mon, 9 May 2016 17:07:17 +0200 Subject: [PATCH 23/24] config.py: Set DEV_MODE=False again ba47972c79 had mistakenly set DEV_MODE=True --- geeknote/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/geeknote/config.py b/geeknote/config.py index 69767dd..0c72985 100644 --- a/geeknote/config.py +++ b/geeknote/config.py @@ -37,8 +37,8 @@ DEF_WIN_EDITOR = "notepad.exe" EDITOR_OPEN = "WRITE" -DEV_MODE = True -DEBUG = False +DEV_MODE = False +APPDEBUG = False # Url view the note NOTE_URL = "https://%domain%/Home.action?#n=%s" From 7dcf81b96735b0f689596662a2dc290c035b28b3 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Tue, 24 May 2016 21:03:35 +0200 Subject: [PATCH 24/24] config.py: Fix stupid typing bug --- geeknote/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geeknote/config.py b/geeknote/config.py index 0c72985..2b5ccf3 100644 --- a/geeknote/config.py +++ b/geeknote/config.py @@ -38,7 +38,7 @@ EDITOR_OPEN = "WRITE" DEV_MODE = False -APPDEBUG = False +DEBUG = False # Url view the note NOTE_URL = "https://%domain%/Home.action?#n=%s"