From 0a589c68e047baea95f640165c155c22beb40277 Mon Sep 17 00:00:00 2001 From: mhadam Date: Mon, 1 Oct 2018 09:46:14 -0400 Subject: [PATCH 1/8] Update PyPI deployment to use Twine (close #207) --- .travis.yml | 1 + .travis/deploy.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7424bc4e..ee02e065 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - '3.4' - '3.5' install: +- pip install -U pip setuptools virtualenv twine - pip install -r requirements-test.txt - pip install release-manager - pip install -e . diff --git a/.travis/deploy.py b/.travis/deploy.py index 7efde113..3ee3e0e5 100755 --- a/.travis/deploy.py +++ b/.travis/deploy.py @@ -13,7 +13,6 @@ HOME = expanduser("~") -DEFAULT_SERVER = 'https://pypi.python.org/pypi' DEFAULT_REPO = 'pypi' PYPIRC_FILE = '%s/.pypirc' % HOME @@ -55,7 +54,6 @@ def write_config(): ' %s\n' % DEFAULT_REPO, '\n', '[%s]\n' % DEFAULT_REPO, - 'repository=%s\n' % DEFAULT_SERVER, 'username=snowplow\n', 'password=%s\n' % PYPI_PASSWORD ] @@ -71,8 +69,8 @@ def deploy_to_pypi(): """Deploys the release to PyPi""" logger.log_start("Deploying to PyPi") os.chdir(TRAVIS_BUILD_DIR) - utils.execute("python setup.py register -r pypi", shell=True) - utils.execute("python setup.py sdist upload -r pypi", shell=True) + utils.execute("python setup.py sdist bdist_wheel", shell=True) + utils.execute("twine upload dist/*", shell=True) logger.log_info("Module deployed to PyPi!") logger.log_done() From ee6fed590f0f4594a10140f3c212e380f58d1ed4 Mon Sep 17 00:00:00 2001 From: mhadam Date: Mon, 1 Oct 2018 09:48:40 -0400 Subject: [PATCH 2/8] Remove Python 3.3 from tests (close #206) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ee02e065..42d762c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ services: - redis-server python: - '2.7' -- '3.3' - '3.4' - '3.5' install: From ca42f4a7683531ee5ea96105b57b87493bf197e4 Mon Sep 17 00:00:00 2001 From: mhadam Date: Mon, 1 Oct 2018 09:52:30 -0400 Subject: [PATCH 3/8] Add version bounds for all dependencies (close #208) --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 88392f47..53a836ec 100644 --- a/setup.py +++ b/setup.py @@ -75,12 +75,12 @@ ], install_requires=[ - "greenlet==0.4.10", - "requests==2.2.1", - "pycontracts==1.7.6", - "celery==3.1.11", - "gevent==1.0.2", - "redis==2.9.1", - "six==1.9.0" + "greenlet>=0.4.10,<1.0", + "requests>=2.2.1,<3.0", + "pycontracts>=1.7.6,<2.0", + "celery>=3.1.11,<4.0", + "gevent>=1.0.2,<2.0", + "redis>=2.9.1,<3.0", + "six>=1.9.0,<2.0" ], ) From cee48e9899dfeac9cd8ec6446b89f698a0a1bfe7 Mon Sep 17 00:00:00 2001 From: mhadam Date: Mon, 1 Oct 2018 10:03:28 -0400 Subject: [PATCH 4/8] Fix date for 0.8.0 release in CHANGELOG (close #183) --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index fe6d5d23..4f54a382 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,4 @@ -Version 0.8.0 (2016-09-xx) +Version 0.8.0 (2016-10-13) -------------------------- Add byte_limit to Emitter (#170) Add support for dvce_sent_tstamp (#159) From b1dff951f6d4e622ce3036f61d005828baf926c2 Mon Sep 17 00:00:00 2001 From: mhadam Date: Mon, 22 Oct 2018 20:07:11 -0400 Subject: [PATCH 5/8] Flush argument shadows the async keyword (#211) --- snowplow_tracker/tracker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/snowplow_tracker/tracker.py b/snowplow_tracker/tracker.py index a6830a68..18516925 100644 --- a/snowplow_tracker/tracker.py +++ b/snowplow_tracker/tracker.py @@ -626,16 +626,16 @@ def track_unstruct_event(self, event_json, context=None, tstamp=None): track_self_describing_event = track_unstruct_event @contract - def flush(self, async=False): + def flush(self, is_async=False): """ Flush the emitter - :param async: Whether the flush is done asynchronously. Default is False - :type async: bool + :param is_async: Whether the flush is done asynchronously. Default is False + :type is_async: bool :rtype: tracker """ for emitter in self.emitters: - if async: + if is_async: emitter.flush() else: emitter.sync_flush() From bcb33481527aead0ebabff9b6ee0a054a06b9206 Mon Sep 17 00:00:00 2001 From: mhadam Date: Mon, 22 Oct 2018 16:09:20 -0400 Subject: [PATCH 6/8] Upgrade Celery to 4.x (close #210) --- setup.py | 2 +- snowplow_tracker/emitters.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 53a836ec..48f7ce13 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ "greenlet>=0.4.10,<1.0", "requests>=2.2.1,<3.0", "pycontracts>=1.7.6,<2.0", - "celery>=3.1.11,<4.0", + "celery>=4.0,<5.0", "gevent>=1.0.2,<2.0", "redis>=2.9.1,<3.0", "six>=1.9.0,<2.0" diff --git a/snowplow_tracker/emitters.py b/snowplow_tracker/emitters.py index 2eccb0fb..ff2380cc 100644 --- a/snowplow_tracker/emitters.py +++ b/snowplow_tracker/emitters.py @@ -31,7 +31,6 @@ from queue import Queue from celery import Celery -from celery.contrib.methods import task import redis import requests from contracts import contract, new_contract @@ -57,9 +56,9 @@ import snowplow_celery_config app = Celery() app.config_from_object(snowplow_celery_config) - except ImportError: # Otherwise configure Celery with default settings + snowplow_celery_config = None app = Celery("Snowplow", broker="redis://guest@localhost//") @@ -172,7 +171,6 @@ def reached_limit(self): else: return self.bytes_queued >= self.byte_limit or len(self.buffer) >= self.buffer_size - @task(name="Flush") def flush(self): """ Sends all events in the buffer to the collector. @@ -387,6 +385,14 @@ def consume(self): self.queue.task_done() +@app.task(bind=True, name='tasks.flush') # the self passed with bind can be used for on_fail/retrying +def flush_emitter(self, emitter): + try: + emitter.flush() + finally: + logger.info("Flush called on emitter") + + class CeleryEmitter(Emitter): """ Uses a Celery worker to send HTTP requests asynchronously. @@ -400,7 +406,7 @@ def flush(self): """ Schedules a flush task """ - super(CeleryEmitter, self).flush.delay() + flush_emitter.delay(self) # passes emitter (self - CeleryEmitter) to task logger.info("Scheduled a Celery task to flush the event queue") From 59b1739b6cac6b55096098e8ae61a9a42f3b33c8 Mon Sep 17 00:00:00 2001 From: Michael Hadam Date: Thu, 29 Nov 2018 19:56:16 -0500 Subject: [PATCH 7/8] Add Python 3.7 to test suite (close #212) --- .travis.yml | 21 +++++++++------- requirements-test.txt | 9 ++++--- run-tests.sh | 56 ++++++++++++++++++++++++------------------- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index 42d762c3..a038d04b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,20 @@ language: python services: -- redis-server + - redis-server python: -- '2.7' -- '3.4' -- '3.5' + - '2.7' + - '3.4' + - '3.5' +matrix: + include: + - python: '3.7' + dist: xenial + sudo: true install: -- pip install -U pip setuptools virtualenv twine -- pip install -r requirements-test.txt -- pip install release-manager -- pip install -e . + - pip install -U pip setuptools virtualenv twine + - pip install -r requirements-test.txt + - pip install -e . + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install release-manager; fi script: pytest --cov=snowplow_tracker after_success: coveralls deploy: diff --git a/requirements-test.txt b/requirements-test.txt index 2741eba4..b4485ca8 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,5 @@ -pytest==3.0.2 +pytest==4.0.1 httmock==1.2.5 -freezegun==0.3.7 -pytest-cov==2.3.1 -coveralls==1.1 - +freezegun==0.3.11 +pytest-cov==2.6.0 +coveralls==1.5.1 diff --git a/run-tests.sh b/run-tests.sh index 27cb18fa..65be0d94 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -15,30 +15,20 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" function deploy { - - # pyenv install 2.7.12 + # pyenv install 2.7.15 if [ ! -f ~/.pyenv/versions/tracker27 ]; then - pyenv virtualenv 2.7.12 tracker27 + pyenv virtualenv 2.7.15 tracker27 pyenv activate tracker27 - pip install -r requirements.txt - pip install -r requirements-test.txt - source deactivate - fi - - # pyenv install 3.3.6 - if [ ! -f ~/.pyenv/versions/tracker33 ]; then - pyenv virtualenv 3.3.6 tracker33 - pyenv activate tracker33 - pip install -r requirements.txt + pip install . pip install -r requirements-test.txt source deactivate fi - # pyenv install 3.4.5 + # pyenv install 3.4.9 if [ ! -f ~/.pyenv/versions/tracker34 ]; then - pyenv virtualenv 3.4.5 tracker34 + pyenv virtualenv 3.4.9 tracker34 pyenv activate tracker34 - pip install -r requirements.txt + pip install . pip install -r requirements-test.txt source deactivate fi @@ -47,32 +37,46 @@ function deploy { if [ ! -f ~/.pyenv/versions/tracker35 ]; then pyenv virtualenv 3.5.2 tracker35 pyenv activate tracker35 - pip install -r requirements.txt + pip install . pip install -r requirements-test.txt source deactivate fi + # pyenv install 3.7.1 + if [ ! -f ~/.pyenv/versions/tracker37 ]; then + pyenv virtualenv 3.7.1 tracker37 + pyenv activate tracker37 + pip install . + pip install -r requirements-test.txt + source deactivate + fi } function run_tests { - pyenv activate tracker27 pytest -s source deactivate - pyenv activate tracker33 - pytest - source deactivate - pyenv activate tracker34 pytest source deactivate pyenv activate tracker35 pytest + source deactivate + + pyenv activate tracker37 + pytest source deactivate -} +} + +function refresh_deploy { + pyenv uninstall -f tracker27 + pyenv uninstall -f tracker34 + pyenv uninstall -f tracker35 + pyenv uninstall -f tracker37 +} case "$1" in @@ -80,9 +84,13 @@ case "$1" in "deploy") echo "Deploying python environments. This can take few minutes" deploy ;; - "test") echo "Running tests" + "test") echo "Running tests" run_tests ;; + "refresh") echo "Refreshing python environments" + refresh_deploy + deploy + ;; *) echo "Unknown subcommand. Specify deploy or test" exit 1 ;; From 2aba5ba4927030c07440c06649095f0b253eb10c Mon Sep 17 00:00:00 2001 From: mhadam Date: Mon, 1 Oct 2018 10:01:23 -0400 Subject: [PATCH 8/8] Prepared for release --- CHANGES.txt | 10 ++++++++++ snowplow_tracker/_version.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4f54a382..c98c9b68 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,13 @@ +Version 0.8.2 (2018-12-01) +-------------------------- +Fix date for 0.8.0 release in CHANGELOG (#183) +Remove Python 3.3 from tests (#206) +Update PyPI deployment to use Twine (#207) +Add version bounds for all dependencies (#208) +Upgrade Celery to 4.x (#210) +Fix flush argument clash with async keyword (#211) +Add Python 3.7 to test suite (close #212) + Version 0.8.0 (2016-10-13) -------------------------- Add byte_limit to Emitter (#170) diff --git a/snowplow_tracker/_version.py b/snowplow_tracker/_version.py index 02e250a1..59f9008f 100644 --- a/snowplow_tracker/_version.py +++ b/snowplow_tracker/_version.py @@ -20,6 +20,6 @@ """ -__version_info__ = (0, 8, 0) +__version_info__ = (0, 8, 2) __version__ = ".".join(str(x) for x in __version_info__) __build_version__ = __version__ + ''