diff --git a/.travis.yml b/.travis.yml index 7424bc4e..a038d04b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,20 @@ language: python services: -- redis-server + - redis-server python: -- '2.7' -- '3.3' -- '3.4' -- '3.5' + - '2.7' + - '3.4' + - '3.5' +matrix: + include: + - python: '3.7' + dist: xenial + sudo: true install: -- pip install -r requirements-test.txt -- pip install release-manager -- pip install -e . + - pip install -U pip setuptools virtualenv twine + - pip install -r requirements-test.txt + - pip install -e . + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install release-manager; fi script: pytest --cov=snowplow_tracker after_success: coveralls deploy: diff --git a/.travis/deploy.py b/.travis/deploy.py index 7efde113..3ee3e0e5 100755 --- a/.travis/deploy.py +++ b/.travis/deploy.py @@ -13,7 +13,6 @@ HOME = expanduser("~") -DEFAULT_SERVER = 'https://pypi.python.org/pypi' DEFAULT_REPO = 'pypi' PYPIRC_FILE = '%s/.pypirc' % HOME @@ -55,7 +54,6 @@ def write_config(): ' %s\n' % DEFAULT_REPO, '\n', '[%s]\n' % DEFAULT_REPO, - 'repository=%s\n' % DEFAULT_SERVER, 'username=snowplow\n', 'password=%s\n' % PYPI_PASSWORD ] @@ -71,8 +69,8 @@ def deploy_to_pypi(): """Deploys the release to PyPi""" logger.log_start("Deploying to PyPi") os.chdir(TRAVIS_BUILD_DIR) - utils.execute("python setup.py register -r pypi", shell=True) - utils.execute("python setup.py sdist upload -r pypi", shell=True) + utils.execute("python setup.py sdist bdist_wheel", shell=True) + utils.execute("twine upload dist/*", shell=True) logger.log_info("Module deployed to PyPi!") logger.log_done() diff --git a/CHANGES.txt b/CHANGES.txt index fe6d5d23..c98c9b68 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,14 @@ -Version 0.8.0 (2016-09-xx) +Version 0.8.2 (2018-12-01) +-------------------------- +Fix date for 0.8.0 release in CHANGELOG (#183) +Remove Python 3.3 from tests (#206) +Update PyPI deployment to use Twine (#207) +Add version bounds for all dependencies (#208) +Upgrade Celery to 4.x (#210) +Fix flush argument clash with async keyword (#211) +Add Python 3.7 to test suite (close #212) + +Version 0.8.0 (2016-10-13) -------------------------- Add byte_limit to Emitter (#170) Add support for dvce_sent_tstamp (#159) diff --git a/requirements-test.txt b/requirements-test.txt index 2741eba4..b4485ca8 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,5 @@ -pytest==3.0.2 +pytest==4.0.1 httmock==1.2.5 -freezegun==0.3.7 -pytest-cov==2.3.1 -coveralls==1.1 - +freezegun==0.3.11 +pytest-cov==2.6.0 +coveralls==1.5.1 diff --git a/run-tests.sh b/run-tests.sh index 27cb18fa..65be0d94 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -15,30 +15,20 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" function deploy { - - # pyenv install 2.7.12 + # pyenv install 2.7.15 if [ ! -f ~/.pyenv/versions/tracker27 ]; then - pyenv virtualenv 2.7.12 tracker27 + pyenv virtualenv 2.7.15 tracker27 pyenv activate tracker27 - pip install -r requirements.txt - pip install -r requirements-test.txt - source deactivate - fi - - # pyenv install 3.3.6 - if [ ! -f ~/.pyenv/versions/tracker33 ]; then - pyenv virtualenv 3.3.6 tracker33 - pyenv activate tracker33 - pip install -r requirements.txt + pip install . pip install -r requirements-test.txt source deactivate fi - # pyenv install 3.4.5 + # pyenv install 3.4.9 if [ ! -f ~/.pyenv/versions/tracker34 ]; then - pyenv virtualenv 3.4.5 tracker34 + pyenv virtualenv 3.4.9 tracker34 pyenv activate tracker34 - pip install -r requirements.txt + pip install . pip install -r requirements-test.txt source deactivate fi @@ -47,32 +37,46 @@ function deploy { if [ ! -f ~/.pyenv/versions/tracker35 ]; then pyenv virtualenv 3.5.2 tracker35 pyenv activate tracker35 - pip install -r requirements.txt + pip install . pip install -r requirements-test.txt source deactivate fi + # pyenv install 3.7.1 + if [ ! -f ~/.pyenv/versions/tracker37 ]; then + pyenv virtualenv 3.7.1 tracker37 + pyenv activate tracker37 + pip install . + pip install -r requirements-test.txt + source deactivate + fi } function run_tests { - pyenv activate tracker27 pytest -s source deactivate - pyenv activate tracker33 - pytest - source deactivate - pyenv activate tracker34 pytest source deactivate pyenv activate tracker35 pytest + source deactivate + + pyenv activate tracker37 + pytest source deactivate -} +} + +function refresh_deploy { + pyenv uninstall -f tracker27 + pyenv uninstall -f tracker34 + pyenv uninstall -f tracker35 + pyenv uninstall -f tracker37 +} case "$1" in @@ -80,9 +84,13 @@ case "$1" in "deploy") echo "Deploying python environments. This can take few minutes" deploy ;; - "test") echo "Running tests" + "test") echo "Running tests" run_tests ;; + "refresh") echo "Refreshing python environments" + refresh_deploy + deploy + ;; *) echo "Unknown subcommand. Specify deploy or test" exit 1 ;; diff --git a/setup.py b/setup.py index 88392f47..48f7ce13 100644 --- a/setup.py +++ b/setup.py @@ -75,12 +75,12 @@ ], install_requires=[ - "greenlet==0.4.10", - "requests==2.2.1", - "pycontracts==1.7.6", - "celery==3.1.11", - "gevent==1.0.2", - "redis==2.9.1", - "six==1.9.0" + "greenlet>=0.4.10,<1.0", + "requests>=2.2.1,<3.0", + "pycontracts>=1.7.6,<2.0", + "celery>=4.0,<5.0", + "gevent>=1.0.2,<2.0", + "redis>=2.9.1,<3.0", + "six>=1.9.0,<2.0" ], ) diff --git a/snowplow_tracker/_version.py b/snowplow_tracker/_version.py index 02e250a1..59f9008f 100644 --- a/snowplow_tracker/_version.py +++ b/snowplow_tracker/_version.py @@ -20,6 +20,6 @@ """ -__version_info__ = (0, 8, 0) +__version_info__ = (0, 8, 2) __version__ = ".".join(str(x) for x in __version_info__) __build_version__ = __version__ + '' diff --git a/snowplow_tracker/emitters.py b/snowplow_tracker/emitters.py index 2eccb0fb..ff2380cc 100644 --- a/snowplow_tracker/emitters.py +++ b/snowplow_tracker/emitters.py @@ -31,7 +31,6 @@ from queue import Queue from celery import Celery -from celery.contrib.methods import task import redis import requests from contracts import contract, new_contract @@ -57,9 +56,9 @@ import snowplow_celery_config app = Celery() app.config_from_object(snowplow_celery_config) - except ImportError: # Otherwise configure Celery with default settings + snowplow_celery_config = None app = Celery("Snowplow", broker="redis://guest@localhost//") @@ -172,7 +171,6 @@ def reached_limit(self): else: return self.bytes_queued >= self.byte_limit or len(self.buffer) >= self.buffer_size - @task(name="Flush") def flush(self): """ Sends all events in the buffer to the collector. @@ -387,6 +385,14 @@ def consume(self): self.queue.task_done() +@app.task(bind=True, name='tasks.flush') # the self passed with bind can be used for on_fail/retrying +def flush_emitter(self, emitter): + try: + emitter.flush() + finally: + logger.info("Flush called on emitter") + + class CeleryEmitter(Emitter): """ Uses a Celery worker to send HTTP requests asynchronously. @@ -400,7 +406,7 @@ def flush(self): """ Schedules a flush task """ - super(CeleryEmitter, self).flush.delay() + flush_emitter.delay(self) # passes emitter (self - CeleryEmitter) to task logger.info("Scheduled a Celery task to flush the event queue") diff --git a/snowplow_tracker/tracker.py b/snowplow_tracker/tracker.py index a6830a68..18516925 100644 --- a/snowplow_tracker/tracker.py +++ b/snowplow_tracker/tracker.py @@ -626,16 +626,16 @@ def track_unstruct_event(self, event_json, context=None, tstamp=None): track_self_describing_event = track_unstruct_event @contract - def flush(self, async=False): + def flush(self, is_async=False): """ Flush the emitter - :param async: Whether the flush is done asynchronously. Default is False - :type async: bool + :param is_async: Whether the flush is done asynchronously. Default is False + :type is_async: bool :rtype: tracker """ for emitter in self.emitters: - if async: + if is_async: emitter.flush() else: emitter.sync_flush()