diff --git a/.github/workflows/anchore.yml b/.github/workflows/anchore.yml index 7eff99f17..69a65ead0 100644 --- a/.github/workflows/anchore.yml +++ b/.github/workflows/anchore.yml @@ -37,13 +37,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout the code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Build the Docker image run: docker pull python:3-alpine; docker build . --file scripts/Docker/Dockerfile --tag localbuild/testimage:latest - name: List the Docker image run: docker image ls - name: Run the Anchore scan action itself with GitHub Advanced Security code scanning integration enabled - uses: anchore/scan-action@1638637db639e0ade3258b51db49a9a137574c3e # 6.5.1 + uses: anchore/scan-action@8d2fce09422cd6037e577f4130e9b925e9a37175 # 7.3.1 id: scan with: image: "localbuild/testimage:latest" diff --git a/.github/workflows/build-xapian.yml b/.github/workflows/build-xapian.yml index 4d6edf6dc..262a542da 100644 --- a/.github/workflows/build-xapian.yml +++ b/.github/workflows/build-xapian.yml @@ -42,11 +42,11 @@ jobs: # if: {{ false }} # continue running if step fails # continue-on-error: true - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # Setup version of Python to use - name: Set Up Python 3.13 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.13 allow-prereleases: true diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 8e88da76c..d382541d5 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -52,14 +52,14 @@ jobs: # Run in all these versions of Python python-version: # - "2.7" + - "3.14" - "3.13" - # - 3.6 run via include on ubuntu 20.04 - # - "3.7" + # - "3.7" run via include for ubuntu-22.04 # - "3.8" run via include for ubuntu-22.04 # - "3.9" - "3.10" # - "3.11" - - "3.12" + # - "3.12" # use for multiple os or ubuntu versions #os: [ubuntu-latest, macos-latest, windows-latest] @@ -74,7 +74,7 @@ jobs: # example: if this version fails the jobs still succeeds # allow-prereleases in setup-python allows alpha/beta # releases to run. Also allow free threaded python testing - - python-version: 3.13t + - python-version: 3.14t os: ubuntu-24.04 experimental: true @@ -116,11 +116,11 @@ jobs: # if: {{ false }} # continue running if step fails # continue-on-error: true - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # Setup version of Python to use - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} allow-prereleases: true @@ -257,6 +257,11 @@ jobs: run: | set -xv pip install zstd || true + # justhtml supports python 3.10 or newer according to pypi + # page, but will install on 3.8 (and maybe 3.9) but formats + # differently. So skip install if before 3.10. + if echo $PYTHON_VERSION | grep '^3\...'; then + pip install justhtml || true; fi if [[ "$PYTHON_VERSION" != "2."* ]]; then pip install Markdown; fi @@ -323,7 +328,7 @@ jobs: - name: Upload coverage to Codecov # see: https://github.com/codecov/codecov-action#usage - uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: verbose: true token: ${{ secrets.CODECOV_TOKEN }} @@ -331,7 +336,7 @@ jobs: - name: Upload coverage to Coveralls # python 2.7 and 3.6 versions of coverage can't produce lcov files. if: matrix.python-version != '2.7' && matrix.python-version != '3.6' - uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6 + uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7 with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: coverage.lcov @@ -367,7 +372,7 @@ jobs: steps: - name: Coveralls Finished - uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6 + uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7 with: github-token: ${{ secrets.github_token }} parallel-finished: true diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0b214cc78..8d4599924 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -49,7 +49,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index c6a5547e3..85622e1ec 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -35,12 +35,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v5.2.1 + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v5.2.1 with: results_file: results.sarif results_format: sarif @@ -62,7 +62,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: SARIF file path: results.sarif diff --git a/CHANGES.txt b/CHANGES.txt index 122e83527..77d59d070 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -23,12 +23,56 @@ Fixed: roundup.cgi.exceptions. Also it now inherits from HTTPException rather than Exception since it is an HTTP exception. (John Rouillard) +- cleaned up repo. Close obsolete branches and close a split head due + to an identical merge in two different working copies. (John + Rouillard) +- in roundup-admin, using 'pragma history_length interactively now + sets readline history length. Using -P history_length=10 on the + command line always worked. (John Rouillard) +- enhanced error reporting for errors in ini style logging + configuration. (John Rouillard) +- fix bogus javascript emitted by user_src_input macro. (John + Rouillard) +- replaced hostname localhost with 127.0.0.1 in docker healthcheck + script. Found/patch by Norbert Schlemmer. (John Rouillard) +- change some internal classes to use __slots__ for hopefully a small + performance improvement. (John Rouillard) +- issue2551413 - Broken MultiLink columns in CSV export. CSV export of + a multilink link "messages" that does not have a 'name' property + causes a crash. (found/fix by cmeerw; commit and better handling of + non-labeled multilink by John Rouillard) +- in cgi/client.py, set self.language attribute when translator passed + into Client(). (John Rouillard) +- issue2551393 - Named searches lose their name in title when next + page is selected. (John Rouillard) Features: - add support for authorized changes. User can be prompted to enter their password to authorize a change. If the user's password is properly entered, the change is committed. (John Rouillard) +- add support for dictConfig style logging configuration. Ini/File + style configs will still be supported. (John Rouillard) +- add 'q' as alias for quit in roundup-admin interactive mode. (John + Rouillard) +- add readline command to roundup-admin to list history, control input + mode etc. Also support bang (!) commands to rerun commands in history + or put them in the input buffer for editing. (John Rouillard) +- add format to logging section in config.ini. Used to set default + logging format. (John Rouillard) +- the default logging format template includes an identifier unique + for a request. This identifier (trace_id) can be use to identify + logs for a specific transaction. Will use nanoid if installed, uses + uuid.uuid4 otherwise. Logging also supports a trace_reason log token + with the url for a web request. The logging format can be changed in + config.ini. (John Rouillard) +- issue2551152 - added basic PGP setup/use info to admin_guide. (John + Rouillard) +- add support for the 'justhtml' html 5 parser library for python >= + 3.10. It is written in pure Python. Used to convert html emails into + plain text. Faster then beautifulsoup4 and it passes the html 5 + standard browser test suite. Beautifulsoup is still supported. (John + Rouillard) 2025-07-13 2.5.0 diff --git a/detectors/irker.py b/detectors/irker.py index 57ce2bfb1..ff330cbbc 100644 --- a/detectors/irker.py +++ b/detectors/irker.py @@ -1,6 +1,6 @@ # This detector sends notification on IRC through an irker daemon -# (http://www.catb.org/esr/irker/) when issues are created or messages -# are added. +# (https://gitlab.com/esr/irker formerly http://www.catb.org/esr/irker/) +# when issues are created or messages are added. # # Written by Ezio Melotti # diff --git a/doc/_static/style.css b/doc/_static/style.css index ef5215caf..60f673a2c 100644 --- a/doc/_static/style.css +++ b/doc/_static/style.css @@ -186,6 +186,8 @@ div[class^=highlight-], div[class^=highlight-] * { width: /* style */ :link { color: rgb(220,0,0); text-decoration: none;} +/* improve contrast to AA */ +.admonition.note :link { color: rgb(170,1,1); text-decoration: none;} :link:hover { text-decoration: underline solid clamp(1px, .3ex, 4px); text-underline-position: under; @@ -438,6 +440,14 @@ dd > ul:first-child { white-space: break-spaces; }*/ +/* improve color contrast to AA against yellowish highlight bg */ +div.highlight .c1 { + color: rgb(3,114,3); +} +div.highlight .na { + color: rgb(220,2,2); +} + /* Forcing wrap in a pre leads to some confusing line breaks. Use a horizontal scroll. Indicate the scroll by using rounded scroll shadows. diff --git a/doc/admin_guide.txt b/doc/admin_guide.txt index 538ad8fbe..4ee64d636 100644 --- a/doc/admin_guide.txt +++ b/doc/admin_guide.txt @@ -46,31 +46,542 @@ There's two "installations" that we talk about when using Roundup: both choosing your "tracker home" and the ``main`` -> ``database`` variable in the tracker's config.ini. +.. _rounduplogging: + +Configuring Roundup Message Logging +=================================== + +You can control how Roundup logs messages using your tracker's +config.ini file. Roundup uses the standard Python logging +implementation. The config file and ``roundup-server`` provide +very basic control over logging. + +``roundup-server``'s logging is controlled from the command +line. You can: + +- specify the location of a log file or +- enable logging using the standard Python logging library under + the tag/channel ``roundup.http`` + +Configuration for "BasicLogging" implementation for your tracker +is done using the settings in the tracker's ``config.ini`` under +the ``logging`` section: + +- ``filename`` setting: specifies the location of a log file +- ``level`` setting: specifies the minimum level to log +- ``disable_loggers`` setting: disable other loggers (e.g. when + running under a wsgi framework) +- ``format`` setting: set the log format template. See + :ref:`logFormat` for more info. + +In either case, if logfile is not specified, logging is sent to +sys.stderr. If level is not set, only ERROR or higher priority +log messages will be reported. + +You can get more control over logging by using the ``config`` +setting in the tracker's ``config.ini``. Using a logging config +file overrides all the rest of the other logging settings in +``config.ini``. You get more control over the logs by supplying +a log config file in ini or json (dictionary) format. + +Using this, you can set different levels by channel. For example +roundup.hyperdb can be set to WARNING while other Roundup log +channels are set to INFO and the roundup.mailgw channel logs at +the DEBUG level. You can also control the distribution of +logs. For example roundup.mailgw logs to syslog, other channels +log to an automatically rotating log file, or are submitted to +your log aggregator over https. + +.. _logFormat: + +Defining the Log Format +----------------------- + +Starting with Roundup 2.6 you can specify the logging format in +config.ini. The ``logging`` -> ``format`` setting of config.ini +supports all of the `standard logging LogRecord attributes +`_ +or Roundup logging attributes. However you must double any ``%`` +format markers. The default value is:: + + %%(asctime)s %%(trace_id)s %%(levelname)s %%(message)s + +Roundup Logging Attributes +-------------------------- + +The `logging package has a number of attributes +`_ +that can be expanded in the format template. In addition to the +ones supplied by Python's logging module, Roundup defines +additional attributes: + + trace_id + a unique string that is generated for each request. It is + unique per thread. + + trace_reason + a string describing the reason for the trace/request. + + * the URL for a web triggered (http, rest, xmlrpc) request + * the email message id for an email triggered request + * the roundup-admin os user and start of command. Only first + two words in command are printed so seting a password will + not be leaked to the logs. + + sinfo + the stack traceback information at the time the log call id + made. + + This must be intentionally activated by using the extras + parameter. For example calling:: + + logging.get_logger('roundup.something').warning( + "I am here\n%(sinfo)s", extra={"sinfo": 2}) + + in the function confirmid() of the file detectors/reauth.py + in your demo tracker will print 2 items on the stack + including the log call. It results in the following (5 lines + total in the log file):: + + 2025-09-14 23:07:58,668 Cm0ZPlBaklLZ3Mm6hAAgoC WARNING I am here + File "[...]/roundup/hyperdb.py", line 1924, in fireAuditors + audit(self.db, self, nodeid, newvalues) + File "demo/detectors/reauth.py", line 7, in confirmid + logging.getLogger('roundup.something').warning( + + Note that the output does not include the arguments to + ``warning`` because they are on the following line. If you + want arguments to the log call included, they have to be on + the same line. + + Setting ``sinfo`` to an integer value N includes N lines up + the stack ending with the logging call. Setting it to 0 + includes all the lines in the stack ending with the logging + call. + + If the value is less than 0, the stack dump doesn't end at + the logging call but continues to the function that + generates the stack report. So it includes functions inside + the logging module. + + Setting it to a number larger than the stack trace will + print the trace down to the log call. So using ``-1000`` + will print up to 1000 stack frames and start at the function + that generates the stack report. + + Setting ``sinfo`` to a non-integer value ``{"sinfo": None}`` + will produce 5 lines of the stack trace ending at the + logging call. + + pct_char + produces a single ``%`` sign in the log. The usual way of + embedding a percent sign in a formatted string is to double + it like: ``%%``. However when the format string is specified + in the config.ini file percent signs are manipulated. So + ``%%(pct_char)s`` can be used in config.ini to print a + percent sign. + +The default logging template is defined in config.ini in the +``logging`` -> ``format`` setting. It includes the ``trace_id``. +When searching logs, you can use the trace_id to see all the log +messages associated with a request. + +If you want to log from a detector, extension or other code, you +can use these tokens in your messages when calling the logging +functions. (Note that doubling ``%`` signs is only required when +defining the log format in a config file, not when defining a +msg.) For example:: + + logging.getLogger('roundup.myextension').error('problem with ' + '%(trace_reason)s') + +will include the url in the message when triggered from the +web. This also works with other log methods: ``warning()``, +``debug()``, .... + +Note you must **not** use positional arguments in your +message. Using:: + + logging.getLogger('roundup.myextension').error( + '%s problem with %(trace_reason)s', "a") + +will not properly substitute the argument. You must use mapping +key based arguments and define the local values as part of the +extra dictionary. For example:: + + logging.getLogger('roundup.myextension').error('%(article)s ' + 'problem with %(trace_reason)', + extra={"article": some_local_variable}) + +Also if you are logging any data supplied by a user, you must not +log it directly. If the variable ``url`` contains the url typed in +by the user, never use: + + logger.info(url) + +or + + logger.info("Url is %s" % url) + +Use: + + logger.info("Url is %s", url) + +or + + logger.info("Url is %(url)s", extra={"url": url) + +This prevents printf style tokens in ``url`` from being processed +where it can raise an exception. This could be used to prevent +the log message from being generated. + +More on trace_id +~~~~~~~~~~~~~~~~ + +The trace_id provides a unique token (a UUID4 encoded to make it +shorter or a nanoid) for each transaction in the database. It is +unique to each thread or transaction. A transaction: -Configuring Roundup's Logging of Messages For Sysadmins -======================================================= + for the web interface is + each web, rest or xmlrpc request -You may configure where Roundup logs messages in your tracker's config.ini -file. Roundup will use the standard Python (2.3+) logging implementation. + for the email interface is + each email request. Using pipe mode will generate one + transaction. Using pop/imap etc can generate multiple + transactions, one for each email. Logging that occurs prior + to processing an email transaction has the default + ``not_set`` value for trace_id -Configuration for standard "logging" module: - - tracker configuration file specifies the location of a logging - configration file as ``logging`` -> ``config`` - - ``roundup-server`` specifies the location of a logging configuration - file on the command line -Configuration for "BasicLogging" implementation: - - tracker configuration file specifies the location of a log file - ``logging`` -> ``filename`` - - tracker configuration file specifies the level to log to as - ``logging`` -> ``level`` - - ``roundup-server`` specifies the location of a log file on the command - line - - ``roundup-server`` specifies the level to log to on the command line + for the roundup-admin interface is + each command in the interactive interface or on the command + line. Plus one transaction when/if a commit happens on + roundup-admin exit. -(``roundup-mailgw`` always logs to the tracker's log file) +When creating scripts written using the roundup package the entry +point should use the ``@gen_trace_id`` decorator. For example to +decorate the entry point that performs one transaction:: -In both cases, if no logfile is specified then logging will simply be sent -to sys.stderr with only logging of ERROR messages. + from roundup.logcontext import gen_trace_id + + # stuff ... + + @gen_trace_id() + def main(...): + ... + +If your script does multiple processing operations, decorate the entry +point for the processing operation:: + + from roundup.logcontext import gen_trace_id + + @gen_trace_id() + def process_one(thing): + ... + + def main(): + for thing in things: + process_one(thing) + +You can change the format of the trace_id if required using the +tracker's interfaces.py file. See the :ref:`module docs for the +logcontext module ` for details. + +Advanced Logging Setup +---------------------- + +If the settings in config.ini are not sufficient for your logging +requirements, you can specify a full logging configuration in one +of two formats: + + * `dictConfig format + `_ + using json with comment support + * `fileConfig format + `_ + in ini style + +The dictConfig format allows more control over configuration +including loading your own log handlers and disabling existing +handlers. If you use the fileConfig format, the ``logging`` -> +``disable_loggers`` flag in the tracker's config is used to +enable/disable pre-existing loggers as there is no way to do this +in the logging config file. + +.. _`dictLogConfig`: + +dictConfig Based Logging Config +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +dictConfigs are specified in JSON format with support for +comments. The file name in the tracker's config for the +``logging`` -> ``config`` setting must end with ``.json`` to +choose the correct processing. + +Comments have to be in one of two forms based on javascript line +comments: + +1. A ``//`` possibly indented with whitespace on a line is + considereda a comment and is stripped from the file before + being passed to the json parser. This is a "line comment". + +2. A ``//`` with at least three white space characters before it + is stripped from the end of the line before being passed to + the json parser. This is an "inline comment". + +Block style comments are not supported. + +Other than this the file is a standard json file that matches the +`Configuration dictionary schema +`_ +defined in the Python documentation. + + +Example dictConfig Logging Config +................................. + +Note that this file is not actually JSON format as it include +comments. However by using javascript style comments, some tools +that treat JSON like javascript (editors, linters, formatters) +might work with it. A command like:: + + sed -e 's#^\s*//.*##' -e 's#\s*\s\s\s//.*##' logging.json + +can be used to strip comments for programs that need it. + +The config below works with the `Waitress wsgi server +`_ configured to use the +roundup.wsgi channel. It also controls the `TransLogger +middleware `_ configured to +use roundup.wsgi.translogger, to produce httpd style combined +logs. + +The log file is specified relative to the current working +directory not the tracker home. The tracker home is the +subdirectory demo under the current working directory. + +The config also expects the ``python-json-logger`` package to be +installed so that it can produce a jsonl (json line) formatted +output log file. This format is useful for sending to log +management/observability platforms like elasticsearch, splunk, +logly, or honeycomb. + +The commented config is:: + + { + "version": 1, // only supported version + "disable_existing_loggers": false, // keep the wsgi loggers + + "formatters": { + // standard format for Roundup messages + "standard": { + "format": "%(asctime)s %(trace_id)s %(levelname)s %(name)s:%(module)s %(message)s" + }, + // Used to dump all log requests in jsonl format. + // Each json object is on one line. Can be pretty printed + // using: + // python -m json.tool --json-lines --sort-keys < roundup.json.log + // jq --slurp --sort-keys . < roundup.json.log + // requires that you pip install python-json-logger + // * does not report the fields in reserved_attrs + // * example to remap a field in the log to traceID in + // the output json. (note trace_id_eg is not defined by + // logging + // * also adds the env atribute to json with the value of demo + "json": { + "()": "pythonjsonlogger.json.JsonFormatter", + "reserved_attrs": ["ROUNDUP_CONTEXT_FILTER_CALLED", + "msg", "pct_char", "relativeCreated"], + "rename_fields": { + "trace_id_eg": "traceID" + }, + "static_fields": { + "env": "demo" + } + }, + // used for waitress wsgi server to produce httpd style logs + "http": { + "format": "%(message)s %(trace_id)" + + } + }, + "handlers": { + // create an access.log style http log file + "access": { + "level": "INFO", + "formatter": "http", + "class": "logging.FileHandler", + "filename": "demo/access.log" + }, + // logging for roundup.* loggers + "roundup": { + "level": "DEBUG", + "formatter": "standard", + "class": "logging.FileHandler", + "filename": "demo/roundup.log" + }, + // handler for json output log file + "roundup_json": { + "level": "DEBUG", // "DEBUG", + "formatter": "json", + "class": "logging.FileHandler", + "filename": "demo/roundup.json.log" + }, + // print to stdout - fall through for other logging + "default": { + "level": "DEBUG", + "formatter": "standard", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout" + } + }, + "loggers": { + "": { + "handlers": [ + "default", + "roundup_json" // add json formatted logging + ], + "level": "DEBUG", + "propagate": false + }, + // used by roundup.* loggers + "roundup": { + "handlers": [ + "roundup", + "roundup_json" + ], + "level": "DEBUG", + "propagate": false // note pytest testing with caplog requires + // this to be true + }, + "roundup.hyperdb": { + "handlers": [ + "roundup" + ], + "level": "INFO", // can be a little noisy use INFO for production + "propagate": false + }, + "roundup.wsgi": { // using the waitress framework + "handlers": [ + "roundup" + ], + "level": "DEBUG", + "propagate": false + }, + "roundup.wsgi.translogger": { // httpd style logging + "handlers": [ + "access" + ], + "level": "DEBUG", + "propagate": false + }, + "root": { + "handlers": [ + "default" + ], + "level": "DEBUG", + "propagate": false + } + } + } + +fileConfig Based Logging Config +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The file config is an older and more limited method of +configuring logging. It is described by the `Configuration file +format +`_ +in the Python documentation. The file name in the tracker's +config for the ``logging`` -> ``config`` setting must end with +``.ini`` to choose the correct processing. + +Example fileConfig LoggingConfig +................................ + +This is an example .ini used with roundup-server configured to use +``roundup.http`` channel. It also includes some custom logging +qualnames/tags/channels for logging schema/permission detector and +extension output:: + + [loggers] + #other keys: roundup.hyperdb.backend + keys=root,roundup,roundup.http,roundup.hyperdb,actions,schema,extension,detector + + [logger_root] + #also for root where channlel is not set (NOTSET) aka all + level=DEBUG + handlers=rotate + + [logger_roundup] + # logger for all roundup.* not otherwise configured + level=DEBUG + handlers=rotate + qualname=roundup + propagate=0 + + [logger_roundup.http] + level=INFO + handlers=rotate_weblog + qualname=roundup.http + propagate=0 + + [logger_roundup.hyperdb] + level=WARNING + handlers=rotate + qualname=roundup.hyperdb + propagate=0 + + [logger_actions] + level=INFO + handlers=rotate + qualname=actions + propagate=0 + + [logger_detector] + level=INFO + handlers=rotate + qualname=detector + propagate=0 + + [logger_schema] + level=DEBUG + handlers=rotate + qualname=schema + propagate=0 + + [logger_extension] + level=INFO + handlers=rotate + qualname=extension + propagate=0 + + [handlers] + keys=basic,rotate,rotate_weblog + + [handler_basic] + class=StreamHandler + args=(sys.stderr,) + formatter=basic + + [handler_rotate] + class=logging.handlers.RotatingFileHandler + args=('roundup.log','a', 5120000, 2) + formatter=basic + + [handler_rotate_weblog] + class=logging.handlers.RotatingFileHandler + args=('httpd.log','a', 1024000, 2) + formatter=plain + + [formatters] + keys=basic,plain + + [formatter_basic] + format=%(asctime)s %(trace_id)s %(process)d %(name)s:%(module)s.%(funcName)s,%(levelname)s: %(message)s + datefmt=%Y-%m-%d %H:%M:%S + + [formatter_plain] + format=%(process)d %(message)s Configuring roundup-server @@ -270,7 +781,7 @@ on the fly, compression is enabled by default, to disable it set:: dynamic_compression = No in the tracker's ``config.ini``. You should disable compression if -your proxy (e.g. nginx or apache) or wsgi server (uwsgi) is configured +your proxy (e.g. nginx or apache) is configured to compress responses on the fly. The python standard library includes gzip support. For brotli or zstd you will need to install packages. See the `installation documentation`_ for details. @@ -432,9 +943,9 @@ There are two ways to add a CSP: Fixed CSP --------- -If you are using a web server (Apache, Nginx) to run Roundup, you can -add a ``Content-Security-Policy`` header using that server. WSGI -servers like uWSGI can also be configured to add headers. An example +If you are using a web server (Apache, Nginx) to run Roundup, you +can add a ``Content-Security-Policy`` header using that +server. WSGI middleware can be written to add headers. An example header would look like:: Content-Security-Policy: default-src 'self' 'unsafe-inline' 'strict-dynamic'; @@ -1362,6 +1873,106 @@ Because environment variables can be inadvertently exposed in logs or process listings, Roundup does not currently support loading secrets from environment variables. +.. _pgpconfig: + +Configuring PGP Email Support +============================= + +.. note:: + This section was written with the help of the Devin/DeepWiki AI. + +You have to install the gpg module using pip. See :ref:`directions for +installing gpg ` +in the upgrading document for more information. + +In your tracker's config.ini configure the following settings in the +``[pgp]`` section:: + + enable = yes + homedir = /path/to/pgp/configdir + roles = admin + +This will allow any user with the admin role to send signed pgp +email. If ``roles`` is not set, all users will need to use signed +emails. If it is not signed it will be rejected. Note that ``homedir`` +must be an absolute path. Unlike other path settings, a relative path +is not interpreted relative to the tracker home. See the documentation +in config.ini for more information and other settings (e.g. to send +encrypted emails from the tracker). + +When PGP is enabled and a message is signed with a valid signature, +the database transaction source (db.tx_Source) is set to +``email-sig-openpgp`` instead of ``email``. This allows you to +restrict certain operations (e.g. changing a private flag) to +authenticated/signed emails. + +Creating GPG Keys for the Tracker +--------------------------------- + +To generate a keypair use:: + + gpg --homedir /path/to/pgp/configdir --gen-key + +where the homedir directory matches the one you set in +config.ini. Note the gpg homedir must be created before you run the +command. You will be prompted for the full name of your tracker and +the email address for your tracker. You also need to do with as the +user who runs roundup (aka the roundup user) and the roundup email +gateway. Do not encrypt the key. + +Roundup has no mechanism for reading the private key if it is +encrypted. So make sure the permissions on the homedir only allow the +roundup user to read the files. + +You can export the public key for use by clients using:: + + gpg --homedir /path/to/pgp/configdir --export -a tracker@example.com > tracker-public.key + +with homedir and email matching the values used to generate the +key. This will allow users to import the public key and encrypt emails +to the tracker. + +The public gpg key for each user's email address must be imported. To +do this, obtain the user's public key for their primary email address +and import it using:: + + gpg --homedir /path/to/tracker/gpg --import user-public-key.asc + +You may also be able to get it from a public keyserver using:: + + gpg --recv-keys KEYID + +where the ``KEYID`` is supplied by the roundup user. + +While Roundup supports multiple addresses for each user, only the +primary address supports PGP signed or encrypted messages. + +You should verify that the public key is sane and has few signatures +attached. You can import a key into a throw away keystore:: + + mkdir throwaway + gpg --homedir throwaway -- import user-public-key.asc + gpg --homedir throwaway --list-sigs + +and verify that the number of sig lines is small (under 10 or so). If +it takes a long time to import you can kill the import without +affecting your production keystore. Large numbers of sig lines can +take a long time to import/access when compressed. See: +https://nvd.nist.gov/vuln/detail/CVE-2022-3219. + +.. comment: + Questions: + + Can roundup send signed emails? (looks like no, why??) + + Why are alternate addresses not supported for receiving PGP emails? + + Does Roundup ever send an email to an alternate email address? + + Should there be some way for a user to upload their own public key? + If so what ui (paste armored asci cert in textbox, upload ascii + file from user page and process)? + Tasks ===== @@ -1808,6 +2419,9 @@ The basic usage is: Commands may be abbreviated as long as the abbreviation matches only one command, e.g. l == li == lis == list. +In interactive mode entering: ``q``, ``quit``, or ``exit`` alone on a +line will exit the program. + One thing to note, The ``-u user`` setting does not currently operate like a user logging in via the web. The user running roundup-admin must have read access to the tracker home directory. As a result the @@ -1898,13 +2512,16 @@ you can put the initialise command and password on the command line. But this allows others on the host to see the password (using the ps command). To initialise a tracker non-interactively without exposing the password, create a file (e.g init_tracker) set to mode -600 (so only the owner can read it) with the contents: +600 (so only the owner can read it) with the contents:: initialise admin_password -and feed it to roundup-admin on standard input. E.G. +and feed it to roundup-admin on standard input. E.G.:: + + cat init_tracker | roundup-admin -i tracker_dir -P history_features=2 - cat init_tracker | roundup-admin -i tracker_dir +setting the pragma ``history_features=2`` prevents storing the command +in the user's history file. (for more details see https://issues.roundup-tracker.org/issue2550789.) @@ -1937,6 +2554,57 @@ https://pythonhosted.org/pyreadline/usage.html#configuration-file. History is saved to the file ``.roundup_admin_history`` in your home directory (for windows usually ``\Users\``. +In Roundup 2.6.0 and newer, you can use the ``readline`` command to +make changes on the fly. + +* ``readline vi`` - change input mode to use vi key binding when + editing. It starts in entry mode. +* ``readline emacs`` - change input mode to emacs key bindings when + editing. This is also the default. +* ``readline reload`` - reloads the ``~/.roundup_admin_rlrc`` file so + you can test and use changes. +* ``readline history`` - dumps the history buffer and numbers all + commands. +* ``readline .inputrc_command_line`` can be used to make on the fly + key and key sequence bindings to readline commands. It can also be + used to change the internal readline settings using a set + command. For example:: + + readline set bell-style none + + will turn off a ``visible`` or ``audible`` bell. Single character + keybindings:: + + readline Control-o: dump-variables + + to list all the variables that can be set are supported. As are + multi-character bindings:: + + readline "\C-o1": "commit" + + will put "commit" on the input line when you type Control-o followed + by 1. See the `readline manual for details + `_ + on the command lines that can be used. + +Also a limited form of ``!`` (bang) history reference was added. The +reference must be at the start of the line. Typing ``!23`` will rerun +command number 23 from your history. + +Typing ``!23:p`` will load command 23 into the buffer so you can edit +and submit it. Using the bang feature will append the command to the +end of the history list. + +Pyreadline3 users can use ``readline history`` and the +bang commands (including ``:p``). Single character bindings can be +done. For example:: + + readline Control-w: history-search-backward + +The commands that are available are limited compared to Unix's +readline or libedit. Setting variables or entry mode (emacs, +vi) switching do not work in testing. + Using with the shell -------------------- diff --git a/doc/customizing.txt b/doc/customizing.txt index 62faac188..c3aa5fb29 100644 --- a/doc/customizing.txt +++ b/doc/customizing.txt @@ -1836,6 +1836,12 @@ contents:: # the user has confirmed their identity return + if db.tx_Source not in ("web"): + # the user is using rest, xmlrpc, command line, + # email (unlikely) which don't support interactive + # verification + return + # if the password or email are changing, require id confirmation if 'password' in newvalues: raise Reauth('Add an optional message to the user') diff --git a/doc/html_extra/accessibility-statement_2025-10-08.html b/doc/html_extra/accessibility-statement_2025-10-08.html new file mode 100644 index 000000000..f4e67e004 --- /dev/null +++ b/doc/html_extra/accessibility-statement_2025-10-08.html @@ -0,0 +1,75 @@ + +

Accessibility Statement for Roundup Issue Tracker Main Website

+

+ This is an accessibility statement from Roundup Issue Tracker. +

+

Conformance status

+

+ The Web Content Accessibility Guidelines (WCAG) defines requirements for designers and developers to improve accessibility for people with disabilities. It defines three levels of conformance: Level A, Level AA, and Level AAA. + Roundup Issue Tracker Main Website + is + partially conformant + with + WCAG 2.2 level AA. + + Partially conformant + means that + some parts of the content do not fully conform to the accessibility standard. + +

+

Feedback

+

+ We welcome your feedback on the accessibility of + Roundup Issue Tracker Main Website. + Please let us know if you encounter accessibility barriers on + Roundup Issue Tracker Main Website: +

+ +

Technical specifications

+

+ Accessibility of + Roundup Issue Tracker Main Website + relies on the following technologies to work with the particular combination of web browser and any assistive technologies or plugins installed on your computer: +

+
    +
  • HTML
    • +
  • WAI-ARIA
    • +
  • CSS
    • +
+

These technologies are relied upon for conformance with the accessibility standards used.

+

Limitations and alternatives

+

+ Despite our best efforts to ensure accessibility of + Roundup Issue Tracker Main Website , there may be some limitations. Below is a description of known limitations, and potential solutions. Please contact us if you observe an issue not listed below. +

+

+ Known limitations for + Roundup Issue Tracker Main Website: +

+
    +
  1. Search input does not have a visible label: The visible label is in the [Search] button next to the input but it not tied to the input directly. because The button acts as a visible label from proximity. The input has an aria-label describing the input.
    2. +
  2. Multi-language tab panel examples do not scroll: Multi-language example panels require two tabs to get to the element that will scroll. The Sphinx tab extension is used to show different language examples for the same item. It adds a focusable container around the example container. So the outer container must be tabbed/focused through to scroll the example when using the keyboard.
    3. +
+

Assessment approach

+

+ Roundup Issue Tracker + assessed the accessibility of + Roundup Issue Tracker Main Website + by the following approaches: +

+
    +
  • Self-evaluation
    • +
+
+

Date

+

+ This statement was created on + 8 November 2025 + using the W3C Accessibility Statement Generator Tool. +

diff --git a/doc/installation.txt b/doc/installation.txt index 14d5b0087..b50c1f00c 100644 --- a/doc/installation.txt +++ b/doc/installation.txt @@ -253,6 +253,10 @@ gpg version 2.0 of gpg from test.pypi.org. See the `gpg install directions in the upgrading document`_. +nanoid + If nanoid_ is installed, it is used to generate short unique + ids to link all logging to a single request. If not installed, + uuid4's from the standard library are used. jinja2 To use the jinja2 template (may still be experimental, check out @@ -307,6 +311,14 @@ polib roundup-gettext, you must install polib_. See the `developer's guide`_ for details on translating your tracker. +beautifulsoup, justhtml + When HTML only email is received, Roundup can convert it into + plain text using the native dehtml parser. To convert HTML + email into plain text, beautifulsoup4_ or justhtml_ can also be + used. You can choose the converter in the tracker's + config. Note that justhtml is pure Python, fast and conforms to + HTML 5 standards but works only for Python 3.10 or newer. + pywin32 - Windows Service You can run Roundup as a Windows service if pywin32_ is installed. Otherwise it must be started manually. @@ -1283,7 +1295,7 @@ FastCGI (Cherokee, Hiawatha, lighttpd) The Hiawatha and lighttpd web servers can run Roundup using FastCGI. Cherokee can run FastCGI but it also supports wsgi directly using a -wsgi server like uWSGI, Gnuicorn etc. +wsgi server like Waitress, Gnuicorn etc. To run Roundup using FastCGI, the flup_ package can be used under Python 2 and Python 3. We don't have a detailed config for this, but @@ -1499,23 +1511,6 @@ including putting it behind a proxy, IPV6 support etc. .. _`See the Waitress docs`: https://docs.pylonsproject.org/projects/waitress/en/stable/ -.. index:: pair: web interface; uWSGI - single: wsgi; uWSGI - -uWSGI Installation -~~~~~~~~~~~~~~~~~~ - -For a basic roundup install using uWSGI behind a front end server, -install uwsgi and the python3 (or python) plugin. Then run:: - - uwsgi --http-socket 127.0.0.1:8917 \ - --plugin python3 --mount=/tracker=wsgi.py \ - --manage-script-name --callable app - -using the same wsgi.py as was used for Gunicorn. If you get path not -found errors, check the mount option. The /tracker entry must match -the path used for the [tracker] web value in the tracker's config.ini. - Configure an Email Interface ============================ @@ -2436,6 +2431,7 @@ the test. .. _`adding MySQL users`: https://dev.mysql.com/doc/refman/8.0/en/creating-accounts.html .. _apache: https://httpd.apache.org/ +.. _beautifulsoup4: https://pypi.org/project/beautifulsoup4/ .. _brotli: https://pypi.org/project/Brotli/ .. _`developer's guide`: developers.html .. _defusedxml: https://pypi.org/project/defusedxml/ @@ -2443,12 +2439,14 @@ the test. .. _flup: https://pypi.org/project/flup/ .. _gpg: https://www.gnupg.org/software/gpgme/index.html .. _jinja2: https://palletsprojects.com/projects/jinja/ +.. _justhtml: https://pypi.org/project/justhtml/ .. _markdown: https://python-markdown.github.io/ .. _markdown2: https://github.com/trentm/python-markdown2 .. _mistune: https://pypi.org/project/mistune/ .. _mod_python: https://github.com/grisha/mod_python .. _mod_wsgi: https://pypi.org/project/mod-wsgi/ .. _MySQLdb: https://pypi.org/project/mysqlclient/ +.. _nanoid: https://pypi.org/project/nanoid/ .. _Olson tz database: https://www.iana.org/time-zones .. _polib: https://polib.readthedocs.io .. _Psycopg2: https://www.psycopg.org/ diff --git a/doc/pydoc.txt b/doc/pydoc.txt index ef7bf849b..36df7250f 100644 --- a/doc/pydoc.txt +++ b/doc/pydoc.txt @@ -1,10 +1,14 @@ -================ -Pydocs from code -================ +================================ +Embedded documentation from code +================================ .. contents:: :local: +The following are embedded documentation selected from the Roundup +code base. You can see the same information using the ``help`` +function after importing the modules. + Client class ============ @@ -40,3 +44,11 @@ Templating Utils class .. autoclass:: roundup.cgi.templating::TemplatingUtils :members: + +Logcontext Module +================= +.. _logcontext_pydoc: + +.. automodule:: roundup.logcontext + :members: + :exclude-members: SimpleSentinel diff --git a/doc/reference.txt b/doc/reference.txt index 9713ae283..48a8151cf 100644 --- a/doc/reference.txt +++ b/doc/reference.txt @@ -1307,7 +1307,7 @@ is added to the db object. As a result, the ``hasattr`` call shown above will return True and the Reauth exception is not raised. (Note that the value of the ``reauth_done`` attribute is True, so ``getattr(db, "reauth_done", False)`` will return True when reauth is -done and the defaul value of False if the attribute is missing. If the +done and the default value of False if the attribute is missing. If the default is not set, `getattr` raises an ``AttributeError`` which might be useful for flow control.) @@ -1321,6 +1321,12 @@ user to submit each sensitive property separately. For example:: 'at the same time is not allowed. Please ' 'submit two changes.') + if db.tx_Source not in ("web"): + # the user is using rest, xmlrpc, command line, + # email (unlikely) which don't support interactive + # verification + return + if 'password' in newvalues and not hasattr(db, 'reauth_done'): raise Reauth() @@ -1362,10 +1368,8 @@ from this directory, at which point it calls ``init(instance)`` from each file supplying itself as a first argument. Note that at this point web interface is not loaded, but -extensions still can register actions for it in the tracker -instance. This may be fixed in Roundup 1.6 by introducing -``init_web(client)`` callback or a more flexible extension point -mechanism. +extensions can register actions for it in the tracker +instance. * ``instance.registerUtil`` is used for adding `templating utilities`_ (see `adding a time log to your issues @@ -1378,8 +1382,9 @@ mechanism. * ``instance.registerAction`` is used to add more actions to the instance and to web interface. See `Defining new web actions`_ - for details. Generic action can be added by inheriting from - ``action.Action`` instead of ``cgi.action.Action``. + for details. Generic action (used by xmlrpc or rest interfaces) can + be added by inheriting from ``action.Action`` instead of + ``cgi.action.Action``. .. _interfaces.py: .. _modifying the core of Roundup: @@ -1815,15 +1820,15 @@ The ``addPermission`` method takes a four optional parameters: ``issue`` class and the ``user`` class include a reference to an ``organization`` class. Users are permitted to view only those issues that are associated with their respective organizations. A - check function or this could look like:: + check function for this could look like:: def view_issue(db, userid, itemid): user = db.user.getnode(userid) if not user.organisation: return False issue = db.issue.getnode(itemid) - if user.organisation == issue.organisation: - return True + return user.organisation == issue.organisation + The corresponding ``filter`` function:: diff --git a/doc/tracker_config.txt b/doc/tracker_config.txt index 1d132660f..01292b547 100644 --- a/doc/tracker_config.txt +++ b/doc/tracker_config.txt @@ -867,6 +867,14 @@ # Default: ERROR level = ERROR + # Format of the logging messages with all '%' signs + # doubled so they are not interpreted by the config file. + # Allowed value: Python LogRecord attribute named formats with % *sign doubled*. + # Also you can include the following attributes: + # %%(trace_id)s %%(trace_reason)s and %%(pct_char)s + # Default: %%(asctime)s %%(trace_id)s %%(levelname)s %%(message)s + format = %%(asctime)s %%(trace_id)s %%(levelname)s %%(message)s + # If set to yes, only the loggers configured in this section will # be used. Yes will disable gunicorn's --access-logfile. # @@ -1112,12 +1120,12 @@ # If an email has only text/html parts, use this module # to convert the html to text. Choose from beautifulsoup 4, - # dehtml - (internal code), or none to disable conversion. - # If 'none' is selected, email without a text/plain part - # will be returned to the user with a message. If - # beautifulsoup is selected but not installed dehtml will - # be used instead. - # Allowed values: beautifulsoup, dehtml, none + # justhtml, dehtml - (internal code), or none to disable + # conversion. If 'none' is selected, email without a text/plain + # part will be returned to the user with a message. If + # beautifulsoup or justhtml is selected but not installed + # dehtml will be used instead. + # Allowed values: beautifulsoup, justhtml, dehtml, none # Default: none convert_htmltotext = none diff --git a/doc/upgrading.txt b/doc/upgrading.txt index 66faca566..c610b22da 100644 --- a/doc/upgrading.txt +++ b/doc/upgrading.txt @@ -108,6 +108,75 @@ your database. Migrating from 2.5.0 to 2.6.0 ============================= +Default Logs Include Unique Request Identifier (info) +----------------------------------------------------- + +The default logging format has been changed from:: + + %(asctime)s %(levelname)s %(message)s + +to:: + + %(asctime)s %(trace_id)s %(levelname)s %(message)s + +So logs now look like:: + + 2025-08-20 03:25:00,308 f6RPbT2s70vvJ2jFb9BQNF DEBUG get user1 cached + +which in the previous format would look like:: + + 2025-08-20 03:25:00,308 DEBUG get user1 cached + +The new format includes ``trace_id`` which is a thread and process +unique identifier for a single request. So you can link together all +of the log lines and determine where a slow down or other +problem occurred. + +The logging format is now a ``config.ini`` parameter in the +``logging`` section with the name ``format``. You can change it if you +would like the old logging format without having to create a logging +configuration file. See :ref:`rounduplogging` for details. + +Make Pagination Links Keep Search Name (optional) +------------------------------------------------- + +When displaying a named search, index templates don't preserve +the name when using the pagination (Next/Prev) links. This is +fixed in the 2.6.0 templates for issues/bugs/tasks. To make the +change to your templates, look for the pagination links (look for +prev or previous case insensitive) in your tracker's html +subdirectory and change:: + + request.indexargs_url(request.classname, + {'@startwith':prev.first, '@pagesize':prev.size})" + +to read:: + + request.indexargs_url(request.classname, + dict({'@dispname': request.dispname} + if request.dispname + else {}, + **{'@startwith':prev.first, '@pagesize':prev.size}))" + +This code will be embedded in templating markup that is not shown +above. The change above is for your previous/prev link. The +change for the next pagination link is similar with:: + + {'@startwith':next.first, '@pagesize':next.size} + +replacing:: + + {'@startwith':prev.first, '@pagesize':prev.size} + +in the example. + +This moves the existing dictionary used to override the URL +arguments to the second argument inside a ``dict()`` call. It +also adds ``**`` before it. This change creates a new override +dictionary that includes an ``@dispname`` parameter if it is set +in the request. If ``@dispname`` is not set, the existing +dictionary contents are used. + Support authorized changes in your tracker (optional) ----------------------------------------------------- @@ -133,6 +202,41 @@ date, text etc.) do not need JavaScript to work. See :ref:`Confirming the User` in the reference manual for details. +Support for dictConfig Logging Configuration (optional) +------------------------------------------------------- + +Roundup's basic log configuration via config.ini has always had the +ability to use an ini style logging configuration to set levels per +log channel, control output file rotation etc. + +With Roundup 2.6 you can use a JSON like file to configure logging +using `dictConfig +`_. The +JSON file format as been enhanced to support comments that are +stripped before being processed by the logging system. + +You can read about the details in the :ref:`admin manual `. + +Fix user.item.html template producing invalid Javascript (optional) +------------------------------------------------------------------- + +The html template ``page.html`` in the classic, devel, minimal, and +responsive tracker templates define a ``user_src_input`` macro. This +macro produces invalid javascript for the ``onblur`` event when used +by ``user.item.html``. The only effect from this bug is a javascript +error reported in the user's browser when the user does not have edit +permissions on the page. It doesn't have any user visible impact. + +If you want to fix this, replace:: + + tal:attributes="onblur python:edit_ok and 'split_name(this)'; + +with:: + + tal:attributes="onblur python:'split_name(this)' if edit_ok else ''; + +in the ``html/page.html`` file in your tracker. + .. index:: Upgrading; 2.4.0 to 2.5.0 Migrating from 2.4.0 to 2.5.0 @@ -721,6 +825,8 @@ request (tu.client.request), the translator for the current language You can find an example in :ref:`dynamic_csp`. +.. _gpginstall: + Directions for installing gpg (optional) ---------------------------------------- @@ -934,9 +1040,9 @@ bug. Splitting the large script into two parts: allows use of ``structure`` on the script with no replaced strings should it be required for your tracker. -.. [#markdown-note] If you are using markdown formatting for your tracker's notes, - the user will see the markdown label rather than the long - (suspicious) URL. You may want to add something like:: +.. [#markdown-note] If you are using markdown formatting for your + tracker's notes, the user will see the markdown label rather than + the long (suspicious) URL. You may want to add something like:: a[href*=\@template]::after { content: ' [' attr(href) ']'; @@ -2173,7 +2279,7 @@ running:: roundup-admin -i table password,id,username Look for lines starting with ``{CRYPT}``. You can reset the user's -password using:: +password using [#history-pragma]_ :: roundup-admin -i roundup> set user16 password=somenewpassword @@ -2184,6 +2290,14 @@ prompt). This prevents the new password from showing up in the output of ps or shell history. The new password will be encrypted using the default encryption method (usually pbkdf2). +.. [#history-pragma] If your version of roundup-admin provides history + support, you should add ``-P history_features=2`` to the command + line or run ``pragma history_features=2`` at the ``roundup>`` + prompt. This will prevent the command line (and password) from being + saved to your history file (usually ``.roundup_admin_history`` in + your user's home directory. You can use ``roundup-admin -i + pragma list`` to see if pragmas are supported. + Enable performance improvement for wsgi mode (optional) ------------------------------------------------------- diff --git a/roundup/admin.py b/roundup/admin.py index 5fd1394bb..446b13f2c 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -34,8 +34,8 @@ import roundup.instance from roundup import __version__ as roundup_version -from roundup import date, hyperdb, init, password, token_r -from roundup.anypy import scandir_ +from roundup import date, hyperdb, init, password, support, token_r +from roundup.anypy import scandir_ # noqa: F401 define os.scandir from roundup.anypy.my_input import my_input from roundup.anypy.strings import repr_export from roundup.configuration import ( @@ -49,7 +49,7 @@ ) from roundup.exceptions import UsageError from roundup.i18n import _, get_translation -from roundup import support +from roundup.logcontext import gen_trace_id, set_processName, store_trace_reason try: from UserDict import UserDict @@ -93,6 +93,13 @@ class AdminTool: Additional help may be supplied by help_*() methods. """ + # import here to make AdminTool.readline accessible or + # mockable from tests. + try: + import readline # noqa: I001, PLC0415 + except ImportError: + readline = None + # Make my_input a property to allow overriding in testing. # my_input is imported in other places, so just set it from # the imported value rather than moving def here. @@ -243,7 +250,7 @@ def help_commands(self): if seq in h: commands.append(' ' + h.split(seq, 1)[1].lstrip()) break - + commands.sort() commands.append(_( """Commands may be abbreviated as long as the abbreviation @@ -1815,6 +1822,115 @@ def do_pragma(self, args): type(self.settings[setting]).__name__) self.settings[setting] = value + # history_length has to be pushed to readline to have any effect. + if setting == "history_length": + self.readline.set_history_length( + self.settings['history_length']) + + def do_readline(self, args): + ''"""Usage: readline initrc_line | 'emacs' | 'history' | 'reload' | 'vi' + + Using 'reload' will reload the file ~/.roundup_admin_rlrc. + 'history' will show (and number) all commands in the history. + + You can change input mode using the 'emacs' or 'vi' parameters. + The default is emacs. This is the same as using:: + + readline set editing-mode emacs + + or:: + + readline set editing-mode vi + + Any command that can be placed in a readline .inputrc file can + be executed using the readline command. You can assign + dump-variables to control O using:: + + readline Control-o: dump-variables + + Assigning multi-key values also works. + + pyreadline3 support on windows: + + Mode switching doesn't work, emacs only. + + Binding single key commands works with:: + + readline Control-w: history-search-backward + + Multiple key sequences don't work. + + Setting values may work. Difficult to tell because the library + has no way to view the live settings. + + """ + + # TODO: allow history 20 # most recent 20 commands + # history 100-200 # show commands 100-200 + + if not self.readline: + print(_("Readline support is not available.")) + return + # The if test allows pyreadline3 settings like: + # bind_exit_key("Control-z") get through to + # parse_and_bind(). It is not obvious that this form of + # command is supported. Pyreadline3 is supposed to parse + # readline style commands, so we use those for emacs/vi. + # Trying set-mode(...) as in the pyreadline3 init file + # didn't work in testing. + + if len(args) == 1 and args[0].find('(') == -1: + if args[0] == "vi": + self.readline.parse_and_bind("set editing-mode vi") + print(_("Enabled vi mode.")) + elif args[0] == "emacs": + self.readline.parse_and_bind("set editing-mode emacs") + print(_("Enabled emacs mode.")) + elif args[0] == "history": + history_size = self.readline.get_current_history_length() + print("history size", history_size) + print('\n'.join([ + "%3d %s" % ((i + 1), + self.readline.get_history_item(i + 1)) + for i in range(history_size) + ])) + elif args[0] == "reload": + try: + # readline is a singleton. In testing previous + # tests using read_init_file are loading from ~ + # not the test directory because it doesn't + # matter. But for reload we want to test with the + # init file under the test directory. Calling + # read_init_file() calls with the ~/.. init + # location and I can't seem to reset it + # or the readline state. + # So call with explicit file here. + self.readline.read_init_file( + self.get_readline_init_file()) + except FileNotFoundError as e: + # If user invoked reload explicitly, report + # if file not found. + # + # DOES NOT WORK with pyreadline3. Exception + # is not raised if file is missing. + # + # Also e.filename is None under cygwin. A + # simple test case does set e.filename + # correctly?? sigh. So I just call + # get_readline_init_file again to get + # filename. + fn = e.filename or self.get_readline_init_file() + print(_("Init file %s not found.") % fn) + else: + print(_("File %s reloaded.") % + self.get_readline_init_file()) + else: + print(_("Unknown readline parameter %s") % args[0]) + return + + self.readline.parse_and_bind(" ".join(args)) + return + designator_re = re.compile('([A-Za-z]+)([0-9]+)$') designator_rng = re.compile('([A-Za-z]+):([0-9]+)-([0-9]+)$') @@ -1968,9 +2084,9 @@ def do_security(self, args): sys.stdout.write(_('New Email users get the Role "%(role)s"\n') % locals()) roles.sort() for _rolename, role in roles: - sys.stdout.write(_('Role "%(name)s":\n') % role.__dict__) + sys.stdout.write(_('Role "%(name)s":\n') % role.props_dict()) for permission in role.permission_list(): - d = permission.__dict__ + d = permission.props_dict() if permission.klass: if permission.properties: sys.stdout.write(_( @@ -2239,6 +2355,9 @@ def usageError_feedback(self, message, function): print(function.__doc__) return 1 + @set_processName("roundup-admin") + @gen_trace_id() + @store_trace_reason('admin') def run_command(self, args): """Run a single command """ @@ -2365,29 +2484,34 @@ def history_features(self, feature): # setting the bit disables the feature, so use not. return not self.settings['history_features'] & features[feature] + def get_readline_init_file(self): + return os.path.join(os.path.expanduser("~"), + ".roundup_admin_rlrc") + def interactive(self): """Run in an interactive mode """ print(_('Roundup %s ready for input.\nType "help" for help.') % roundup_version) - initfile = os.path.join(os.path.expanduser("~"), - ".roundup_admin_rlrc") + initfile = self.get_readline_init_file() histfile = os.path.join(os.path.expanduser("~"), ".roundup_admin_history") - try: - import readline + if self.readline: + # clear any history that might be left over from caller + # when reusing AdminTool from tests or program. + self.readline.clear_history() try: if self.history_features('load_rc'): - readline.read_init_file(initfile) - except IOError: # FileNotFoundError under python3 + self.readline.read_init_file(initfile) + except FileNotFoundError: # file is optional pass try: if self.history_features('load_history'): - readline.read_history_file(histfile) + self.readline.read_history_file(histfile) except IOError: # FileNotFoundError under python3 # no history file yet pass @@ -2397,36 +2521,94 @@ def interactive(self): # Pragma history_length allows setting on a per # invocation basis at startup if self.settings['history_length'] != -1: - readline.set_history_length( + self.readline.set_history_length( self.settings['history_length']) - except ImportError: - readline = None - print(_('Note: command history and editing not available')) + if hasattr(self.readline, 'backend'): + # FIXME after min 3.13 version; no backend prints pyreadline3 + print(_("Readline enabled using %s.") % self.readline.backend) + else: + print(_("Readline enabled using unknown library.")) + + else: + print(_('Command history and line editing not available')) + + autosave_enabled = sys.stdin.isatty() and sys.stdout.isatty() while 1: try: command = self.my_input('roundup> ') + # clear an input hook in case it was used to prefill + # buffer. + self.readline.set_pre_input_hook() except EOFError: print(_('exit...')) break if not command: continue # noqa: E701 + if command.startswith('!'): # Pull numbered command from history + print_only = command.endswith(":p") + try: + hist_num = int(command[1:]) \ + if not print_only else int(command[1:-2]) + command = self.readline.get_history_item(hist_num) + except ValueError: + # pass the unknown command + pass + else: + if autosave_enabled and \ + hasattr(self.readline, "replace_history_item"): + # history has the !23 input. Replace it if possible. + # replace_history_item not supported by pyreadline3 + # so !23 will show up in history not the command. + self.readline.replace_history_item( + self.readline.get_current_history_length() - 1, + command) + + if print_only: + # fill the edit buffer with the command + # the user selected. + + # from https://stackoverflow.com/questions/8505163/is-it-possible-to-prefill-a-input-in-python-3s-command-line-interface + # This triggers: + # B023 Function definition does not bind loop variable + # `command` + # in ruff. command will be the value of the command + # variable at the time the function is run. + # Not the value at define time. This is ok since + # hook is run before command is changed by the + # return from (readline) input. + def hook(): + self.readline.insert_text(command) # noqa: B023 + self.readline.redisplay() + self.readline.set_pre_input_hook(hook) + # we clear the hook after the next line is read. + continue + + if not autosave_enabled: + # needed to make testing work and also capture + # commands received on stdin from file/other command + # output. Disable saving with pragma on command line: + # -P history_features=2. + self.readline.add_history(command) + try: args = token_r.token_split(command) except ValueError: continue # Ignore invalid quoted token if not args: continue # noqa: E701 - if args[0] in ('quit', 'exit'): break # noqa: E701 + if args[0] in ('q', 'quit', 'exit') and len(args) == 1: + break # noqa: E701 self.run_command(args) # exit.. check for transactions if self.db and self.db_uncommitted: commit = self.my_input(_('There are unsaved changes. Commit them (y/N)? ')) if commit and commit[0].lower() == 'y': - self.db.commit() + self.run_command(["commit"]) # looks like histfile is saved with mode 600 - if readline and self.history_features('save_history'): - readline.write_history_file(histfile) + if self.readline and self.history_features('save_history'): + self.readline.write_history_file(histfile) + return 0 def main(self): # noqa: PLR0912, PLR0911 @@ -2496,7 +2678,7 @@ def main(self): # noqa: PLR0912, PLR0911 self.interactive() else: ret = self.run_command(args) - if self.db: self.db.commit() # noqa: E701 + if self.db: self.run_command(["commit"]) # noqa: E701 return ret finally: if self.db: diff --git a/roundup/anypy/cgi_.py b/roundup/anypy/cgi_.py index 6df75c7fd..72df69762 100644 --- a/roundup/anypy/cgi_.py +++ b/roundup/anypy/cgi_.py @@ -1,7 +1,10 @@ # ruff: noqa: F401 - unused imports +import warnings try: # used for python2 and python 3 < 3.13 - import cgi + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=DeprecationWarning) + import cgi from cgi import FieldStorage, MiniFieldStorage except ImportError: # use for python3 >= 3.13 diff --git a/roundup/backends/sessions_dbm.py b/roundup/backends/sessions_dbm.py index 5eca055a4..996084846 100644 --- a/roundup/backends/sessions_dbm.py +++ b/roundup/backends/sessions_dbm.py @@ -201,6 +201,10 @@ def updateTimestamp(self, sessid): def clean(self): ''' Remove session records that haven't been used for a week. ''' + ''' Note: deletion of old keys must be completed when this method + returns. Calling code must not have any expired keys present + after this returns or expired keys could be used to validate + a user. This can mean a long delay when expiring but ....''' now = time.time() week = 60*60*24*7 a_week_ago = now - week diff --git a/roundup/cgi/actions.py b/roundup/cgi/actions.py index 7d12af4a6..236cb7211 100644 --- a/roundup/cgi/actions.py +++ b/roundup/cgi/actions.py @@ -1688,7 +1688,9 @@ def fct(arg): if isinstance(props[col], hyperdb.Multilink): cname = props[col].classname cclass = self.db.getclass(cname) - represent[col] = repr_list(cclass, 'name') + # Use id by default to handle cases like messages + # which have no useful label field issue2551413 + represent[col] = repr_list(cclass, 'id') if not self.hasPermission(self.permissionType, classname=cname): represent[col] = repr_no_right(cclass, 'name') else: diff --git a/roundup/cgi/client.py b/roundup/cgi/client.py index ed091c507..faa59bd9e 100644 --- a/roundup/cgi/client.py +++ b/roundup/cgi/client.py @@ -56,9 +56,9 @@ class SysCallError(Exception): UsageError, ) -from roundup.mlink_expr import ExpressionError - +from roundup.logcontext import gen_trace_id, store_trace_reason from roundup.mailer import Mailer, MessageSendError +from roundup.mlink_expr import ExpressionError logger = logging.getLogger('roundup') @@ -105,6 +105,7 @@ def add_message(msg_list, msg, escape=True): msg_list.append(msg) return msg_list # for unittests + # if set to False via interfaces.py do not log a warning when # xmlrpc is used and defusedxml is not installed. WARN_FOR_MISSING_DEFUSEDXML = True @@ -434,6 +435,8 @@ class Client: # content-type. precompressed_mime_types = ["image/png", "image/jpeg"] + @gen_trace_id() + @store_trace_reason('client') def __init__(self, instance, request, env, form=None, translator=None): # re-seed the random number generator. Is this is an instance of # random.SystemRandom it has no effect. @@ -449,9 +452,12 @@ def __init__(self, instance, request, env, form=None, translator=None): self.env = env if translator is not None: self.setTranslator(translator) - # XXX we should set self.language to "translator"'s language, - # but how to get it ? - self.language = "" + # set self.language to "translator"'s language + try: + self.language = translator.info()["language"] + except (AttributeError, KeyError): + # info() missing or no language key + self.language = "" else: self.setTranslator(TranslationService.NullTranslationService()) self.language = "" # as is the default from determine_language @@ -581,6 +587,8 @@ def setTranslator(self, translator=None): self._ = self.gettext = translator.gettext self.ngettext = translator.ngettext + @gen_trace_id() + @store_trace_reason('client_main') def main(self): """ Wrap the real main in a try/finally so we always close off the db. """ @@ -1391,8 +1399,9 @@ def check_anonymous_access(self): # allow Anonymous to view the "user" "register" template if they're # allowed to register - if (self.db.security.hasPermission('Register', self.userid, 'user') - and self.classname == 'user' and self.template == 'register'): + if (self.template == 'register' and self.classname == 'user' + and self.db.security.hasPermission('Register', + self.userid, 'user')): return # otherwise for everything else @@ -1926,19 +1935,19 @@ def reauth(self, exception): Can be overridden by code in tracker's interfaces.py. """ - - from roundup.anypy.vendored.cgi import MiniFieldStorage + + from roundup.anypy.cgi_ import MiniFieldStorage original_action = self.form['@action'].value if '@action' \ in self.form else "" original_template = self.template self.template = 'reauth' - self.form.list = [ x for x in self.form.list - if x.name not in ('@action', - '@csrf', - '@template' - )] + self.form.list = [x for x in self.form.list + if x.name not in ('@action', + '@csrf', + '@template' + )] # save the action and template used when the Reauth as # triggered. Will be used to resolve the change by the reauth diff --git a/roundup/cgi/wsgi_handler.py b/roundup/cgi/wsgi_handler.py index f730503dc..46458891d 100644 --- a/roundup/cgi/wsgi_handler.py +++ b/roundup/cgi/wsgi_handler.py @@ -13,6 +13,7 @@ from roundup.anypy.strings import s2b from roundup.cgi import TranslationService from roundup.cgi.client import BinaryFieldStorage +from roundup.logcontext import gen_trace_id, set_processName, store_trace_reason BaseHTTPRequestHandler = http_.server.BaseHTTPRequestHandler DEFAULT_ERROR_MESSAGE = http_.server.DEFAULT_ERROR_MESSAGE @@ -96,12 +97,19 @@ def __init__(self, home, debug=False, timing=False, lang=None, else: self.translator = None - if "cache_tracker" not in self.feature_flags or \ - self.feature_flags["cache_tracker"] is not False: + if self.use_cached_tracker(): self.tracker = roundup.instance.open(self.home, not self.debug) else: self.preload() + def use_cached_tracker(self): + return ( + "cache_tracker" not in self.feature_flags or + self.feature_flags["cache_tracker"] is not False) + + @set_processName("wsgi_handler") + @gen_trace_id() + @store_trace_reason("wsgi") def __call__(self, environ, start_response): """Initialize with `apache.Request` object""" request = RequestHandler(environ, start_response) @@ -132,8 +140,7 @@ def __call__(self, environ, start_response): else: form = BinaryFieldStorage(fp=environ['wsgi.input'], environ=environ) - if "cache_tracker" not in self.feature_flags or \ - self.feature_flags["cache_tracker"] is not False: + if self.use_cached_tracker(): client = self.tracker.Client(self.tracker, request, environ, form, self.translator) try: diff --git a/roundup/configuration.py b/roundup/configuration.py index 421d91bcd..1c470b4d9 100644 --- a/roundup/configuration.py +++ b/roundup/configuration.py @@ -17,12 +17,14 @@ import smtplib import sys import time +import traceback import roundup.date from roundup.anypy import random_ from roundup.anypy.strings import b2s from roundup.backends import list_backends from roundup.i18n import _ +from roundup.logcontext import gen_trace_id, get_context_info if sys.version_info[0] > 2: import configparser # Python 3 @@ -43,6 +45,16 @@ def __str__(self): return self.args[0] +class LoggingConfigError(ConfigurationError): + def __init__(self, message, **attrs): + super().__init__(message) + for key, value in attrs.items(): + self.__setattr__(key, value) + + def __str__(self): + return self.args[0] + + class NoConfigError(ConfigurationError): """Raised when configuration loading fails @@ -263,7 +275,8 @@ def load_ini(self, config): try: if config.has_option(self.section, self.setting): self.set(config.get(self.section, self.setting)) - except configparser.InterpolationSyntaxError as e: + except (configparser.InterpolationSyntaxError, + configparser.InterpolationMissingOptionError) as e: raise ParsingOptionError( _("Error in %(filepath)s with section [%(section)s] at " "option %(option)s: %(message)s") % { @@ -371,17 +384,17 @@ class HtmlToTextOption(Option): """What module should be used to convert emails with only text/html parts into text for display in roundup. Choose from beautifulsoup - 4, dehtml - the internal code or none to disable html to text - conversion. If beautifulsoup chosen but not available, dehtml will - be used. + 4, justhtml, dehtml - the internal code or none to disable html to + text conversion. If beautifulsoup or justhtml is chosen but not + available, dehtml will be used. """ - class_description = "Allowed values: beautifulsoup, dehtml, none" + class_description = "Allowed values: beautifulsoup, justhtml, dehtml, none" def str2value(self, value): _val = value.lower() - if _val in ("beautifulsoup", "dehtml", "none"): + if _val in ("beautifulsoup", "justhtml", "dehtml", "none"): return _val else: raise OptionValueError(self, value, self.class_description) @@ -565,6 +578,52 @@ def get(self): return None +class LoggingFormatOption(Option): + """Escape/unescape logging format string '%(' <-> '%%(' + + Config file parsing allows variable interpolation using + %(keyname)s. However this is exactly the format that we need + for creating a logging format string. So we tell the user to + quote the string using %%(...). Then we turn %%( -> %( when + retrieved and turn %( into %%( when saving the file. + """ + + class_description = ("Allowed value: Python LogRecord attribute named " + "formats with % *sign doubled*.\n" + "Also you can include the following attributes:\n" + " %%(trace_id)s %%(trace_reason)s and %%(pct_char)s" + ) + + def str2value(self, value): + """Validate and convert value of logging format string. + + This does a dirty check to see if a token is missing a + specifier. So "%%(ascdate)s %%(level) " would fail because of + the 's' missing after 'level)'. But "%%(ascdate)s %%(level)s" + would pass. + + Note that '%(foo)s' (i.e. unescaped substitution) generates + a error from the ini parser with a less than wonderful message. + """ + unquoted_val = value.replace("%%(", "%(") + + # regexp matches all current logging record object attribute names. + scanned_result = re.sub(r'%\([A-Za-z_]+\)\S', '', unquoted_val) + if scanned_result.find('%(') != -1: + raise OptionValueError( + self, unquoted_val, + "Check that all substitution tokens have a format " + "specifier after the ). Unrecognized use of %%(...) in: " + "%s" % scanned_result) + + return str(unquoted_val) + + def _value2str(self, value): + """Replace %( with %%( to quote the format substitution param. + """ + return value.replace("%(", "%%(") + + class OriginHeadersListOption(Option): """List of space seperated origin header values. @@ -1604,6 +1663,10 @@ def str2value(self, value): "Minimal severity level of messages written to log file.\n" "If above 'config' option is set, this option has no effect.\n" "Allowed values: DEBUG, INFO, WARNING, ERROR"), + (LoggingFormatOption, "format", + "%(asctime)s %(trace_id)s %(levelname)s %(message)s", + "Format of the logging messages with all '%' signs\n" + "doubled so they are not interpreted by the config file."), (BooleanOption, "disable_loggers", "no", "If set to yes, only the loggers configured in this section will\n" "be used. Yes will disable gunicorn's --access-logfile.\n"), @@ -1748,11 +1811,11 @@ def str2value(self, value): (HtmlToTextOption, "convert_htmltotext", "none", "If an email has only text/html parts, use this module\n" "to convert the html to text. Choose from beautifulsoup 4,\n" - "dehtml - (internal code), or none to disable conversion.\n" - "If 'none' is selected, email without a text/plain part\n" - "will be returned to the user with a message. If\n" - "beautifulsoup is selected but not installed dehtml will\n" - "be used instead."), + "justhtml, dehtml - (internal code), or none to disable\n" + "conversion. If 'none' is selected, email without a text/plain\n" + "part will be returned to the user with a message. If\n" + "beautifulsoup or justhtml is selected but not installed\n" + "dehtml will be used instead."), (BooleanOption, "keep_real_from", "no", "When handling emails ignore the Resent-From:-header\n" "and use the original senders From:-header instead.\n" @@ -2322,6 +2385,18 @@ def _get_unset_options(self): def _get_name(self): return self["TRACKER_NAME"] + @gen_trace_id() + def _logging_test(self, sinfo, msg="test %(a)s\n%(sinfo)s", args=None): + """Test method for logging formatting. + + Not used in production. + """ + logger = logging.getLogger('roundup') + if args: + logger.info(msg, *args) + else: + logger.info(msg, extra={"a": "a_var", "sinfo": sinfo}) + def reset(self): Config.reset(self) if self.ext: @@ -2330,22 +2405,324 @@ def reset(self): self.detectors.reset() self.init_logging() + def gather_callstack(self, keep_full_stack=False): + # Locate logging call in stack + stack = traceback.extract_stack() + if keep_full_stack: + last_frame_index = len(stack) + else: + for last_frame_index, frame in enumerate(stack): + # Walk from the top of stack looking for + # "logging" in filename (really + # filepath). Can't use /logging/ as + # windows uses \logging\. + # + # Filtering by looking for "logging" in + # the filename isn't great. + if "logging" in frame.filename: + break + if not keep_full_stack: + stack = stack[0:last_frame_index] + return (stack, last_frame_index) + + def context_filter(self, record): + """Add context to record, expand context references in record.msg + Define pct_char as '%' + """ + + # This method can be called multiple times on different handlers. + # However it modifies the record on the first call and the changes + # persist in the record. So we only want to be called once per + # record. + if hasattr(record, "ROUNDUP_CONTEXT_FILTER_CALLED"): + return True + + for name, value in get_context_info(): + if not hasattr(record, name): + setattr(record, name, value) + continue + if (name == "processName" and + isinstance(value, str) and + getattr(record, name) == "MainProcess"): + setattr(record, name, value) + + record.pct_char = "%" + record.ROUNDUP_CONTEXT_FILTER_CALLED = True + + if hasattr(record, "sinfo"): + # sinfo has to be set via extras argument to logging commands + # to activate this. + # + # sinfo value can be: + # non-integer: "", None etc. Print 5 elements of + # stack before logging call + # integer N > 0: print N elements of stack before + # logging call + # 0: print whole stack before logging call + # integer N < 0: undocumented print stack starting at + # extract_stack() below. I.E. do not set bottom of + # stack to the logging call. + # if |N| is greater than stack height, print whole stack. + stack_height = record.sinfo + keep_full_stack = False + + if isinstance(stack_height, int): + if stack_height < 0: + keep_full_stack = True + stack_height = abs(stack_height) + if stack_height == 0: + # None will set value to actual stack height. + stack_height = None + else: + stack_height = 5 + + stack, last_frame_index = self.gather_callstack(keep_full_stack) + + if stack_height is None: + stack_height = last_frame_index + elif stack_height > last_frame_index: + stack_height = last_frame_index # start at frame 0 + + # report the stack info + record.sinfo = "".join( + traceback.format_list( + stack[last_frame_index - stack_height:last_frame_index] + ) + ) + + # if args are present, just return. Logging will + # expand the arguments. + if record.args: + return True + + # Since args not present, try formatting msg using + # named arguments. + try: + record.msg = record.msg % record.__dict__ + except (ValueError, TypeError): + # ValueError: means there is a % sign in the msg + # somewhere that is not meant to be a format token. So + # just leave msg unexpanded. + # + # TypeError - a positional format string is being + # handled without setting record.args. E.G. + # .info("result is %f") + # Leave message unexpanded. + pass + return True + + def add_logging_context_filter(self): + """Update log record with contextvar values and expand msg + + The contextvar values are stored as attributes on the log + record object in record.__dict__. They should not exist + when this is called. Do not overwrite them if they do exist + as they can be set in a logger call using:: + + logging.warning("a message", extra = {"trace_id": foo}) + + the extra argument/parameter. + + Attempt to expand msg using the variables in + record.__dict__. This makes:: + + logging.warning("the URL was: %(trace_reason)s") + + work and replaces the ``%(trace_reason)s`` token with the value. + Note that you can't use positional params and named params + together. For example:: + + logging.warning("user = %s and URL was: %(trace_reason)s", user) + + will result in an exception in logging when it formats the + message. + + Also ``%(pct_char)`` is defined to allow the addition of % + characters in the format string as bare % chars can't make + it past the configparser and %% encoded ones run into issue + with the format verifier. + + Calling a logger (.warning() etc.) with the argument:: + + extra={"sinfo": an_int} + + will result in a stack trace starting at the logger call + and going up the stack for `an_int` frames. Using "True" + in place of `an_int` will print only the call to the logger. + + Note that logging untrusted strings in the msg set by user + (untrusted) may be an issue. So don't do something like: + + .info("%s" % web_url) + + as web_url could include '%(trace_id)s'. Instead use: + + .info("%(url)s", extra=("url": web_url)) + + Even in the problem case, I think the damage is contained since: + + * data doesn't leak as the log string is not exposed to the + user. + + * the log string isn't executed or used internally. + + * log formating can raise an exception. But this won't + affect the application as the exception is swallowed in + the logging package. The raw message would be printed by + the fallback logging handler. + + but if it is a concern, make sure user data is added using + the extra dict when calling one of the logging functions. + """ + loggers = [logging.getLogger(name) for name in + logging.root.manager.loggerDict] + # append the root logger as it is not listed in loggerDict + loggers.append(logging.getLogger()) + for logger in loggers: + for hdlr in logger.handlers: + hdlr.addFilter(self.context_filter) + + def load_config_dict_from_json_file(self, filename): + import json + comment_re = re.compile( + r"""^\s*//#.* # comment at beginning of line possibly indented. + | # or + ^(.*)\s\s\s\//.* # comment char preceeded by at least three spaces. + """, re.VERBOSE) + + config_list = [] + with open(filename) as config_file: + for line in config_file: + match = comment_re.search(line) + if match: + if match.lastindex: + config_list.append(match.group(1) + "\n") + else: + # insert blank line for comment line to + # keep line numbers in sync. + config_list.append("\n") + continue + config_list.append(line) + + try: + config_dict = json.loads("".join(config_list)) + except json.decoder.JSONDecodeError as e: + error_at_doc_line = e.lineno + # subtract 1 - zero index on config_list + # remove '\n' for display + try: + line = config_list[error_at_doc_line - 1][:-1] + except IndexError: + line = _("Error found at end of file. Maybe missing a " + "block closing '}'.") + + hint = "" + if line.find('//') != -1: + hint = "\nMaybe bad inline comment, 3 spaces needed before //." + + raise LoggingConfigError( + 'Error parsing json logging dict (%(file)s) ' + 'near \n\n %(line)s\n\n' + '%(msg)s: line %(lineno)s column %(colno)s.%(hint)s' % + {"file": filename, + "line": line, + "msg": e.msg, + "lineno": error_at_doc_line, + "colno": e.colno, + "hint": hint}, + config_file=self.filepath, + source="json.loads" + ) + + return config_dict + def init_logging(self): _file = self["LOGGING_CONFIG"] if _file and os.path.isfile(_file): - logging.config.fileConfig( - _file, - disable_existing_loggers=self["LOGGING_DISABLE_LOGGERS"]) + if _file.endswith(".ini"): + try: + logging.config.fileConfig( + _file, + disable_existing_loggers=self[ + "LOGGING_DISABLE_LOGGERS"]) + except (ValueError, RuntimeError, configparser.ParsingError, + KeyError, NameError, ModuleNotFoundError) as e: + # configparser.DuplicateOptionError includes + # filename, line number and a useful error. + # so we don't have to augment it. + context = [] + if hasattr(e, '__context__') and e.__context__: + # get additional error info. + context.append(str(e.__context__)) + if hasattr(e, '__doc__') and e.__doc__: + context.append(e.__doc__) + + if isinstance(e, KeyError): + context.append("No section found with this name.") + if not context: + context = ["No additional information available."] + + raise LoggingConfigError( + "Error loading logging config from %(file)s.\n\n" + " %(msg)s\n\n%(context)s\n" % { + "file": _file, + "msg": type(e).__name__ + ": " + str(e), + "context": " ".join(context), + } + ) + elif _file.endswith(".json"): + config_dict = self.load_config_dict_from_json_file(_file) + try: + logging.config.dictConfig(config_dict) + except ValueError as e: + # docs say these exceptions: + # ValueError, TypeError, AttributeError, ImportError + # could be raised, but + # looking through the code, it looks like + # configure() maps all exceptions (including + # ImportError, TypeError) raised by functions to + # ValueError. + context = "No additional information available." + if hasattr(e, '__context__') and e.__context__: + # get additional error info. E.G. if INFO + # is replaced by MANGO, context is: + # ValueError("Unknown level: 'MANGO'") + # while str(e) is "Unable to configure handler 'access'" + context = e.__context__ + + raise LoggingConfigError( + 'Error loading logging dict from %(file)s.\n' + '%(msg)s\n%(context)s\n' % { + "file": _file, + "msg": type(e).__name__ + ": " + str(e), + "context": context + }, + config_file=self.filepath, + source="dictConfig" + ) + else: + raise OptionValueError( + self.options['LOGGING_CONFIG'], + _file, + "Unable to load logging config file. " + "File extension must be '.ini' or '.json'.\n" + ) + + self.add_logging_context_filter() return + if _file: + raise OptionValueError(self.options['LOGGING_CONFIG'], + _file, + "Unable to find logging config file.") + _file = self["LOGGING_FILENAME"] # set file & level on the roundup logger logger = logging.getLogger('roundup') hdlr = logging.FileHandler(_file) if _file else \ logging.StreamHandler(sys.stdout) - formatter = logging.Formatter( - '%(asctime)s %(levelname)s %(message)s') + formatter = logging.Formatter(self["LOGGING_FORMAT"]) hdlr.setFormatter(formatter) # no logging API to remove all existing handlers!?! for h in logger.handlers: @@ -2353,6 +2730,15 @@ def init_logging(self): logger.removeHandler(hdlr) logger.handlers = [hdlr] logger.setLevel(self["LOGGING_LEVEL"] or "ERROR") + if 'pytest' not in sys.modules: + # logger.propatgate is True by default. This is + # needed so that pytest caplog code will work. In + # production leaving it set to True relogs everything + # using the root logger with logging BASIC_FORMAT: + # "%(level)s:%(name)s:%(message)s + logger.propagate = False + + self.add_logging_context_filter() def validator(self, options): """ Validate options once all options are loaded. diff --git a/roundup/date.py b/roundup/date.py index 6515ee320..4bcd90ab9 100644 --- a/roundup/date.py +++ b/roundup/date.py @@ -324,6 +324,9 @@ class Date: >>> test_fin(u) ''' + __slots__ = ("year", "month", "day", "hour", "minute", "second", + "_", "ngettext", "translator") + def __init__(self, spec='.', offset=0, add_granularity=False, translator=i18n): """Construct a date given a specification and a time zone offset. diff --git a/roundup/dehtml.py b/roundup/dehtml.py index cd913a0d1..ec73fb406 100644 --- a/roundup/dehtml.py +++ b/roundup/dehtml.py @@ -5,6 +5,10 @@ from roundup.anypy.strings import u2s, uchr +# ruff PLC0415 ignore imports not at top of file +# ruff RET505 ignore else after return +# ruff: noqa: PLC0415 RET505 + _pyver = sys.version_info[0] @@ -28,6 +32,108 @@ def html2text(html): return u2s(soup.get_text("\n", strip=True)) + self.html2text = html2text + elif converter == "justhtml": + from justhtml import stream + + def html2text(html): + # The below does not work. + # Using stream parser since I couldn't seem to strip + # 'script' and 'style' blocks. But stream doesn't + # have error reporting or stripping of text nodes + # and dropping empty nodes. Also I would like to try + # its GFM markdown output too even though it keeps + # tables as html and doesn't completely covert as + # this would work well for those supporting markdown. + # + # ctx used for for testing since I have a truncated + # test doc. It eliminates error from missing DOCTYPE + # and head. + # + #from justhtml import JustHTML + # from justhtml.context import FragmentContext + # + #ctx = FragmentContext('html') + #justhtml = JustHTML(html,collect_errors=True, + # fragment_context=ctx) + # I still have the text output inside style/script tags. + # with :not(style, script). I do get text contents + # with query("style, script"). + # + #return u2s("\n".join( + # [elem.to_text(separator="\n", strip=True) + # for elem in justhtml.query(":not(style, script)")]) + # ) + + # define inline elements so I can accumulate all unbroken + # text in a single line with embedded inline elements. + # 'br' is inline but should be treated it as a line break + # and element before/after should not be accumulated + # together. + inline_elements = ( + "a", + "address", + "b", + "cite", + "code", + "em", + "i", + "img", + "mark", + "q", + "s", + "small", + "span", + "strong", + "sub", + "sup", + "time") + + # each line is appended and joined at the end + text = [] + # the accumulator for all text in inline elements + text_accumulator = "" + # if set skip all lines till matching end tag found + # used to skip script/style blocks + skip_till_endtag = None + # used to force text_accumulator into text with added + # newline so we have a blank line between paragraphs. + _need_parabreak = False + + for event, data in stream(html): + if event == "end" and skip_till_endtag == data: + skip_till_endtag = None + continue + if skip_till_endtag: + continue + if (event == "start" and + data[0] in ('script', 'style')): + skip_till_endtag = data[0] + continue + if (event == "start" and + text_accumulator and + data[0] not in inline_elements): + # add accumulator to "text" + text.append(text_accumulator) + text_accumulator = "" + _need_parabreak = False + elif event == "text": + if not data.isspace(): + text_accumulator = text_accumulator + data + _need_parabreak = True + elif (_need_parabreak and + event == "start" and + data[0] == "p"): + text.append(text_accumulator + "\n") + text_accumulator = "" + _need_parabreak = False + + # save anything left in the accumulator at end of document + if text_accumulator: + # add newline to match dehtml and beautifulsoup + text.append(text_accumulator + "\n") + return u2s("\n".join(text)) + self.html2text = html2text else: raise ImportError @@ -96,6 +202,16 @@ def html2text(html): if __name__ == "__main__": + # ruff: noqa: B011 S101 + + try: + assert False + except AssertionError: + pass + else: + print("Error, assertions turned off. Test fails") + sys.exit(1) + html = """