cd logfmt1 && ./setup.py bdist_wheel
whl_up:
	twine upload dist/logfmt*
docs:
	pygmentize -S pastie -f html > logfmt1/docs/syntax.css
	cd logfmt1 ; PYTHONPATH=. mkdocs build -v -v -v
	sed -i 's/table\.docutils/table/g' logfmt1/html/css/theme.css 
	sed -i 's/[{};]/&\n/g' logfmt1/html/css/theme.css
%.1:	%.md
	pandoc --standalone -f markdown+pandoc_title_block -t man $< -o $@
man:	logfmt1/manpage/logex.1 logfmt1/manpage/update-logfmt.1

Public Domain (CC0)

## html2mallard / mkdocs-mallard

Extremely crude HTML to [mallard help](http://projectmallard.org/) conversion.
Specifically for output from mkdocs 1.x with RTD theme. It's a very basic regex
extraction and filtering process, that only retains some structural
elements (headlines, paragraphs, tables, lists, notes). Doesn't even
attempt to gather any topic relation/structure from the navigation list.

Really only suitable for one-time/initial conversion, and requires some
editing to get pages to validate. (Though they probably "work" in yelp
as is). Links certainly require manual cleanup. And API docs are least
convertible.


## html2mallard

Simple command line tool to convert a single .html file:


    html2mallard site/index.html > help/index.page


## mkdocs-mallard

Converts a list of mkdocs output files to *.page files. Requires an extra
`mallard_dir` in the `mallard.xml` config.

    mkdocs-mallard

Sample config:

    site_name: logfmt1
    docs_dir: docs
    site_dir: html
    mallard_dir: mallard
    use_directory_urls: false
    nav:
      - Intro: index.md
    theme:
      name: readthedocs
      highlightjs: false
    repo_url: https://...
    markdown_extensions:
      - admonition
      - codehilite
      - attr_list
      - def_list
      - tables
      - markdown.extensions.codehilite:
          guess_lang: true
    plugins:
      - mkdocstrings

Note the `mallard_dir` and `use_directory_urls`. The script only scans
one level of `*.html` files.


## Adaption

The first two `rewrite` rules likely require changes for other HTML sources
or templates. Specifically `"^.+?</nav>"` should strip the initial
boilerplate, else might need expansion.


### from `project` import `meta`

| meta           | info                                                            |
|:---------------|:----------------------------------------------------------------|
| depends        | -                                                               |
| compat         | Python ≥3.6                                                     |
| compliancy     | !pep8, ~mallard, !doap                                          |
| system usage   | -                                                               |
| paths          | -                                                               |
| testing        | `None`                                                          |
| docs           | -                                                               |
| activity       | abandoned                                                       |
| state          | alpha                                                           |
| support        | `None`                                                          |
| contrib        | -                                                               |
| announce       | -                                                               |

#!/usr/bin/env python3
# api: cli
# encoding: utf-8
# type: transform
# title: HTML to mallard
# description: convert mkdocs´ html output to mallard/yelp xml
# category: documentation
# keywords: mkdocs mallard
# version: 0.2
# depends: python (>= 3.6)
# license: Public Domain
# url: https://fossil.include-once.org/modseccfg/wiki/html2mallard
# 
# Poor transformation approach, mostly salvaging some HTML structures
# and reshuffling document body into mallard <page> and allowed
# inline markup.
# XSLT might have been easier, but doesn't work on HTML.
#


import os, sys
import re
from textwrap import dedent, indent
from glob import glob
import yaml


# output
template = dedent("""
    <page
        xmlns="http://projectmallard.org/1.0/"
        type="guide"
        id="{id}">

        <info>
            <link type="guide" xref="index#nav"/>
            {links}
            <desc>{desc}</desc>
        </info>

        <title>{title}</title>

        {body}

    </page>
""").lstrip()

# regex all the way
extract = {
    "mkdocs_page_name = \"(.*?)\";": "title",
    "<title>(.+?)</title>": "title",
    '<a class="reference internal" href="(\w+).html">.+?</a>': "links",
}
rewrite = {
    # trim and cleanup
    "^.+?</nav>": "",
    "^.+?<div\srole=\"main\">": "",   # mkdocs RTD template
    "<script.+?</script>": "",
    "<head>.+?</head>": "",
    "</body>|</html>": "",
    "<span></span>": "",
    '<footer>.+\\Z': "",    # mkdocs footer
    '<footer\sid="fossil-footer">.+\\Z': "", # fossil footer
    '\\A.+<main[^>]*>': "", # strip fossil wiki header
    'Next\s<span\sclass="icon\sicon-circle-arrow-right"></span>.+\\Z': "",
    "&rarrq;": "→",
    "&nbsp;": "␣",
    #"&quot;": "\"",
    #"&apos;": "\'"",
    "&(?!lt|gt|amp)\w+;": "",
    
    # actual conversions
    "<div\sclass=\"admonition\s(?:note|abstract|summary|tldr)\">(.+?)</div>": "<note style=\"tip\">\\1</note>",
    "<div\sclass=\"admonition\s(?:todo|seealso)\">(.+?)</div>": "<note style=\"advanced\">\\1</note>",
    "<div\sclass=\"admonition\s(?:danger|error|failure|fail|missing|bug)\">(.+?)</div>": "<note style=\"bug\">\\1</note>",
    "<div\sclass=\"admonition\s(?:info|todo)\">(.+?)</div>": "<note style=\"important\">\\1</note>",
    "<div\sclass=\"admonition\s(?:example|quote|cite)\">(.+?)</div>": "<note style=\"plain\">\\1</note>",
    "<div\sclass=\"admonition\s(?:question|help|faq)\">(.+?)</div>": "<note style=\"sidebar\">\\1</note>",
    "<div\sclass=\"admonition\s(?:notes|tip|hint|important)\">(.+?)</div>": "<note style=\"tip\">\\1</note>",
    "<div\sclass=\"admonition\s(?:warning|caution|attention)\">(.+?)</div>": "<note style=\"warning\">\\1</note>",
    "<div\sclass=\"admonition(?:\s\w+)?\">(.+?)</div>": "<note style=\"tip\">\\1</note>",
    "<p\sclass=\"admonition-title\">(.+?)</p>": "<subtitle>\\1</subtitle>",
    # headlines
    "(<h\d[^>]*>.+?(?<!\s))\s*(?=<h\d|<footer|</body|\Z)": "\n<section>\n\\1\n</section>\n",
    "<(?:h1|h2)[^>]*>(.+?)</(?:h1|h2)>": "<title>\\1</title>",
    "<(?:h3|h4)[^>]*>(.+?)</(?:h3|h4)>": "<subtitle>\\1</subtitle>",
    "<(?:h5|h6)[^>]*>(.+?)</(?:h5|h7)>": "<em>\\1</em>",
    "<strong>(.+?)</strong>": "<em style=\"strong\">\\1</em>",
    # lists
    "<ol>(.+?)</ol>": "<steps>\\1</steps>",
    "<ul>(.+?)</ul>": "<list>\\1</list>",
    "<li>(.+?)</li>": "<item><p>\\1</p></item>",
    "<dl>(.+?)</dl>": "<terms>\\1</terms>",
    "<dt>(.+?)</dt>": "<item><title>\\1</title>",
    "<dd>(.+?)</dd>": "<p>\\1</p></item>",
    # fix nested list   \1         \2                 \3                      \4    
    "(<(?:item|steps|terms)>)<p> ([^<]+(?<!\s)) \s* <(list|steps|terms)> \s* (.+?) </\\3>":
        "\\1<p>\\2</p>\n <\\3>\n<item><p>\\4 </\\3>\n</item>",
    # links
    "<a\shref=\"([^\">]+)\.html\">(.+?)</a>": "<link type=\"seealso\" xref=\"\\1\">\\2</link>",
    "<a\shref=\"(\w+://[^\">]+)\">(.+?)</a>": "<link type=\"seealso\" href=\"\\1\">\\2</link>",
    # media
    "<img[^>]+src=\"(.+?)\"[^>]*>": "<media type=\"image\" src=\"\\1\" mime=\"image/png\" />",
    # tables
    "</?tbody>": "",
    "<table[^>]*>": "<table shade=\"rows cols\" rules=\"rows cols\"><tbody>",
    "</table>": "</tbody></table>",
    "<tr[^>]*>": "<tr>",
    "<(td|th)\\b[^>]*>": "    <td><p>",
    "</(td|th)\\b[^>]*>": "</p></td>",

    # strip codehilite markup
    "<span\sclass=\"\w{1,2}\">(.+?)</span>": "<span>\\1</span>",
    
    # strip any remaining non-mallard tags, except: |div|revision|thead
    """</? 
       (?!(?:page|section|info|credit|link|link|title|desc|title|keywords|license|desc|
       years|email|name|links|code|media|p|screen|quote|comment|example|figure|listing|
       note|synopsis|list|item|steps|item|terms|item|tree|item|table|col|colgroup|tr|
       tbody|tfoot|td|th|title|subtitle|desc|cite|app|code|cmd|output|em|file|gui|guiseq|hi|
       link|media|keyseq|key|span|sys|input|var)\\b)
       \w+[^>]* >""": "",

    # prettify sections
    "(<section>)(.+?)(</section>)": lambda m: f"{m[1]}\n{indent(m[2].strip(), prefix=' ')}\n{m[3]}",
    # strip lone </section>, empty spans
    "(<section>.+?</section>)|</section>": "\\1",
    "(<span[^>]*></span>)": "",
}


def convert(html, fn):

    # prepare snippets for .format kwargs
    kw = {
        "id": re.sub("^.+/|\.\w+$", "", fn),
        "desc": "",
        "title": "",
        "body": "",
        "links": "",
    }
    for rx, name in extract.items():
        m = re.search(rx, html)
        if m and not kw[name]:
            if name == "links":
                kw[name] = re.findall(rx, html)
            else:
                kw[name] = m.group(1)
    if kw["links"]:
        kw["links"] = "\n        ".join(f"<link type=\"guide\" xref=\"{id}\"/>" for id in kw["links"])
        
    # simplify/convert html
    for rx, repl in rewrite.items():
        html = re.sub(rx, repl, html, 0, re.X|re.M|re.S|re.I)
    kw["body"] = html
    
    # return converted
    return template.format(**kw)


def cnv_file(fn):
    with open(fn, "r", encoding="utf-8") as f:
        return convert(f.read(), fn)

def mkdocs():
    src = open("mkdocs.yml", "r") # mkdocs config should be in current directory
    cfg = yaml.load(src, Loader=yaml.Loader)
    srcdir = cfg["site_dir"]
    target = cfg["mallard_dir"]
    if not os.path.exists(target):
        os.makedirs(target)
    for fn in glob(f"{srcdir}/*.html"):
        page = cnv_file(fn)
        fn = re.sub(".+/", "", fn)
        fn = re.sub("\.html", ".page", fn)
        with open(f"{target}/{fn}", "w", encoding="utf-8") as f:
            f.write(page)

def main():
    if len(sys.argv) == 2:
        print(cnv_file(sys.argv[1])) # e.g. "site/index.html"
    else:
        mkdocs() # iterate through site/*html

if __name__ == "__main__":
    main()
    

#!/usr/bin/env python3
# encoding: utf-8
# api: pip
# type: build
# title: config for setuptools
#
# Always prefer setuptools over distutils
#

from pluginconf.setup import setup

setup(
    debug=1,
    fn="html2mallard.py",
    long_description="README.md",
    packages=[""],
    package_dir={"": "."},
    entry_points={
        "console_scripts": [
            "html2mallard=html2mallard:main",
            "mkdocs-mallard=html2mallard:mkdocs",
        ]
    }
)

# depends: python:pysimplegui (>= 3.0), python:pluginconf (>= 0.7.3),
#    python:appdirs (>= 1.3), python:logfmt1 (>= 0.4),
#    python (>= 3.6), deb:python3-tk, bin:sshfs
# priority: core
# url: https://fossil.include-once.org/modseccfg/
# faq: https://fossil.include-once.org/modseccfg/doc/trunk/FAQ.md
# category: config
# keywords: modsecurity mod-security mod_security apache config desktop sshfs
# classifiers: x11, http
#
# Correlates mod_security SecRules to logs, and simplifies
# disabling unneeded rules. It's very basic and not gonna
# win any usability awards.
# BE WARNED THAT ALPHA RELEASES MAY DAMAGE YOUR APACHE SETUP.
#

<page
    xmlns="http://projectmallard.org/1.0/"
    type="guide"
    id="index">

    <info>
        <link type="guide" xref="index#nav"/>
        
        <desc></desc>
    </info>

    <title>modseccfg: modseccfg</title>

    
          
<section>
 <title>modseccfg</title>
</section>

<section>
 <subtitle>mod_security config GUI</subtitle>
 <list>
 <item><p>GUI to define SecRuleRemoveById settings on a vhost-basis</p></item>
 <item><p>Tries to suggest false positives from error and audit logs</p></item>
 <item><p>And configure mod_security and CoreRuleSet variables.</p></item>
 <item><p>Runs locally, via <code class="prettyprint prettyprinted" style=""><span class="pln">ssh </span><span class="pun">-</span><span class="pln">X</span></code> forwarding, or per <link type="seealso" xref="remoting"><code class="prettyprint prettyprinted" style=""><span class="pln">modseccfg ssh</span><span class="pun">:/</span></code>
 remoting</link>.</p></item>
 </list>

 <p><media type="image" src="https://fossil.include-once.org/modseccfg/raw/80ea9c62eece1?=image/gif"
mime="image/gif" /></p>


  WARNING: THIS IS ALPHA STAGE QUALITY AND WILL MOST CERTAINLY DELETE YOUR APACHE CONFIGURATION - It doesn't, but: no warranty and such. - Also, hasn't many features yet.
</section>

<section>
 <title>Installation</title>
 <list>
 <item><p><p>You can install this package locally or on a server:</p>

 <code class="prettyprint"><span class="pln">pip3 install </span><span class="pun">-</span><span class="pln">U modseccfg</span></code></p></item>
 <item><p><p>And your distro must provide a full Python installaton and mod_security:</p>

 <code class="prettyprint"><span class="pln">sudo apt install python3</span><span class="pun">-</span><span class="pln">tk ttf</span><span class="pun">-</span><span class="pln">unifont libapache2</span><span class="pun">-</span><span class="pln">mod</span><span class="pun">-</span><span class="pln">security2</span></code></p></item>
 </list>
</section>

<section>
 <title>Start options</title>
 <list>
 <item><p>To run the GUI locally / on test setups:</p>

 <p><code class="prettyprint"><span class="pln">modseccfg</span></code></p></item>

 <item><p>Or to <link type="seealso" xref="remoting">connect to a remote</link> server:</p>

 <p><code class="prettyprint"><span class="pln">modseccfg root@vps5</span><span class="pun">:/</span></code></p></item>
 </list>

 <p>   Takes a bit longer on startup, but is heaps better than X11 forwarding.</p>
</section>

<section>
 <title>Usage</title>
 <p>You obviously should have Apache + mod_security + CRS set up
 and running already (in DetectionOnly mode initially), to allow for log
 inspection and adapting rules.</p>

 <steps>
 <item><p>Start modseccfg (<code class="prettyprint prettyprinted" style=""><span class="pln">python3 </span><span class="pun">-</span><span class="pln">m modseccfg</span></code>)</p></item>
 <item><p>Select a configuration/vhost file to inspect + work on.</p></item>
 <item><p>Pick the according error.log</p></item>
 <item><p>Inspect the rules with a high error count.</p></item>
 <item><p>[Disable] offending rules</p>
  <list>
 <item><p><em style="strong">Don't just go by the error count however!</em></p></item>
 <item><p>Make sure you don't disable essential or heuristic rules.</p></item>
 <item><p>Compare error with access log details.</p></item>
 <item><p>Else craft an exception rule ([Modify] or →Recipes).</p></item>
  </list>
 </item>
 <item><p>Thenceforth restart Apache (after testing changes: <code class="prettyprint prettyprinted" style=""><span class="pln">apache2ctl </span><span class="pun">-</span><span class="pln">t</span></code>).</p></item>
 </steps>

 <p>See also:</p>

 <list>
 <item><p><link type="seealso" xref="usage">usage</link></p></item>
 <item><p><link type="seealso" xref="remoting">remoting</link></p></item>
 <item><p><link type="seealso" xref="preconf">preconf setup</link> and <link type="seealso" xref="recipe">recipes</link></p></item>
 <item><p><link type="seealso" xref="scripts">log scripts/</link></p></item>
 <item><p>or the <link type="seealso" href="https://fossil.include-once.org/modseccfg/doc/trunk/FAQ.md">"FAQ"</link></p></item>
 </list>
</section>

<section>
 <subtitle>Notes</subtitle>
 <list>
 <item><p>Preferrably do not edit default <code class="prettyprint prettyprinted" style=""><span class="str">/etc/</span><span class="pln">apache</span><span class="pun">*</span></code> files</p></item>
 <item><p>Work on separated <code class="prettyprint prettyprinted" style=""><span class="str">/srv/</span><span class="pln">web</span><span class="pun">/</span><span class="pln">conf</span><span class="pun">.</span><span class="pln">d</span><span class="com">/*</span></code> configuration, if available</p></item>
 <item><p>And keep vhost settings in e.g. <code class="prettyprint prettyprinted" style=""><span class="pln">vhost</span><span class="pun">.*.</span><span class="pln">dir</span></code> files, rather than
 multiple <code class="prettyprint prettyprinted" style=""><span class="tag">&lt;VirtualHost&gt;</span></code> in one <code class="prettyprint prettyprinted" style=""><span class="pun">*.</span><span class="pln">conf</span></code> (else only the first section
 will be augmented).</p></item>
 <item><p>Use the editor (F4) to verify more complex settings.</p></item>
 </list>
</section>

<section>
 <subtitle>Missing features</subtitle>
 <list>
 <item><p>Rule [modify] is still unimplemented.</p></item>
 <item><p>Recipes are not worth using yet.</p></item>
 <item><p>No sudo usage.</p></item>
 </list>
</section>


</page>

<page
    xmlns="http://projectmallard.org/1.0/"
    type="guide"
    id="preconf">

    <info>
        <link type="guide" xref="index#nav"/>
        
        <desc></desc>
    </info>

    <title>modseccfg: preconf</title>

    
          
<section>
 <title>preconf</title>
          





 Help
 History

 
</section>

<section>
 <title>*.preconf scheme</title>

 Quick setup: File → Install → setup_preconf_scheme.py.


 <p>Due to how mod_security works, the definition order of rules and exceptions
 is important (the rule IDs are meaningless to the execution order). 
 In particular conditional SecRules (ctl:removeRule*) have to be declared before
 the CRS rules.</p>

 <p>So for recipes/macros to work you have to have one <code class="prettyprint prettyprinted" style=""><span class="pun">*.</span><span class="pln">preconf</span></code> per vhost.  And
 this has to be configured in a way they actually run before the rest of the CRS
 rules. 
 To do so:</p>

 <list>
 <item><p><p>Select <code class="prettyprint prettyprinted" style=""><span class="str">/etc/</span><span class="pln">modsecurity</span><span class="pun">/</span><span class="pln">crs</span><span class="pun">/</span><span class="pln">REQUEST</span><span class="pun">-</span><span class="lit">900</span><span class="pun">-</span><span class="pln">EXCLUSION</span><span class="pun">-</span><span class="pln">RULES</span><span class="pun">-</span><span class="pln">BEFORE</span><span class="pun">-</span><span class="pln">CRS</span><span class="pun">.</span><span class="pln">conf</span></code>
  in the vhost/conf dropdown.</p></p></item>
 <item><p><p>Add the Recipe → Setup → <em style="strong">CRS preconfig includes</em>.
  Change the <code class="prettyprint prettyprinted" style=""><span class="typ">IncludeOptional</span></code> path to where your normal <code class="prettyprint prettyprinted" style=""><span class="pln">vhost</span><span class="pun">.*.</span><span class="pln">conf</span></code> files reside,
  if it wasn't autodetected.</p></p></item>
 <item><p><p>Lastly open File → <em style="strong">Settings</em> (F12)
  and enable  ☑ Create and use *.preconf globally</p></p></item>
 </list>

 <p>Whenever you declared a recipe that belongs in a *.preconf file, modseccfg would
 automatically redirect the directive, or create the according preconf first.</p>

 <p>So once set up, you can just select the regular *.conf file in the vhost/conf dropdown,
 and modseccfg will redirect any settings if need be.</p>
</section>

<section>
 <subtitle>What's the resulting file structure?</subtitle>
 <p>Normally you'd have one <em>.preconf per vhost.</em>.conf:</p>

 <list>
 <item><p>/etc/apache2/sites-enabled/examplecom.preconf</p></item>
 <item><p>/etc/apache2/sites-enabled/examplecom.conf</p></item>
 <item><p>/srv/www/etc/exampleorg.preconf</p></item>
 <item><p>/srv/www/etc/exampleorg.conf</p></item>
 </list>

 <p>They also show up atop the vhost/conf dropdown, when the 900-EXCLUSION/IncludeOptional
 was defined correctly. The dropdown matches the order in which Apache sees any
 configuration files.</p>
</section>

<section>
 <subtitle>Do I really need this?</subtitle>
 <p>If you're only using <link type="seealso" href="https://fossil.include-once.org/modseccfg/wiki/modseccfg">modseccfg</link> for disabling existing rules, then no;
 the standard vhost.conf files are sufficient. You won't need *.preconf files for that.</p>

 <p>Currently it only pertains a handful of recipes and setting CRS variables.</p>
</section>

<section>
 <subtitle>One global *.preconf for all sites?</subtitle>
 <p>Auto-created *.preconf files are bound to one DocumentRoot per vhost.
 So if you want to have one preconf-file for all (or multiple) vhosts,
 you'll have to remove the <code class="prettyprint prettyprinted" style=""><span class="tag">&lt;Directory&gt;</span></code> wrapper; and manually <em style="strong">symlink</em>
 the filenames to one central file.</p>
</section>

<section>
 <subtitle>Implementation notes</subtitle>
 <list>
 <item><p>For auto-creation, file extensions .conf, .dir, .vhost are recognized,
 else .preconf is just appended</p></item>
 <item><p>For writer insertions, rx.end ought to look for <code class="prettyprint prettyprinted" style=""><span class="tag">&lt;/Directory&gt;</span></code> in place
 of <code class="prettyprint prettyprinted" style=""><span class="tag">&lt;/VirtualHost&gt;</span></code> section markers, and insert before that.</p></item>
 <item><p>Appending should still work for .dir files (single global preconf).</p></item>
 </list>
</section>

<section>
 <subtitle>Alternative: …/late-enabled/</subtitle>
 <p>Instead of preconf files, you could also set up the mod_security rules to
 load after any vhosts, with e.g. a sites- oder modules-late-enabled/ directory.
 Though that's gonna make simpler SecRuleRemove directives largely disfunct.</p>
</section>

  




  



</page>

<page
    xmlns="http://projectmallard.org/1.0/"
    type="guide"
    id="recipe">

    <info>
        <link type="guide" xref="index#nav"/>
        
        <desc></desc>
    </info>

    <title>modseccfg: recipe</title>

    
          
<section>
 <title>recipe</title>
          





 Help
 History

 
</section>

<section>
 <title>Recipe</title>
 <p>There's a small collection of SecRule/directive samples in Recipe→….
 Nothing novel, but just common rule exception schemes, and a few mod_security and apache
 defaults to consider.</p>

 <p>Note that most recipes take the currently selected rule id into account. But some
 work best with a relevant log entry selected (for the request_path and detected rule/data).</p>

 <table shade="rows cols" rules="rows cols"><tbody>

   <tr>
         <td><p>Menu entry</p></td>
         <td><p>What does it do</p></td>
         <td><p>Source</p></td>
   </tr>


   <tr>
         <td><p>❮Wrap❯ Exclusions</p></td>
         <td><p>SecRuleRemoveById</p></td>
         <td><p>-</p></td>
   </tr>
   <tr>
         <td><p>→ Locaton</p></td>
         <td><p>- wrapped in ❮Location❯</p></td>
         <td><p>rule+log</p></td>
   </tr>
   <tr>
         <td><p>→ Directory</p></td>
         <td><p>- wrapped in ❮Directory❯</p></td>
         <td><p>rule+log</p></td>
   </tr>
   <tr>
         <td><p>→ FilesMatch</p></td>
         <td><p>- wrapped in ❮FilesMatch❯</p></td>
         <td><p>rule</p></td>
   </tr>
   <tr>
         <td><p>Exclude Param</p></td>
         <td><p>SecRuleUpdateTargetByID</p></td>
         <td><p>rule</p></td>
   </tr>
   <tr>
         <td><p>Rule  DetectOnly</p></td>
         <td><p>SecRuleUpdateActionById</p></td>
         <td><p>rule</p></td>
   </tr>
   <tr>
         <td><p>URL  DetectOnly</p></td>
         <td><p>SecRule ctl:</p></td>
         <td><p>log</p></td>
   </tr>
   <tr>
         <td><p>Whitelist</p></td>
         <td><p>-</p></td>
         <td><p>-</p></td>
   </tr>
   <tr>
         <td><p>→ RREMOTE_ADDR</p></td>
         <td><p>SecRule with $remote_addr</p></td>
         <td><p>log</p></td>
   </tr>
   <tr>
         <td><p>→ IP File</p></td>
         <td><p>SecRule from $confn.whitelist</p></td>
         <td><p>confn</p></td>
   </tr>
   <tr>
         <td><p>Macros</p></td>
         <td><p>mod_macro definitions</p></td>
         <td><p>-</p></td>
   </tr>
   <tr>
         <td><p>Setup</p></td>
         <td><p>-</p></td>
         <td><p>-</p></td>
   </tr>
   <tr>
         <td><p>→ CRS *.preconf</p></td>
         <td><p>IncludeOptional .../*.preconf</p></td>
         <td><p>vhosts</p></td>
   </tr>
   <tr>
         <td><p>→ CldFl IP2L</p></td>
         <td><p>SetEnvIf + SecRule setvar:</p></td>
         <td><p>-</p></td>
   </tr>
   <tr>
         <td><p>→ CldFl RemoteIP</p></td>
         <td><p>RemoteIPTrustedProxy + SecRule TX.IS_CLOUDFLARE</p></td>
         <td><p>-</p></td>
   </tr>
   <tr>
         <td><p>→ LogFormat</p></td>
         <td><p>LogFormats</p></td>
         <td><p>-</p></td>
   </tr>
   <tr>
         <td><p>→ preconf_stub</p></td>
         <td><p>(autocreated) ❮Directory❯ wrapper for *.preconf files</p></td>
         <td><p>vhosts</p></td>
   </tr>

 </tbody></table>

 <p>Now the mini-editor window isn't very legible without syntax highlighting. But
 presumably this isn't an overly fascinating feature anyway. Hence there's also
 no recipe/*.txt directory for adding new ones yet.</p>

 <p>Some of these rule exclusion snippets require the <link type="seealso" xref="preconf">preconf</link> scheme to
 be configured, btw.</p>
</section>

  




  



</page>

<page
    xmlns="http://projectmallard.org/1.0/"
    type="guide"
    id="remoting">

    <info>
        <link type="guide" xref="index#nav"/>
        
        <desc></desc>
    </info>

    <title>modseccfg: remoting</title>

    
          
<section>
 <title>remoting</title>
          





 Help
 History

 
</section>

<section>
 <title>remoting ssh:/</title>
 <p>In order to use <link type="seealso" href="https://fossil.include-once.org/modseccfg/wiki/modseccfg">modseccfg</link> with a remote server, the
 easiest option is to start it with a ssh: servername:</p>

 <code class="prettyprint"><span class="pln">modseccfg vps</span><span class="pun">:/</span></code>

 <p>This obviously requires that you have an according <link type="seealso" href="https://kb.iu.edu/d/aews">Host entry in your
 <code class="prettyprint prettyprinted" style=""><span class="pun">~</span><span class="str">/.ssh/</span><span class="pln">config</span></code></link> with RSA key.</p>

 <p>You can of course use a full hostname and user prefix:</p>

 <code class="prettyprint"><span class="pln">modseccfg www</span><span class="pun">-</span><span class="pln">data@ssh</span><span class="pun">.</span><span class="pln">example</span><span class="pun">.</span><span class="pln">com</span><span class="pun">:/</span></code>

 <p>But if you don't have a key configured, the terminal would bring up the
 ssh password prompt quite frequently.</p>
</section>

<section>
 <subtitle>How does this work?</subtitle>
 <p>modseccfg internally starts <code class="prettyprint prettyprinted" style=""><span class="pln">sshfs</span></code> and binds the remote system to
 <code class="prettyprint prettyprinted" style=""><span class="pun">~</span><span class="str">/mnt/</span><span class="pln">sshname</span><span class="pun">:/</span></code>. That mount will remain active while modseccfg is
 running, and unmounted on exit. All *.conf file access and most log
 scanning uses this filesystem mount point.</p>

 <p>Other functions (concurrent audit log reading, or package installs)
 will open another ssh channel however.</p>
</section>

<section>
 <subtitle>Alternatives</subtitle>
 <p>In theory you could also use X11 forwarding, when modseccfg was installed
 on the server:</p>

 <code class="prettyprint"><span class="pln"> ssh vps </span><span class="pun">-</span><span class="pln">X modseccfg</span></code>

 <p>In practice, this is unbearably slow. Python/Tkinter over SSH is no fun,
 but a fallback alternative.</p>

 <p>So alternatively, you might want to install <link type="seealso" href="https://xpra.org/">xpra</link>:</p>

 <code class="prettyprint"><span class="pln"> xpra </span><span class="pun">--</span><span class="pln">start ssh</span><span class="pun">:</span><span class="pln">vps5 </span><span class="pun">--</span><span class="pln">start</span><span class="pun">=</span><span class="pln">modseccfg</span></code>

 <p>That's much faster. But still requires both xpra and <code class="prettyprint prettyprinted" style=""><span class="pln">pip3 install modseccfg</span></code>
 on the server beforehand.</p>
</section>

<section>
 <subtitle>Pros &amp; Cons</subtitle>
 <p>The sshfs-remoting might be slower on startup, as each Apache *.conf file
 has to be read individually. (And ssh/encryption does indeed have a
 performance price).
 But overall it's easier to use than installing modseccfg on the server.</p>

 <p>Care should be taken that the <code class="prettyprint prettyprinted" style=""><span class="pun">~</span><span class="str">/mnt/</span></code> point isn't accessed by other tools
 (backup scripts) while modseccfg is running.</p>

 <p>Alternatively you can change the mount point in 
 File →  Settings → Utils, of course.</p>

 <p>There's also an sshfs_opts config option. Which you don't normally need
 however.</p>
</section>

  




  



</page>

<page
    xmlns="http://projectmallard.org/1.0/"
    type="guide"
    id="scripts">

    <info>
        <link type="guide" xref="index#nav"/>
        
        <desc></desc>
    </info>

    <title>modseccfg: scripts</title>

    
          
<section>
 <title>scripts</title>
          





 Help
 History

 
</section>

<section>
 <title>Log processors / filters</title>
 <p>There's a small sample of log filter scripts/ in Log →
 Report and → Preprocess.</p>


 <p>There aren't many yet.  Because this is slightly hamstrung by the virtually
 unparsable Apache log formats.  In particular any "extended" log formats
 aren't extractable just with cut/awk and similar shell constructs, because
 the log field order is arbitrary and thus incompatible across different
 setups.  (And no, the standard combined format itself isn't useful to
 scan for more interesting attributes. Nor is the mish-mash of delimiters
 and the mid-90s datetime format.)</p>


 <p>Nonetheless, you can probably use some of the reports. In particular the
 netnea rulereport scripts. They basically craft similar exception rules,
 as modseccfg recipes, but show them all at once - to cherrypick from.</p>

 <p>You can keep the rule report window open alongside the main window. Making
 it slightly more useful than the log tab even.</p>

 <p>Report and preprocess scripts are just bundled with modseccfg for convenience.
 And you can easily copy them out from the scripts/ directory, if you want
 to keep using them independently. (If anyone wants an installer .deb bundle,
 please drop a mail.)</p>
</section>

<section>
 <subtitle>Add your own</subtitle>
 <p>Use <code class="prettyprint prettyprinted" style=""><span class="pln">locate modseccfg</span><span class="pun">/</span><span class="pln">scripts</span></code> to find the scripts/ directory. But <em style="strong">do not</em> save
 your own scripts there, but just symlink them in. (Else they will get deleted with
 any update.)</p>

 <p>Each script (even if just a slim shell wrapper), requires a little description
 block atop, at least:</p>

 <code class="prettyprint"><span class="com"># type: report</span><span class="pln">
 </span><span class="com"># category: mytools</span><span class="pln">
 </span><span class="com"># title: search for common issue...</span><span class="pln">

 grep </span><span class="str">"error"</span><span class="pln"> $1  </span><span class="com"># or whatever</span></code>

 <p>You can parameterize the scripts likewise, or have a <code class="prettyprint prettyprinted" style=""><span class="com"># type: image</span></code> report
 even.
 See the <code class="prettyprint prettyprinted" style=""><span class="pln">scripts</span><span class="pun">/</span><span class="pln">__init__</span><span class="pun">.</span><span class="pln">py</span></code> documentation block for details. 
 Filters are probably the easiest to craft.</p>
</section>

  




  



</page>

<page
    xmlns="http://projectmallard.org/1.0/"
    type="guide"
    id="usage">

    <info>
        <link type="guide" xref="index#nav"/>
        
        <desc></desc>
    </info>

    <title>modseccfg: usage</title>

    
          
<section>
 <title>usage</title>
          





 Help
 History

 
</section>

<section>
 <title>Usage</title>
</section>

<section>
 <subtitle>Select a vhost/conf</subtitle>
 <p>Select which config file will be shown and edited through the vhost/conf dropdown:</p>

 <p><media type="image" src="https://fossil.include-once.org/modseccfg/raw/97fbb5bb014" mime="image/png" /></p>

 <p>This will change the status icons shown next to the rules, if anything is configured in that vhost/conf file.</p>

 <p><media type="image" src="https://fossil.include-once.org/modseccfg/raw/229483a1bea" mime="image/png" /></p>

 <p>And Disable or Enable will influence that very rule state.
 Browse for high error counts to check on rules which <em style="strong"><em>might</em></em> be false positives.
 Verify what the rules do with the Info dialog.</p>

 <p><media type="image" src="https://fossil.include-once.org/modseccfg/raw/32085eff9eb6b" mime="image/png" /></p>

 <p>Take particular note to the recent log entries there. Recent events will give
 a clue if a rule really blocked concrete intrusion attempts, or expected requests.</p>
</section>

<section>
 <subtitle>Select log file to show</subtitle>
 <p>Switching vhosts should automatically select the according log file.
  Else use the dropdown box:</p>

 <p><media type="image" src="https://fossil.include-once.org/modseccfg/raw/f23b415f9707d3" mime="image/png" /></p>

 <p>For rule scoring it's best to use the error.log.
 Audit.logs take much longer to process. (In particular non-JSON audit logs,
 or reading concurrent/directory-stored ones.)
 Browse through the entries to see more detail in the logview box:</p>

 <p><media type="image" src="https://fossil.include-once.org/modseccfg/raw/1a0c173d78" mime="image/png" /></p>

 <p>Use the search feature above the log dropdown to filter events by common messages.</p>

 <p>See also <link type="seealso" xref="scripts">Log → Reports / Preprocessors</link></p>

 <p><media type="image" src="https://fossil.include-once.org/modseccfg/raw/1d01294f4f2d" mime="image/png" /></p>

 <p>A handful of common log errors are explained via Log → Advise</p>
</section>

<section>
 <subtitle>Install</subtitle>
 <list>
 <item><p>There's a few packages/scripts in File → Install</p></item>
 <item><p>Any entry will bring up a "terminal" prompt before excuting the commands.</p></item>
 <item><p>Notably the installation will work on remote servers. If you want to apply the
 same package locally, you might need to restart modseccfg without <link type="seealso" xref="remoting">remoting</link>.</p></item>
 </list>
</section>

<section>
 <subtitle>File → Settings</subtitle>
 <p><media type="image" src="https://fossil.include-once.org/modseccfg/raw/cf8c04316a8f6" mime="image/png" /></p>

 <p>There's a few notable options for modseccfg itself, that change default behaviours
 and even how config files are updated.</p>

 <list>
 <item><p>Most notably the backup options (albeit there are failsafes, it's still beta software).</p></item>
 <item><p>Or where to mount remote filesystems.</p></item>
 <item><p>And how to filter logs.</p></item>
 </list>
</section>

<section>
 <subtitle>File → SecOptions</subtitle>
 <p>This dialog updates core mod_security directives. Most of those you want to change
 in the global mod_security.conf (selected as vhost/conf), or a customized
 /www/etc/security.conf if you have such.</p>

 <p>But you can of course change these directives on a per-vhost basis. Most notably
 SecRuleEngine to DetectionOnly whilst testing the rules.</p>

 <p>Note that each option will yield a lengthier tooltip explanation.</p>
</section>

<section>
 <subtitle>File → CoreRuleSet options</subtitle>
 <p>The CoreRuleSet comes with its own set of runtime variables (tx.varname). Generally
 you want to edit the crs-setup.conf file globally, if possible.</p>

 <p>Some vhosts might need customized handling however. And this is where it gets complicated.
 You will need <link type="seealso" xref="preconf">preconf</link> enabled. And keep in mind that you'll be preempting
 setvar: expressions from crs-setup.conf. Which is why the dialog offers an "id" and a
 "fn" option atop. 
 When overriding variables, the according entry from crs-setup needs to be stopped from
 running (because it's executed after the *.preconf rule). Hence the CRS options dialog
 will usually use id:5999 and a ctl:removeRule= list for each variable.</p>

 <p>However, when invoking the dialog on a freshly created preconf file, all the
 usual fields will be empty.  That's ok for boolean and numberic flags, which are quick
 to fill in.  But the tx.allowed_request_content_types for example requires appending
 on the original list. </p>

 <list>
 <item><p>So, you either want to use an text editor in parallel for the long fields,</p></item>
 <item><p>Or temporarily enable Settings → CRS options → use defaults to have
 them filled with standard values.</p></item>
 <item><p>Or select the global crs-setup.conf as vhost/conf first, then start the CRS options
 dialog, and set "id" to 5999 and "fn" to the vhost.*.preconf file you actually want
 to update.</p></item>
 </list>

 <p>So the dialog is more of a gimmick here. Editing crs-setup.conf directly is often more
 practical, unless there are stark contrasts between vhosts.</p>
</section>

<section>
 <subtitle>Recipes</subtitle>
 <p>See also <link type="seealso" xref="recipe">recipe</link> on other/conditional SecRule* constructs to control
 rules.</p>
</section>

<section>
 <subtitle>Editor</subtitle>
 <p>F4 will bring up the .conf file editor. Because some things are best handled with
 a keyboard after all.</p>

 <p>And F3 will show the editor (in read-only mode) for the current log file instead.</p>
</section>


</page>

# [Disable] and [Enable] or Recipes will update the currently
# selected vhost.conf
#



import sys, os, re, json, subprocess
from modseccfg import utils, icons, data, vhosts, logs, writer, editor, ruleinfo, recipe, modify, install, scripts
from modseccfg.utils import srvroot, conf, inject, log
import tkinter as tk, PySimpleGUI as sg, warnings

#-- init
sg.theme(conf.theme)
log.init.info("initialize modules")
vhosts.scan_all()

                return self.status("Needs a rule selected")
            func(self, data)
        return mask


#-- widget structure
menu = [
    ["File", ["Edit conf/vhost file (F4)", "---", "Settings (F12)", "SecEngine options", "CoreRuleSet options", "Rescan configs", "---", "Test", "Debug", "Help", "About", "---", "Exit"]],
    ["Rule", ["Info (F1)", "Disable", "Enable", "Modify", "<Wrap>", "Masquerade"]],
    ["Recipe"],
    ["Log", ["Advise", "Log View (F3)", "Reread log"]],
]
layout = [
    [
        sg.Column([

            values = re.grep(rx, logs.state.prev)
        )

    # File: About
    def about(self, data):
        m = utils.pluginconf.plugin_meta(module="__init__")
        sg.popup(f"{m['title']} {m['version']}\n{m['description']}\n\n{m['doc']}\n")
    # File: Help
    def help(self, uu):
        os.system(f"yelp '{data.dir}/help/index.page' &")

    # File: Debug
    def debug(self, data):
        sg.show_debugger_window()  # we need the window handle for the event loop
        w = sys.modules["PySimpleGUI.PySimpleGUI"]._Debugger.debugger.watcher_window
        self.win_register(w, lambda *x: None)

        }
        for k,v in data.items():
            if v and isinstance(k, str) and k.find("[") >= 0:
                name, key, *uu = re.split("[\[\]]", k)
                if name=="transforms":
                    for key in v: # already a list
                        opts["flags"][key] = 1
                #elif key == "logdata":
                #    opts[key] = v # properties, but also params{}
                elif key == "msg":
                    # only assign legitimate/new message, not faux @SecAction or VARS+PATTERN
                    if not self.r.msg_stub or v != self.r.msg:
                        opts[key] = v
                else:
                    opts[name][key] = v
        return opts