GUI editor to tame mod_security rules

⌈⌋ branch:  modseccfg


Check-in [fbaa7c8587]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:add WrapDirective detection, and SecRuleUpdate* collection
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: fbaa7c8587f6d8a182d35e4d9859d98ee0b11f621c62bb88698faca2ff5edb25
User & Date: mario 2020-11-23 18:56:02
Context
2020-11-24
18:24
Change `fn` to select with existing vhost/conf files check-in: 2660ee756b user: mario tags: trunk
2020-11-23
18:56
add WrapDirective detection, and SecRuleUpdate* collection check-in: fbaa7c8587 user: mario tags: trunk
18:54
Prepare audit/*/*/*/* path collection (find+cat per ssh-pipe) and snippet extraction. check-in: ba60b0748a user: mario tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to modseccfg/vhosts.py.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# api: modseccfg
# encoding: utf-8
# title: *.conf scanner
# description: Compiles a list of relevant apache/vhost files and Sec* settings
# type: tokenizer
# category: apache
# version: 0.5
# config:
#    { name: envvars, value: "/etc/default/apache2", type: str, description: "Look up APACHE_ENV vars from shell script", help: "Mostly applies to Debian derivates. Other distros usually embed SetEnv directives for log paths." }
# license: ASL
#
# Runs once to scan for an vhost* and mod_security config files.
# Uses `apache2ctl -t -D DUMP_INCLUDES` to find all includes,
# and regexes for Sec*Rules or *Log locations and ServerNames.






|







1
2
3
4
5
6
7
8
9
10
11
12
13
14
# api: modseccfg
# encoding: utf-8
# title: *.conf scanner
# description: Compiles a list of relevant apache/vhost files and Sec* settings
# type: tokenizer
# category: apache
# version: 0.6
# config:
#    { name: envvars, value: "/etc/default/apache2", type: str, description: "Look up APACHE_ENV vars from shell script", help: "Mostly applies to Debian derivates. Other distros usually embed SetEnv directives for log paths." }
# license: ASL
#
# Runs once to scan for an vhost* and mod_security config files.
# Uses `apache2ctl -t -D DUMP_INCLUDES` to find all includes,
# and regexes for Sec*Rules or *Log locations and ServerNames.
57
58
59
60
61
62
63

64


65
66
67
68

69
70
71
72
73
74
75
76
        re.M|re.I|re.X
    )
    # extract directive line including line continuations (<\><NL>)
    configline = re.compile(
        """ ^
        [\ \\t]*                          # whitespace \h*
        # (?:Use \s{1,4})?                  # optional: `Use␣` to find custom macros like `Use SecRuleRemoveByPath…`

        (\w+)                             # alphanumeric directive 


          [\ \\t]+                        # whitespace \h+
        (
          (?: [^\\n\\\\]+ | [\\\\]. )*    # literals, or backslash + anything
        )

        $ """,
        re.M|re.S|re.X
    )
    # to strip <\><NL>
    escnewline = re.compile(
        """[\\\\][\\n]\s*"""              # escaped linkebreaks
    )
    # handle quoted/unquoted directive arguments (not entirely sure if Apache does \" escaped quotes within)







>
|
>
>




>
|







57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
        re.M|re.I|re.X
    )
    # extract directive line including line continuations (<\><NL>)
    configline = re.compile(
        """ ^
        [\ \\t]*                          # whitespace \h*
        # (?:Use \s{1,4})?                  # optional: `Use␣` to find custom macros like `Use SecRuleRemoveByPath…`
        (
          \w+ |                           # alphanumeric directive 
          </?(?:File|Loc|Dir|If)\w*\\b    # or <Wrap> section
        )
          [\ \\t]+                        # whitespace \h+
        (
          (?: [^\\n\\\\]+ | [\\\\]. )*    # literals, or backslash + anything
        )
        (?: $ | >.*$ )                    # account for line end or closing >
        """,
        re.M|re.S|re.X
    )
    # to strip <\><NL>
    escnewline = re.compile(
        """[\\\\][\\n]\s*"""              # escaped linkebreaks
    )
    # handle quoted/unquoted directive arguments (not entirely sure if Apache does \" escaped quotes within)
161
162
163
164
165
166
167


168
169
170
171
172
173
174
175
176
177
178
179
180
181
182


183
184

185
186
187
188
189
190
191
192
193
194

195
196










197
198
199
200
201
202
203
                List of error/access.log filenames
        cfg : dict
                SecOption directives
        rulestate : dict
                SecRuleRemove* states (id→0)
        ruledecl : dict
                Map contained SecRules id into vhosts.rules{}


        linemap : dict
                Line number → RuleID (for looking up chained rules in error.log)
    """

    # split *.conf directives, dispatch onto assignment/extract methods
    def __init__(self, fn, src, cfg_only=False):

        # vhost properties
        self.fn = fn
        self.t = "cfg"
        self.name = ""
        self.logs = []
        self.cfg = {}
        self.rulestate = {}
        self.ruledecl = {}


        self.linemap = {} 
        self.mk_linemap(src)   # fill .linemap{}

        
        # extract directive lines
        for dir,args  in rx.configline.findall(src):    # or .finditer()? to record positions right away?
            dir = dir.lower()
            #print(dir, args)
            if hasattr(self, dir):
                if cfg_only: #→ if run from secoptions, we don't actually want rules collected
                    continue
                func = getattr(self, dir)
                func(self.split_args(args))

            elif dir.startswith("sec"):
                self.cfg[dir] = args










        # determine config file type
        if self.name:
            self.t = "vhost"
        elif len(self.rulestate) >= 5:
            self.t = "cfg"
        elif len(self.ruledecl) >= 5:
            self.t = "rules"







>
>















>
>


>










>


>
>
>
>
>
>
>
>
>
>







165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
                List of error/access.log filenames
        cfg : dict
                SecOption directives
        rulestate : dict
                SecRuleRemove* states (id→0)
        ruledecl : dict
                Map contained SecRules id into vhosts.rules{}
        update : dict
                Map of SecRuleUpdate…By…  { id→{vars:[],action:[]} }
        linemap : dict
                Line number → RuleID (for looking up chained rules in error.log)
    """

    # split *.conf directives, dispatch onto assignment/extract methods
    def __init__(self, fn, src, cfg_only=False):

        # vhost properties
        self.fn = fn
        self.t = "cfg"
        self.name = ""
        self.logs = []
        self.cfg = {}
        self.rulestate = {}
        self.ruledecl = {}
        self.update = {}   # SecRuleUpdate… map
        # internal state
        self.linemap = {} 
        self.mk_linemap(src)   # fill .linemap{}
        self.wrap = []     # state of <Wrap> section
        
        # extract directive lines
        for dir,args  in rx.configline.findall(src):    # or .finditer()? to record positions right away?
            dir = dir.lower()
            #print(dir, args)
            if hasattr(self, dir):
                if cfg_only: #→ if run from secoptions, we don't actually want rules collected
                    continue
                func = getattr(self, dir)
                func(self.split_args(args))
            # .cfg option
            elif dir.startswith("sec"):
                self.cfg[dir] = args
                print(self.cfg)
            # .wrap state
            elif dir.startswith("<"):
                if dir.startswith("</"):
                    if self.wrap:
                        self.wrap.pop(0)
                    else:
                        print("ERROR IN CONFIG STRUCTURE(?): tried to pop a </Wrap> directive without being within a section")
                else:
                    self.wrap.insert(0, dir[1:])
        # determine config file type
        if self.name:
            self.t = "vhost"
        elif len(self.rulestate) >= 5:
            self.t = "cfg"
        elif len(self.ruledecl) >= 5:
            self.t = "rules"
230
231
232
233
234
235
236


237
238
239
240
241
242
243
244



245
246
247
248
249
250
251
252
253
254
255
256
257
258
259


260








261





262
263
264
265
266
267
268
269
            tmp.last_rule_id = r.id
        # add a float id (+0.1) to chained rules (no idea how virtual `7f9aa85dec58` rule ids are generated, lackluster docs / no mailing list / no IRC response)
        elif rules.get(last_id) and "chain" in rules[last_id].flags:
            tmp.last_rule_id = round(tmp.last_rule_id + 0.1, 1)
            r.id = tmp.last_rule_id
            r.chained_to = int(last_id) # primary parent
        rules[r.id] = self.ruledecl[r.id] = r


        #print(r.__dict__)

    # modsec: just a secrule without conditions
    def secaction(self, args):
        self.secrule(["@SecAction", "setvar:", args[0]])
    
    # modsec: SecRuleRemoveById 900001 900002 900003
    def secruleremovebyid(self, args):



        for a in args:
            if re.match("^\d+-\d+$", a):   # are ranges still allowed?
                a = [int(x) for x in a.split("-")]
                for i in range(*a):
                    if i in rules:    # only apply state for known/existing rules, not the whole range()
                        self.rulestate[i] = 0
            elif re.match("^\d+$", a):
                self.rulestate[int(a)] = 0
            else:
                self.rulestate[a] = 0  # from tag

    # modsec: SecRuleRemoveByTag sqli app-name
    def secruleremovebytag(self, args):
        self.secruleremovebyid(args)



    # these need to be mapped onto existing rules (if within t==cfg)








    # · SecRuleUpdateTargetById





    # · SecRuleUpdateActionById

    # modssec: irrelevant (not caring about skipAfter rules)
    def secmarker(self, args):
        pass

    # v3-connector: Include
    def modsecurity_rules_file(self, args):







>
>








>
>
>





|

|

|





>
>
|
>
>
>
>
>
>
>
>
|
>
>
>
>
>
|







250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
            tmp.last_rule_id = r.id
        # add a float id (+0.1) to chained rules (no idea how virtual `7f9aa85dec58` rule ids are generated, lackluster docs / no mailing list / no IRC response)
        elif rules.get(last_id) and "chain" in rules[last_id].flags:
            tmp.last_rule_id = round(tmp.last_rule_id + 0.1, 1)
            r.id = tmp.last_rule_id
            r.chained_to = int(last_id) # primary parent
        rules[r.id] = self.ruledecl[r.id] = r
        if self.wrap: # now, rule declarations shouldn't be conditional, and we're not really gonna use this; just record it
            rules[r.id].wrap = True #self.wrap[0]
        #print(r.__dict__)

    # modsec: just a secrule without conditions
    def secaction(self, args):
        self.secrule(["@SecAction", "setvar:", args[0]])
    
    # modsec: SecRuleRemoveById 900001 900002 900003
    def secruleremovebyid(self, args):
        state = 0
        if self.wrap:  # record if within <Dir|File|If|Wrap> section
            state = -1
        for a in args:
            if re.match("^\d+-\d+$", a):   # are ranges still allowed?
                a = [int(x) for x in a.split("-")]
                for i in range(*a):
                    if i in rules:    # only apply state for known/existing rules, not the whole range()
                        self.rulestate[i] = state
            elif re.match("^\d+$", a):
                self.rulestate[int(a)] = state
            else:
                self.rulestate[a] = state  # from tag

    # modsec: SecRuleRemoveByTag sqli app-name
    def secruleremovebytag(self, args):
        self.secruleremovebyid(args)

    # modsec: SecRuleUpdateTargetById, SecRuleUpdateActionById
    #
    # These shouldo be mapped ~~onto existing rules (if within t==cfg)~~
    # But just store them into vhost.update{} for now. They don't necessarily apply globally.
    def secruleupdatetargetbyid(self, args):
        self._secruleupdate("vars", *args)
    def secruleupdateactionbyid(self, args):
        self._secruleupdate("action", *args)
    def _secruleupdate(self, cls, id, arg, *repl):
        if re.match("\d+"):
            id = int(id)
        if repl:  # merge third parameter from `SecRuleUpdateTarget 123456 NEW_TARGET REMOVE_VAR`
            arg = f"!{repl[0]},{arg}"
        if not self.update.get(id):
            self.update[id] = {"vars":[], "action":[]}
        self.update[id][cls].append(arg)
        # We don't really use the detail. This is just to record that any one rule has been "modified".
        

    # modssec: irrelevant (not caring about skipAfter rules)
    def secmarker(self, args):
        pass

    # v3-connector: Include
    def modsecurity_rules_file(self, args):
321
322
323
324
325
326
327


328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343

344
345
346
347
348
349
350
                Any servar:name=val from rule actions
        vars : str
                e.g. ARGS|PARAMS or &TX.VAR
        pattern : str
                e.g. '@rx ^.*$'
        hidden : bool
                Mark pure control rules / SecActions


    """
    
    def __init__(self, args):
        # secrule properties
        self.id = 0
        self.chained_to = 0
        self.msg = ""
        self.flags = []
        self.params = {}
        self.tags = []
        self.tag_primary = ""
        self.ctl = {}
        self.setvar = {}
        self.vars = "REQ*"
        self.pattern = "@rx ..."
        self.hidden = False

        # args must contain 3 bits for a relevant SecRule
        if len(args) != 3:
            #print("UNEXPECTED NUMBER OF ARGS:", args)
            return
        self.vars, self.pattern, actions = args
        #print(args)
        # split up actions,attributes:…







>
>
















>







361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
                Any servar:name=val from rule actions
        vars : str
                e.g. ARGS|PARAMS or &TX.VAR
        pattern : str
                e.g. '@rx ^.*$'
        hidden : bool
                Mark pure control rules / SecActions
        wrap : bool
                Mark if option occured in <If|Match|Etc> section
    """
    
    def __init__(self, args):
        # secrule properties
        self.id = 0
        self.chained_to = 0
        self.msg = ""
        self.flags = []
        self.params = {}
        self.tags = []
        self.tag_primary = ""
        self.ctl = {}
        self.setvar = {}
        self.vars = "REQ*"
        self.pattern = "@rx ..."
        self.hidden = False
        self.wrap = False
        # args must contain 3 bits for a relevant SecRule
        if len(args) != 3:
            #print("UNEXPECTED NUMBER OF ARGS:", args)
            return
        self.vars, self.pattern, actions = args
        #print(args)
        # split up actions,attributes:…