Skip to content

Commit c42a95f

Browse files
authored
Merge pull request #391 from nexB/backport-v4-filters
Backport v4 Jinja filters
2 parents b9c8532 + 469a061 commit c42a95f

File tree

11 files changed

+193
-259
lines changed

11 files changed

+193
-259
lines changed

docs/CHANGELOG.rst

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
2018-01-03
2+
3+
Release 3.3.1
4+
5+
* Add new Jinja2 custom template filters to multi_sort to sort and
6+
unique_together to compute unique lists. Both filter take an attributes
7+
list of attribute names and use all these attribute names to sort or
8+
compute unique values.
9+
* Use saneyaml library to dump and load YAML
10+
11+
112
2018-11-15
213

314
Release 3.3.0

setup.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ def read(*names, **kwargs):
2424

2525
setup(
2626
name='aboutcode-toolkit',
27-
version='3.3.0',
27+
version='3.3.1',
2828
license='Apache-2.0',
2929
description=(
3030
'AboutCode-toolkit is a tool to document the provenance (origin and license) of '
@@ -70,7 +70,8 @@ def read(*names, **kwargs):
7070
'jinja2 >= 2.9, < 3.0',
7171
'click >= 6.7, < 7.0',
7272
"backports.csv ; python_version<'3.6'",
73-
'PyYAML >= 3.0, < 4.0',
73+
'PyYAML >= 3.11, <=3.13',
74+
'saneyaml',
7475
'boolean.py >= 3.5, < 4.0',
7576
'license_expression >= 0.94, < 1.0',
7677
],

src/attributecode/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
basestring = str # Python 3 #NOQA
2828

2929

30-
__version__ = '3.3.0'
30+
__version__ = '3.3.1'
3131

3232

3333
__about_spec_version__ = '3.1'

src/attributecode/attrib.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import attributecode
3333
from attributecode import ERROR
3434
from attributecode import Error
35+
from attributecode.attrib_util import get_template
3536
from attributecode.licenses import COMMON_LICENSES
3637
from attributecode.model import parse_license_expression
3738
from attributecode.util import add_unc
@@ -46,7 +47,7 @@ def generate(abouts, template_string=None, vartext_dict=None):
4647
syntax_error = check_template(template_string)
4748
if syntax_error:
4849
return 'Template validation error at line: %r: %r' % (syntax_error)
49-
template = jinja2.Template(template_string)
50+
template = get_template(template_string)
5051

5152
try:
5253
captured_license = []
@@ -119,13 +120,14 @@ def generate(abouts, template_string=None, vartext_dict=None):
119120
return rendered
120121

121122

123+
122124
def check_template(template_string):
123125
"""
124126
Check the syntax of a template. Return an error tuple (line number,
125127
message) if the template is invalid or None if it is valid.
126128
"""
127129
try:
128-
jinja2.Template(template_string)
130+
get_template(template_string)
129131
except (jinja2.TemplateSyntaxError, jinja2.TemplateAssertionError) as e:
130132
return e.lineno, e.message
131133

src/attributecode/attrib_util.py

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env python
2+
# -*- coding: utf8 -*-
3+
4+
# ============================================================================
5+
# Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved.
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ============================================================================
16+
17+
from __future__ import absolute_import
18+
from __future__ import print_function
19+
from __future__ import unicode_literals
20+
21+
from jinja2 import Environment
22+
from jinja2.filters import environmentfilter
23+
from jinja2.filters import make_attrgetter
24+
from jinja2.filters import ignore_case
25+
from jinja2.filters import FilterArgumentError
26+
27+
28+
"""
29+
Extra JINJA2 custom filters and other template utilities.
30+
"""
31+
32+
33+
def get_template(template_text):
34+
"""
35+
Return a template built from a text string.
36+
Register custom templates as needed.
37+
"""
38+
env = Environment(autoescape=True)
39+
# register our custom filters
40+
env.filters.update(dict(
41+
unique_together=unique_together,
42+
multi_sort=multi_sort))
43+
return env.from_string(template_text)
44+
45+
46+
@environmentfilter
47+
def multi_sort(environment, value, reverse=False, case_sensitive=False,
48+
attributes=None):
49+
"""
50+
Sort an iterable using an "attributes" list of attribute names available on
51+
each iterable item. Sort ascending unless reverse is "true". Ignore the case
52+
of strings unless "case_sensitive" is "true".
53+
54+
.. sourcecode:: jinja
55+
56+
{% for item in iterable|multi_sort(attributes=['date', 'name']) %}
57+
...
58+
{% endfor %}
59+
"""
60+
if not attributes:
61+
raise FilterArgumentError(
62+
'The multi_sort filter requires a list of attributes as argument, '
63+
'such as in: '
64+
"for item in iterable|multi_sort(attributes=['date', 'name'])")
65+
66+
# build a list of attribute getters, one for each attribute
67+
do_ignore_case = ignore_case if not case_sensitive else None
68+
attribute_getters = []
69+
for attribute in attributes:
70+
ag = make_attrgetter(environment, attribute, postprocess=do_ignore_case)
71+
attribute_getters.append(ag)
72+
73+
# build a key function that has runs all attribute getters
74+
def key(v):
75+
return [a(v) for a in attribute_getters]
76+
77+
return sorted(value, key=key, reverse=reverse)
78+
79+
80+
@environmentfilter
81+
def unique_together(environment, value, case_sensitive=False, attributes=None):
82+
"""
83+
Return a list of unique items from an iterable. Unicity is checked when
84+
considering together all the values of an "attributes" list of attribute
85+
names available on each iterable item.. The items order is preserved. Ignore
86+
the case of strings unless "case_sensitive" is "true".
87+
.. sourcecode:: jinja
88+
89+
{% for item in iterable|unique_together(attributes=['date', 'name']) %}
90+
...
91+
{% endfor %}
92+
93+
"""
94+
if not attributes:
95+
raise FilterArgumentError(
96+
'The unique_together filter requires a list of attributes as argument, '
97+
'such as in: '
98+
"{% for item in iterable|unique_together(attributes=['date', 'name']) %} ")
99+
100+
# build a list of attribute getters, one for each attribute
101+
do_ignore_case = ignore_case if not case_sensitive else None
102+
attribute_getters = []
103+
for attribute in attributes:
104+
ag = make_attrgetter(environment, attribute, postprocess=do_ignore_case)
105+
attribute_getters.append(ag)
106+
107+
# build a unique_key function that has runs all attribute getters
108+
# and returns a hashable tuple
109+
def unique_key(v):
110+
return tuple(repr(a(v)) for a in attribute_getters)
111+
112+
unique = []
113+
seen = set()
114+
for item in value:
115+
key = unique_key(item)
116+
if key not in seen:
117+
seen.add(key)
118+
unique.append(item)
119+
return unique

src/attributecode/model.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,14 @@
5353
from urllib.error import HTTPError # NOQA
5454

5555
from license_expression import Licensing
56+
import saneyaml
5657

5758
from attributecode import CRITICAL
5859
from attributecode import ERROR
5960
from attributecode import INFO
6061
from attributecode import WARNING
6162
from attributecode import api
6263
from attributecode import Error
63-
from attributecode import saneyaml
6464
from attributecode import util
6565
from attributecode.util import add_unc
6666
from attributecode.util import copy_license_notice_files
@@ -1015,8 +1015,6 @@ def load(self, location, use_mapping=False, mapping_file=None):
10151015
loc = add_unc(loc)
10161016
with codecs.open(loc, encoding='utf-8') as txt:
10171017
input_text = txt.read()
1018-
# Check for duplicated key
1019-
yaml.load(input_text, Loader=util.NoDuplicateLoader)
10201018
"""
10211019
The running_inventory defines if the current process is 'inventory' or not.
10221020
This is used for the validation of the path of the 'about_resource'.
@@ -1030,7 +1028,9 @@ def load(self, location, use_mapping=False, mapping_file=None):
10301028
# wrap the value of the boolean field in quote to avoid
10311029
# automatically conversion from yaml.load
10321030
input = util.wrap_boolean_value(input_text) # NOQA
1033-
errs = self.load_dict(saneyaml.load(input), base_dir, running_inventory, use_mapping, mapping_file)
1031+
data = saneyaml.load(input, allow_duplicate_keys=False)
1032+
1033+
errs = self.load_dict(data, base_dir, running_inventory, use_mapping, mapping_file)
10341034
errors.extend(errs)
10351035
except Exception as e:
10361036
msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n' + str(e)
@@ -1081,7 +1081,7 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e
10811081
If with_absent, include absent (not present) fields.
10821082
If with_empty, include empty fields.
10831083
"""
1084-
about_data = {}
1084+
about_data = OrderedDict()
10851085
# Group the same license information (name, url, file) together
10861086
license_key = []
10871087
license_name = []
@@ -1110,7 +1110,7 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e
11101110
# Group the same license information in a list
11111111
license_group = list(zip_longest(license_key, license_name, license_file, license_url))
11121112
for lic_group in license_group:
1113-
lic_dict = {}
1113+
lic_dict = OrderedDict()
11141114
if lic_group[0]:
11151115
lic_dict['key'] = lic_group[0]
11161116
if lic_group[1]:
@@ -1121,7 +1121,8 @@ def dumps(self, use_mapping=False, mapping_file=False, with_absent=False, with_e
11211121
lic_dict['url'] = lic_group[3]
11221122
about_data.setdefault('licenses', []).append(lic_dict)
11231123
formatted_about_data = util.format_output(about_data, use_mapping, mapping_file)
1124-
return saneyaml.dump(formatted_about_data)
1124+
1125+
return saneyaml.dump(formatted_about_data, indent=2)
11251126

11261127
def dump(self, location, use_mapping=False, mapping_file=False, with_absent=False, with_empty=True):
11271128
"""

0 commit comments

Comments
 (0)