diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4ec2a0a..93af3c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,6 +40,11 @@ jobs: pip install ".[test,pandas,spark]" fi + - name: Lint with pre-commit + run: | + pip install pre-commit + pre-commit run --all-files --show-diff-on-failure + - name: Test with pytest run: | pytest tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a396a1e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.6 + hooks: + - id: ruff + args: [--fix] + exclude: notebooks/ + - id: ruff-format + exclude: notebooks/ diff --git a/CHANGES.rst b/CHANGES.rst index 73dc147..39fda8f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,10 @@ Release notes ============= +Version 1.1.0, Dec 2024 +----------------------- +* Removed all ROOT, cpp and cuda code, it was no longer supported. + Version 1.0.34, Dec 2024 ------------------------ * Fix typo in build pipeline Python versions config list. diff --git a/README.rst b/README.rst index 7c50ce9..9fd458f 100644 --- a/README.rst +++ b/README.rst @@ -11,18 +11,24 @@ At its core histogrammar is a suite of data aggregation primitives designed for In the simplest case, you can use this to compute histograms, but the generality of the primitives allows much more. -Several common histogram types can be plotted in Matplotlib, Bokeh and PyROOT with a single method call. +Several common histogram types can be plotted in Matplotlib and Bokeh with a single method call. If Numpy or Pandas is available, histograms and other aggregators can be filled from arrays ten to a hundred times -more quickly via Numpy commands, rather than Python for loops. If PyROOT is available, histograms and other -aggregators can be filled from ROOT TTrees hundreds of times more quickly by JIT-compiling a specialized C++ filler. -Histograms and other aggregators may also be converted into CUDA code for inclusion in a GPU workflow. And if -PyCUDA is available, they can also be filled from Numpy arrays by JIT-compiling the CUDA code. +more quickly via Numpy commands, rather than Python for loops. This Python implementation of histogrammar been tested to guarantee compatibility with its Scala implementation. -Latest Python release: v1.0.34 (Dec 2024). +Latest Python release: v1.1.0 (Dec 2024). Latest update: Dec 2024. +References +========== + +Histogrammar is a core component of `popmon `_, a package by ING bank +that allows one to check the stability of a dataset. popmon works with both pandas and spark datasets, +largely thanks to Histogrammar. + + + Announcements ============= diff --git a/docs/conf.py b/docs/conf.py index 32cd699..d88907e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Histogrammar documentation build configuration file. # @@ -10,37 +9,48 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os, inspect, importlib +import importlib +import inspect # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.napoleon', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.linkcode'] +extensions = [ + "sphinx.ext.autodoc", + "sphinxcontrib.napoleon", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.linkcode", +] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'Histogrammar' -copyright = u'2016, DIANA-HEP' +project = "Histogrammar" +copyright = "2016, DIANA-HEP" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -53,44 +63,44 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'sphinxdoc' +html_theme = "sphinxdoc" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -98,126 +108,120 @@ html_theme_options = {"nosidebar": True} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['static'] +html_static_path = ["static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'Histogrammardoc' +htmlhelp_basename = "Histogrammardoc" # -- Options for LaTeX output -------------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # # Additional stuff for the LaTeX preamble. + # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'Histogrammar.tex', u'Histogrammar Documentation', - u'DIANA-HEP', 'manual'), + ("index", "Histogrammar.tex", "Histogrammar Documentation", "DIANA-HEP", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'histogrammar', u'Histogrammar Documentation', - [u'DIANA-HEP'], 1) -] +man_pages = [("index", "histogrammar", "Histogrammar Documentation", ["DIANA-HEP"], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -226,85 +230,94 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Histogrammar', u'Histogrammar Documentation', - u'DIANA-HEP', 'Histogrammar', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "Histogrammar", + "Histogrammar Documentation", + "DIANA-HEP", + "Histogrammar", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u'Histogrammar' -epub_author = u'DIANA-HEP' -epub_publisher = u'DIANA-HEP' -epub_copyright = u'2016, DIANA-HEP' +epub_title = "Histogrammar" +epub_author = "DIANA-HEP" +epub_publisher = "DIANA-HEP" +epub_copyright = "2016, DIANA-HEP" # The language of the text. It defaults to the language option # or en if the language is not set. -#epub_language = '' +# epub_language = '' # The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' +# epub_scheme = '' # The unique identifier of the text. This can be a ISBN number # or the project homepage. -#epub_identifier = '' +# epub_identifier = '' # A unique identification for the text. -#epub_uid = '' +# epub_uid = '' # A tuple containing the cover image and cover page html template filenames. -#epub_cover = () +# epub_cover = () # HTML files that should be inserted before the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_pre_files = [] +# epub_pre_files = [] # HTML files shat should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_post_files = [] +# epub_post_files = [] # A list of files that should not be packed into the epub file. -#epub_exclude_files = [] +# epub_exclude_files = [] # The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 +# epub_tocdepth = 3 # Allow duplicate toc entries. -#epub_tocdup = True +# epub_tocdup = True # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = {"http://docs.python.org/": None} + def skip(app, what, name, obj, skip, options): if name == "__init__": return False return skip + def setup(app): app.connect("autodoc-skip-member", skip) + def linkcode_resolve(domain, info): - if domain != 'py': + if domain != "py": return None - if not info['module']: + if not info["module"]: return None - fileName = info['module'].replace('.', '/') + fileName = info["module"].replace(".", "/") try: lineNumber = inspect.getsourcelines(getattr(importlib.import_module(info["module"]), info["fullname"]))[1] - except: + except: # noqa: E722 return None else: return "https://github.com/histogrammar/histogrammar-python/blob/%s/%s.py#L%d" % (release, fileName, lineNumber) diff --git a/generate-grammars/grammar_to_ply.py b/generate-grammars/grammar_to_ply.py deleted file mode 100755 index 640a14d..0000000 --- a/generate-grammars/grammar_to_ply.py +++ /dev/null @@ -1,448 +0,0 @@ -#!/usr/bin/env python - -# Written by Andrew Dalke -# Copyright (c) 2008 by Dalke Scientific, AB -# Modified by Jim Pivarski, 2016 -# -# (This is the MIT License with the serial numbers scratched off and my -# name written in in crayon. I would prefer "share and enjoy" but -# apparently that isn't a legally acceptable.) -# -# Copyright (c) 2008 Andrew Dalke -# Dalke Scientific Software, AB -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -"""This program converts Python's 'Grammar' file into a PLY grammar - -The Grammar file is pretty simple but not designed for LALR(1) and -similar parsers. This program tweaks the grammar slightly and -flattens the results to a more usable form. It might prove useful for -other parsers. -""" - -import sys -import itertools -import importlib -import time -import datetime - -import histogrammar -from histogrammar.pycparser.ply import lex, yacc - -inputGrammar, grammarActions, inputLex, outputPLY = sys.argv[1:] - -outputTable = outputPLY.split("/")[-1].replace(".py", "") - -literal_to_name = importlib.import_module(inputLex, histogrammar).literal_to_name - -grammarActionsDict = {} -exec open(grammarActions).read() in grammarActionsDict -actions = grammarActionsDict["actions"] -asts = grammarActionsDict["asts"] - -duplicates = { - "varargslist_star2": "varargslist_star", - "print_stmt_star": "print_stmt_plus", - "import_from_star": "import_from_plus", -} - -tokens = ("LEXER_NAME", "PARSER_NAME", "STRING", - "NL", "LPAR", "RPAR", "COLON") - -def t_comment(t): - r"\#.*" - pass - -t_ignore = " \t" - -def t_NL(t): - r"\n" - t.value = t.lexer.lineno - t.lexer.lineno += 1 - if getattr(t.lexer, "paren_depth", 0) == 0: - return t - -def t_word(t): - r"[a-zA-Z_0-9]+" - if t.value == t.value.upper(): - t.type = "LEXER_NAME" - return t - if t.value == t.value.lower(): - t.type = "PARSER_NAME" - return t - raise AssertionError("Unknown word: %r" % t.value) - -t_STRING = r"'[^']+'" - -def t_LPAR(t): - r"\(" - t.lexer.paren_depth = getattr(t.lexer, "paren_depth", 0)+1 - return t - -def t_RPAR(t): - r"\)" - t.lexer.paren_depth = getattr(t.lexer, "paren_depth", 0)-1 - assert t.lexer.paren_depth >= 0 - return t - -def t_COLON(t): - r":" - t.value = t.lexer.lineno - return t - -literals = ('[', ']', '|', '+', '*') - -def t_error(t): - raise AssertionError(t) - -lexer = lex.lex() - -class Definition(object): - def __init__(self, name, expr, first_line, last_line): - self.name = name - self.expr = expr - self.first_line = first_line - self.last_line = last_line - def __repr__(self): - return "Definition(%r, %r, %r, %r)" % ( - self.name, self.expr, self.first_line, self.last_line) - -class Star(object): - def __init__(self, child): - self.child = child - def __repr__(self): - return "Star(%r)" % (self.child,) - -class Plus(object): - def __init__(self, child): - self.child = child - def __repr__(self): - return "Plus(%r)" % (self.child,) - -class Opt(object): - def __init__(self, child): - self.child = child - def __repr__(self): - return "Opt(%r)" % (self.child,) - -class Or(object): - def __init__(self, left, right): - self.left = left - self.right = right - def __repr__(self): - return "Or(%r, %r)" % (self.left, self.right) - -class Seq(object): - def __init__(self, first, next): - self.first = first - self.next = next - def __repr__(self): - return "Seq(%r, %r)" % (self.first, self.next) - -def p_datafile1(p): - """datafile : definition - | datafile definition""" - if len(p) == 2: - p[0] = [p[1]] - else: - p[0] = p[1] + [p[2]] - -def p_datafile(p): - """datafile : NL - | datafile NL""" - if len(p) == 3: - p[0] = p[1] - else: - p[0] = [] - -def p_definition(p): - """definition : PARSER_NAME COLON expr NL""" - p[0] = Definition(p[1], p[3], p[2], p[4]) - -def p_expr(p): - """expr : sequential_terms - | expr '|' sequential_terms""" - if len(p) == 2: - p[0] = p[1] - else: - p[0] = Or(p[1], p[3]) - -def p_sequential_terms(p): - """sequential_terms : term - | sequential_terms term""" - if len(p) == 2: - p[0] = p[1] - else: - p[0] = Seq(p[1], p[2]) - -def p_term(p): - """term : element '*' - | element '+' - | element - """ - if len(p) == 3: - if p[2] == "+": - p[0] = Plus(p[1]) - elif p[2] == "*": - p[0] = Star(p[1]) - else: - raise AssertionError(p[2]) - else: - p[0] = p[1] # no repeat - -def p_element(p): - """element : '[' expr ']' - | LPAR expr RPAR - | STRING - | LEXER_NAME - | PARSER_NAME""" - if len(p) == 4: - if p[1] == '[': - p[0] = Opt(p[2]) - else: - p[0] = p[2] # no repeat - elif p[1].startswith("'"): - # Quoted string; turn into a token name - literal = p[1][1:-1] - p[0] = literal_to_name[literal] - else: - p[0] = p[1] - -def p_error(p): - raise AssertionError(p) - -yacc.yacc(debug=False, write_tables=False) - -s = open(inputGrammar).read() - -# Both of these map to NOTEQUAL -# Easiest way to fix it is to patch the grammar -grammar_text = s.replace("'<>'|'!='", "'!='") - -definition_list = yacc.parse(grammar_text) - -def add_flattened_definition(name, flat_expr): - print name, ":", flat_expr - -_seen_names = set() -def new_name(name): - if name in _seen_names: - for i in itertools.count(2): - name2 = name + str(i) - if name2 not in _seen_names: - break - name = name2 - _seen_names.add(name) - return name - -def flatten(name, expr, need_list): - if isinstance(expr, Seq): - for first_terms in flatten(name, expr.first, need_list): - for next_terms in flatten(name, expr.next, need_list): - yield first_terms + next_terms - - elif isinstance(expr, Or): - for left_terms in flatten(name, expr.left, need_list): - yield left_terms - for right_terms in flatten(name, expr.right, need_list): - yield right_terms - - elif isinstance(expr, Star): - yield [] - child_name = new_name(name + "_star") - yield [child_name] - need_list.append( (child_name, expr.child) ) - - elif isinstance(expr, Plus): - child_name = new_name(name + "_plus") - yield [child_name] - need_list.append( (child_name, expr.child) ) - - elif isinstance(expr, Opt): - yield [] - for term in flatten(name, expr.child, need_list): - yield term - - elif isinstance(expr, str): - yield [expr] - - else: - raise AssertionError(expr) - -f = open(outputPLY, "w") -def W(s): - f.write(s + "\n") - -W('''#!/usr/bin/env python -# generated at %s by "python %s" - -import re -import ast -import inspect - -from histogrammar.pycparser.ply import yacc''' % (datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%dT%H:%M:%S'), " ".join(sys.argv[1:]))) - -W('''from %s import PythonLexer, tokens -''' % inputLex) - -for x in asts: - W(x) - -W('''def inherit_lineno(p0, px, alt=True): - if isinstance(px, dict): - p0.lineno = px["lineno"] - p0.col_offset = px["col_offset"] - else: - if alt and hasattr(px, "alt"): - p0.lineno = px.alt["lineno"] - p0.col_offset = px.alt["col_offset"] - else: - p0.lineno = px.lineno - p0.col_offset = px.col_offset - -def ctx_to_store(obj, store=ast.Store): - if isinstance(obj, list): - for i, x in enumerate(obj): - obj[i] = ctx_to_store(x, store) - return obj - elif isinstance(obj, (ast.Attribute, ast.Subscript)): - obj.ctx = store() - return obj - elif isinstance(obj, ast.AST): - for attrib in obj._fields: - value = getattr(obj, attrib) - if isinstance(value, ast.Load): - setattr(obj, attrib, store()) - elif isinstance(value, ast.Param): - setattr(obj, attrib, store()) - elif isinstance(value, list): - for i, x in enumerate(value): - value[i] = ctx_to_store(x, store) - elif isinstance(value, ast.AST): - setattr(obj, attrib, ctx_to_store(value, store)) - return obj - else: - return obj - -def iskeyword(x): return isinstance(x, ast.keyword) -def notkeyword(x): return not isinstance(x, ast.keyword) - -def unwrap_left_associative(args, rule, alt=False): - out = ast.BinOp(args[0], args[1], args[2], rule=rule) - inherit_lineno(out, args[0]) - args = args[3:] - while len(args) > 0: - out = ast.BinOp(out, args[0], args[1], rule=rule) - inherit_lineno(out, out.left) - if alt: - out.alt = {"lineno": out.lineno, "col_offset": out.col_offset} - inherit_lineno(out, out.op) - args = args[2:] - return out - -def unpack_trailer(atom, power_star): - out = atom - for trailer in power_star: - if isinstance(trailer, ast.Call): - trailer.func = out - inherit_lineno(trailer, out) - out = trailer - elif isinstance(trailer, ast.Attribute): - trailer.value = out - inherit_lineno(trailer, out, alt=False) - if hasattr(out, "alt"): - trailer.alt = out.alt - out = trailer - elif isinstance(trailer, ast.Subscript): - trailer.value = out - inherit_lineno(trailer, out) - out = trailer - else: - assert False - return out -''') - -def numbers(s): - return " ".join(("%" + str(len(x)) + "d") % (i+1) for i, x in enumerate(s.split())) - -def format_function(name, rules): - if len(rules) == 1: - W("def p_%s(p):" % name) - W(" '''%s : %s'''" % (name, rules[0])) - W(" # %s %s" % (" " * len(name), numbers(rules[0]))) - r = "%s : %s" % (name, rules[0]) - if r in actions: - W(actions[r]) - else: - W(" raise NotImplementedError") - else: - for i, rule in enumerate(rules): - W("def p_%s_%d(p):" % (name, i+1)) - W(" '''%s : %s'''" % (name, rule)) - W(" # %s %s" % (" " * len(name), numbers(rule))) - r = "%s : %s" % (name, rule) - if r in actions: - W(actions[r]) - else: - W(" raise NotImplementedError") - -grammar_lines = grammar_text.splitlines() - -for definition in definition_list: - if definition.name in ("single_input", "eval_input"): - continue - - rules = [] - need_list = [] - for terms in flatten(definition.name, definition.expr, need_list): - terms = [duplicates.get(term, term) for term in terms] - rules.append( " ".join(terms) ) - - W("\n# " + - "\n# ".join(grammar_lines[definition.first_line-1:definition.last_line])) - format_function(definition.name, rules) - - while need_list: - name, expr = need_list.pop(0) - if name in duplicates: - continue - rules = [] - for terms in flatten(name, expr, need_list): - terms = [duplicates.get(term, term) for term in terms] - rules.append( " ".join(terms) ) - rules = rules + [name + " " + rule for rule in rules] - W("") - format_function(name, rules) - -W('''def p_error(p): - lexer = p.lexer - if isinstance(lexer, PythonLexer): - lexer = lexer.lexer - pos = lexer.kwds(lexer.lexpos) - line = re.split("\\r?\\n", lexer.source)[max(pos["lineno"] - 3, 0):pos["lineno"]] - indicator = "-" * pos["col_offset"] + "^" - raise SyntaxError("invalid syntax\\n File " + lexer.fileName + ", line " + str(pos["lineno"]) + " col " + str(pos["col_offset"]) + "\\n " + "\\n ".join(line) + "\\n----" + indicator) - -def parse(source, fileName=""): - lexer = PythonLexer(fileName=fileName) - parser = yacc.yacc(debug=False, write_tables=True, tabmodule="%s_table", errorlog=yacc.NullLogger()) - return parser.parse(source + "\\n", lexer=lexer) -''' % outputTable) diff --git a/generate-grammars/python-awk/hgawk_test.py b/generate-grammars/python-awk/hgawk_test.py deleted file mode 100755 index 71edc2c..0000000 --- a/generate-grammars/python-awk/hgawk_test.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python - -import ast - -import histogrammar -import histogrammar.hgawk_grammar as grammar - -expectFunction = set([ - # fundamental aggregators - ("Count", 0), - ("Count", "quantity"), - ("Sum", 0), - ("Sum", "quantity"), - ("Average", 0), - ("Average", "quantity"), - ("Deviate", 0), - ("Deviate", "quantity"), - ("Minimize", 0), - ("Minimize", "quantity"), - ("Maximize", 0), - ("Maximize", "quantity"), - ("Bag", 0), - ("Bag", "quantity"), - ("Bin", 3), - ("Bin", "quantity"), - ("SparselyBin", 1), - ("SparselyBin", "quantity"), - ("CentrallyBin", 1), - ("CentrallyBin", "quantity"), - ("IrregularlyBin", 1), - ("IrregularlyBin", "quantity"), - ("Categorize", 0), - ("Categorize", "quantity"), - ("Fraction", 0), - ("Fraction", "quantity"), - ("Stack", 1), - ("Stack", "quantity"), - ("Select", 0), - ("Select", "quantity"), - - # convenience functions - ("Histogram", 3), - ("Histogram", 4), - ("Histogram", "quantity"), - ("Histogram", "selection"), - ("SparselyHistogram", 1), - ("SparselyHistogram", 2), - ("SparselyHistogram", "quantity"), - ("SparselyHistogram", "selection"), - ("Profile", 3), - ("Profile", 4), - ("Profile", 5), - ("Profile", "binnedQuantity"), - ("Profile", "averagedQuantity"), - ("Profile", "selection"), - ("SparselyProfile", 1), - ("SparselyProfile", 2), - ("SparselyProfile", 3), - ("SparselyProfile", "binnedQuantity"), - ("SparselyProfile", "averagedQuantity"), - ("SparselyProfile", "selection"), - ("ProfileErr", 3), - ("ProfileErr", 4), - ("ProfileErr", 5), - ("ProfileErr", "binnedQuantity"), - ("ProfileErr", "averagedQuantity"), - ("ProfileErr", "selection"), - ("SparselyProfileErr", 1), - ("SparselyProfileErr", 2), - ("SparselyProfileErr", 3), - ("SparselyProfileErr", "binnedQuantity"), - ("SparselyProfileErr", "averagedQuantity"), - ("SparselyProfileErr", "selection"), - ("TwoDimensionallyHistogram", 3), - ("TwoDimensionallyHistogram", 7), - ("TwoDimensionallyHistogram", 8), - ("TwoDimensionallyHistogram", "xquantity"), - ("TwoDimensionallyHistogram", "yquantity"), - ("TwoDimensionallyHistogram", "selection"), - ("TwoDimensionallySparselyHistogram", 1), - ("TwoDimensionallySparselyHistogram", 3), - ("TwoDimensionallySparselyHistogram", 4), - ("TwoDimensionallySparselyHistogram", "xquantity"), - ("TwoDimensionallySparselyHistogram", "yquantity"), - ("TwoDimensionallySparselyHistogram", "selection"), - ]) - -def highestDollar(node): - if isinstance(node, grammar.DollarNumber): - return node.n - elif isinstance(node, ast.AST): - return max([0] + [highestDollar(getattr(node, field)) for field in node._fields]) - elif isinstance(node, list): - return max([0] + [highestDollar(x) for x in node]) - else: - return 0 - -def dollarToArg(node): - if isinstance(node, grammar.DollarNumber): - out = ast.Subscript(ast.Name("$args", ast.Load()), ast.Index(ast.Num(node.n - 1)), ast.Load()) - # ASTs need to have line numbers to be compilable by Python - out.lineno, out.col_offset = node.lineno, node.col_offset - out.value.lineno, out.value.col_offset = node.lineno, node.col_offset - out.value.ctx.lineno, out.value.ctx.col_offset = node.lineno, node.col_offset - out.slice.lineno, out.slice.col_offset = node.lineno, node.col_offset - out.slice.value.lineno, out.slice.value.col_offset = node.lineno, node.col_offset - out.ctx.lineno, out.ctx.col_offset = node.lineno, node.col_offset - return out - elif isinstance(node, ast.AST): - for field in node._fields: - setattr(node, field, dollarToArg(getattr(node, field))) - return node - elif isinstance(node, list): - for i, x in enumerate(node): - node[i] = dollarToArg(x) - return node - else: - return node - -def functionalize(node): - # changes node in-place, but returns it anyway - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): - for i, x in enumerate(node.args): - if (node.func.id, i) in expectFunction: - numargs = highestDollar(x) - if numargs > 0: - # the parameter name "$args" can't conflict with any valid Python names - out = ast.Lambda(ast.arguments([ast.Name("$args", ast.Param())], None, None, []), dollarToArg(x)) - out.lineno, out.col_offset = x.lineno, x.col_offset - out.args.lineno, out.args.col_offset = x.lineno, x.col_offset - out.args.args[0].lineno, out.args.args[0].col_offset = x.lineno, x.col_offset - out.args.args[0].ctx.lineno, out.args.args[0].ctx.col_offset = x.lineno, x.col_offset - node.args[i] = out - - for keyword in node.keywords: - if (node.func.id, keyword.arg) in expectFunction: - x = keyword.value - numargs = highestDollar(x) - if numargs > 0: - out = ast.Lambda(ast.arguments([ast.Name("$args", ast.Param())], None, None, []), dollarToArg(x)) - out.lineno, out.col_offset = x.lineno, x.col_offset - out.args.lineno, out.args.col_offset = x.lineno, x.col_offset - out.args.args[0].lineno, out.args.args[0].col_offset = x.lineno, x.col_offset - out.args.args[0].ctx.lineno, out.args.args[0].ctx.col_offset = x.lineno, x.col_offset - keyword.value = out - - for field in node._fields: - subnode = getattr(node, field) - if isinstance(subnode, ast.AST): - functionalize(subnode) - elif isinstance(subnode, list): - for x in subnode: - if isinstance(x, ast.AST): - functionalize(x) - - return node - -def assignLast(node): - # also changes node in-place and returns it anyway - assert isinstance(node, ast.Module) - assert len(node.body) > 0 - last = node.body[-1] - if not isinstance(last, ast.Expr): - raise SyntaxError("last line must be an expression (the aggregator)") - # the variable name "$h" can't conflict with any valid Python names - out = ast.Assign([ast.Name("$h", ast.Store())], last.value) - out.lineno, out.col_offset = last.lineno, last.col_offset - out.targets[0].lineno, out.targets[0].col_offset = last.lineno, last.col_offset - out.targets[0].ctx.lineno, out.targets[0].ctx.col_offset = last.lineno, last.col_offset - node.body[-1] = out - return node - -def executeWithDollars(source): - namespace = {} - namespace.update(histogrammar.__dict__) - exec(compile(assignLast(functionalize(grammar.parse(source))), "", "exec")) in namespace - return namespace["$h"] - -if __name__ == "__main__": - h = executeWithDollars("Bin(10, -5.0, 5.0, $1 + $2)") - h.fill((1.0, 2.0)) - h.fill((-2.0, -1.0)) - print(h.toJsonString()) - - h = executeWithDollars(''' -def addEmUp(x, y): - return x + y -Bin(10, -5.0, 5.0, addEmUp($1, $2)) -''') - h.fill((1.0, 2.0)) - h.fill((-2.0, -1.0)) - print(h.toJsonString()) - - h = executeWithDollars("Bin(quantity=$1 + $2, num=10, low=-5.0, high=5.0)") - h.fill((1.0, 2.0)) - h.fill((-2.0, -1.0)) - print(h.toJsonString()) - - h = executeWithDollars("Bin(10, -5.0, 5.0, lambda xs: xs[0] + xs[1])") - h.fill((1.0, 2.0)) - h.fill((-2.0, -1.0)) - print(h.toJsonString()) diff --git a/generate-grammars/python-awk/python3.g b/generate-grammars/python-awk/python3.g deleted file mode 100644 index 99fcea0..0000000 --- a/generate-grammars/python-awk/python3.g +++ /dev/null @@ -1,150 +0,0 @@ -# Grammar for Python - -# Note: Changing the grammar specified in this file will most likely -# require corresponding changes in the parser module -# (../Modules/parsermodule.c). If you can't make the changes to -# that module yourself, please co-ordinate the required changes -# with someone who can; ask around on python-dev for help. Fred -# Drake will probably be listening there. - -# NOTE WELL: You should also follow all the steps listed at -# https://docs.python.org/devguide/grammar.html - -# Start symbols for the grammar: -# single_input is a single interactive statement; -# file_input is a module or sequence of commands read from an input file; -# eval_input is the input for the eval() functions. -# NB: compound_stmt in single_input is followed by extra NEWLINE! -single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE -file_input: (NEWLINE | stmt)* ENDMARKER -eval_input: testlist NEWLINE* ENDMARKER - -decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE -decorators: decorator+ -decorated: decorators (classdef | funcdef | async_funcdef) - -async_funcdef: ASYNC funcdef -funcdef: 'def' NAME parameters ['->' test] ':' suite - -parameters: '(' [typedargslist] ')' -typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' - ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] - | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) -tfpdef: NAME [':' test] -varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' - ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] - | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) -vfpdef: NAME - -stmt: simple_stmt | compound_stmt -simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | - import_stmt | global_stmt | nonlocal_stmt | assert_stmt) -expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) -testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] -augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | - '<<=' | '>>=' | '**=' | '//=') -# For normal assignments, additional restrictions enforced by the interpreter -del_stmt: 'del' exprlist -pass_stmt: 'pass' -flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt -break_stmt: 'break' -continue_stmt: 'continue' -return_stmt: 'return' [testlist] -yield_stmt: yield_expr -raise_stmt: 'raise' [test ['from' test]] -import_stmt: import_name | import_from -import_name: 'import' dotted_as_names -# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS -import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) - 'import' ('*' | '(' import_as_names ')' | import_as_names)) -import_as_name: NAME ['as' NAME] -dotted_as_name: dotted_name ['as' NAME] -import_as_names: import_as_name (',' import_as_name)* [','] -dotted_as_names: dotted_as_name (',' dotted_as_name)* -dotted_name: NAME ('.' NAME)* -global_stmt: 'global' NAME (',' NAME)* -nonlocal_stmt: 'nonlocal' NAME (',' NAME)* -assert_stmt: 'assert' test [',' test] - -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt -async_stmt: ASYNC (funcdef | with_stmt | for_stmt) -if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] -while_stmt: 'while' test ':' suite ['else' ':' suite] -for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] -try_stmt: ('try' ':' suite - ((except_clause ':' suite)+ - ['else' ':' suite] - ['finally' ':' suite] | - 'finally' ':' suite)) -with_stmt: 'with' with_item (',' with_item)* ':' suite -with_item: test ['as' expr] -# NB compile.c makes sure that the default except clause is last -except_clause: 'except' [test ['as' NAME]] -suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT - -test: or_test ['if' or_test 'else' test] | lambdef -test_nocond: or_test | lambdef_nocond -lambdef: 'lambda' [varargslist] ':' test -lambdef_nocond: 'lambda' [varargslist] ':' test_nocond -or_test: and_test ('or' and_test)* -and_test: not_test ('and' not_test)* -not_test: 'not' not_test | comparison -comparison: expr (comp_op expr)* -# <> isn't actually a valid comparison operator in Python. It's here for the -# sake of a __future__ import described in PEP 401 (which really works :-) -comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -star_expr: '*' expr -expr: xor_expr ('|' xor_expr)* -xor_expr: and_expr ('^' and_expr)* -and_expr: shift_expr ('&' shift_expr)* -shift_expr: arith_expr (('<<'|'>>') arith_expr)* -arith_expr: term (('+'|'-') term)* -term: factor (('*'|'@'|'/'|'%'|'//') factor)* -factor: ('+'|'-'|'~') factor | power -power: atom_expr ['**' factor] -atom_expr: [AWAIT] atom trailer* -atom: ('(' [yield_expr|testlist_comp] ')' | - '[' [testlist_comp] ']' | - '{' [dictorsetmaker] '}' | - NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') -testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) -trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -subscriptlist: subscript (',' subscript)* [','] -subscript: test | [test] ':' [test] [sliceop] -sliceop: ':' [test] -exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] -testlist: test (',' test)* [','] -dictorsetmaker: ( ((test ':' test | '**' expr) - (comp_for | (',' (test ':' test | '**' expr))* [','])) | - ((test | star_expr) - (comp_for | (',' (test | star_expr))* [','])) ) - -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite - -arglist: argument (',' argument)* [','] - -# The reason that keywords are test nodes instead of NAME is that using NAME -# results in an ambiguity. ast.c makes sure it's a NAME. -# "test '=' test" is really "keyword '=' test", but we have no such token. -# These need to be in a single rule to avoid grammar that is ambiguous -# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, -# we explicitly match '*' here, too, to give it proper precedence. -# Illegal combinations and orderings are blocked in ast.c: -# multiple (test comp_for) arguements are blocked; keyword unpackings -# that precede iterable unpackings are blocked; etc. -argument: ( test [comp_for] | - test '=' test | - '**' test | - '*' test ) - -comp_iter: comp_for | comp_if -comp_for: 'for' exprlist 'in' or_test [comp_iter] -comp_if: 'if' test_nocond [comp_iter] - -# not used in grammar, but may appear in "node" passed from Parser to Compiler -encoding_decl: NAME - -yield_expr: 'yield' [yield_arg] -yield_arg: 'from' test | testlist diff --git a/generate-grammars/python-awk/python3_actions.py b/generate-grammars/python-awk/python3_actions.py deleted file mode 100644 index cbf7ba7..0000000 --- a/generate-grammars/python-awk/python3_actions.py +++ /dev/null @@ -1,476 +0,0 @@ -#!/usr/bin/env python - -actions = {} -actions['''file_input : ENDMARKER'''] = ''' p[0] = ast.Module([], rule=inspect.currentframe().f_code.co_name, lineno=0, col_offset=0)''' -actions['''file_input : file_input_star ENDMARKER'''] = ''' p[0] = ast.Module(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1][0])''' -actions['''file_input_star : NEWLINE'''] = ''' p[0] = ast.Module([], rule=inspect.currentframe().f_code.co_name, lineno=0, col_offset=0)''' -actions['''file_input_star : stmt'''] = ''' p[0] = p[1]''' -actions['''file_input_star : file_input_star NEWLINE'''] = ''' p[0] = ast.Module(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1][0])''' -actions['''file_input_star : file_input_star stmt'''] = ''' p[0] = p[1] + p[2]''' -actions['''decorator : AT dotted_name NEWLINE'''] = ''' p[0] = p[2] - p[0].alt = p[1][1]''' -actions['''decorator : AT dotted_name LPAR RPAR NEWLINE'''] = ''' p[0] = ast.Call(p[2], [], [], None, None, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1][1])''' -actions['''decorator : AT dotted_name LPAR arglist RPAR NEWLINE'''] = ''' p[4].func = p[2] - p[0] = p[4] - inherit_lineno(p[0], p[2]) - p[0].alt = p[1][1]''' -actions['''decorators : decorators_plus'''] = ''' p[0] = p[1]''' -actions['''decorators_plus : decorator'''] = ''' p[0] = [p[1]]''' -actions['''decorators_plus : decorators_plus decorator'''] = ''' p[0] = p[1] + [p[2]]''' -actions['''decorated : decorators classdef'''] = ''' p[2].decorator_list = p[1] - p[0] = p[2] - inherit_lineno(p[0], p[1][0])''' -actions['''decorated : decorators funcdef'''] = ''' p[2].decorator_list = p[1] - p[0] = p[2] - inherit_lineno(p[0], p[1][0])''' -actions['''funcdef : DEF NAME parameters COLON suite'''] = ''' p[0] = ast.FunctionDef(p[2][0], p[3], p[5], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''parameters : LPAR RPAR'''] = ''' p[0] = ast.arguments([], None, None, [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''stmt : simple_stmt'''] = ''' p[0] = p[1]''' -actions['''stmt : compound_stmt'''] = ''' p[0] = p[1]''' -actions['''simple_stmt : small_stmt NEWLINE'''] = ''' p[0] = [p[1]]''' -actions['''simple_stmt : small_stmt SEMI NEWLINE'''] = ''' p[0] = [p[1]]''' -actions['''simple_stmt : small_stmt simple_stmt_star NEWLINE'''] = ''' p[0] = [p[1]] + p[2]''' -actions['''simple_stmt : small_stmt simple_stmt_star SEMI NEWLINE'''] = ''' p[0] = [p[1]] + p[2]''' -actions['''simple_stmt_star : SEMI small_stmt'''] = ''' p[0] = [p[2]]''' -actions['''simple_stmt_star : simple_stmt_star SEMI small_stmt'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''small_stmt : expr_stmt'''] = ''' p[0] = p[1]''' -actions['''small_stmt : del_stmt'''] = ''' p[0] = p[1]''' -actions['''small_stmt : pass_stmt'''] = ''' p[0] = p[1]''' -actions['''small_stmt : flow_stmt'''] = ''' p[0] = p[1]''' -actions['''small_stmt : import_stmt'''] = ''' p[0] = p[1]''' -actions['''small_stmt : global_stmt'''] = ''' p[0] = p[1]''' -actions['''small_stmt : assert_stmt'''] = ''' p[0] = p[1]''' -actions['''expr_stmt : testlist_star_expr'''] = ''' p[0] = ast.Expr(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''expr_stmt_star : EQUAL yield_expr'''] = ''' p[0] = [p[2]]''' -actions['''expr_stmt_star : expr_stmt_star EQUAL yield_expr'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''testlist_star_expr : test'''] = ''' p[0] = p[1]''' -actions['''augassign : PLUSEQUAL'''] = ''' p[0] = ast.Add(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : MINEQUAL'''] = ''' p[0] = ast.Sub(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : STAREQUAL'''] = ''' p[0] = ast.Mult(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : SLASHEQUAL'''] = ''' p[0] = ast.Div(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : PERCENTEQUAL'''] = ''' p[0] = ast.Mod(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : AMPEREQUAL'''] = ''' p[0] = ast.BitAnd(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : VBAREQUAL'''] = ''' p[0] = ast.BitOr(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : CIRCUMFLEXEQUAL'''] = ''' p[0] = ast.BitXor(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : LEFTSHIFTEQUAL'''] = ''' p[0] = ast.LShift(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : RIGHTSHIFTEQUAL'''] = ''' p[0] = ast.RShift(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : DOUBLESTAREQUAL'''] = ''' p[0] = ast.Pow(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''augassign : DOUBLESLASHEQUAL'''] = ''' p[0] = ast.FloorDiv(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''del_stmt : DEL exprlist'''] = ''' ctx_to_store(p[2], ast.Del) - if isinstance(p[2], ast.Tuple) and not p[2].paren: - p[0] = ast.Delete(p[2].elts, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - else: - p[0] = ast.Delete([p[2]], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''pass_stmt : PASS'''] = ''' p[0] = ast.Pass(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''flow_stmt : break_stmt'''] = ''' p[0] = p[1]''' -actions['''flow_stmt : continue_stmt'''] = ''' p[0] = p[1]''' -actions['''flow_stmt : return_stmt'''] = ''' p[0] = p[1]''' -actions['''flow_stmt : raise_stmt'''] = ''' p[0] = p[1]''' -actions['''flow_stmt : yield_stmt'''] = ''' p[0] = ast.Expr(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''break_stmt : BREAK'''] = ''' p[0] = ast.Break(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''continue_stmt : CONTINUE'''] = ''' p[0] = ast.Continue(rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''return_stmt : RETURN'''] = ''' p[0] = ast.Return(None, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''return_stmt : RETURN testlist'''] = ''' p[0] = ast.Return(p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''yield_stmt : yield_expr'''] = ''' p[0] = p[1]''' -actions['''raise_stmt : RAISE'''] = ''' p[0] = ast.Raise(None, None, None, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''raise_stmt : RAISE test'''] = ''' p[0] = ast.Raise(p[2], None, None, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_stmt : import_name'''] = ''' p[0] = p[1]''' -actions['''import_stmt : import_from'''] = ''' p[0] = p[1]''' -actions['''import_name : IMPORT dotted_as_names'''] = ''' p[0] = ast.Import(p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from : FROM dotted_name IMPORT STAR'''] = ''' dotted = [] - last = p[2] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), [ast.alias("*", None, rule=inspect.currentframe().f_code.co_name, **p[3][1])], 0, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from : FROM dotted_name IMPORT LPAR import_as_names RPAR'''] = ''' dotted = [] - last = p[2] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), p[5], 0, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from : FROM dotted_name IMPORT import_as_names'''] = ''' dotted = [] - last = p[2] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), p[4], 0, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from : FROM import_from_plus dotted_name IMPORT STAR'''] = ''' dotted = [] - last = p[3] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), [ast.alias("*", None, rule=inspect.currentframe().f_code.co_name, **p[4][1])], p[2], **p[1][1])''' -actions['''import_from : FROM import_from_plus dotted_name IMPORT LPAR import_as_names RPAR'''] = ''' dotted = [] - last = p[3] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), p[6], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from : FROM import_from_plus dotted_name IMPORT import_as_names'''] = ''' dotted = [] - last = p[3] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), p[5], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from : FROM import_from_plus IMPORT STAR'''] = ''' p[0] = ast.ImportFrom(None, [ast.alias("*", None, rule=inspect.currentframe().f_code.co_name, **p[3][1])], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from : FROM import_from_plus IMPORT LPAR import_as_names RPAR'''] = ''' p[0] = ast.ImportFrom(None, p[5], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from : FROM import_from_plus IMPORT import_as_names'''] = ''' p[0] = ast.ImportFrom(None, p[4], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_from_plus : DOT'''] = ''' p[0] = 1''' -actions['''import_from_plus : import_from_plus DOT'''] = ''' p[0] = p[1] + 1''' -actions['''import_as_name : NAME'''] = ''' p[0] = ast.alias(p[1][0], None, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''import_as_name : NAME AS NAME'''] = ''' p[0] = ast.alias(p[1][0], p[3][0], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''dotted_as_name : dotted_name'''] = ''' dotted = [] - last = p[1] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.alias(".".join(dotted), None, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''dotted_as_name : dotted_name AS NAME'''] = ''' dotted = [] - last = p[1] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.alias(".".join(dotted), p[3][0], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''import_as_names : import_as_name'''] = ''' p[0] = [p[1]]''' -actions['''import_as_names : import_as_name COMMA'''] = ''' p[0] = [p[1]]''' -actions['''import_as_names : import_as_name import_as_names_star'''] = ''' p[0] = [p[1]] + p[2]''' -actions['''import_as_names : import_as_name import_as_names_star COMMA'''] = ''' p[0] = [p[1]] + p[2]''' -actions['''import_as_names_star : COMMA import_as_name'''] = ''' p[0] = [p[2]]''' -actions['''import_as_names_star : import_as_names_star COMMA import_as_name'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''dotted_as_names : dotted_as_name'''] = ''' p[0] = [p[1]]''' -actions['''dotted_as_names : dotted_as_name dotted_as_names_star'''] = ''' p[0] = [p[1]] + p[2]''' -actions['''dotted_as_names_star : COMMA dotted_as_name'''] = ''' p[0] = [p[2]]''' -actions['''dotted_as_names_star : dotted_as_names_star COMMA dotted_as_name'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''dotted_name : NAME'''] = ''' p[0] = ast.Name(p[1][0], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''dotted_name : NAME dotted_name_star'''] = ''' last = p[2] - if isinstance(last, ast.Attribute): - inherit_lineno(last, p[1][1]) - while isinstance(last.value, ast.Attribute): - last = last.value - inherit_lineno(last, p[1][1]) - last.value = ast.Attribute(ast.Name(p[1][0], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1]), last.value, ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1]) - p[0] = p[2] - else: - p[0] = ast.Attribute(ast.Name(p[1][0], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''dotted_name_star : DOT NAME'''] = ''' p[0] = p[2][0]''' -actions['''dotted_name_star : dotted_name_star DOT NAME'''] = ''' p[0] = ast.Attribute(p[1], p[3][0], ast.Load(), rule=inspect.currentframe().f_code.co_name)''' -actions['''global_stmt : GLOBAL NAME'''] = ''' p[0] = ast.Global([p[2][0]], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''global_stmt : GLOBAL NAME global_stmt_star'''] = ''' p[0] = ast.Global([p[2][0]] + p[3], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''global_stmt_star : COMMA NAME'''] = ''' p[0] = [p[2][0]]''' -actions['''global_stmt_star : global_stmt_star COMMA NAME'''] = ''' p[0] = p[1] + [p[3][0]]''' -actions['''assert_stmt : ASSERT test'''] = ''' p[0] = ast.Assert(p[2], None, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''assert_stmt : ASSERT test COMMA test'''] = ''' p[0] = ast.Assert(p[2], p[4], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''compound_stmt : if_stmt'''] = ''' p[0] = [p[1]]''' -actions['''compound_stmt : while_stmt'''] = ''' p[0] = [p[1]]''' -actions['''compound_stmt : for_stmt'''] = ''' p[0] = [p[1]]''' -actions['''compound_stmt : try_stmt'''] = ''' p[0] = [p[1]]''' -actions['''compound_stmt : with_stmt'''] = ''' p[0] = [p[1]]''' -actions['''compound_stmt : funcdef'''] = ''' p[0] = [p[1]]''' -actions['''compound_stmt : classdef'''] = ''' p[0] = [p[1]]''' -actions['''compound_stmt : decorated'''] = ''' p[0] = [p[1]]''' -actions['''if_stmt : IF test COLON suite'''] = ''' p[0] = ast.If(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''if_stmt : IF test COLON suite ELSE COLON suite'''] = ''' p[0] = ast.If(p[2], p[4], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''if_stmt : IF test COLON suite if_stmt_star'''] = ''' p[0] = ast.If(p[2], p[4], [p[5]], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''if_stmt : IF test COLON suite if_stmt_star ELSE COLON suite'''] = ''' last = p[5] - while len(last.orelse) > 0: - last = last.orelse[0] - last.orelse.extend(p[8]) - p[0] = ast.If(p[2], p[4], [p[5]], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''if_stmt_star : ELIF test COLON suite'''] = ''' p[0] = ast.If(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[2])''' -actions['''if_stmt_star : if_stmt_star ELIF test COLON suite'''] = ''' last = p[1] - while len(last.orelse) > 0: - last = last.orelse[0] - last.orelse.append(ast.If(p[3], p[5], [], rule=inspect.currentframe().f_code.co_name)) - inherit_lineno(last.orelse[-1], p[3]) - p[0] = p[1]''' -actions['''while_stmt : WHILE test COLON suite'''] = ''' p[0] = ast.While(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''while_stmt : WHILE test COLON suite ELSE COLON suite'''] = ''' p[0] = ast.While(p[2], p[4], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''for_stmt : FOR exprlist IN testlist COLON suite'''] = ''' ctx_to_store(p[2]) - p[0] = ast.For(p[2], p[4], p[6], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''for_stmt : FOR exprlist IN testlist COLON suite ELSE COLON suite'''] = ''' ctx_to_store(p[2]) - p[0] = ast.For(p[2], p[4], p[6], p[9], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''try_stmt : TRY COLON suite try_stmt_plus'''] = ''' p[0] = ast.TryExcept(p[3], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''try_stmt : TRY COLON suite try_stmt_plus FINALLY COLON suite'''] = ''' p[0] = ast.TryFinally([ast.TryExcept(p[3], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''try_stmt : TRY COLON suite try_stmt_plus ELSE COLON suite'''] = ''' p[0] = ast.TryExcept(p[3], p[4], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''try_stmt : TRY COLON suite try_stmt_plus ELSE COLON suite FINALLY COLON suite'''] = ''' p[0] = ast.TryFinally([ast.TryExcept(p[3], p[4], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1])], p[10], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''try_stmt : TRY COLON suite FINALLY COLON suite'''] = ''' p[0] = ast.TryFinally(p[3], p[6], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''try_stmt_plus : except_clause COLON suite'''] = ''' p[1].body = p[3] - p[0] = [p[1]]''' -actions['''try_stmt_plus : try_stmt_plus except_clause COLON suite'''] = ''' p[2].body = p[4] - p[0] = p[1] + [p[2]]''' -actions['''with_stmt : WITH with_item COLON suite'''] = ''' p[2].body = p[4] - p[0] = p[2]''' -actions['''with_stmt : WITH with_item with_stmt_star COLON suite'''] = ''' p[2].body.append(p[3]) - last = p[2] - while len(last.body) > 0: - last = last.body[0] - last.body = p[5] - p[0] = p[2]''' -actions['''with_stmt_star : COMMA with_item'''] = ''' p[0] = p[2]''' -actions['''with_stmt_star : with_stmt_star COMMA with_item'''] = ''' last = p[1] - while len(last.body) > 0: - last = last.body[0] - last.body.append(p[3]) - p[0] = p[1]''' -actions['''with_item : test'''] = ''' p[0] = ast.With(p[1], None, [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''with_item : test AS expr'''] = ''' ctx_to_store(p[3]) - p[0] = ast.With(p[1], p[3], [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''except_clause : EXCEPT'''] = ''' p[0] = ast.ExceptHandler(None, None, [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''except_clause : EXCEPT test'''] = ''' p[0] = ast.ExceptHandler(p[2], None, [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''suite : simple_stmt'''] = ''' p[0] = p[1]''' -actions['''suite : NEWLINE INDENT suite_plus DEDENT'''] = ''' p[0] = p[3]''' -actions['''suite_plus : stmt'''] = ''' p[0] = p[1]''' -actions['''suite_plus : suite_plus stmt'''] = ''' p[0] = p[1] + p[2]''' -actions['''test : or_test'''] = ''' p[0] = p[1]''' -actions['''test : or_test IF or_test ELSE test'''] = ''' p[0] = ast.IfExp(p[3], p[1], p[5], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''test : lambdef'''] = ''' p[0] = p[1]''' -actions['''lambdef : LAMBDA COLON test'''] = ''' p[0] = ast.Lambda(ast.arguments([], None, None, [], rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''lambdef : LAMBDA varargslist COLON test'''] = ''' p[0] = ast.Lambda(p[2], p[4], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''or_test : and_test'''] = ''' p[0] = p[1]''' -actions['''or_test : and_test or_test_star'''] = ''' theor = ast.Or(rule=inspect.currentframe().f_code.co_name) - inherit_lineno(theor, p[2][0]) - p[0] = ast.BoolOp(theor, [p[1]] + p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''or_test_star : OR and_test'''] = ''' p[0] = [p[2]]''' -actions['''or_test_star : or_test_star OR and_test'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''and_test : not_test'''] = ''' p[0] = p[1]''' -actions['''and_test : not_test and_test_star'''] = ''' theand = ast.And(rule=inspect.currentframe().f_code.co_name) - inherit_lineno(theand, p[2][0]) - p[0] = ast.BoolOp(theand, [p[1]] + p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''and_test_star : AND not_test'''] = ''' p[0] = [p[2]]''' -actions['''and_test_star : and_test_star AND not_test'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''not_test : NOT not_test'''] = ''' thenot = ast.Not(rule=inspect.currentframe().f_code.co_name) - inherit_lineno(thenot, p[2]) - p[0] = ast.UnaryOp(thenot, p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''not_test : comparison'''] = ''' p[0] = p[1]''' -actions['''comparison : expr'''] = ''' p[0] = p[1]''' -actions['''comparison : expr comparison_star'''] = ''' ops, exprs = p[2] - p[0] = ast.Compare(p[1], ops, exprs, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''comparison_star : comp_op expr'''] = ''' inherit_lineno(p[1], p[2]) - p[0] = ([p[1]], [p[2]])''' -actions['''comparison_star : comparison_star comp_op expr'''] = ''' ops, exprs = p[1] - inherit_lineno(p[2], p[3]) - p[0] = (ops + [p[2]], exprs + [p[3]])''' -actions['''comp_op : LESS'''] = ''' p[0] = ast.Lt(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : GREATER'''] = ''' p[0] = ast.Gt(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : EQEQUAL'''] = ''' p[0] = ast.Eq(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : GREATEREQUAL'''] = ''' p[0] = ast.GtE(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : LESSEQUAL'''] = ''' p[0] = ast.LtE(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : NOTEQUAL'''] = ''' p[0] = ast.NotEq(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : IN'''] = ''' p[0] = ast.In(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : NOT IN'''] = ''' p[0] = ast.NotIn(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : IS'''] = ''' p[0] = ast.Is(rule=inspect.currentframe().f_code.co_name)''' -actions['''comp_op : IS NOT'''] = ''' p[0] = ast.IsNot(rule=inspect.currentframe().f_code.co_name)''' -actions['''expr : xor_expr'''] = ''' p[0] = p[1]''' -actions['''expr : xor_expr expr_star'''] = ''' p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2)''' -actions['''expr_star : VBAR xor_expr'''] = ''' p[0] = [ast.BitOr(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''expr_star : expr_star VBAR xor_expr'''] = ''' p[0] = p[1] + [ast.BitOr(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''xor_expr : and_expr'''] = ''' p[0] = p[1]''' -actions['''xor_expr : and_expr xor_expr_star'''] = ''' p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2)''' -actions['''xor_expr_star : CIRCUMFLEX and_expr'''] = ''' p[0] = [ast.BitXor(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''xor_expr_star : xor_expr_star CIRCUMFLEX and_expr'''] = ''' p[0] = p[1] + [ast.BitXor(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''and_expr : shift_expr'''] = ''' p[0] = p[1]''' -actions['''and_expr : shift_expr and_expr_star'''] = ''' p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 0)''' -actions['''and_expr_star : AMPER shift_expr'''] = ''' p[0] = [ast.BitAnd(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''and_expr_star : and_expr_star AMPER shift_expr'''] = ''' p[0] = p[1] + [ast.BitAnd(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''shift_expr : arith_expr'''] = ''' p[0] = p[1]''' -actions['''shift_expr : arith_expr shift_expr_star'''] = ''' p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2)''' -actions['''shift_expr_star : LEFTSHIFT arith_expr'''] = ''' p[0] = [ast.LShift(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''shift_expr_star : RIGHTSHIFT arith_expr'''] = ''' p[0] = [ast.RShift(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''shift_expr_star : shift_expr_star LEFTSHIFT arith_expr'''] = ''' p[0] = p[1] + [ast.LShift(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''shift_expr_star : shift_expr_star RIGHTSHIFT arith_expr'''] = ''' p[0] = p[1] + [ast.RShift(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''arith_expr : term'''] = ''' p[0] = p[1]''' -actions['''arith_expr : term arith_expr_star'''] = ''' p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2)''' -actions['''arith_expr_star : PLUS term'''] = ''' p[0] = [ast.Add(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''arith_expr_star : MINUS term'''] = ''' p[0] = [ast.Sub(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''arith_expr_star : arith_expr_star PLUS term'''] = ''' p[0] = p[1] + [ast.Add(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''arith_expr_star : arith_expr_star MINUS term'''] = ''' p[0] = p[1] + [ast.Sub(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''term : factor'''] = ''' p[0] = p[1]''' -actions['''term : factor term_star'''] = ''' p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2)''' -actions['''term_star : STAR factor'''] = ''' p[0] = [ast.Mult(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''term_star : SLASH factor'''] = ''' p[0] = [ast.Div(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''term_star : PERCENT factor'''] = ''' p[0] = [ast.Mod(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''term_star : DOUBLESLASH factor'''] = ''' p[0] = [ast.FloorDiv(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]]''' -actions['''term_star : term_star STAR factor'''] = ''' p[0] = p[1] + [ast.Mult(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''term_star : term_star SLASH factor'''] = ''' p[0] = p[1] + [ast.Div(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''term_star : term_star PERCENT factor'''] = ''' p[0] = p[1] + [ast.Mod(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''term_star : term_star DOUBLESLASH factor'''] = ''' p[0] = p[1] + [ast.FloorDiv(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]]''' -actions['''factor : PLUS factor'''] = ''' op = ast.UAdd(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - p[0] = ast.UnaryOp(op, p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], op)''' -actions['''factor : MINUS factor'''] = ''' op = ast.USub(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - p[0] = ast.UnaryOp(op, p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], op)''' -actions['''factor : TILDE factor'''] = ''' op = ast.Invert(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - p[0] = ast.UnaryOp(op, p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], op)''' -actions['''factor : power'''] = ''' p[0] = p[1]''' -actions['''power : atom_expr'''] = ''' p[0] = p[1]''' -actions['''atom_expr : atom'''] = ''' p[0] = p[1]''' -actions['''atom : LPAR RPAR'''] = ''' p[0] = ast.Tuple([], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=True, **p[1][1])''' -actions['''atom : LPAR yield_expr RPAR'''] = ''' p[0] = p[2] - if isinstance(p[0], ast.Tuple): - p[0].paren = True - p[0].alt = p[1][1]''' -actions['''atom : LPAR testlist_comp RPAR'''] = ''' p[0] = p[2] - if isinstance(p[0], ast.Tuple): - p[0].paren = True - p[0].alt = p[1][1]''' -actions['''atom : LSQB RSQB'''] = ''' p[0] = ast.List([], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''atom : LBRACE RBRACE'''] = ''' p[0] = ast.Dict([], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''atom : LBRACE dictorsetmaker RBRACE'''] = ''' if isinstance(p[2], (ast.SetComp, ast.DictComp)): - p[0] = p[2] - p[0].alt = p[1][1] - else: - keys, values = p[2] - if keys is None: - p[0] = ast.Set(values, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - else: - p[0] = ast.Dict(keys, values, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''atom : NAME'''] = ''' p[0] = ast.Name(p[1][0], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''atom : NUMBER'''] = ''' p[0] = ast.Num(p[1][0], rule=inspect.currentframe().f_code.co_name, **p[1][2])''' -actions['''atom : atom_plus'''] = ''' p[0] = p[1]''' -actions['''atom_plus : STRING'''] = ''' p[0] = ast.Str(p[1][0], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''atom_plus : atom_plus STRING'''] = ''' p[1].s = p[1].s + p[2][0] - p[0] = p[1]''' -actions['''testlist_comp : test comp_for'''] = ''' p[0] = ast.GeneratorExp(p[1], p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''testlist_comp : test'''] = ''' p[0] = p[1]''' -actions['''testlist_comp : test COMMA'''] = ''' p[0] = ast.Tuple([p[1]], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''testlist_comp : test testlist_comp_star'''] = ''' p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''testlist_comp : test testlist_comp_star COMMA'''] = ''' p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''testlist_comp_star : COMMA test'''] = ''' p[0] = [p[2]]''' -actions['''testlist_comp_star : testlist_comp_star COMMA test'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''trailer : LPAR RPAR'''] = ''' p[0] = ast.Call(None, [], [], None, None, rule=inspect.currentframe().f_code.co_name)''' -actions['''trailer : LPAR arglist RPAR'''] = ''' p[0] = p[2]''' -actions['''trailer : LSQB subscriptlist RSQB'''] = ''' p[0] = ast.Subscript(None, p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name)''' -actions['''trailer : DOT NAME'''] = ''' p[0] = ast.Attribute(None, p[2][0], ast.Load(), rule=inspect.currentframe().f_code.co_name)''' -actions['''subscriptlist : subscript'''] = ''' p[0] = p[1]''' -actions['''subscriptlist : subscript COMMA'''] = ''' if isinstance(p[1], ast.Index): - tup = ast.Tuple([p[1].value], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(tup, p[1].value) - p[0] = ast.Index(tup, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], tup) - else: - p[0] = ast.ExtSlice([p[1]], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''subscriptlist : subscript subscriptlist_star'''] = ''' args = [p[1]] + p[2] - if all(isinstance(x, ast.Index) for x in args): - tup = ast.Tuple([x.value for x in args], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(tup, args[0].value) - p[0] = ast.Index(tup, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], tup) - else: - p[0] = ast.ExtSlice(args, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''subscriptlist : subscript subscriptlist_star COMMA'''] = ''' args = [p[1]] + p[2] - if all(isinstance(x, ast.Index) for x in args): - tup = ast.Tuple([x.value for x in args], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(tup, args[0].value) - p[0] = ast.Index(tup, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], tup) - else: - p[0] = ast.ExtSlice(args, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''subscriptlist_star : COMMA subscript'''] = ''' p[0] = [p[2]]''' -actions['''subscriptlist_star : subscriptlist_star COMMA subscript'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''subscript : test'''] = ''' p[0] = ast.Index(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''subscript : COLON'''] = ''' p[0] = ast.Slice(None, None, None, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''subscript : COLON sliceop'''] = ''' p[0] = ast.Slice(None, None, p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''subscript : COLON test'''] = ''' p[0] = ast.Slice(None, p[2], None, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''subscript : COLON test sliceop'''] = ''' p[0] = ast.Slice(None, p[2], p[3], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''subscript : test COLON'''] = ''' p[0] = ast.Slice(p[1], None, None, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''subscript : test COLON sliceop'''] = ''' p[0] = ast.Slice(p[1], None, p[3], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''subscript : test COLON test'''] = ''' p[0] = ast.Slice(p[1], p[3], None, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''subscript : test COLON test sliceop'''] = ''' p[0] = ast.Slice(p[1], p[3], p[4], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''sliceop : COLON'''] = ''' p[0] = ast.Name("None", ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''sliceop : COLON test'''] = ''' p[0] = p[2]''' -actions['''exprlist : expr'''] = ''' p[0] = p[1]''' -actions['''exprlist : expr COMMA'''] = ''' p[0] = ast.Tuple([p[1]], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''exprlist : expr exprlist_star'''] = ''' p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''exprlist : expr exprlist_star COMMA'''] = ''' p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''exprlist_star : COMMA expr'''] = ''' p[0] = [p[2]]''' -actions['''exprlist_star : exprlist_star COMMA expr'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''testlist : test'''] = ''' p[0] = p[1]''' -actions['''testlist : test COMMA'''] = ''' p[0] = ast.Tuple([p[1]], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''testlist : test testlist_star'''] = ''' p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''testlist : test testlist_star COMMA'''] = ''' p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1])''' -actions['''testlist_star : COMMA test'''] = ''' p[0] = [p[2]]''' -actions['''testlist_star : testlist_star COMMA test'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''dictorsetmaker : test COLON test comp_for'''] = ''' p[0] = ast.DictComp(p[1], p[3], p[4], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''dictorsetmaker : test COLON test'''] = ''' p[0] = ([p[1]], [p[3]])''' -actions['''dictorsetmaker : test COLON test COMMA'''] = ''' p[0] = ([p[1]], [p[3]])''' -actions['''dictorsetmaker : test COLON test dictorsetmaker_star'''] = ''' keys, values = p[4] - p[0] = ([p[1]] + keys, [p[3]] + values)''' -actions['''dictorsetmaker : test COLON test dictorsetmaker_star COMMA'''] = ''' keys, values = p[4] - p[0] = ([p[1]] + keys, [p[3]] + values)''' -actions['''dictorsetmaker : test comp_for'''] = ''' p[0] = ast.SetComp(p[1], p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''dictorsetmaker : test'''] = ''' p[0] = (None, [p[1]])''' -actions['''dictorsetmaker : test COMMA'''] = ''' p[0] = (None, [p[1]])''' -actions['''dictorsetmaker_star : COMMA test COLON test'''] = ''' p[0] = ([p[2]], [p[4]])''' -actions['''dictorsetmaker_star : dictorsetmaker_star COMMA test COLON test'''] = ''' keys, values = p[1] - p[0] = (keys + [p[3]], values + [p[5]])''' -actions['''classdef : CLASS NAME COLON suite'''] = ''' p[0] = ast.ClassDef(p[2][0], [], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''classdef : CLASS NAME LPAR RPAR COLON suite'''] = ''' p[0] = ast.ClassDef(p[2][0], [], p[6], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])''' -actions['''arglist : argument'''] = ''' if notkeyword(p[1]): - p[0] = ast.Call(None, [p[1]], [], None, None, rule=inspect.currentframe().f_code.co_name) - else: - p[0] = ast.Call(None, [], [p[1]], None, None, rule=inspect.currentframe().f_code.co_name)''' -actions['''arglist : argument COMMA'''] = ''' if notkeyword(p[1]): - p[0] = ast.Call(None, [p[1]], [], None, None, rule=inspect.currentframe().f_code.co_name) - else: - p[0] = ast.Call(None, [], [p[1]], None, None, rule=inspect.currentframe().f_code.co_name)''' -actions['''arglist_star : COMMA argument'''] = ''' p[0] = [p[2]]''' -actions['''arglist_star : arglist_star COMMA argument'''] = ''' p[0] = p[1] + [p[3]]''' -actions['''argument : test'''] = ''' p[0] = p[1]''' -actions['''argument : test comp_for'''] = ''' p[0] = ast.GeneratorExp(p[1], p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''argument : test EQUAL test'''] = ''' p[0] = ast.keyword(p[1].id, p[3], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1])''' -actions['''comp_iter : comp_for'''] = ''' p[0] = ([], p[1])''' -actions['''comp_iter : comp_if'''] = ''' p[0] = p[1]''' -actions['''comp_for : FOR exprlist IN or_test'''] = ''' ctx_to_store(p[2]) - p[0] = [ast.comprehension(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])]''' -actions['''comp_for : FOR exprlist IN or_test comp_iter'''] = ''' ctx_to_store(p[2]) - ifs, iters = p[5] - p[0] = [ast.comprehension(p[2], p[4], ifs, rule=inspect.currentframe().f_code.co_name, **p[1][1])] + iters''' -actions['''encoding_decl : NAME'''] = ''' p[0] = p[1]''' -actions['''yield_expr : YIELD'''] = ''' p[0] = ast.Yield(None, rule=inspect.currentframe().f_code.co_name, **p[1][1])''' diff --git a/histogrammar/__init__.py b/histogrammar/__init__.py index a99ed6f..90717eb 100644 --- a/histogrammar/__init__.py +++ b/histogrammar/__init__.py @@ -16,32 +16,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -from histogrammar.defs import Factory, Container - +# handy monkey patch functions for pandas and spark dataframes +import histogrammar.dfinterface +from histogrammar.convenience import ( + Histogram, + Profile, + ProfileErr, + SparselyHistogram, + SparselyProfile, + SparselyProfileErr, + TwoDimensionallyHistogram, + TwoDimensionallySparselyHistogram, +) +from histogrammar.defs import Container, Factory from histogrammar.primitives.average import Average from histogrammar.primitives.bag import Bag from histogrammar.primitives.bin import Bin from histogrammar.primitives.categorize import Categorize from histogrammar.primitives.centrallybin import CentrallyBin -from histogrammar.primitives.collection import Collection, Branch, Index, Label, UntypedLabel +from histogrammar.primitives.collection import ( + Branch, + Collection, + Index, + Label, + UntypedLabel, +) from histogrammar.primitives.count import Count from histogrammar.primitives.deviate import Deviate from histogrammar.primitives.fraction import Fraction from histogrammar.primitives.irregularlybin import IrregularlyBin -from histogrammar.primitives.minmax import Minimize, Maximize +from histogrammar.primitives.minmax import Maximize, Minimize from histogrammar.primitives.select import Select from histogrammar.primitives.sparselybin import SparselyBin from histogrammar.primitives.stack import Stack from histogrammar.primitives.sum import Sum - -from histogrammar.convenience import Histogram -from histogrammar.convenience import SparselyHistogram -from histogrammar.convenience import Profile -from histogrammar.convenience import SparselyProfile -from histogrammar.convenience import ProfileErr -from histogrammar.convenience import SparselyProfileErr -from histogrammar.convenience import TwoDimensionallyHistogram -from histogrammar.convenience import TwoDimensionallySparselyHistogram - -# handy monkey patch functions for pandas and spark dataframes -import histogrammar.dfinterface diff --git a/histogrammar/convenience.py b/histogrammar/convenience.py index 33b5eb6..0a7bbd6 100644 --- a/histogrammar/convenience.py +++ b/histogrammar/convenience.py @@ -20,11 +20,11 @@ from histogrammar.defs import identity, unweighted from histogrammar.primitives.average import Average from histogrammar.primitives.bin import Bin +from histogrammar.primitives.categorize import Categorize from histogrammar.primitives.count import Count from histogrammar.primitives.deviate import Deviate -from histogrammar.primitives.sparselybin import SparselyBin -from histogrammar.primitives.categorize import Categorize from histogrammar.primitives.select import Select +from histogrammar.primitives.sparselybin import SparselyBin def Histogram(num, low, high, quantity=identity): @@ -54,7 +54,10 @@ def HistogramCut(num, low, high, quantity=identity, selection=unweighted): selection (function returning boolean): function that computes if data point is accepted or not. default is: lamba x: True """ - return Select.ing(selection, Bin.ing(num, low, high, quantity, Count.ing(), Count.ing(), Count.ing(), Count.ing())) + return Select.ing( + selection, + Bin.ing(num, low, high, quantity, Count.ing(), Count.ing(), Count.ing(), Count.ing()), + ) def SparselyHistogram(binWidth, quantity=identity, origin=0.0): @@ -114,6 +117,10 @@ def TwoDimensionallyHistogram(xnum, xlow, xhigh, xquantity, ynum, ylow, yhigh, y def TwoDimensionallySparselyHistogram(xbinWidth, xquantity, ybinWidth, yquantity, xorigin=0.0, yorigin=0.0): """Convenience function for creating a sparsely binned, two-dimensional histogram.""" - return SparselyBin.ing(xbinWidth, xquantity, - SparselyBin.ing(ybinWidth, yquantity, Count.ing(), Count.ing(), yorigin), - Count.ing(), xorigin) + return SparselyBin.ing( + xbinWidth, + xquantity, + SparselyBin.ing(ybinWidth, yquantity, Count.ing(), Count.ing(), yorigin), + Count.ing(), + xorigin, + ) diff --git a/histogrammar/defs.py b/histogrammar/defs.py index 2af8388..57afb4d 100644 --- a/histogrammar/defs.py +++ b/histogrammar/defs.py @@ -14,68 +14,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 -import datetime import json as jsonlib -import math -import random -import re - -try: - from collections import OrderedDict -except ImportError: - class OrderedDict(object): - def __init__(self): - self.pairs = {} - self.keys = [] - - def __setitem__(self, key, value): - self.pairs[key] = value - if key not in self.keys: - self.keys.append(key) - - def __getitem__(self, key): - return self.pairs[key] - - def values(self): - return [self.pairs[k] for k in self.keys] - - def items(self): - return [(k, self.pairs[k]) for k in self.keys] - - def __iter__(self): - return iter(self.keys) - - def __len__(self): - return len(self.keys) - -from histogrammar.util import FillMethod, PlotMethod, basestring, xrange, named -from histogrammar.parsing import C99SourceToAst -from histogrammar.parsing import C99AstToSource -from histogrammar.pycparser import c_ast +from pathlib import Path + +import numpy + import histogrammar.version +from histogrammar.util import FillMethod, PlotMethod, basestring, named class ContainerException(Exception): """Exception type for improperly configured containers.""" - pass class InvalidJsonException(Exception): """Exception type for strings that cannot be parsed because they are not proper JSON.""" def __init__(self, message): - super(InvalidJsonException, self).__init__("invalid JSON: {0}".format(message)) + super().__init__(f"invalid JSON: {message}") class JsonFormatException(Exception): """Exception type for unexpected JSON structure, thrown by ``fromJson`` methods.""" def __init__(self, x, context): - super(JsonFormatException, self).__init__("wrong JSON format for {0}: {1}".format(context, jsonlib.dumps(x))) + super().__init__(f"wrong JSON format for {context}: {jsonlib.dumps(x)}") -class Factory(object): +class Factory: """Interface for a container factory, always named as imperative verbs, such as "Count" and "Bin". Each factory has: @@ -121,6 +87,7 @@ def specialize(self): # MB 20220517: warning, this is a slow function. # Adding functions to each object, ideally avoid this. import histogrammar.specialized + histogrammar.specialized.addImplicitMethods(self) except (ImportError, AttributeError): pass @@ -140,7 +107,8 @@ def fromJsonFragment(json, nameFromParent): @staticmethod def fromJsonFile(fileName): - return Factory.fromJson(jsonlib.load(open(fileName))) + path = Path(fileName) + return Factory.fromJson(jsonlib.load(path.open())) @staticmethod def fromJsonString(json): @@ -157,8 +125,10 @@ def fromJson(json): if isinstance(json["version"], basestring): if not histogrammar.version.compatible(json["version"]): raise ContainerException( - "cannot read a Histogrammar {0} document with histogrammar-python version {1}".format( - json["version"], histogrammar.version.version)) + "cannot read a Histogrammar {} document with histogrammar-python version {}".format( + json["version"], histogrammar.version.version + ) + ) else: raise JsonFormatException(json["version"], "Factory.version") @@ -168,16 +138,17 @@ def fromJson(json): raise JsonFormatException(json["type"], "Factory.type") if name not in Factory.registered: - raise JsonFormatException(json, "unrecognized container (is it a custom container " - "that hasn't been registered?): {0}".format(name)) + raise JsonFormatException( + json, + "unrecognized container (is it a custom container " f"that hasn't been registered?): {name}", + ) return Factory.registered[name].fromJsonFragment(json["data"], None) - else: - raise JsonFormatException(json, "Factory") + raise JsonFormatException(json, "Factory") -class Container(object): +class Container: """Interface for classes that contain aggregated data, such as "Count" or "Bin". Containers are monoids: they have a neutral element (``zero``) and an associative operator (``+``). @@ -195,7 +166,7 @@ def factory(self): return self.__class__ def zero(self): - """Create an empty container with the same parameters as this one. The original is unaffected. """ + """Create an empty container with the same parameters as this one. The original is unaffected.""" raise NotImplementedError def __add__(self, other): @@ -237,7 +208,7 @@ def plot(self, httpServer=None, **parameters): def __getstate__(self): # used by pickling state = dict(self.__dict__) - for s in ['fill', 'plot']: + for s in ["fill", "plot"]: # these states are set dynamically by FillMethod and PlotMethod, in factory.specialize(). # MB 20220517: turned off specialize() for Count objects, # for which specialized fill and plot methods are not needed. @@ -252,7 +223,7 @@ def __setstate__(self, dict): self.plot = PlotMethod(self, self.plot) def copy(self): - """Copy this container, making a clone with no reference to the original. """ + """Copy this container, making a clone with no reference to the original.""" return self + self.zero() @property @@ -265,14 +236,15 @@ def _checkForCrossReferences(self, memo=None): if memo is None: memo = set() if any(x is self for x in memo): - raise ContainerException("cannot fill a tree that contains the same aggregator twice: {0}".format(self)) + raise ContainerException(f"cannot fill a tree that contains the same aggregator twice: {self}") memo.add(self) for child in self.children: child._checkForCrossReferences(memo) self._checkedForCrossReferences = True def toJsonFile(self, fileName): - return jsonlib.dump(self.toJson(), open(fileName, "w")) + path = Path(fileName) + return jsonlib.dump(self.toJson(), path.open(mode="w")) def toJsonString(self): return jsonlib.dumps(self.toJson()) @@ -282,7 +254,11 @@ def toJson(self): Note that the dicts and lists can be turned into a string with ``json.dumps``. """ - return {"type": self.name, "data": self.toJsonFragment(False), "version": histogrammar.version.specification} + return { + "type": self.name, + "data": self.toJsonFragment(False), + "version": histogrammar.version.specification, + } def toJsonFragment(self, suppressName): """Used internally to convert the container to JSON without its ``"type"`` header.""" @@ -296,1067 +272,11 @@ def toImmutable(self): """ return Factory.fromJson(self.toJson()) - _clingClassNameNumber = 0 - - def fillroot(self, ttree, start=-1, end=-1, debug=False, debugOnError=True, **exprs): - self._checkForCrossReferences() - - if not hasattr(self, "_clingFiller"): - import ROOT - - parser = C99SourceToAst() - generator = C99AstToSource() - - inputFieldNames = {} - inputFieldTypes = {} - for branch in ttree.GetListOfBranches(): - if branch.GetClassName() == "": - for leaf in branch.GetListOfLeaves(): - if leaf.IsA() == ROOT.TLeafO.Class(): - inputFieldTypes[leaf.GetName()] = "bool" - elif leaf.IsA() == ROOT.TLeafB.Class() and leaf.IsUnsigned(): - inputFieldTypes[leaf.GetName()] = "unsigned char" - elif leaf.IsA() == ROOT.TLeafB.Class(): - inputFieldTypes[leaf.GetName()] = "char" - elif leaf.IsA() == ROOT.TLeafS.Class() and leaf.IsUnsigned(): - inputFieldTypes[leaf.GetName()] = "unsigned short" - elif leaf.IsA() == ROOT.TLeafS.Class(): - inputFieldTypes[leaf.GetName()] = "short" - elif leaf.IsA() == ROOT.TLeafI.Class() and leaf.IsUnsigned(): - inputFieldTypes[leaf.GetName()] = "UInt_t" - elif leaf.IsA() == ROOT.TLeafI.Class(): - inputFieldTypes[leaf.GetName()] = "Int_t" - elif leaf.IsA() == ROOT.TLeafL.Class() and leaf.IsUnsigned(): - inputFieldTypes[leaf.GetName()] = "ULong64_t" - elif leaf.IsA() == ROOT.TLeafL.Class(): - inputFieldTypes[leaf.GetName()] = "Long64_t" - elif leaf.IsA() == ROOT.TLeafF.Class(): - inputFieldTypes[leaf.GetName()] = "float" - elif leaf.IsA() == ROOT.TLeafD.Class(): - inputFieldTypes[leaf.GetName()] = "double" - elif leaf.IsA() == ROOT.TLeafC.Class(): - raise NotImplementedError("TODO: TLeafC (string)") - elif leaf.IsA() == ROOT.TLeafElement.Class(): - raise NotImplementedError("TODO: TLeafElement") - elif leaf.IsA() == ROOT.TLeafObject.Class(): - raise NotImplementedError("TODO: TLeafObject") - else: - raise NotImplementedError("unknown leaf type: " + repr(leaf)) - - inputFieldTypes[leaf.GetName()] += "*" * leaf.GetTitle().count("[") - - else: - inputFieldTypes[branch.GetName()] = branch.GetClassName() + "*" - - derivedFieldTypes = {} - derivedFieldExprs = {} - - storageStructs = OrderedDict() - initCode = [] - fillCode = [] - weightVars = ["weight_0"] - weightVarStack = ("weight_0",) - tmpVarTypes = {} - - for name, expr in exprs.items(): - self._clingAddExpr( - parser, - generator, - name, - expr, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs) - - self._cppGenerateCode( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - (("var", - "storage"), - ), - 4, - fillCode, - (("var", - "storage"), - ), - 6, - weightVars, - weightVarStack, - tmpVarTypes) - - className = "HistogrammarClingFiller_" + str(Container._clingClassNameNumber) - Container._clingClassNameNumber += 1 - classCode = """class {0} {{ -public: -{1} -{2}{3} -{4}{5} {6} storage; - - void init() {{ -{7} - weight_0 = 1.0; - }} - - void fillall(TTree* ttree, Long64_t start, Long64_t end) {{ - ttree->SetBranchStatus("*", 0); -{11} - init(); -{8} - if (start < 0) start = 0; - if (end < 0) end = ttree->GetEntries(); - for (; start < end; ++start) {{ - ttree->GetEntry(start); -{9}{10} - }} - - ttree->ResetBranchAddresses(); - }} -}}; -""".format(className, - "".join(storageStructs.values()), - "".join(" double " + n + ";\n" for n in weightVars), - "".join(" " + t + " " + self._cppNormalizeInputName(n) + ";\n" for n, - t in inputFieldTypes.items() if self._cppNormalizeInputName(n) in inputFieldNames), - "".join(" " + t + " " + n + ";\n" for n, t in derivedFieldTypes.items() if t != "auto"), - "".join(" " + t + " " + n + ";\n" for n, t in tmpVarTypes.items()), - self._cppStorageType(), - "\n".join(initCode), - "".join(" ttree->SetBranchAddress(" + jsonlib.dumps(key) + - ", &" + n + ");\n" for n, key in inputFieldNames.items()), - "".join(x for x in derivedFieldExprs.values()), - "\n".join(fillCode), - "".join(" ttree->SetBranchStatus(\"" + key + "\", 1);\n" for key in inputFieldNames.values())) - - if debug: - print("line |") - print("\n".join("{0:4d} | {1}".format(i + 1, line) for i, line in enumerate(classCode.split("\n")))) - if not ROOT.gInterpreter.Declare(classCode): - if debug: - raise SyntaxError("Could not compile the above") - elif debugOnError: - raise SyntaxError( - "Could not compile the following:\n\n" + - "\n".join( - "{0:4d} | {1}".format( - i + - 1, - line) for i, - line in enumerate( - classCode.split("\n")))) - else: - raise SyntaxError("Could not compile (rerun with debug=True to see the generated C++ code)") - - self._clingFiller = getattr(ROOT, className)() - - # we already have a _clingFiller; just fill - self._clingFiller.fillall(ttree, start, end) - self._clingUpdate(self._clingFiller, ("var", "storage")) - - _cudaNamespaceNumber = 0 - - def cuda(self, namespace=True, namespaceName=None, writeSize=False, commentMain=True, - split=False, testData=[round(random.gauss(0, 1), 2) for x in xrange(10)], **exprs): - parser = C99SourceToAst() - generator = C99AstToSource() - - inputFieldNames = OrderedDict() - inputFieldTypes = {} - derivedFieldTypes = {} - derivedFieldExprs = OrderedDict() - storageStructs = OrderedDict() - initCode = [] - fillCode = [] - combineCode = [] - jsonCode = [] - weightVars = ["weight_0"] - weightVarStack = ("weight_0",) - tmpVarTypes = {} - - for name, expr in exprs.items(): - self._cudaAddExpr( - parser, - generator, - name, - expr, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs) - - self._cudaGenerateCode( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - (("var", - "(*aggregator)"), - ), - 4, - fillCode, - (("var", - "(*aggregator)"), - ), - 4, - combineCode, - (("var", - "(*total)"), - ), - (("var", - "(*item)"), - ), - 4, - jsonCode, - (("var", - "(*aggregator)"), - ), - 4, - weightVars, - weightVarStack, - tmpVarTypes, - False) - - if namespaceName is None: - namespaceName = "HistogrammarCUDA_" + str(Container._cudaNamespaceNumber) - Container._cudaNamespaceNumber += 1 - - out = '''// Auto-generated on {timestamp:%Y-%m-%d %H:%M:%S} -// If you edit this file, it will be hard to swap it out for another auto-generated copy. - -#ifndef {NS} -#define {NS} - -#include -#include - -/////////////////////////////////////////////////////////////////// declarations - -namespace {ns} {{ - // How the aggregator is laid out in memory (CPU main memory and GPU shared memory).{typedefs} - // Convenient name for the whole aggregator. - typedef {lastStructName} Aggregator; - - // Specific logic of how to zero out the aggregator. - __device__ void zero(Aggregator* aggregator); - - // Specific logic of how to increment the aggregator with input values. - __device__ void increment(Aggregator* aggregator{comma}{inputArgList}); - - // Specific logic of how to combine two aggregators. - __device__ void combine(Aggregator* total, Aggregator* item); - - // Used by toJson. - __host__ void floatToJson(FILE* out, float x); - - // Specific logic of how to print out the aggregator. - __host__ void toJson(Aggregator* aggregator, FILE* out); - - // Generic blockId calculation (3D is the most general). - __device__ int blockId(); - - // Generic blockSize calculation (3D is the most general). - __device__ int blockSize(); - - // Generic threadId calculation (3D is the most general). - __device__ int threadId(); - - // Wrapper for CUDA calls to report errors. - void errorCheck(cudaError_t code); - - // User-level API for initializing the aggregator (from CPU or GPU). - // - // aggregators: array of aggregators to fill in parallel. - // numAggregators: number of aggregators to fill. - // - __global__ void initialize(Aggregator* aggregators, int numAggregators); - - // User-level API for filling the aggregator with a single value (from GPU). - // - // aggregators: array of aggregators to fill in parallel. - // numAggregators: number of aggregators to fill. - // input_*: the input variables you used in the aggregator\'s fill rule. - // - __device__ void fill(Aggregator* aggregators, int numAggregators{comma}{inputArgList}); - - // User-level API for filling the aggregator with arrays of values (from CPU or GPU). - // - // aggregators: array of aggregators to fill in parallel. - // numAggregators: number of aggregators to fill. - // input_*: the input variables you used in the aggregator\'s fill rule. - // numDataPoints: the number of values in each input_* array. - // - __global__ void fillAll(Aggregator* aggregators, int numAggregators{comma}{inputArgStarList}, int numDataPoints); - - // User-level API for combining all aggregators in a block (from CPU or GPU). - // - // Assumes that at least `numAggregators` threads are all running at the same time (can be mutually - // synchronized with `__syncthreads()`. Generally, this means that `extract` should be called on - // no more than one block, which suggests that `numAggregators` ought to be equal to the maximum - // number of threads per block. - // - // aggregators: array of aggregators to fill in parallel. - // numAggregators: number of aggregators to fill. - // result: single output - // - __global__ void extract(Aggregator* aggregators, int numAggregators, Aggregator* result); - - // Test function provides an example of how to use the API. - // - // numAggregators: number of aggregators to fill. - // numBlocks: number of independent blocks to run. - // numThreadsPerBlock: number of threads to run in each block. - // input_*: the input variables you used in the aggregator\'s fill rule. - // numDataPoints: the number of values in each input_* array. - // - void test(int numAggregators, int numBlocks, int numThreadsPerBlock, {inputArgStarList}{comma}int numDataPoints); -}} - -/////////////////////////////////////////////////////////////////// implementations - -namespace {ns} {{{writeSize} - // Specific logic of how to zero out the aggregator. - __device__ void zero(Aggregator* aggregator) {{ -{tmpVarDeclarations}{initCode} - }} - - // Specific logic of how to increment the aggregator with input values. - __device__ void increment(Aggregator* aggregator{comma}{inputArgList}) {{ - const float weight_0 = 1.0f; -{weightVarDeclarations}{tmpVarDeclarations}{quantities} -{fillCode} - }} - - // Specific logic of how to combine two aggregators. - __device__ void combine(Aggregator* total, Aggregator* item) {{ -{tmpVarDeclarations}{combineCode} - }} - - // Used by toJson. - __host__ void floatToJson(FILE* out, float x) {{ - if (isnan(x)) - fprintf(out, "\\"nan\\""); - else if (isinf(x) && x > 0.0f) - fprintf(out, "\\"inf\\""); - else if (isinf(x)) - fprintf(out, "\\"-inf\\""); - else - fprintf(out, "%g", x); - }} - - // Specific logic of how to print out the aggregator. - __host__ void toJson(Aggregator* aggregator, FILE* out) {{ -{tmpVarDeclarations} fprintf(out, "{{\\"version\\": \\"{specVersion}\\", \\"type\\": \\"{factoryName}\\", \\"data\\": "); -{jsonCode} - fprintf(out, "}}\\n"); - }} - - // Generic blockId calculation (3D is the most general). - __device__ int blockId() {{ - return blockIdx.x + blockIdx.y * gridDim.x + blockIdx.z * gridDim.x * gridDim.y; - }} - - // Generic blockSize calculation (3D is the most general). - __device__ int blockSize() {{ - return blockDim.x * blockDim.y * blockDim.z; - }} - - // Generic threadId calculation (3D is the most general). - __device__ int threadId() {{ - return threadIdx.x + threadIdx.y * blockDim.x + threadIdx.z * blockDim.x * blockDim.y; - }} - - // Wrapper for CUDA calls to report errors. - void errorCheck(cudaError_t code) {{ - if (code != cudaSuccess) {{ - fprintf(stderr, "CUDA error: %s\\n", cudaGetErrorString(code)); - exit(code); - }} - }} - - // User-level API for initializing the aggregator (from CPU or GPU). - // - // aggregators: array of aggregators to fill in parallel. - // numAggregators: number of aggregators to fill. - // - __global__ void initialize(Aggregator* aggregators, int numAggregators) {{ - zero(&aggregators[(threadId() + blockId() * blockSize()) % numAggregators]); - }} - - // User-level API for filling the aggregator with a single value (from GPU). - // - // aggregators: array of aggregators to fill in parallel. - // numAggregators: number of aggregators to fill. - // input_*: the input variables you used in the aggregator\'s fill rule. - // - __device__ void fill(Aggregator* aggregators, int numAggregators{comma}{inputArgList}) {{ - increment(&aggregators[(threadId() + blockId() * blockSize()) % numAggregators]{comma}{inputList}); - }} - - // User-level API for filling the aggregator with arrays of values (from CPU or GPU). - // - // aggregators: array of aggregators to fill in parallel. - // numAggregators: number of aggregators to fill. - // input_*: the input variables you used in the aggregator\'s fill rule. - // numDataPoints: the number of values in each input_* array. - // - __global__ void fillAll(Aggregator* aggregators, int numAggregators{comma}{inputArgStarList}, int numDataPoints) {{ - int id = threadId() + blockId() * blockSize(); - if (id < numDataPoints) - fill(aggregators, numAggregators{comma}{inputListId}); - }} - - // User-level API for combining all aggregators in a block (from CPU or GPU). - // - // Assumes that at least `numAggregators` threads are all running at the same time (can be mutually - // synchronized with `__syncthreads()`. Generally, this means that `extract` should be called on - // no more than one block, which suggests that `numAggregators` ought to be equal to the maximum - // number of threads per block. - // - // aggregators: array of aggregators to fill in parallel. - // numAggregators: number of aggregators to fill. - // result: single output - // - __global__ void extract(Aggregator* aggregators, int numAggregators, Aggregator* result) {{ - // merge down in log(N) steps until the thread with id == 0 has the total for this block - int id = threadId(); - - // i should be the first power of 2 larger than numAggregators/2 - int i = 1; - while (2*i < numAggregators) i <<= 1; - - // iteratively split the sample and combine the upper half into the lower half - while (i != 0) {{ - if (id < i && id + i < numAggregators) {{ - Aggregator* ours = &aggregators[id % numAggregators]; - Aggregator* theirs = &aggregators[(id + i) % numAggregators]; - combine(ours, theirs); - }} - - // every iteration should be in lock-step across threads in this block - __syncthreads(); - i >>= 1; - }} - - // return the result, which is in thread 0\'s copy (aggregators[0]) - if (id == 0) {{ - Aggregator* blockLocal = &result[blockId()]; - memcpy(blockLocal, aggregators, sizeof(Aggregator)); - }} - }} - - // Test function provides an example of how to use the API. - // - // numAggregators: number of aggregators to fill. - // numBlocks: number of independent blocks to run. - // numThreadsPerBlock: number of threads to run in each block. - // input_*: the input variables you used in the aggregator\'s fill rule. - // numDataPoints: the number of values in each input_* array. - // - void test(int numAggregators, int numBlocks, int numThreadsPerBlock, {inputArgStarList}{comma}int numDataPoints) {{ - // Create the aggregators and call initialize on them. - Aggregator* aggregators; - cudaMalloc((void**)&aggregators, numAggregators * sizeof(Aggregator)); - initialize<<<1, numThreadsPerBlock>>>(aggregators, numAggregators); - errorCheck(cudaPeekAtLastError()); - errorCheck(cudaDeviceSynchronize()); - -{copyTestData} - // Call fill next, using the same number of blocks, threads per block, and memory allocation. - // fillAll is a __global__ function that takes arrays; fill is a __device__ function that - // takes single entries. Use the latter if filling from your GPU application. - fillAll<<>>(aggregators, numAggregators{comma}{gpuList}, numDataPoints); - errorCheck(cudaPeekAtLastError()); - errorCheck(cudaDeviceSynchronize()); - -{freeTestData} - // Call extract and give it an Aggregator to overwrite. - Aggregator* resultGPU; - cudaMalloc((void**)&resultGPU, sizeof(Aggregator)); - extract<<<1, numThreadsPerBlock>>>(aggregators, numAggregators, resultGPU); - - // Now you can free the collection of subaggregators from the GPU. - cudaFree(aggregators); - - Aggregator resultCPU; - cudaMemcpy(&resultCPU, resultGPU, sizeof(Aggregator), cudaMemcpyDeviceToHost); - - // Now you can free the total aggregator from the GPU. - cudaFree(resultGPU); - - // This Aggregator can be written to stdout as JSON for other Histogrammar programs to interpret - // (and plot). - toJson(&resultCPU, stdout); - }} -}} - -// Optional main runs a tiny test. -{startComment} -int main(int argc, char** argv) {{ - int numAggregators = 5; - int numBlocks = 2; - int numThreadsPerBlock = 5; - - int numDataPoints = 10; -{initTestData} - {ns}::test(numAggregators, numBlocks, numThreadsPerBlock, {inputList}{comma}numDataPoints); -}} -{endComment} - -#endif // {NS} -'''.format(timestamp=datetime.datetime.now(), - ns=namespaceName, - NS=namespaceName.upper(), - writeSize=""" __global__ void write_size(size_t *output) {{ - *output = sizeof(Aggregator); - }} -""" if writeSize else "", - specVersion=histogrammar.version.specification, - factoryName=self.name, - typedefs="".join(storageStructs.values()), - lastStructName="float" if self._c99StructName() == "Ct" else self._c99StructName(), - initCode="\n".join(initCode), - fillCode="\n".join(fillCode), - combineCode="\n".join(combineCode), - quantities="".join(derivedFieldExprs.values()), - jsonCode="\n".join(jsonCode), - comma=", " if len(inputFieldNames) > 0 else "", - inputList=", ".join(norm for norm, name in inputFieldNames.items()), - gpuList=", ".join("gpu_" + name for norm, name in inputFieldNames.items()), - inputListId=", ".join(norm + "[id]" for norm, name in inputFieldNames.items()), - inputArgList=", ".join(inputFieldTypes[name] + " " + norm for norm, name in inputFieldNames.items()), - inputArgStarList=", ".join(inputFieldTypes[name] + "* " + norm for norm, name in inputFieldNames.items()), - weightVarDeclarations="".join(" float " + n + ";\n" for n in weightVars if n != "weight_0"), - tmpVarDeclarations="".join(" " + t + " " + n + ";\n" for n, t in tmpVarTypes.items()), - copyTestData="".join(''' float* gpu_{1}; - errorCheck(cudaMalloc((void**)&gpu_{1}, numDataPoints * sizeof(float))); - errorCheck(cudaMemcpy(gpu_{1}, {0}, numDataPoints * sizeof(float), cudaMemcpyHostToDevice)); -'''.format(norm, name) for norm, name in inputFieldNames.items()), - freeTestData="".join(''' errorCheck(cudaFree(gpu_{0})); -'''.format(name) for name in inputFieldNames.values()), - initTestData="".join(''' float {0}[10] = {{{1}}}; -'''.format(norm, ", ".join(str(float(x)) + "f" - for x in (testData[name] if isinstance(testData, dict) else testData))) - for norm, name in inputFieldNames.items()), - startComment="/*" if commentMain else "", - endComment="*/" if commentMain else "" - ) - - if split: - ifndefIndex = out.find("#ifndef") - splitIndex = out.find("/////////////////////////////////////////////////////////////////// implementations") - endifIndex = out.find("#endif") - doth = out[:splitIndex] + out[endifIndex:] - dotcu = out[:ifndefIndex] + "#include \"" + namespaceName + ".h\"\n\n" + out[splitIndex:endifIndex] - return doth, dotcu - - else: - return out - - def fillpycuda(self, length=None, **exprs): - import numpy - import pycuda.autoinit - import pycuda.driver - import pycuda.compiler - import pycuda.gpuarray - - parser = C99SourceToAst() - generator = C99AstToSource() - - inputFieldNames = OrderedDict() - inputFieldTypes = {} - derivedFieldTypes = {} - derivedFieldExprs = OrderedDict() - storageStructs = OrderedDict() - initCode = [] - fillCode = [] - combineCode = [] - jsonCode = [] - weightVars = ["weight_0"] - weightVarStack = ("weight_0",) - tmpVarTypes = {} - - inputArrays = {} - for name, expr in exprs.items(): - if isinstance(expr, pycuda.driver.DeviceAllocation): - inputArrays[name] = expr - elif isinstance(expr, numpy.ndarray): - inputArrays[name] = expr.astype(numpy.float32) - if len(inputArrays[name].shape) != 1: - raise ValueError("Numpy arrays must be one-dimensional") - elif not isinstance(expr, basestring) and hasattr(expr, "__iter__"): - inputArrays[name] = numpy.array(expr).astype(numpy.float32) - else: - self._cudaAddExpr( - parser, - generator, - name, - expr, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs) - - self._cudaGenerateCode( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - (("var", - "(*aggregator)"), - ), - 4, - fillCode, - (("var", - "(*aggregator)"), - ), - 4, - combineCode, - (("var", - "(*total)"), - ), - (("var", - "(*item)"), - ), - 4, - jsonCode, - (("var", - "(*aggregator)"), - ), - 6, - weightVars, - weightVarStack, - tmpVarTypes, - False) - - arguments = [] - for name in inputFieldNames.values(): - if name not in inputArrays: - raise ValueError("no input supplied for \"" + name + "\"") - if isinstance(inputArrays[name], numpy.ndarray): - if length is None or inputArrays[name].shape[0] < length: - length = inputArrays[name].shape[0] - arguments.append(pycuda.driver.In(inputArrays[name])) - else: - arguments.append(inputArrays[name]) - - if length is None: - raise ValueError( - "no arrays specified as input fields in the aggregator to get length from " - "(and length not specified explicitly)") - - module = pycuda.compiler.SourceModule(self.cuda(namespace=False, writeSize=True)) - - numThreadsPerBlock = min(pycuda.driver.Context.get_device().get_attribute( - pycuda.driver.device_attribute.MAX_THREADS_PER_BLOCK), length) - numBlocks = int(math.ceil(float(length) / float(numThreadsPerBlock))) - numAggregators = numThreadsPerBlock - - write_size = module.get_function("write_size") - initialize = module.get_function("initialize") - fillAll = module.get_function("fillAll") - extract = module.get_function("extract") - - aggregatorSize = pycuda.gpuarray.empty((), dtype=numpy.uintp) - write_size(aggregatorSize, block=(1, 1, 1), grid=(1, 1)) - pycuda.driver.Context.synchronize() - aggregatorSize = int(aggregatorSize.get()) - - aggregators = pycuda.driver.InOut(numpy.zeros(numAggregators * aggregatorSize, dtype=numpy.uint8)) - result = numpy.zeros(aggregatorSize, dtype=numpy.uint8) - - initialize(aggregators, numpy.intc(numAggregators), block=(numThreadsPerBlock, 1, 1), grid=(1, 1)) - - fillAll(aggregators, numpy.intc(numAggregators), *(arguments + [numpy.intc(length)]), - block=(numThreadsPerBlock, 1, 1), grid=(numBlocks, 1)) - - extract( - aggregators, - numpy.intc(numAggregators), - pycuda.driver.Out(result), - block=( - numThreadsPerBlock, - 1, - 1), - grid=( - 1, - 1)) - - pycuda.driver.Context.synchronize() - self._cudaUnpackAndFill(result.tostring(), False, 4) # TODO: determine bigendian, alignment and use them! - - def _cppExpandPrefix(self, *prefix): - return self._c99ExpandPrefix(*prefix) - - def _c99ExpandPrefix(self, *prefix): - out = "" - for t, x in prefix: - if t == "var": - if len(out) == 0: - out += x - else: - out += "." + x - elif t == "index": - out += "[" + str(x) + "]" - else: - raise Exception((t, x)) - return out - - def _clingExpandPrefix(self, obj, *prefix): - for t, x in prefix: - if t == "var": - obj = getattr(obj, x) - elif t == "index": - obj = obj.__getitem__(x) - elif t == "func": - name = x[0] - args = x[1:] - obj = getattr(obj, name)(*args) - else: - raise NotImplementedError((t, x)) - return obj - - def _cppNormalizeInputName(self, key): - return self._c99NormalizeInputName(key) - - def _c99NormalizeInputName(self, key): - if re.match("^[a-zA-Z0-9]*$", key) is not None: - return "input_" + key - else: - return "input_" + base64.b64encode(key).replace("+", "_1").replace("/", "_2").replace("=", "") - - def _cppNormalizeExpr(self, ast, inputFieldNames, inputFieldTypes, weightVar): - return self._c99NormalizeExpr(ast, inputFieldNames, inputFieldTypes, weightVar) - - def _c99NormalizeExpr(self, ast, inputFieldNames, inputFieldTypes, weightVar): - # interpret raw identifiers as tree field names IF they're in the tree (otherwise, leave them alone) - if isinstance(ast, c_ast.ID): - if weightVar is not None and ast.name == "weight": - ast.name = weightVar - elif ast.name in inputFieldTypes: - norm = self._cppNormalizeInputName(ast.name) - inputFieldNames[norm] = ast.name - if inputFieldTypes[ast.name].endswith("*"): - norm = "(" + ("*" * inputFieldTypes[ast.name].count("*")) + norm + ")" - ast.name = norm - - elif isinstance(ast, c_ast.FuncCall): - # t("field name") for field names that aren't valid C identifiers - if isinstance(ast.name, c_ast.ID) and ast.name.name == "t" and isinstance(ast.args, c_ast.ExprList) and \ - len(ast.args.exprs) == 1 and isinstance(ast.args.exprs[0], c_ast.Constant) and \ - ast.args.exprs[0].type == "string": - ast = self._cppNormalizeExpr( - c_ast.ID( - jsonlib.loads( - ast.args.exprs[0].value)), - inputFieldNames, - inputFieldTypes, - weightVar) - # ordinary function: don't translate the name (let function names live in - # a different namespace from variables) - elif isinstance(ast.name, c_ast.ID): - if ast.args is not None: - ast.args = self._cppNormalizeExpr(ast.args, inputFieldNames, inputFieldTypes, weightVar) - # weird function: calling the result of an evaluation, probably an overloaded operator() in C++ - else: - ast.name = self._cppNormalizeExpr(ast.name, inputFieldNames, inputFieldTypes, weightVar) - if ast.args is not None: - ast.args = self._cppNormalizeExpr(ast.args, inputFieldNames, inputFieldTypes, weightVar) - - # only the top (x) of a dotted expression (x.y.z) should be interpreted as a field name - elif isinstance(ast, c_ast.StructRef): - self._cppNormalizeExpr(ast.name, inputFieldNames, inputFieldTypes, weightVar) - - # anything else - else: - for fieldName, fieldValue in ast.children(): - m = re.match(r"([^[]+)\[([0-9]+)\]", fieldName) - if m is not None: - tmp = getattr(ast, m.group(1)) - tmp.__setitem__(int(m.group(2)), self._cppNormalizeExpr( - fieldValue, inputFieldNames, inputFieldTypes, weightVar)) - else: - setattr( - ast, - fieldName, - self._cppNormalizeExpr( - fieldValue, - inputFieldNames, - inputFieldTypes, - weightVar)) - - return ast - - def _cudaNormalizeExpr(self, ast, inputFieldNames, inputFieldTypes, weightVar, derivedFieldExprs, intermediates): - if isinstance(ast, c_ast.ID): - if weightVar is not None and ast.name == "weight": - ast.name = weightVar - elif ast.name in derivedFieldExprs: - norm = "quantity_" + ast.name - ast.name = norm - elif ast.name not in intermediates: - norm = "input_" + ast.name - inputFieldNames[norm] = ast.name - inputFieldTypes[ast.name] = "float" - ast.name = norm - - elif isinstance(ast, c_ast.Decl): - intermediates.add(ast.name) - self._cudaNormalizeExpr( - ast.init, - inputFieldNames, - inputFieldTypes, - weightVar, - derivedFieldExprs, - intermediates) - - elif isinstance(ast, c_ast.FuncCall): - # ordinary function: don't translate the name (let function names live in - # a different namespace from variables) - if isinstance(ast.name, c_ast.ID): - if ast.args is not None: - ast.args = self._cudaNormalizeExpr( - ast.args, - inputFieldNames, - inputFieldTypes, - weightVar, - derivedFieldExprs, - intermediates) - # weird function: calling the result of an evaluation, probably an overloaded operator() in C++ - else: - ast.name = self._cudaNormalizeExpr( - ast.name, - inputFieldNames, - inputFieldTypes, - weightVar, - derivedFieldExprs, - intermediates) - if ast.args is not None: - ast.args = self._cudaNormalizeExpr( - ast.args, - inputFieldNames, - inputFieldTypes, - weightVar, - derivedFieldExprs, - intermediates) - - # only the top (x) of a dotted expression (x.y.z) should be interpreted as a field name - elif isinstance(ast, c_ast.StructRef): - self._cudaNormalizeExpr( - ast.name, - inputFieldNames, - inputFieldTypes, - weightVar, - derivedFieldExprs, - intermediates) - - # anything else - else: - for fieldName, fieldValue in ast.children(): - m = re.match(r"([^[]+)\[([0-9]+)\]", fieldName) - if m is not None: - tmp = getattr(ast, m.group(1)) - tmp.__setitem__(int(m.group(2)), - self._cudaNormalizeExpr(fieldValue, inputFieldNames, inputFieldTypes, weightVar, - derivedFieldExprs, intermediates)) - else: - setattr( - ast, - fieldName, - self._cudaNormalizeExpr( - fieldValue, - inputFieldNames, - inputFieldTypes, - weightVar, - derivedFieldExprs, - intermediates)) - - return ast - - def _cppQuantityExpr(self, parser, generator, inputFieldNames, inputFieldTypes, - derivedFieldTypes, derivedFieldExprs, weightVar): - return self._c99QuantityExpr(parser, generator, inputFieldNames, inputFieldTypes, - derivedFieldTypes, derivedFieldExprs, weightVar) - - def _c99QuantityExpr(self, parser, generator, inputFieldNames, inputFieldTypes, - derivedFieldTypes, derivedFieldExprs, weightVar): - if weightVar is not None: - if not isinstance(self.transform.expr, basestring): - raise ContainerException("Count.transform must be provided as a C99 string when used with Cling") - try: - ast = parser(self.transform.expr) - except Exception as err: - raise SyntaxError("""Couldn't parse C99 expression "{0}": {1}""".format(self.transform.expr, str(err))) - else: - if not isinstance(self.quantity.expr, basestring): - raise ContainerException(self.name + ".quantity must be provided as a C99 string when used with Cling") - try: - ast = parser(self.quantity.expr) - except Exception as err: - raise SyntaxError("""Couldn't parse C99 expression "{0}": {1}""".format(self.quantity.expr, str(err))) - - ast = [self._cppNormalizeExpr(x, inputFieldNames, inputFieldTypes, weightVar) for x in ast] - - if len(ast) == 1 and isinstance(ast[0], c_ast.ID): - return generator(ast) - - else: - normexpr = generator(ast) - derivedFieldName = None - for name, expr in derivedFieldExprs.items(): - if expr == normexpr: - derivedFieldName = name - break - - if derivedFieldName is None: - derivedFieldName = "quantity_" + str(len(derivedFieldExprs)) - if len(ast) > 1: - derivedFieldExprs[derivedFieldName] = " {\n " + ";\n ".join( - generator(x) for x in ast[:-1]) + ";\n " + \ - derivedFieldName + " = " + \ - generator(ast[-1]) + ";\n }\n" - else: - derivedFieldExprs[derivedFieldName] = " " + derivedFieldName + " = " + normexpr + ";\n" - - if self.name == "Categorize": - derivedFieldTypes[derivedFieldName] = "std::string" - elif self.name == "Bag": - if self.range == "S": - derivedFieldTypes[derivedFieldName] = "std::string" - elif self.range == "N": - derivedFieldTypes[derivedFieldName] = "double" - else: - derivedFieldTypes[derivedFieldName] = self.range - else: - derivedFieldTypes[derivedFieldName] = "double" - - return derivedFieldName - - def _cudaQuantityExpr(self, parser, generator, inputFieldNames, inputFieldTypes, - derivedFieldTypes, derivedFieldExprs, weightVar): - if weightVar is not None: - if not isinstance(self.transform.expr, basestring): - raise ContainerException("Count.transform must be provided as a C99 string when used with CUDA") - try: - ast = parser(self.transform.expr) - except Exception as err: - raise SyntaxError("""Couldn't parse C99 expression "{0}": {1}""".format(self.transform.expr, str(err))) - else: - if not isinstance(self.quantity.expr, basestring): - raise ContainerException(self.name + ".quantity must be provided as a C99 string when used with CUDA") - try: - ast = parser(self.quantity.expr) - except Exception as err: - raise SyntaxError("""Couldn't parse C99 expression "{0}": {1}""".format(self.quantity.expr, str(err))) - - intermediates = set() - ast = [ - self._cudaNormalizeExpr( - x, - inputFieldNames, - inputFieldTypes, - weightVar, - derivedFieldExprs, - intermediates) for x in ast] - - if len(ast) == 1 and isinstance(ast[0], c_ast.ID): - return generator(ast) - - else: - normexpr = generator(ast) - derivedFieldName = None - for name, expr in derivedFieldExprs.items(): - if expr == normexpr: - derivedFieldName = name - break - - if derivedFieldName is None: - derivedFieldName = "quantity_" + str(len(derivedFieldExprs)) - if len(ast) > 1: - derivedFieldExprs[derivedFieldName] = " float " + derivedFieldName + \ - ";\n {\n " + ";\n ".join( - generator(x) for x in ast[:-1]) + ";\n " + derivedFieldName + " = " + \ - generator(ast[-1]) + ";\n }\n" - else: - derivedFieldExprs[derivedFieldName] = " float " + derivedFieldName + " = " + normexpr + ";\n" - - derivedFieldTypes[derivedFieldName] = "float" - - return derivedFieldName - - def _clingAddExpr(self, parser, generator, name, expr, inputFieldNames, - inputFieldTypes, derivedFieldTypes, derivedFieldExprs): - if not isinstance(expr, basestring): - raise ContainerException("expressions like {0} must be provided as a C99 string".format(name)) - try: - ast = parser(expr) - except Exception as err: - raise SyntaxError("""Couldn't parse C99 expression "{0}": {1}""".format(expr, str(err))) - - ast = [self._cppNormalizeExpr(x, inputFieldNames, inputFieldTypes, None) for x in ast] - - if len(ast) > 1: - derivedFieldExprs[name] = " auto " + name + " = [this]{\n " + ";\n ".join( - generator(x) for x in ast[:-1]) + ";\n return " + generator(ast[-1]) + ";\n }();\n" - else: - derivedFieldExprs[name] = " auto " + name + " = " + generator(ast[0]) + ";\n" - derivedFieldTypes[name] = "auto" - - def _cudaAddExpr(self, parser, generator, name, expr, inputFieldNames, - inputFieldTypes, derivedFieldTypes, derivedFieldExprs): - if not isinstance(expr, basestring): - raise ContainerException("expressions like {0} must be provided as a C99 string".format(name)) - try: - ast = parser(expr) - except Exception as err: - raise SyntaxError("""Couldn't parse C99 expression "{0}": {1}""".format(expr, str(err))) - - intermediates = set() - ast = [ - self._cudaNormalizeExpr( - x, - inputFieldNames, - inputFieldTypes, - None, - derivedFieldExprs, - intermediates) for x in ast] - - if len(ast) > 1: - derivedFieldExprs[name] = " float quantity_" + name + ";\n {\n " + ";\n ".join( - generator(x) for x in ast[:-1]) + ";\n quantity_" + name + " = " + \ - generator(ast[-1]) + ";\n }\n" - else: - derivedFieldExprs[name] = " float quantity_" + name + " = " + generator(ast[0]) + ";\n" - derivedFieldTypes[name] = "float" - - def _cppStorageType(self): - return self._c99StorageType() - - def _c99StorageType(self): - return self._c99StructName() - - def _cudaStorageType(self): - return self._c99StructName() - def fillnumpy(self, data, weights=1.0): self._checkForCrossReferences() self._numpy(data, weights, shape=[None]) def _checkNPQuantity(self, q, shape): - import numpy if isinstance(q, (list, tuple)): q = numpy.array(q) assert isinstance(q, numpy.ndarray) @@ -1367,17 +287,14 @@ def _checkNPQuantity(self, q, shape): assert q.shape[0] == shape[0] def _checkNPWeights(self, weights, shape): - import numpy if isinstance(weights, numpy.ndarray): assert len(weights.shape) == 1 assert weights.shape[0] == shape[0] def _makeNPWeights(self, weights, shape): - import numpy if isinstance(weights, numpy.ndarray): return weights - else: - return weights * numpy.ones(shape, dtype=numpy.float64) + return weights * numpy.ones(shape, dtype=numpy.float64) def fillsparksql(self, df): converter = df._sc._jvm.org.dianahep.histogrammar.sparksql.pyspark.AggregatorConverter() @@ -1386,6 +303,7 @@ def fillsparksql(self, df): delta = Factory.fromJson(jsonlib.loads(result.toJsonString())) self += delta + # useful functions @@ -1414,6 +332,7 @@ def combine(container1, container2): """ return container1 + container2 + # symbols for Branch paths (Index paths use integers, Label/UntypedLabel paths use strings) diff --git a/histogrammar/dfinterface/__init__.py b/histogrammar/dfinterface/__init__.py index 878a19b..7bf6c6c 100644 --- a/histogrammar/dfinterface/__init__.py +++ b/histogrammar/dfinterface/__init__.py @@ -21,14 +21,16 @@ try: from pyspark.sql import DataFrame as sdf + # add function to create histogrammar histograms - add_sparksql_methods(cls=sdf, prefix='hg_') + add_sparksql_methods(cls=sdf, prefix="hg_") except (ModuleNotFoundError, AttributeError): pass try: from pandas import DataFrame as pdf + # add function to create histogrammar histograms - add_pandas_methods(cls=pdf, prefix='hg_') + add_pandas_methods(cls=pdf, prefix="hg_") except (ModuleNotFoundError, AttributeError): pass diff --git a/histogrammar/dfinterface/addmethods.py b/histogrammar/dfinterface/addmethods.py index aadb636..48d9b72 100644 --- a/histogrammar/dfinterface/addmethods.py +++ b/histogrammar/dfinterface/addmethods.py @@ -14,61 +14,70 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import json import types -import inspect +from ..convenience import CategorizeHistogram as hg_CategorizeHistogram +from ..convenience import Histogram as hg_Histogram +from ..convenience import Profile as hg_Profile +from ..convenience import ProfileErr as hg_ProfileErr +from ..convenience import SparselyHistogram as hg_SparselyHistogram +from ..convenience import SparselyProfile as hg_SparselyProfile +from ..convenience import SparselyProfileErr as hg_SparselyProfileErr +from ..convenience import TwoDimensionallyHistogram as hg_TwoDimensionallyHistogram +from ..convenience import ( + TwoDimensionallySparselyHistogram as hg_TwoDimensionallySparselyHistogram, +) +from ..defs import Factory from ..primitives.average import Average as hg_Average from ..primitives.bag import Bag as hg_Bag from ..primitives.bin import Bin as hg_Bin from ..primitives.categorize import Categorize as hg_Categorize from ..primitives.centrallybin import CentrallyBin as hg_CentrallyBin -from ..primitives.collection import Label as hg_Label -from ..primitives.collection import UntypedLabel as hg_UntypedLabel from ..primitives.collection import Branch as hg_Branch from ..primitives.collection import Index as hg_Index +from ..primitives.collection import Label as hg_Label +from ..primitives.collection import UntypedLabel as hg_UntypedLabel from ..primitives.count import Count as hg_Count from ..primitives.deviate import Deviate as hg_Deviate from ..primitives.fraction import Fraction as hg_Fraction from ..primitives.irregularlybin import IrregularlyBin as hg_IrregularlyBin -from ..primitives.minmax import Maximize as hg_Maximize, Minimize as hg_Minimize +from ..primitives.minmax import Maximize as hg_Maximize +from ..primitives.minmax import Minimize as hg_Minimize from ..primitives.select import Select as hg_Select from ..primitives.sparselybin import SparselyBin as hg_SparselyBin from ..primitives.stack import Stack as hg_Stack from ..primitives.sum import Sum as hg_Sum - -from ..convenience import Histogram as hg_Histogram -from ..convenience import SparselyHistogram as hg_SparselyHistogram -from ..convenience import CategorizeHistogram as hg_CategorizeHistogram -from ..convenience import Profile as hg_Profile -from ..convenience import SparselyProfile as hg_SparselyProfile -from ..convenience import ProfileErr as hg_ProfileErr -from ..convenience import SparselyProfileErr as hg_SparselyProfileErr -from ..convenience import TwoDimensionallyHistogram as hg_TwoDimensionallyHistogram -from ..convenience import TwoDimensionallySparselyHistogram as hg_TwoDimensionallySparselyHistogram - - -from ..defs import Factory from .make_histograms import make_histograms -def add_sparksql_methods(cls, prefix=''): +def add_sparksql_methods(cls, prefix=""): add_methods(cls=cls, hg=hg_fill_sparksql, prefix=prefix) -def add_pandas_methods(cls, prefix=''): +def add_pandas_methods(cls, prefix=""): add_methods(cls=cls, hg=hg_fill_numpy, prefix=prefix) -def add_methods(cls, hg, prefix='hg_'): +def add_methods(cls, hg, prefix="hg_"): def Average(self, quantity): return self.histogrammar(hg_Average(quantity)) def Bag(self, quantity, range): return self.histogrammar(hg_Bag(quantity, range)) - def Bin(self, num, low, high, quantity, value=hg_Count(), - underflow=hg_Count(), overflow=hg_Count(), nanflow=hg_Count()): + def Bin( + self, + num, + low, + high, + quantity, + value=hg_Count(), + underflow=hg_Count(), + overflow=hg_Count(), + nanflow=hg_Count(), + ): return self.histogrammar(hg_Bin(num, low, high, quantity, value, underflow, overflow, nanflow)) def Categorize(self, quantity, value=hg_Count()): @@ -89,7 +98,7 @@ def Index(self, *values): def Branch(self, *values): return self.histogrammar(hg_Branch(*values)) - def Count(self): # TODO: handle transform + def Count(self): # TODO: handle transform return self.histogrammar(hg_Count()) def Deviate(self, quantity): @@ -142,78 +151,99 @@ def SparselyProfileErr(self, binWidth, binnedQuantity, averagedQuantity, origin= return self.histogrammar(hg_SparselyProfileErr(binWidth, binnedQuantity, averagedQuantity, origin)) def TwoDimensionallyHistogram(self, xnum, xlow, xhigh, xquantity, ynum, ylow, yhigh, yquantity): - return self.histogrammar(hg_TwoDimensionallyHistogram(xnum, xlow, xhigh, xquantity, ynum, ylow, yhigh, - yquantity)) + return self.histogrammar( + hg_TwoDimensionallyHistogram(xnum, xlow, xhigh, xquantity, ynum, ylow, yhigh, yquantity) + ) def TwoDimensionallySparselyHistogram(self, xbinWidth, xquantity, ybinWidth, yquantity, xorigin=0.0, yorigin=0.0): - return self.histogrammar(hg_TwoDimensionallySparselyHistogram(xbinWidth, xquantity, ybinWidth, yquantity, - xorigin, yorigin)) + return self.histogrammar( + hg_TwoDimensionallySparselyHistogram(xbinWidth, xquantity, ybinWidth, yquantity, xorigin, yorigin) + ) if inspect.isclass(cls): # pure class, not instantiated - setattr(cls, 'histogrammar', hg) - setattr(cls, prefix + 'Average', Average) - setattr(cls, prefix + 'Bag', Bag) - setattr(cls, prefix + 'Bin', Bin) - setattr(cls, prefix + 'Categorize', Categorize) - setattr(cls, prefix + 'CentrallyBin', CentrallyBin) - setattr(cls, prefix + 'Label', Label) - setattr(cls, prefix + 'UntypedLabel', UntypedLabel) - setattr(cls, prefix + 'Index', Index) - setattr(cls, prefix + 'Branch', Branch) - setattr(cls, prefix + 'Count', Count) - setattr(cls, prefix + 'Deviate', Deviate) - setattr(cls, prefix + 'Fraction', Fraction) - setattr(cls, prefix + 'IrregularlyBin', IrregularlyBin) - setattr(cls, prefix + 'Minimize', Minimize) - setattr(cls, prefix + 'Maximize', Maximize) - setattr(cls, prefix + 'Select', Select) - setattr(cls, prefix + 'SparselyBin', SparselyBin) - setattr(cls, prefix + 'Stack', Stack) - setattr(cls, prefix + 'Sum', Sum) - setattr(cls, prefix + 'make_histograms', make_histograms) - setattr(cls, prefix + 'Histogram', Histogram) - setattr(cls, prefix + 'SparselyHistogram', SparselyHistogram) - setattr(cls, prefix + 'CategorizeHistogram', CategorizeHistogram) - setattr(cls, prefix + 'Profile', Profile) - setattr(cls, prefix + 'SparselyProfile', SparselyProfile) - setattr(cls, prefix + 'ProfileErr', ProfileErr) - setattr(cls, prefix + 'SparselyProfileErr', SparselyProfileErr) - setattr(cls, prefix + 'TwoDimensionallyHistogram', TwoDimensionallyHistogram) - setattr(cls, prefix + 'TwoDimensionallySparselyHistogram', TwoDimensionallySparselyHistogram) + setattr(cls, "histogrammar", hg) + setattr(cls, prefix + "Average", Average) + setattr(cls, prefix + "Bag", Bag) + setattr(cls, prefix + "Bin", Bin) + setattr(cls, prefix + "Categorize", Categorize) + setattr(cls, prefix + "CentrallyBin", CentrallyBin) + setattr(cls, prefix + "Label", Label) + setattr(cls, prefix + "UntypedLabel", UntypedLabel) + setattr(cls, prefix + "Index", Index) + setattr(cls, prefix + "Branch", Branch) + setattr(cls, prefix + "Count", Count) + setattr(cls, prefix + "Deviate", Deviate) + setattr(cls, prefix + "Fraction", Fraction) + setattr(cls, prefix + "IrregularlyBin", IrregularlyBin) + setattr(cls, prefix + "Minimize", Minimize) + setattr(cls, prefix + "Maximize", Maximize) + setattr(cls, prefix + "Select", Select) + setattr(cls, prefix + "SparselyBin", SparselyBin) + setattr(cls, prefix + "Stack", Stack) + setattr(cls, prefix + "Sum", Sum) + setattr(cls, prefix + "make_histograms", make_histograms) + setattr(cls, prefix + "Histogram", Histogram) + setattr(cls, prefix + "SparselyHistogram", SparselyHistogram) + setattr(cls, prefix + "CategorizeHistogram", CategorizeHistogram) + setattr(cls, prefix + "Profile", Profile) + setattr(cls, prefix + "SparselyProfile", SparselyProfile) + setattr(cls, prefix + "ProfileErr", ProfileErr) + setattr(cls, prefix + "SparselyProfileErr", SparselyProfileErr) + setattr(cls, prefix + "TwoDimensionallyHistogram", TwoDimensionallyHistogram) + setattr( + cls, + prefix + "TwoDimensionallySparselyHistogram", + TwoDimensionallySparselyHistogram, + ) else: # instantiated class - setattr(cls, 'histogrammar', types.MethodType(hg, cls)) - setattr(cls, prefix + 'Average', types.MethodType(Average, cls)) - setattr(cls, prefix + 'Bag', types.MethodType(Bag, cls)) - setattr(cls, prefix + 'Bin', types.MethodType(Bin, cls)) - setattr(cls, prefix + 'Categorize', types.MethodType(Categorize, cls)) - setattr(cls, prefix + 'CentrallyBin', types.MethodType(CentrallyBin, cls)) - setattr(cls, prefix + 'Label', types.MethodType(Label, cls)) - setattr(cls, prefix + 'UntypedLabel', types.MethodType(UntypedLabel, cls)) - setattr(cls, prefix + 'Index', types.MethodType(Index, cls)) - setattr(cls, prefix + 'Branch', types.MethodType(Branch, cls)) - setattr(cls, prefix + 'Count', types.MethodType(Count, cls)) - setattr(cls, prefix + 'Deviate', types.MethodType(Deviate, cls)) - setattr(cls, prefix + 'Fraction', types.MethodType(Fraction, cls)) - setattr(cls, prefix + 'IrregularlyBin', types.MethodType(IrregularlyBin, cls)) - setattr(cls, prefix + 'Minimize', types.MethodType(Minimize, cls)) - setattr(cls, prefix + 'Maximize', types.MethodType(Maximize, cls)) - setattr(cls, prefix + 'Select', types.MethodType(Select, cls)) - setattr(cls, prefix + 'SparselyBin', types.MethodType(SparselyBin, cls)) - setattr(cls, prefix + 'Stack', types.MethodType(Stack, cls)) - setattr(cls, prefix + 'Sum', types.MethodType(Sum, cls)) - setattr(cls, prefix + 'make_histograms', types.MethodType(make_histograms, cls)) - setattr(cls, prefix + 'Histogram', types.MethodType(Histogram, cls)) - setattr(cls, prefix + 'SparselyHistogram', types.MethodType(SparselyHistogram, cls)) - setattr(cls, prefix + 'CategorizeHistogram', types.MethodType(CategorizeHistogram, cls)) - setattr(cls, prefix + 'Profile', types.MethodType(Profile, cls)) - setattr(cls, prefix + 'SparselyProfile', types.MethodType(SparselyProfile, cls)) - setattr(cls, prefix + 'ProfileErr', types.MethodType(ProfileErr, cls)) - setattr(cls, prefix + 'SparselyProfileErr', types.MethodType(SparselyProfileErr, cls)) - setattr(cls, prefix + 'TwoDimensionallyHistogram', types.MethodType(TwoDimensionallyHistogram, cls)) - setattr(cls, prefix + 'TwoDimensionallySparselyHistogram', - types.MethodType(TwoDimensionallySparselyHistogram, cls)) + setattr(cls, "histogrammar", types.MethodType(hg, cls)) + setattr(cls, prefix + "Average", types.MethodType(Average, cls)) + setattr(cls, prefix + "Bag", types.MethodType(Bag, cls)) + setattr(cls, prefix + "Bin", types.MethodType(Bin, cls)) + setattr(cls, prefix + "Categorize", types.MethodType(Categorize, cls)) + setattr(cls, prefix + "CentrallyBin", types.MethodType(CentrallyBin, cls)) + setattr(cls, prefix + "Label", types.MethodType(Label, cls)) + setattr(cls, prefix + "UntypedLabel", types.MethodType(UntypedLabel, cls)) + setattr(cls, prefix + "Index", types.MethodType(Index, cls)) + setattr(cls, prefix + "Branch", types.MethodType(Branch, cls)) + setattr(cls, prefix + "Count", types.MethodType(Count, cls)) + setattr(cls, prefix + "Deviate", types.MethodType(Deviate, cls)) + setattr(cls, prefix + "Fraction", types.MethodType(Fraction, cls)) + setattr(cls, prefix + "IrregularlyBin", types.MethodType(IrregularlyBin, cls)) + setattr(cls, prefix + "Minimize", types.MethodType(Minimize, cls)) + setattr(cls, prefix + "Maximize", types.MethodType(Maximize, cls)) + setattr(cls, prefix + "Select", types.MethodType(Select, cls)) + setattr(cls, prefix + "SparselyBin", types.MethodType(SparselyBin, cls)) + setattr(cls, prefix + "Stack", types.MethodType(Stack, cls)) + setattr(cls, prefix + "Sum", types.MethodType(Sum, cls)) + setattr(cls, prefix + "make_histograms", types.MethodType(make_histograms, cls)) + setattr(cls, prefix + "Histogram", types.MethodType(Histogram, cls)) + setattr(cls, prefix + "SparselyHistogram", types.MethodType(SparselyHistogram, cls)) + setattr( + cls, + prefix + "CategorizeHistogram", + types.MethodType(CategorizeHistogram, cls), + ) + setattr(cls, prefix + "Profile", types.MethodType(Profile, cls)) + setattr(cls, prefix + "SparselyProfile", types.MethodType(SparselyProfile, cls)) + setattr(cls, prefix + "ProfileErr", types.MethodType(ProfileErr, cls)) + setattr( + cls, + prefix + "SparselyProfileErr", + types.MethodType(SparselyProfileErr, cls), + ) + setattr( + cls, + prefix + "TwoDimensionallyHistogram", + types.MethodType(TwoDimensionallyHistogram, cls), + ) + setattr( + cls, + prefix + "TwoDimensionallySparselyHistogram", + types.MethodType(TwoDimensionallySparselyHistogram, cls), + ) def hg_fill_sparksql(self, hist): diff --git a/histogrammar/dfinterface/filling_utils.py b/histogrammar/dfinterface/filling_utils.py index ed1c739..c0e34c6 100644 --- a/histogrammar/dfinterface/filling_utils.py +++ b/histogrammar/dfinterface/filling_utils.py @@ -83,23 +83,12 @@ def to_str(val): """ if isinstance(val, str): return val - elif isinstance(val, pd.Series): + if isinstance(val, pd.Series): # Note: at this point, data type of pd.series has already been inferred as being of type object (mixed) return val.astype(str).values - elif hasattr(val, "__iter__"): - return np.asarray( - list( - map( - lambda s: s - if isinstance(s, str) - else str(s) - if hasattr(s, "__str__") - else "", - val, - ) - ) - ) - elif hasattr(val, "__str__"): + if hasattr(val, "__iter__"): + return np.asarray([(s if isinstance(s, str) else str(s) if hasattr(s, "__str__") else "") for s in val]) + if hasattr(val, "__str__"): return str(val) return "None" @@ -113,11 +102,11 @@ def only_str(val): """ if isinstance(val, str): return val - elif isinstance(val, pd.Series): + if isinstance(val, pd.Series): # at this point, data type of pd.series has already been inferred as *to be* 'string' dtype = np.dtype(val.dtype).type return val.values if dtype in [str, np.str_, np.bytes_] else val.astype(str).values - elif hasattr(val, "__iter__"): + if hasattr(val, "__iter__"): return np.asarray([s if isinstance(s, str) else "None" for s in val]) return "None" @@ -131,12 +120,10 @@ def only_bool(val): """ if isinstance(val, (np.bool_, bool)): return val - elif isinstance(val, pd.Series) and val.dtype in [np.bool_, bool]: + if isinstance(val, pd.Series) and val.dtype in [np.bool_, bool]: return val.values - elif hasattr(val, "__iter__") and not isinstance(val, str): - return np.asarray( - [s if isinstance(s, (np.bool_, bool)) else "NaN" for s in val] - ) + if hasattr(val, "__iter__") and not isinstance(val, str): + return np.asarray([s if isinstance(s, (np.bool_, bool)) else "NaN" for s in val]) return "NaN" @@ -149,12 +136,10 @@ def only_int(val): """ if isinstance(val, (np.int64, int)): return val - elif isinstance(val, pd.Series) and val.dtype in [np.int64, int]: + if isinstance(val, pd.Series) and val.dtype in [np.int64, int]: return val.values - elif hasattr(val, "__iter__") and not isinstance(val, str): - return np.asarray( - [s if isinstance(s, (np.int64, int)) else np.nan for s in val] - ) + if hasattr(val, "__iter__") and not isinstance(val, str): + return np.asarray([s if isinstance(s, (np.int64, int)) else np.nan for s in val]) return np.nan @@ -167,12 +152,10 @@ def only_float(val): """ if isinstance(val, (np.float64, float)): return val - elif isinstance(val, pd.Series) and val.dtype in [np.float64, float]: + if isinstance(val, pd.Series) and val.dtype in [np.float64, float]: return val.values - elif hasattr(val, "__iter__") and not isinstance(val, str): - return np.asarray( - [s if isinstance(s, (np.float64, float)) else np.nan for s in val] - ) + if hasattr(val, "__iter__") and not isinstance(val, str): + return np.asarray([s if isinstance(s, (np.float64, float)) else np.nan for s in val]) return np.nan @@ -207,8 +190,7 @@ def value_to_bin_index(val, **kwargs): # NOTE this notation also works for timestamps bin_width = kwargs.get("binWidth", kwargs.get("bin_width", 1)) bin_offset = kwargs.get("origin", kwargs.get("bin_offset", 0)) - bin_index = int(np.floor((val - bin_offset) / bin_width)) - return bin_index + return int(np.floor((val - bin_offset) / bin_width)) except BaseException: pass return val diff --git a/histogrammar/dfinterface/histogram_filler_base.py b/histogrammar/dfinterface/histogram_filler_base.py index cfd01e1..b658425 100644 --- a/histogrammar/dfinterface/histogram_filler_base.py +++ b/histogrammar/dfinterface/histogram_filler_base.py @@ -1,5 +1,4 @@ -""" -Copyright Eskapade: +"""Copyright Eskapade: License Apache-2: https://github.com/KaveIO/Eskapade-Core/blob/master/LICENSE Reference link: https://github.com/KaveIO/Eskapade/blob/master/python/eskapade/analysis/histogram_filling.py @@ -26,11 +25,10 @@ from ..primitives.sparselybin import SparselyBin from ..primitives.stack import Stack from ..primitives.sum import Sum - from .filling_utils import check_column, normalize_dtype -class HistogramFillerBase(object): +class HistogramFillerBase: """Base class link to fill histograms. Timestamp features are @@ -105,7 +103,7 @@ def __init__( features = features or [] self.features = [check_column(c) for c in features] - if not any([binning == opt for opt in ["auto", "unit"]]): + if not any(binning == opt for opt in ["auto", "unit"]): raise TypeError('binning should be "auto" or "unit".') self.binning = binning self.bin_specs = bin_specs or {} @@ -176,16 +174,14 @@ def _auto_n_bins(self, c): del c[0] if len(c) <= 1: return self._auto_n_bins_1d - elif len(c) == 2: + if len(c) == 2: return self._auto_n_bins_2d - elif len(c) == 3: - return self._auto_n_bins_3d - else: + if len(c) == 3: return self._auto_n_bins_3d + return self._auto_n_bins_3d def _execute(self, df): - """ - _execute() does five things: + """_execute() does five things: * check presence and data type of requested features * timestamp variables are converted to nanosec (integers) @@ -212,9 +208,7 @@ def _execute(self, df): self.auto_complete_bin_specs(idf, cols_by_type) # 5. do the actual histogram/counter filling - self.logger.info( - f"Filling {len(self.features)} specified histograms. {self.binning}-binning." - ) + self.logger.info(f"Filling {len(self.features)} specified histograms. {self.binning}-binning.") self.fill_histograms(idf) return self._hists @@ -229,19 +223,17 @@ def assign_and_check_features(self, df, cols_by_type): no_initial_features = len(self.features) == 0 all_cols = ( - list(cols_by_type["num"]) + - list(cols_by_type["dt"]) + - list(cols_by_type["bool"]) + - list(cols_by_type["str"]) + list(cols_by_type["num"]) + + list(cols_by_type["dt"]) + + list(cols_by_type["bool"]) + + list(cols_by_type["str"]) ) # 1. assign / figure out a time axis if isinstance(self.time_axis, str) and len(self.time_axis) > 0: # a) specified time axis if self.time_axis not in all_cols: - raise RuntimeError( - f'Specified time-axis "{self.time_axis}" not found in dataframe.' - ) + raise RuntimeError(f'Specified time-axis "{self.time_axis}" not found in dataframe.') elif isinstance(self.time_axis, bool) and self.time_axis: # b) try to figure out time axis self.time_axis = "" @@ -251,9 +243,7 @@ def assign_and_check_features(self, df, cols_by_type): self.time_axis = list(cols_by_type["dt"])[0] self.logger.info(f'Time-axis automatically set to "{self.time_axis}"') elif num == 0: - self.logger.warning( - "No obvious time-axes found to choose from. So not used." - ) + self.logger.warning("No obvious time-axes found to choose from. So not used.") else: self.logger.warning( f'Found {num} time-axes: {cols_by_type["dt"]}. Set *one* time_axis manually! Now NOT used.' @@ -266,11 +256,7 @@ def assign_and_check_features(self, df, cols_by_type): if no_initial_features: if len(self.time_axis) > 0: # time-axis is selected: make histograms of all columns in dataframe vs time-axis - self.features = [ - [self.time_axis, c] - for c in sorted(self.get_features(df)) - if c != self.time_axis - ] + self.features = [[self.time_axis, c] for c in sorted(self.get_features(df)) if c != self.time_axis] else: # make histograms of all columns in dataframe self.features = [[c] for c in sorted(self.get_features(df))] @@ -294,15 +280,9 @@ def assign_and_check_features(self, df, cols_by_type): # we're the boss. we're not going to histogram this ... huge_cats.append(c) else: # debug mode - self.logger.warning( - f"Column {c} has {nuniq[c]} unique entries (large). Really histogram it?" - ) + self.logger.warning(f"Column {c} has {nuniq[c]} unique entries (large). Really histogram it?") # scrub self.features of huge categories. - self.features = [ - cols - for cols in self.features - if not any([c in huge_cats for c in cols]) - ] + self.features = [cols for cols in self.features if not any(c in huge_cats for c in cols)] def auto_complete_bin_specs(self, df, cols_by_type): """auto complete the bin-specs that have not been provided @@ -347,9 +327,7 @@ def auto_complete_bin_specs(self, df, cols_by_type): # specs for Bin and Sparselybin histograms if q[1] == q[0]: # in case of highly imbalanced data it can happen that q05=q95. If so use min and max instead. - q = (self.get_quantiles(df, quantiles=[0.0, 1.0], columns=[c]))[ - c - ] + q = (self.get_quantiles(df, quantiles=[0.0, 1.0], columns=[c]))[c] qdiff = (q[1] - q[0]) * (1.0 / 0.9) if q[1] > q[0] else 1.0 bin_width = qdiff / float(n_bins) bin_offset = q[0] - qdiff * 0.05 @@ -361,9 +339,7 @@ def auto_complete_bin_specs(self, df, cols_by_type): high = quantiles_i[c][1] bin_width = np.max((np.round((high - low) / float(n_bins)), 1.0)) bin_offset = low = np.floor(low - 0.5) + 0.5 - n_bins = int((high - low) // bin_width) + int( - (high - low) % bin_width > 0.0 - ) + n_bins = int((high - low) // bin_width) + int((high - low) % bin_width > 0.0) high = low + n_bins * bin_width if c == self.time_axis and idx == 0: @@ -398,9 +374,7 @@ def categorize_features(self, df): # sort features into numerical, timestamp and category based cols_by_type = defaultdict(set) - features = ( - self.features if self.features else [[c] for c in self.get_features(df)] - ) + features = self.features if self.features else [[c] for c in self.get_features(df)] for col_list in features: for col in col_list: @@ -414,7 +388,7 @@ def categorize_features(self, df): self.var_dtype[col] = dt # metadata indicates decimal - if hasattr(dt_col, 'metadata') and dt_col.metadata is not None and dt_col.metadata["decimal"]: + if hasattr(dt_col, "metadata") and dt_col.metadata is not None and dt_col.metadata["decimal"]: cols_by_type["decimal"].add(col) if np.issubdtype(dt, np.integer): @@ -430,11 +404,7 @@ def categorize_features(self, df): colset = cols_by_type["str"] colset.add(col) - self.logger.debug( - 'Data type of column "{col}" is "{type}".'.format( - col=col, type=self.var_dtype[col] - ) - ) + self.logger.debug(f'Data type of column "{col}" is "{self.var_dtype[col]}".') return cols_by_type def var_bin_specs(self, c, idx=0): @@ -451,9 +421,7 @@ def var_bin_specs(self, c, idx=0): # determine default bin specs dt = np.dtype(self.var_dtype[c[idx]]) is_timestamp = isinstance(dt.type(), np.datetime64) - default = ( - self._unit_bin_specs if not is_timestamp else self._unit_timestamp_specs - ) + default = self._unit_bin_specs if not is_timestamp else self._unit_timestamp_specs # get bin specs if n in self.bin_specs and len(c) > 1 and len(c) == len(self.bin_specs[n]): @@ -511,8 +479,8 @@ def get_hist_bin(self, hist, features, quant, col, dt): # numbers and timestamps are put in a sparse binned histogram if "binWidth" in specs or "bin_width" in specs: hist = SparselyBin( - binWidth=specs.get("binWidth", specs.get("bin_width", 1.)), - origin=specs.get("origin", specs.get("bin_offset", 0.)), + binWidth=specs.get("binWidth", specs.get("bin_width", 1.0)), + origin=specs.get("origin", specs.get("bin_offset", 0.0)), quantity=quant, value=hist, ) @@ -526,7 +494,7 @@ def get_hist_bin(self, hist, features, quant, col, dt): ) elif "edges" in specs or "bin_edges" in specs: hist = IrregularlyBin( - edges=specs.get('edges', specs.get('bin_edges', [])), + edges=specs.get("edges", specs.get("bin_edges", [])), quantity=quant, value=hist, ) @@ -542,19 +510,19 @@ def get_hist_bin(self, hist, features, quant, col, dt): hist = Sum(quantity=quant) elif "centers" in specs or "bin_centers" in specs: hist = CentrallyBin( - centers=specs.get('centers', specs.get('bin_centers', [])), + centers=specs.get("centers", specs.get("bin_centers", [])), quantity=quant, value=hist, ) elif "thresholds" in specs: hist = Stack( - thresholds=specs['thresholds'], + thresholds=specs["thresholds"], quantity=quant, value=hist, ) elif "bag" in specs or "range" in specs: hist = Bag( - range=specs.get('range', 'N'), + range=specs.get("range", "N"), quantity=quant, ) elif "fraction" in specs: @@ -569,11 +537,10 @@ def get_hist_bin(self, hist, features, quant, col, dt): ) else: raise RuntimeError("Do not know how to interpret bin specifications.") + elif not is_bool and ("bag" in specs or "range" in specs): + hist = Bag(range=specs.get("range", "S"), quantity=quant) else: - if not is_bool and ("bag" in specs or "range" in specs): - hist = Bag(range=specs.get('range', 'S'), quantity=quant) - else: - # string and booleans are treated as categories - hist = Categorize(quantity=quant, value=hist) + # string and booleans are treated as categories + hist = Categorize(quantity=quant, value=hist) return hist diff --git a/histogrammar/dfinterface/make_histograms.py b/histogrammar/dfinterface/make_histograms.py index ad3d29b..3727c62 100644 --- a/histogrammar/dfinterface/make_histograms.py +++ b/histogrammar/dfinterface/make_histograms.py @@ -34,16 +34,15 @@ from ..primitives.deviate import Deviate from ..primitives.fraction import Fraction from ..primitives.irregularlybin import IrregularlyBin -from ..primitives.minmax import Minimize, Maximize +from ..primitives.minmax import Maximize, Minimize from ..primitives.select import Select from ..primitives.sparselybin import SparselyBin from ..primitives.stack import Stack from ..primitives.sum import Sum - +from ..util import _get_sub_hist +from .filling_utils import normalize_dtype from .pandas_histogrammar import PandasHistogrammar from .spark_histogrammar import SparkHistogrammar -from .filling_utils import normalize_dtype -from ..util import _get_sub_hist logger = logging.getLogger() @@ -54,8 +53,8 @@ def make_histograms( binning="auto", bin_specs=None, time_axis="", - time_width='30d', - time_offset='2010-01-04', + time_width="30d", + time_offset="2010-01-04", var_dtype=None, ret_specs=False, nbins_1d=40, @@ -125,15 +124,9 @@ def make_histograms( :return: dict of created histogrammar histograms """ # basic checks on presence of time_axis - if (not isinstance(time_axis, (str, bool))) or ( - isinstance(time_axis, bool) and not time_axis - ): + if (not isinstance(time_axis, (str, bool))) or (isinstance(time_axis, bool) and not time_axis): raise TypeError("time_axis needs to be a string, or a bool set to True") - if ( - isinstance(time_axis, str) and - len(time_axis) > 0 and - time_axis not in df.columns - ): + if isinstance(time_axis, str) and len(time_axis) > 0 and time_axis not in df.columns: raise ValueError(f'time_axis "{time_axis}" not found in columns of dataframe.') if isinstance(time_axis, bool): time_axes = get_time_axes(df) @@ -142,20 +135,16 @@ def make_histograms( time_axis = time_axes[0] logger.info(f'Time-axis automatically set to "{time_axis}"') elif num == 0: - raise RuntimeError( - "No obvious time-axes found. Cannot generate stability report." - ) + raise RuntimeError("No obvious time-axes found. Cannot generate stability report.") else: - raise RuntimeError( - f"Found {num} time-axes: {time_axes}. Set *one* time_axis manually!" - ) + raise RuntimeError(f"Found {num} time-axes: {time_axes}. Set *one* time_axis manually!") # if time_axis present, interpret time_width and time_offset if ( - isinstance(time_axis, str) and - len(time_axis) > 0 and - isinstance(time_width, (str, int, float)) and - isinstance(time_offset, (str, int, float)) + isinstance(time_axis, str) + and len(time_axis) > 0 + and isinstance(time_width, (str, int, float)) + and isinstance(time_offset, (str, int, float)) ): if not isinstance(bin_specs, (type(None), dict)): raise RuntimeError("bin_specs object is not a dictionary") @@ -168,9 +157,7 @@ def make_histograms( } bin_specs[time_axis] = time_specs else: - warnings.warn( - f'time-axis "{time_axis}" already found in binning specifications. not overwriting.' - ) + warnings.warn(f'time-axis "{time_axis}" already found in binning specifications. not overwriting.') cls = PandasHistogrammar if isinstance(df, pd.DataFrame) else SparkHistogrammar hist_filler = cls( @@ -211,7 +198,7 @@ def get_data_type(df, col): # spark conversions to numpy or python equivalent if dt == "string": dt = "str" - elif dt == "timestamp" or dt == "date": + elif dt in ("timestamp", "date"): dt = np.datetime64 elif dt == "boolean": dt = bool @@ -229,11 +216,7 @@ def get_time_axes(df): :param df: input dataframe (pandas or spark) :return: list of time-axis columns """ - return [ - c - for c in df.columns - if np.issubdtype(normalize_dtype(get_data_type(df, c)), np.datetime64) - ] + return [c for c in df.columns if np.issubdtype(normalize_dtype(get_data_type(df, c)), np.datetime64)] def has_one_time_axis(df): @@ -270,33 +253,33 @@ def _get_bin_specs(h): if isinstance(h, Categorize): bin_specs.append({}) elif isinstance(h, Bin): - bin_specs.append(dict(num=h.num, low=h.low, high=h.high)) + bin_specs.append({"num": h.num, "low": h.low, "high": h.high}) elif isinstance(h, SparselyBin): - bin_specs.append(dict(binWidth=h.binWidth, origin=h.origin)) + bin_specs.append({"binWidth": h.binWidth, "origin": h.origin}) elif isinstance(h, IrregularlyBin): - bin_specs.append(dict(edges=h.edges[1:])) # ignore -inf + bin_specs.append({"edges": h.edges[1:]}) # ignore -inf elif isinstance(h, CentrallyBin): - bin_specs.append(dict(centers=h.centers)) + bin_specs.append({"centers": h.centers}) elif isinstance(h, Stack): - bin_specs.append(dict(thresholds=h.thresholds[1:])) # ignore -inf + bin_specs.append({"thresholds": h.thresholds[1:]}) # ignore -inf elif isinstance(h, Maximize): - bin_specs.append(dict(maximize=True)) + bin_specs.append({"maximize": True}) elif isinstance(h, Minimize): - bin_specs.append(dict(minimize=True)) + bin_specs.append({"minimize": True}) elif isinstance(h, Average): - bin_specs.append(dict(average=True)) + bin_specs.append({"average": True}) elif isinstance(h, Deviate): - bin_specs.append(dict(deviate=True)) + bin_specs.append({"deviate": True}) elif isinstance(h, Sum): - bin_specs.append(dict(sum=True)) + bin_specs.append({"sum": True}) elif isinstance(h, Bag): - bin_specs.append(dict(bag=True, range=h.range)) + bin_specs.append({"bag": True, "range": h.range}) elif isinstance(h, Sum): - bin_specs.append(dict(sum=True)) + bin_specs.append({"sum": True}) elif isinstance(h, Fraction): - bin_specs.append(dict(fraction=True)) + bin_specs.append({"fraction": True}) elif isinstance(h, Select): - bin_specs.append(dict(cut=True)) + bin_specs.append({"cut": True}) # histogram may have a sub-histogram. Extract it and recurse hist = _get_sub_hist(h) @@ -316,7 +299,7 @@ def _match_first_key(skip_first_axis=None, feature=""): rest_key = ":".join(karr[1:]) if isinstance(skip_first_axis, bool): return skip_first_axis, rest_key if skip_first_axis else feature - elif isinstance(skip_first_axis, str) and len(skip_first_axis) > 0: + if isinstance(skip_first_axis, str) and len(skip_first_axis) > 0: match = begin == skip_first_axis return match, rest_key if match else feature return False, feature diff --git a/histogrammar/dfinterface/pandas_histogrammar.py b/histogrammar/dfinterface/pandas_histogrammar.py index 8448a5b..940779a 100644 --- a/histogrammar/dfinterface/pandas_histogrammar.py +++ b/histogrammar/dfinterface/pandas_histogrammar.py @@ -1,18 +1,18 @@ -""" -Copyright Eskapade: +"""Copyright Eskapade: License Apache-2: https://github.com/KaveIO/Eskapade-Core/blob/master/LICENSE Reference link: https://github.com/KaveIO/Eskapade/blob/master/python/eskapade/analysis/links/hist_filler.py All modifications copyright ING WBAA. """ -import histogrammar as hg import numpy as np import pandas as pd -from tqdm import tqdm from pandas.api.types import infer_dtype +from tqdm import tqdm -from .filling_utils import to_ns, QUANTITY +import histogrammar as hg + +from .filling_utils import QUANTITY, to_ns from .histogram_filler_base import HistogramFillerBase @@ -129,20 +129,17 @@ def get_data_type(self, df, col): raise KeyError(f'column "{col:s}" not in input dataframe') inferred = infer_dtype(df[col], skipna=True) - if inferred in 'string': - data_type = 'str' - elif inferred == 'integer': - data_type = 'int' - elif inferred == 'boolean': - data_type = 'bool' - elif inferred in {'decimal', 'floating', 'mixed-integer-float'}: + if inferred in "string": + data_type = "str" + elif inferred == "integer": + data_type = "int" + elif inferred == "boolean": + data_type = "bool" + elif inferred in {"decimal", "floating", "mixed-integer-float"}: # decimal needs preprocessing (cast), signal this in metadata - if inferred == "decimal": - data_type = np.dtype('float', metadata={"decimal": True}) - else: - data_type = "float" - elif inferred in {'date', 'datetime', 'datetime64'}: - data_type = 'datetime64' + data_type = np.dtype("float", metadata={"decimal": True}) if inferred == "decimal" else "float" + elif inferred in {"date", "datetime", "datetime64"}: + data_type = "datetime64" else: # categorical, mixed, etc -> object uses to_string() data_type = np.object_ @@ -158,8 +155,7 @@ def get_quantiles(self, df, quantiles=[0.05, 0.95], columns=[]): if len(columns) == 0: return {} qdf = df[columns].quantile(quantiles) - qd = {c: qdf[c].values.tolist() for c in columns} - return qd + return {c: qdf[c].values.tolist() for c in columns} def get_nunique(self, df, columns=[]): """return dict with number of unique entries for given columns @@ -185,11 +181,7 @@ def process_features(self, df, cols_by_type): # make temp df for value counting (used below) idf = df[list(cols_by_type["num"]) + list(cols_by_type["str"]) + list(cols_by_type["bool"])].copy() for col in cols_by_type["dt"]: - self.logger.debug( - 'Converting column "{col}" of type "{type}" to nanosec.'.format( - col=col, type=self.var_dtype[col] - ) - ) + self.logger.debug(f'Converting column "{col}" of type "{self.var_dtype[col]}" to nanosec.') idf[col] = df[col].apply(to_ns) # treat decimal as float, as decimal is not supported by .quantile @@ -244,14 +236,9 @@ def construct_empty_hist(self, features): # processing function, e.g. only accept booleans during filling f = QUANTITY[dt] - if len(features) == 1: - # df[col] is a pd.series - quant = lambda x, fnc=f: fnc(x) # noqa - else: - # df[features] is a pd.Dataframe - # fix column to col - quant = lambda x, fnc=f, clm=col: fnc(x[clm]) # noqa - + # if len(features) == 1: df[col] is a pd.series + # else: df[features] is a pd.Dataframe, so fix column to col + quant = (lambda x, fnc=f: fnc(x)) if len(features) == 1 else (lambda x, fnc=f, clm=col: fnc(x[clm])) hist = self.get_hist_bin(hist, features, quant, col, dt) return hist diff --git a/histogrammar/dfinterface/spark_histogrammar.py b/histogrammar/dfinterface/spark_histogrammar.py index 57b836a..154ce31 100644 --- a/histogrammar/dfinterface/spark_histogrammar.py +++ b/histogrammar/dfinterface/spark_histogrammar.py @@ -1,21 +1,21 @@ -""" -Copyright Eskapade: +"""Copyright Eskapade: License Apache-2: https://github.com/KaveIO/Eskapade-Core/blob/master/LICENSE Reference link: https://github.com/KaveIO/Eskapade-Spark/blob/master/python/eskapadespark/links/spark_histogrammar_filler.py All modifications copyright ING WBAA. """ -import histogrammar as hg import numpy as np from tqdm import tqdm +import histogrammar as hg + from .histogram_filler_base import HistogramFillerBase try: from pyspark.sql import DataFrame - from pyspark.sql.functions import approxCountDistinct from pyspark.sql import functions as f + from pyspark.sql.functions import approxCountDistinct except (ModuleNotFoundError, AttributeError): pass @@ -106,8 +106,7 @@ def __init__( max_nunique, ) self._unit_timestamp_specs = { - k: float(self._unit_timestamp_specs[k]) - for i, k in enumerate(self._unit_timestamp_specs) + k: float(self._unit_timestamp_specs[k]) for i, k in enumerate(self._unit_timestamp_specs) } def assert_dataframe(self, df): @@ -117,7 +116,7 @@ def assert_dataframe(self, df): """ if not isinstance(df, DataFrame): raise TypeError("retrieved object not of type Spark DataFrame") - assert not len(df.head(1)) == 0, "input dataframe is empty" + assert len(df.head(1)) != 0, "input dataframe is empty" return df def get_features(self, df): @@ -137,8 +136,7 @@ def get_quantiles(self, df, quantiles=[0.05, 0.95], columns=[]): if len(columns) == 0: return {} qsl = df.approxQuantile(columns, quantiles, 0.25) - qd = {c: qs for c, qs in zip(columns, qsl)} - return qd + return dict(zip(columns, qsl)) def get_nunique(self, df, columns=[]): """return dict with number of unique entries for given columns @@ -189,11 +187,7 @@ def process_features(self, df, cols_by_type): # timestamp variables are converted here to ns since 1970-1-1 # histogrammar does not (yet) support long integers, so convert timestamps to float for col in cols_by_type["dt"]: - self.logger.debug( - 'Converting column "{col}" of type "{type}" to nanosec.'.format( - col=col, type=self.var_dtype[col] - ) - ) + self.logger.debug(f'Converting column "{col}" of type "{self.var_dtype[col]}" to nanosec.') # first cast to timestamp (in case column is stored as date) to_ns = f.col(col).cast("timestamp").cast("float") * 1e9 idf = idf.withColumn(col, to_ns) @@ -202,10 +196,11 @@ def process_features(self, df, cols_by_type): # in columns that have them, replace by nones by nans for col in cols_by_type["num"]: if len(idf.where(f.col(col).isNull()).limit(1).collect()) > 0: - self.logger.debug( - 'In numeric column "{col}" converting each None to NaN.'.format(col=col) + self.logger.debug(f'In numeric column "{col}" converting each None to NaN.') + idf = idf.withColumn( + col, + f.when(f.col(col).isNotNull(), f.col(col)).otherwise(float("nan")), ) - idf = idf.withColumn(col, f.when(f.col(col).isNotNull(), f.col(col)).otherwise(float('nan'))) return idf @@ -241,9 +236,7 @@ def fill_histograms(self, idf): :param idf: input data frame used for filling histogram """ for cols in tqdm(self.features, ncols=100): - self.logger.debug( - 'Processing feature "{cols}".'.format(cols=":".join(cols)) - ) + self.logger.debug('Processing feature "{cols}".'.format(cols=":".join(cols))) self.fill_histogram(idf, cols) def fill_histogram(self, idf, features): diff --git a/histogrammar/hgawk_grammar.py b/histogrammar/hgawk_grammar.py deleted file mode 100644 index a7e6499..0000000 --- a/histogrammar/hgawk_grammar.py +++ /dev/null @@ -1,3032 +0,0 @@ -#!/usr/bin/env python -# generated at 2016-10-20T23:27:42 by "python -# generate-grammars/python-awk/python2_hgawk.g -# generate-grammars/python-awk/python2_actions.py -# histogrammar.python23_lex histogrammar/hgawk_grammar.py" - -import re -import ast -import inspect - -from histogrammar.pycparser.ply import yacc -from histogrammar.python23_lex import PythonLexer - - -class DollarNumber(ast.expr): - _fields = ("n",) - - def __init__(self, n, **kwds): - self.n = n - self.__dict__.update(kwds) - - -def inherit_lineno(p0, px, alt=True): - if isinstance(px, dict): - p0.lineno = px["lineno"] - p0.col_offset = px["col_offset"] - else: - if alt and hasattr(px, "alt"): - p0.lineno = px.alt["lineno"] - p0.col_offset = px.alt["col_offset"] - else: - p0.lineno = px.lineno - p0.col_offset = px.col_offset - - -def ctx_to_store(obj, store=ast.Store): - if isinstance(obj, list): - for i, x in enumerate(obj): - obj[i] = ctx_to_store(x, store) - return obj - elif isinstance(obj, (ast.Attribute, ast.Subscript)): - obj.ctx = store() - return obj - elif isinstance(obj, ast.AST): - for attrib in obj._fields: - value = getattr(obj, attrib) - if isinstance(value, ast.Load): - setattr(obj, attrib, store()) - elif isinstance(value, ast.Param): - setattr(obj, attrib, store()) - elif isinstance(value, list): - for i, x in enumerate(value): - value[i] = ctx_to_store(x, store) - elif isinstance(value, ast.AST): - setattr(obj, attrib, ctx_to_store(value, store)) - return obj - else: - return obj - - -def iskeyword(x): - return isinstance(x, ast.keyword) - - -def notkeyword(x): - return not isinstance(x, ast.keyword) - - -def unwrap_left_associative(args, rule, alt=False): - out = ast.BinOp(args[0], args[1], args[2], rule=rule) - inherit_lineno(out, args[0]) - args = args[3:] - while len(args) > 0: - out = ast.BinOp(out, args[0], args[1], rule=rule) - inherit_lineno(out, out.left) - if alt: - out.alt = {"lineno": out.lineno, "col_offset": out.col_offset} - inherit_lineno(out, out.op) - args = args[2:] - return out - - -def unpack_trailer(atom, power_star): - out = atom - for trailer in power_star: - if isinstance(trailer, ast.Call): - trailer.func = out - inherit_lineno(trailer, out) - out = trailer - elif isinstance(trailer, ast.Attribute): - trailer.value = out - inherit_lineno(trailer, out, alt=False) - if hasattr(out, "alt"): - trailer.alt = out.alt - out = trailer - elif isinstance(trailer, ast.Subscript): - trailer.value = out - inherit_lineno(trailer, out) - out = trailer - else: - assert False - return out - - -# file_input: (NEWLINE | stmt)* ENDMARKER -def p_file_input_1(p): - '''file_input : ENDMARKER''' - # 1 - p[0] = ast.Module([], rule=inspect.currentframe().f_code.co_name, lineno=0, col_offset=0) - - -def p_file_input_2(p): - '''file_input : file_input_star ENDMARKER''' - # 1 2 - p[0] = ast.Module(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1][0]) - - -def p_file_input_star_1(p): - '''file_input_star : NEWLINE''' - # 1 - p[0] = ast.Module([], rule=inspect.currentframe().f_code.co_name, lineno=0, col_offset=0) - - -def p_file_input_star_2(p): - '''file_input_star : stmt''' - # 1 - p[0] = p[1] - - -def p_file_input_star_3(p): - '''file_input_star : file_input_star NEWLINE''' - # 1 2 - p[0] = ast.Module(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1][0]) - - -def p_file_input_star_4(p): - '''file_input_star : file_input_star stmt''' - # 1 2 - p[0] = p[1] + p[2] - -# decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE - - -def p_decorator_1(p): - '''decorator : AT dotted_name NEWLINE''' - # 1 2 3 - p[0] = p[2] - p[0].alt = p[1][1] - - -def p_decorator_2(p): - '''decorator : AT dotted_name LPAR RPAR NEWLINE''' - # 1 2 3 4 5 - p[0] = ast.Call(p[2], [], [], None, None, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1][1]) - - -def p_decorator_3(p): - '''decorator : AT dotted_name LPAR arglist RPAR NEWLINE''' - # 1 2 3 4 5 6 - p[4].func = p[2] - p[0] = p[4] - inherit_lineno(p[0], p[2]) - p[0].alt = p[1][1] - -# decorators: decorator+ - - -def p_decorators(p): - '''decorators : decorators_plus''' - # 1 - p[0] = p[1] - - -def p_decorators_plus_1(p): - '''decorators_plus : decorator''' - # 1 - p[0] = [p[1]] - - -def p_decorators_plus_2(p): - '''decorators_plus : decorators_plus decorator''' - # 1 2 - p[0] = p[1] + [p[2]] - -# decorated: decorators (classdef | funcdef) - - -def p_decorated_1(p): - '''decorated : decorators classdef''' - # 1 2 - p[2].decorator_list = p[1] - p[0] = p[2] - inherit_lineno(p[0], p[1][0]) - - -def p_decorated_2(p): - '''decorated : decorators funcdef''' - # 1 2 - p[2].decorator_list = p[1] - p[0] = p[2] - inherit_lineno(p[0], p[1][0]) - -# funcdef: 'def' NAME parameters ':' suite - - -def p_funcdef(p): - '''funcdef : DEF NAME parameters COLON suite''' - # 1 2 3 4 5 - p[0] = ast.FunctionDef(p[2][0], p[3], p[5], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# parameters: '(' [varargslist] ')' - - -def p_parameters_1(p): - '''parameters : LPAR RPAR''' - # 1 2 - p[0] = ast.arguments([], None, None, [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_parameters_2(p): - '''parameters : LPAR varargslist RPAR''' - # 1 2 3 - p[0] = p[2] - - -def p_varargslist_1(p): - '''varargslist : fpdef COMMA STAR NAME''' - # 1 2 3 4 - p[0] = ast.arguments([p[1]], p[4][0], None, [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_2(p): - '''varargslist : fpdef COMMA STAR NAME COMMA DOUBLESTAR NAME''' - # 1 2 3 4 5 6 7 - p[0] = ast.arguments([p[1]], p[4][0], p[7][0], [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_3(p): - '''varargslist : fpdef COMMA DOUBLESTAR NAME''' - # 1 2 3 4 - p[0] = ast.arguments([p[1]], None, p[4][0], [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_4(p): - '''varargslist : fpdef''' - # 1 - p[0] = ast.arguments([p[1]], None, None, [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_5(p): - '''varargslist : fpdef COMMA''' - # 1 2 - p[0] = ast.arguments([p[1]], None, None, [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_6(p): - '''varargslist : fpdef varargslist_star COMMA STAR NAME''' - # 1 2 3 4 5 - p[2].args.insert(0, p[1]) - p[2].vararg = p[5][0] - p[0] = p[2] - - -def p_varargslist_7(p): - '''varargslist : fpdef varargslist_star COMMA STAR NAME COMMA DOUBLESTAR NAME''' - # 1 2 3 4 5 6 7 8 - p[2].args.insert(0, p[1]) - p[2].vararg = p[5][0] - p[2].kwarg = p[8][0] - p[0] = p[2] - - -def p_varargslist_8(p): - '''varargslist : fpdef varargslist_star COMMA DOUBLESTAR NAME''' - # 1 2 3 4 5 - p[2].args.insert(0, p[1]) - p[2].kwarg = p[5][0] - p[0] = p[2] - - -def p_varargslist_9(p): - '''varargslist : fpdef varargslist_star''' - # 1 2 - p[2].args.insert(0, p[1]) - p[0] = p[2] - - -def p_varargslist_10(p): - '''varargslist : fpdef varargslist_star COMMA''' - # 1 2 3 - p[2].args.insert(0, p[1]) - p[0] = p[2] - - -def p_varargslist_11(p): - '''varargslist : fpdef EQUAL test COMMA STAR NAME''' - # 1 2 3 4 5 6 - p[0] = ast.arguments([p[1]], p[6][0], None, [p[3]], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_12(p): - '''varargslist : fpdef EQUAL test COMMA STAR NAME COMMA DOUBLESTAR NAME''' - # 1 2 3 4 5 6 7 8 9 - p[0] = ast.arguments([p[1]], p[6][0], p[9][0], [p[3]], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_13(p): - '''varargslist : fpdef EQUAL test COMMA DOUBLESTAR NAME''' - # 1 2 3 4 5 6 - p[0] = ast.arguments([p[1]], None, p[6][0], [p[3]], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_14(p): - '''varargslist : fpdef EQUAL test''' - # 1 2 3 - p[0] = ast.arguments([p[1]], None, None, [p[3]], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_15(p): - '''varargslist : fpdef EQUAL test COMMA''' - # 1 2 3 4 - p[0] = ast.arguments([p[1]], None, None, [p[3]], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_varargslist_16(p): - '''varargslist : fpdef EQUAL test varargslist_star COMMA STAR NAME''' - # 1 2 3 4 5 6 7 - p[4].args.insert(0, p[1]) - p[4].vararg = p[7][0] - p[4].defaults.insert(0, p[3]) - p[0] = p[4] - - -def p_varargslist_17(p): - '''varargslist : fpdef EQUAL test varargslist_star COMMA STAR NAME COMMA DOUBLESTAR NAME''' - # 1 2 3 4 5 6 7 8 9 10 - p[4].args.insert(0, p[1]) - p[4].vararg = p[7][0] - p[4].kwarg = p[10][0] - p[4].defaults.insert(0, p[3]) - p[0] = p[4] - - -def p_varargslist_18(p): - '''varargslist : fpdef EQUAL test varargslist_star COMMA DOUBLESTAR NAME''' - # 1 2 3 4 5 6 7 - p[4].args.insert(0, p[1]) - p[4].kwarg = p[7][0] - p[4].defaults.insert(0, p[3]) - p[0] = p[4] - - -def p_varargslist_19(p): - '''varargslist : fpdef EQUAL test varargslist_star''' - # 1 2 3 4 - p[4].args.insert(0, p[1]) - p[4].defaults.insert(0, p[3]) - p[0] = p[4] - - -def p_varargslist_20(p): - '''varargslist : fpdef EQUAL test varargslist_star COMMA''' - # 1 2 3 4 5 - p[4].args.insert(0, p[1]) - p[4].defaults.insert(0, p[3]) - p[0] = p[4] - - -def p_varargslist_21(p): - '''varargslist : STAR NAME''' - # 1 2 - p[0] = ast.arguments([], p[2][0], None, [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[2][1]) - - -def p_varargslist_22(p): - '''varargslist : STAR NAME COMMA DOUBLESTAR NAME''' - # 1 2 3 4 5 - p[0] = ast.arguments([], p[2][0], p[5][0], [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[2][1]) - - -def p_varargslist_23(p): - '''varargslist : DOUBLESTAR NAME''' - # 1 2 - p[0] = ast.arguments([], None, p[2][0], [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[2][1]) - - -def p_varargslist_star_1(p): - '''varargslist_star : COMMA fpdef''' - # 1 2 - p[0] = ast.arguments([p[2]], None, None, [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[2]) - - -def p_varargslist_star_2(p): - '''varargslist_star : COMMA fpdef EQUAL test''' - # 1 2 3 4 - p[0] = ast.arguments([p[2]], None, None, [p[4]], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[2]) - - -def p_varargslist_star_3(p): - '''varargslist_star : varargslist_star COMMA fpdef''' - # 1 2 3 - p[1].args.append(p[3]) - p[0] = p[1] - - -def p_varargslist_star_4(p): - '''varargslist_star : varargslist_star COMMA fpdef EQUAL test''' - # 1 2 3 4 5 - p[1].args.append(p[3]) - p[1].defaults.append(p[5]) - p[0] = p[1] - - -def p_fpdef_1(p): - '''fpdef : NAME''' - # 1 - p[0] = ast.Name(p[1][0], ast.Param(), rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_fpdef_2(p): - '''fpdef : LPAR fplist RPAR''' - # 1 2 3 - if isinstance(p[2], ast.Tuple): - p[2].paren = True - ctx_to_store(p[2]) - p[0] = p[2] - - -def p_fplist_1(p): - '''fplist : fpdef''' - # 1 - p[0] = p[1] - - -def p_fplist_2(p): - '''fplist : fpdef COMMA''' - # 1 2 - p[0] = ast.Tuple([p[1]], ast.Param(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_fplist_3(p): - '''fplist : fpdef fplist_star''' - # 1 2 - p[2].elts.insert(0, p[1]) - p[0] = p[2] - inherit_lineno(p[0], p[1]) - - -def p_fplist_4(p): - '''fplist : fpdef fplist_star COMMA''' - # 1 2 3 - p[2].elts.insert(0, p[1]) - p[0] = p[2] - inherit_lineno(p[0], p[1]) - - -def p_fplist_star_1(p): - '''fplist_star : COMMA fpdef''' - # 1 2 - p[0] = ast.Tuple([p[2]], ast.Param(), rule=inspect.currentframe().f_code.co_name, paren=False) - - -def p_fplist_star_2(p): - '''fplist_star : fplist_star COMMA fpdef''' - # 1 2 3 - p[1].elts.append(p[3]) - p[0] = p[1] - -# stmt: simple_stmt | compound_stmt - - -def p_stmt_1(p): - '''stmt : simple_stmt''' - # 1 - p[0] = p[1] - - -def p_stmt_2(p): - '''stmt : compound_stmt''' - # 1 - p[0] = p[1] - - -def p_simple_stmt_1(p): - '''simple_stmt : small_stmt NEWLINE''' - # 1 2 - p[0] = [p[1]] - - -def p_simple_stmt_2(p): - '''simple_stmt : small_stmt SEMI NEWLINE''' - # 1 2 3 - p[0] = [p[1]] - - -def p_simple_stmt_3(p): - '''simple_stmt : small_stmt simple_stmt_star NEWLINE''' - # 1 2 3 - p[0] = [p[1]] + p[2] - - -def p_simple_stmt_4(p): - '''simple_stmt : small_stmt simple_stmt_star SEMI NEWLINE''' - # 1 2 3 4 - p[0] = [p[1]] + p[2] - - -def p_simple_stmt_star_1(p): - '''simple_stmt_star : SEMI small_stmt''' - # 1 2 - p[0] = [p[2]] - - -def p_simple_stmt_star_2(p): - '''simple_stmt_star : simple_stmt_star SEMI small_stmt''' - # 1 2 3 - p[0] = p[1] + [p[3]] - - -def p_small_stmt_1(p): - '''small_stmt : expr_stmt''' - # 1 - p[0] = p[1] - - -def p_small_stmt_2(p): - '''small_stmt : print_stmt''' - # 1 - p[0] = p[1] - - -def p_small_stmt_3(p): - '''small_stmt : del_stmt''' - # 1 - p[0] = p[1] - - -def p_small_stmt_4(p): - '''small_stmt : pass_stmt''' - # 1 - p[0] = p[1] - - -def p_small_stmt_5(p): - '''small_stmt : flow_stmt''' - # 1 - p[0] = p[1] - - -def p_small_stmt_6(p): - '''small_stmt : import_stmt''' - # 1 - p[0] = p[1] - - -def p_small_stmt_7(p): - '''small_stmt : global_stmt''' - # 1 - p[0] = p[1] - - -def p_small_stmt_8(p): - '''small_stmt : exec_stmt''' - # 1 - p[0] = p[1] - - -def p_small_stmt_9(p): - '''small_stmt : assert_stmt''' - # 1 - p[0] = p[1] - - -def p_expr_stmt_1(p): - '''expr_stmt : testlist augassign yield_expr''' - # 1 2 3 - ctx_to_store(p[1]) - p[0] = ast.AugAssign(p[1], p[2], p[3], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_expr_stmt_2(p): - '''expr_stmt : testlist augassign testlist''' - # 1 2 3 - ctx_to_store(p[1]) - p[0] = ast.AugAssign(p[1], p[2], p[3], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_expr_stmt_3(p): - '''expr_stmt : testlist''' - # 1 - p[0] = ast.Expr(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_expr_stmt_4(p): - '''expr_stmt : testlist expr_stmt_star''' - # 1 2 - everything = [p[1]] + p[2] - targets, value = everything[:-1], everything[-1] - ctx_to_store(targets) - p[0] = ast.Assign(targets, value, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], targets[0]) - - -def p_expr_stmt_star_1(p): - '''expr_stmt_star : EQUAL yield_expr''' - # 1 2 - p[0] = [p[2]] - - -def p_expr_stmt_star_2(p): - '''expr_stmt_star : EQUAL testlist''' - # 1 2 - p[0] = [p[2]] - - -def p_expr_stmt_star_3(p): - '''expr_stmt_star : expr_stmt_star EQUAL yield_expr''' - # 1 2 3 - p[0] = p[1] + [p[3]] - - -def p_expr_stmt_star_4(p): - '''expr_stmt_star : expr_stmt_star EQUAL testlist''' - # 1 2 3 - p[0] = p[1] + [p[3]] - - -def p_augassign_1(p): - '''augassign : PLUSEQUAL''' - # 1 - p[0] = ast.Add(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_2(p): - '''augassign : MINEQUAL''' - # 1 - p[0] = ast.Sub(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_3(p): - '''augassign : STAREQUAL''' - # 1 - p[0] = ast.Mult(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_4(p): - '''augassign : SLASHEQUAL''' - # 1 - p[0] = ast.Div(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_5(p): - '''augassign : PERCENTEQUAL''' - # 1 - p[0] = ast.Mod(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_6(p): - '''augassign : AMPEREQUAL''' - # 1 - p[0] = ast.BitAnd(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_7(p): - '''augassign : VBAREQUAL''' - # 1 - p[0] = ast.BitOr(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_8(p): - '''augassign : CIRCUMFLEXEQUAL''' - # 1 - p[0] = ast.BitXor(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_9(p): - '''augassign : LEFTSHIFTEQUAL''' - # 1 - p[0] = ast.LShift(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_10(p): - '''augassign : RIGHTSHIFTEQUAL''' - # 1 - p[0] = ast.RShift(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_11(p): - '''augassign : DOUBLESTAREQUAL''' - # 1 - p[0] = ast.Pow(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_augassign_12(p): - '''augassign : DOUBLESLASHEQUAL''' - # 1 - p[0] = ast.FloorDiv(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_1(p): - '''print_stmt : PRINT''' - # 1 - p[0] = ast.Print(None, [], True, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_2(p): - '''print_stmt : PRINT test''' - # 1 2 - p[0] = ast.Print(None, [p[2]], True, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_3(p): - '''print_stmt : PRINT test COMMA''' - # 1 2 3 - p[0] = ast.Print(None, [p[2]], False, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_4(p): - '''print_stmt : PRINT test print_stmt_plus''' - # 1 2 3 - p[0] = ast.Print(None, [p[2]] + p[3], True, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_5(p): - '''print_stmt : PRINT test print_stmt_plus COMMA''' - # 1 2 3 4 - p[0] = ast.Print(None, [p[2]] + p[3], False, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_6(p): - '''print_stmt : PRINT RIGHTSHIFT test''' - # 1 2 3 - p[0] = ast.Print(p[3], [], True, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_7(p): - '''print_stmt : PRINT RIGHTSHIFT test print_stmt_plus''' - # 1 2 3 4 - p[0] = ast.Print(p[3], p[4], True, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_8(p): - '''print_stmt : PRINT RIGHTSHIFT test print_stmt_plus COMMA''' - # 1 2 3 4 5 - p[0] = ast.Print(p[3], p[4], False, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_print_stmt_plus_1(p): - '''print_stmt_plus : COMMA test''' - # 1 2 - p[0] = [p[2]] - - -def p_print_stmt_plus_2(p): - '''print_stmt_plus : print_stmt_plus COMMA test''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# del_stmt: 'del' exprlist - - -def p_del_stmt(p): - '''del_stmt : DEL exprlist''' - # 1 2 - # interesting fact: evaluating Delete nodes with ctx=Store() causes a segmentation fault in Python! - ctx_to_store(p[2], ast.Del) - if isinstance(p[2], ast.Tuple) and not p[2].paren: - p[0] = ast.Delete(p[2].elts, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - else: - p[0] = ast.Delete([p[2]], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_pass_stmt(p): - '''pass_stmt : PASS''' - # 1 - p[0] = ast.Pass(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt - - -def p_flow_stmt_1(p): - '''flow_stmt : break_stmt''' - # 1 - p[0] = p[1] - - -def p_flow_stmt_2(p): - '''flow_stmt : continue_stmt''' - # 1 - p[0] = p[1] - - -def p_flow_stmt_3(p): - '''flow_stmt : return_stmt''' - # 1 - p[0] = p[1] - - -def p_flow_stmt_4(p): - '''flow_stmt : raise_stmt''' - # 1 - p[0] = p[1] - - -def p_flow_stmt_5(p): - '''flow_stmt : yield_stmt''' - # 1 - p[0] = ast.Expr(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - -# break_stmt: 'break' - - -def p_break_stmt(p): - '''break_stmt : BREAK''' - # 1 - p[0] = ast.Break(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# continue_stmt: 'continue' - - -def p_continue_stmt(p): - '''continue_stmt : CONTINUE''' - # 1 - p[0] = ast.Continue(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# return_stmt: 'return' [testlist] - - -def p_return_stmt_1(p): - '''return_stmt : RETURN''' - # 1 - p[0] = ast.Return(None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_return_stmt_2(p): - '''return_stmt : RETURN testlist''' - # 1 2 - p[0] = ast.Return(p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# yield_stmt: yield_expr - - -def p_yield_stmt(p): - '''yield_stmt : yield_expr''' - # 1 - p[0] = p[1] - -# raise_stmt: 'raise' [test [',' test [',' test]]] - - -def p_raise_stmt_1(p): - '''raise_stmt : RAISE''' - # 1 - p[0] = ast.Raise(None, None, None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_raise_stmt_2(p): - '''raise_stmt : RAISE test''' - # 1 2 - p[0] = ast.Raise(p[2], None, None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_raise_stmt_3(p): - '''raise_stmt : RAISE test COMMA test''' - # 1 2 3 4 - p[0] = ast.Raise(p[2], p[4], None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_raise_stmt_4(p): - '''raise_stmt : RAISE test COMMA test COMMA test''' - # 1 2 3 4 5 6 - p[0] = ast.Raise(p[2], p[4], p[6], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# import_stmt: import_name | import_from - - -def p_import_stmt_1(p): - '''import_stmt : import_name''' - # 1 - p[0] = p[1] - - -def p_import_stmt_2(p): - '''import_stmt : import_from''' - # 1 - p[0] = p[1] - -# import_name: 'import' dotted_as_names - - -def p_import_name(p): - '''import_name : IMPORT dotted_as_names''' - # 1 2 - p[0] = ast.Import(p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# import_from: ('from' ('.'* dotted_name | '.'+) -# 'import' ('*' | '(' import_as_names ')' | import_as_names)) - - -def p_import_from_1(p): - '''import_from : FROM dotted_name IMPORT STAR''' - # 1 2 3 4 - dotted = [] - last = p[2] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), - [ast.alias("*", - None, - rule=inspect.currentframe().f_code.co_name, - **p[3][1])], - 0, - rule=inspect.currentframe().f_code.co_name, - **p[1][1]) - - -def p_import_from_2(p): - '''import_from : FROM dotted_name IMPORT LPAR import_as_names RPAR''' - # 1 2 3 4 5 6 - dotted = [] - last = p[2] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), p[5], 0, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_import_from_3(p): - '''import_from : FROM dotted_name IMPORT import_as_names''' - # 1 2 3 4 - dotted = [] - last = p[2] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), p[4], 0, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_import_from_4(p): - '''import_from : FROM import_from_plus dotted_name IMPORT STAR''' - # 1 2 3 4 5 - dotted = [] - last = p[3] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom( - ".".join(dotted), [ - ast.alias( - "*", None, rule=inspect.currentframe().f_code.co_name, **p[4][1])], p[2], **p[1][1]) - - -def p_import_from_5(p): - '''import_from : FROM import_from_plus dotted_name IMPORT LPAR import_as_names RPAR''' - # 1 2 3 4 5 6 7 - dotted = [] - last = p[3] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), p[6], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_import_from_6(p): - '''import_from : FROM import_from_plus dotted_name IMPORT import_as_names''' - # 1 2 3 4 5 - dotted = [] - last = p[3] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.ImportFrom(".".join(dotted), p[5], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_import_from_7(p): - '''import_from : FROM import_from_plus IMPORT STAR''' - # 1 2 3 4 - p[0] = ast.ImportFrom(None, [ast.alias("*", None, rule=inspect.currentframe().f_code.co_name, - **p[3][1])], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_import_from_8(p): - '''import_from : FROM import_from_plus IMPORT LPAR import_as_names RPAR''' - # 1 2 3 4 5 6 - p[0] = ast.ImportFrom(None, p[5], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_import_from_9(p): - '''import_from : FROM import_from_plus IMPORT import_as_names''' - # 1 2 3 4 - p[0] = ast.ImportFrom(None, p[4], p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_import_from_plus_1(p): - '''import_from_plus : DOT''' - # 1 - p[0] = 1 - - -def p_import_from_plus_2(p): - '''import_from_plus : import_from_plus DOT''' - # 1 2 - p[0] = p[1] + 1 - -# import_as_name: NAME ['as' NAME] - - -def p_import_as_name_1(p): - '''import_as_name : NAME''' - # 1 - p[0] = ast.alias(p[1][0], None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_import_as_name_2(p): - '''import_as_name : NAME AS NAME''' - # 1 2 3 - p[0] = ast.alias(p[1][0], p[3][0], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# dotted_as_name: dotted_name ['as' NAME] - - -def p_dotted_as_name_1(p): - '''dotted_as_name : dotted_name''' - # 1 - dotted = [] - last = p[1] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.alias(".".join(dotted), None, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_dotted_as_name_2(p): - '''dotted_as_name : dotted_name AS NAME''' - # 1 2 3 - dotted = [] - last = p[1] - while isinstance(last, ast.Attribute): - dotted.insert(0, last.attr) - last = last.value - dotted.insert(0, last.id) - p[0] = ast.alias(".".join(dotted), p[3][0], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - -# import_as_names: import_as_name (',' import_as_name)* [','] - - -def p_import_as_names_1(p): - '''import_as_names : import_as_name''' - # 1 - p[0] = [p[1]] - - -def p_import_as_names_2(p): - '''import_as_names : import_as_name COMMA''' - # 1 2 - p[0] = [p[1]] - - -def p_import_as_names_3(p): - '''import_as_names : import_as_name import_as_names_star''' - # 1 2 - p[0] = [p[1]] + p[2] - - -def p_import_as_names_4(p): - '''import_as_names : import_as_name import_as_names_star COMMA''' - # 1 2 3 - p[0] = [p[1]] + p[2] - - -def p_import_as_names_star_1(p): - '''import_as_names_star : COMMA import_as_name''' - # 1 2 - p[0] = [p[2]] - - -def p_import_as_names_star_2(p): - '''import_as_names_star : import_as_names_star COMMA import_as_name''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# dotted_as_names: dotted_as_name (',' dotted_as_name)* - - -def p_dotted_as_names_1(p): - '''dotted_as_names : dotted_as_name''' - # 1 - p[0] = [p[1]] - - -def p_dotted_as_names_2(p): - '''dotted_as_names : dotted_as_name dotted_as_names_star''' - # 1 2 - p[0] = [p[1]] + p[2] - - -def p_dotted_as_names_star_1(p): - '''dotted_as_names_star : COMMA dotted_as_name''' - # 1 2 - p[0] = [p[2]] - - -def p_dotted_as_names_star_2(p): - '''dotted_as_names_star : dotted_as_names_star COMMA dotted_as_name''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# dotted_name: NAME ('.' NAME)* - - -def p_dotted_name_1(p): - '''dotted_name : NAME''' - # 1 - p[0] = ast.Name(p[1][0], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_dotted_name_2(p): - '''dotted_name : NAME dotted_name_star''' - # 1 2 - last = p[2] - if isinstance(last, ast.Attribute): - inherit_lineno(last, p[1][1]) - while isinstance(last.value, ast.Attribute): - last = last.value - inherit_lineno(last, p[1][1]) - last.value = ast.Attribute( - ast.Name( - p[1][0], - ast.Load(), - rule=inspect.currentframe().f_code.co_name, - **p[1][1]), - last.value, - ast.Load(), - rule=inspect.currentframe().f_code.co_name, - **p[1][1]) - p[0] = p[2] - else: - p[0] = ast.Attribute( - ast.Name( - p[1][0], - ast.Load(), - rule=inspect.currentframe().f_code.co_name, - **p[1][1]), - p[2], - ast.Load(), - rule=inspect.currentframe().f_code.co_name, - **p[1][1]) - - -def p_dotted_name_star_1(p): - '''dotted_name_star : DOT NAME''' - # 1 2 - p[0] = p[2][0] - - -def p_dotted_name_star_2(p): - '''dotted_name_star : dotted_name_star DOT NAME''' - # 1 2 3 - p[0] = ast.Attribute(p[1], p[3][0], ast.Load(), rule=inspect.currentframe().f_code.co_name) - -# global_stmt: 'global' NAME (',' NAME)* - - -def p_global_stmt_1(p): - '''global_stmt : GLOBAL NAME''' - # 1 2 - p[0] = ast.Global([p[2][0]], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_global_stmt_2(p): - '''global_stmt : GLOBAL NAME global_stmt_star''' - # 1 2 3 - p[0] = ast.Global([p[2][0]] + p[3], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_global_stmt_star_1(p): - '''global_stmt_star : COMMA NAME''' - # 1 2 - p[0] = [p[2][0]] - - -def p_global_stmt_star_2(p): - '''global_stmt_star : global_stmt_star COMMA NAME''' - # 1 2 3 - p[0] = p[1] + [p[3][0]] - -# exec_stmt: 'exec' expr ['in' test [',' test]] - - -def p_exec_stmt_1(p): - '''exec_stmt : EXEC expr''' - # 1 2 - p[0] = ast.Exec(p[2], None, None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_exec_stmt_2(p): - '''exec_stmt : EXEC expr IN test''' - # 1 2 3 4 - p[0] = ast.Exec(p[2], p[4], None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_exec_stmt_3(p): - '''exec_stmt : EXEC expr IN test COMMA test''' - # 1 2 3 4 5 6 - p[0] = ast.Exec(p[2], p[4], p[6], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# assert_stmt: 'assert' test [',' test] - - -def p_assert_stmt_1(p): - '''assert_stmt : ASSERT test''' - # 1 2 - p[0] = ast.Assert(p[2], None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_assert_stmt_2(p): - '''assert_stmt : ASSERT test COMMA test''' - # 1 2 3 4 - p[0] = ast.Assert(p[2], p[4], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated - - -def p_compound_stmt_1(p): - '''compound_stmt : if_stmt''' - # 1 - p[0] = [p[1]] - - -def p_compound_stmt_2(p): - '''compound_stmt : while_stmt''' - # 1 - p[0] = [p[1]] - - -def p_compound_stmt_3(p): - '''compound_stmt : for_stmt''' - # 1 - p[0] = [p[1]] - - -def p_compound_stmt_4(p): - '''compound_stmt : try_stmt''' - # 1 - p[0] = [p[1]] - - -def p_compound_stmt_5(p): - '''compound_stmt : with_stmt''' - # 1 - p[0] = [p[1]] - - -def p_compound_stmt_6(p): - '''compound_stmt : funcdef''' - # 1 - p[0] = [p[1]] - - -def p_compound_stmt_7(p): - '''compound_stmt : classdef''' - # 1 - p[0] = [p[1]] - - -def p_compound_stmt_8(p): - '''compound_stmt : decorated''' - # 1 - p[0] = [p[1]] - -# if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] - - -def p_if_stmt_1(p): - '''if_stmt : IF test COLON suite''' - # 1 2 3 4 - p[0] = ast.If(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_if_stmt_2(p): - '''if_stmt : IF test COLON suite ELSE COLON suite''' - # 1 2 3 4 5 6 7 - p[0] = ast.If(p[2], p[4], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_if_stmt_3(p): - '''if_stmt : IF test COLON suite if_stmt_star''' - # 1 2 3 4 5 - p[0] = ast.If(p[2], p[4], [p[5]], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_if_stmt_4(p): - '''if_stmt : IF test COLON suite if_stmt_star ELSE COLON suite''' - # 1 2 3 4 5 6 7 8 - last = p[5] - while len(last.orelse) > 0: - last = last.orelse[0] - last.orelse.extend(p[8]) - p[0] = ast.If(p[2], p[4], [p[5]], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_if_stmt_star_1(p): - '''if_stmt_star : ELIF test COLON suite''' - # 1 2 3 4 - p[0] = ast.If(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[2]) - - -def p_if_stmt_star_2(p): - '''if_stmt_star : if_stmt_star ELIF test COLON suite''' - # 1 2 3 4 5 - last = p[1] - while len(last.orelse) > 0: - last = last.orelse[0] - last.orelse.append(ast.If(p[3], p[5], [], rule=inspect.currentframe().f_code.co_name)) - inherit_lineno(last.orelse[-1], p[3]) - p[0] = p[1] - -# while_stmt: 'while' test ':' suite ['else' ':' suite] - - -def p_while_stmt_1(p): - '''while_stmt : WHILE test COLON suite''' - # 1 2 3 4 - p[0] = ast.While(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_while_stmt_2(p): - '''while_stmt : WHILE test COLON suite ELSE COLON suite''' - # 1 2 3 4 5 6 7 - p[0] = ast.While(p[2], p[4], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] - - -def p_for_stmt_1(p): - '''for_stmt : FOR exprlist IN testlist COLON suite''' - # 1 2 3 4 5 6 - ctx_to_store(p[2]) - p[0] = ast.For(p[2], p[4], p[6], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_for_stmt_2(p): - '''for_stmt : FOR exprlist IN testlist COLON suite ELSE COLON suite''' - # 1 2 3 4 5 6 7 8 9 - ctx_to_store(p[2]) - p[0] = ast.For(p[2], p[4], p[6], p[9], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# try_stmt: ('try' ':' suite -# ((except_clause ':' suite)+ -# ['else' ':' suite] -# ['finally' ':' suite] | -# 'finally' ':' suite)) - - -def p_try_stmt_1(p): - '''try_stmt : TRY COLON suite try_stmt_plus''' - # 1 2 3 4 - p[0] = ast.TryExcept(p[3], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_try_stmt_2(p): - '''try_stmt : TRY COLON suite try_stmt_plus FINALLY COLON suite''' - # 1 2 3 4 5 6 7 - p[0] = ast.TryFinally([ast.TryExcept(p[3], p[4], [], rule=inspect.currentframe().f_code.co_name, - **p[1][1])], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_try_stmt_3(p): - '''try_stmt : TRY COLON suite try_stmt_plus ELSE COLON suite''' - # 1 2 3 4 5 6 7 - p[0] = ast.TryExcept(p[3], p[4], p[7], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_try_stmt_4(p): - '''try_stmt : TRY COLON suite try_stmt_plus ELSE COLON suite FINALLY COLON suite''' - # 1 2 3 4 5 6 7 8 9 10 - p[0] = ast.TryFinally([ast.TryExcept(p[3], p[4], p[7], rule=inspect.currentframe().f_code.co_name, - **p[1][1])], p[10], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_try_stmt_5(p): - '''try_stmt : TRY COLON suite FINALLY COLON suite''' - # 1 2 3 4 5 6 - p[0] = ast.TryFinally(p[3], p[6], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_try_stmt_plus_1(p): - '''try_stmt_plus : except_clause COLON suite''' - # 1 2 3 - p[1].body = p[3] - p[0] = [p[1]] - - -def p_try_stmt_plus_2(p): - '''try_stmt_plus : try_stmt_plus except_clause COLON suite''' - # 1 2 3 4 - p[2].body = p[4] - p[0] = p[1] + [p[2]] - -# with_stmt: 'with' with_item (',' with_item)* ':' suite - - -def p_with_stmt_1(p): - '''with_stmt : WITH with_item COLON suite''' - # 1 2 3 4 - p[2].body = p[4] - p[0] = p[2] - - -def p_with_stmt_2(p): - '''with_stmt : WITH with_item with_stmt_star COLON suite''' - # 1 2 3 4 5 - p[2].body.append(p[3]) - last = p[2] - while len(last.body) > 0: - last = last.body[0] - last.body = p[5] - p[0] = p[2] - - -def p_with_stmt_star_1(p): - '''with_stmt_star : COMMA with_item''' - # 1 2 - p[0] = p[2] - - -def p_with_stmt_star_2(p): - '''with_stmt_star : with_stmt_star COMMA with_item''' - # 1 2 3 - last = p[1] - while len(last.body) > 0: - last = last.body[0] - last.body.append(p[3]) - p[0] = p[1] - -# with_item: test ['as' expr] - - -def p_with_item_1(p): - '''with_item : test''' - # 1 - p[0] = ast.With(p[1], None, [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_with_item_2(p): - '''with_item : test AS expr''' - # 1 2 3 - ctx_to_store(p[3]) - p[0] = ast.With(p[1], p[3], [], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - -# except_clause: 'except' [test [('as' | ',') test]] - - -def p_except_clause_1(p): - '''except_clause : EXCEPT''' - # 1 - p[0] = ast.ExceptHandler(None, None, [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_except_clause_2(p): - '''except_clause : EXCEPT test''' - # 1 2 - p[0] = ast.ExceptHandler(p[2], None, [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_except_clause_3(p): - '''except_clause : EXCEPT test AS test''' - # 1 2 3 4 - ctx_to_store(p[4]) - p[0] = ast.ExceptHandler(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_except_clause_4(p): - '''except_clause : EXCEPT test COMMA test''' - # 1 2 3 4 - ctx_to_store(p[4]) - p[0] = ast.ExceptHandler(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT - - -def p_suite_1(p): - '''suite : simple_stmt''' - # 1 - p[0] = p[1] - - -def p_suite_2(p): - '''suite : NEWLINE INDENT suite_plus DEDENT''' - # 1 2 3 4 - p[0] = p[3] - - -def p_suite_plus_1(p): - '''suite_plus : stmt''' - # 1 - p[0] = p[1] - - -def p_suite_plus_2(p): - '''suite_plus : suite_plus stmt''' - # 1 2 - p[0] = p[1] + p[2] - -# testlist_safe: old_test [(',' old_test)+ [',']] - - -def p_testlist_safe_1(p): - '''testlist_safe : old_test''' - # 1 - p[0] = p[1] - - -def p_testlist_safe_2(p): - '''testlist_safe : old_test testlist_safe_plus''' - # 1 2 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist_safe_3(p): - '''testlist_safe : old_test testlist_safe_plus COMMA''' - # 1 2 3 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist_safe_plus_1(p): - '''testlist_safe_plus : COMMA old_test''' - # 1 2 - p[0] = [p[2]] - - -def p_testlist_safe_plus_2(p): - '''testlist_safe_plus : testlist_safe_plus COMMA old_test''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# old_test: or_test | old_lambdef - - -def p_old_test_1(p): - '''old_test : or_test''' - # 1 - p[0] = p[1] - - -def p_old_test_2(p): - '''old_test : old_lambdef''' - # 1 - p[0] = p[1] - -# old_lambdef: 'lambda' [varargslist] ':' old_test - - -def p_old_lambdef_1(p): - '''old_lambdef : LAMBDA COLON old_test''' - # 1 2 3 - p[0] = ast.Lambda( - ast.arguments( - [], - None, - None, - [], - rule=inspect.currentframe().f_code.co_name, - **p[2][1]), - p[3], - rule=inspect.currentframe().f_code.co_name, - **p[1][1]) - - -def p_old_lambdef_2(p): - '''old_lambdef : LAMBDA varargslist COLON old_test''' - # 1 2 3 4 - p[0] = ast.Lambda(p[2], p[4], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# test: or_test ['if' or_test 'else' test] | lambdef - - -def p_test_1(p): - '''test : or_test''' - # 1 - p[0] = p[1] - - -def p_test_2(p): - '''test : or_test IF or_test ELSE test''' - # 1 2 3 4 5 - p[0] = ast.IfExp(p[3], p[1], p[5], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_test_3(p): - '''test : lambdef''' - # 1 - p[0] = p[1] - -# or_test: and_test ('or' and_test)* - - -def p_or_test_1(p): - '''or_test : and_test''' - # 1 - p[0] = p[1] - - -def p_or_test_2(p): - '''or_test : and_test or_test_star''' - # 1 2 - theor = ast.Or(rule=inspect.currentframe().f_code.co_name) - inherit_lineno(theor, p[2][0]) - p[0] = ast.BoolOp(theor, [p[1]] + p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_or_test_star_1(p): - '''or_test_star : OR and_test''' - # 1 2 - p[0] = [p[2]] - - -def p_or_test_star_2(p): - '''or_test_star : or_test_star OR and_test''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# and_test: not_test ('and' not_test)* - - -def p_and_test_1(p): - '''and_test : not_test''' - # 1 - p[0] = p[1] - - -def p_and_test_2(p): - '''and_test : not_test and_test_star''' - # 1 2 - theand = ast.And(rule=inspect.currentframe().f_code.co_name) - inherit_lineno(theand, p[2][0]) - p[0] = ast.BoolOp(theand, [p[1]] + p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_and_test_star_1(p): - '''and_test_star : AND not_test''' - # 1 2 - p[0] = [p[2]] - - -def p_and_test_star_2(p): - '''and_test_star : and_test_star AND not_test''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# not_test: 'not' not_test | comparison - - -def p_not_test_1(p): - '''not_test : NOT not_test''' - # 1 2 - thenot = ast.Not(rule=inspect.currentframe().f_code.co_name) - inherit_lineno(thenot, p[2]) - p[0] = ast.UnaryOp(thenot, p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_not_test_2(p): - '''not_test : comparison''' - # 1 - p[0] = p[1] - -# comparison: expr (comp_op expr)* - - -def p_comparison_1(p): - '''comparison : expr''' - # 1 - p[0] = p[1] - - -def p_comparison_2(p): - '''comparison : expr comparison_star''' - # 1 2 - ops, exprs = p[2] - p[0] = ast.Compare(p[1], ops, exprs, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_comparison_star_1(p): - '''comparison_star : comp_op expr''' - # 1 2 - inherit_lineno(p[1], p[2]) - p[0] = ([p[1]], [p[2]]) - - -def p_comparison_star_2(p): - '''comparison_star : comparison_star comp_op expr''' - # 1 2 3 - ops, exprs = p[1] - inherit_lineno(p[2], p[3]) - p[0] = (ops + [p[2]], exprs + [p[3]]) - -# comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' - - -def p_comp_op_1(p): - '''comp_op : LESS''' - # 1 - p[0] = ast.Lt(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_2(p): - '''comp_op : GREATER''' - # 1 - p[0] = ast.Gt(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_3(p): - '''comp_op : EQEQUAL''' - # 1 - p[0] = ast.Eq(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_4(p): - '''comp_op : GREATEREQUAL''' - # 1 - p[0] = ast.GtE(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_5(p): - '''comp_op : LESSEQUAL''' - # 1 - p[0] = ast.LtE(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_6(p): - '''comp_op : NOTEQUAL''' - # 1 - p[0] = ast.NotEq(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_7(p): - '''comp_op : IN''' - # 1 - p[0] = ast.In(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_8(p): - '''comp_op : NOT IN''' - # 1 2 - p[0] = ast.NotIn(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_9(p): - '''comp_op : IS''' - # 1 - p[0] = ast.Is(rule=inspect.currentframe().f_code.co_name) - - -def p_comp_op_10(p): - '''comp_op : IS NOT''' - # 1 2 - p[0] = ast.IsNot(rule=inspect.currentframe().f_code.co_name) - -# expr: xor_expr ('|' xor_expr)* - - -def p_expr_1(p): - '''expr : xor_expr''' - # 1 - p[0] = p[1] - - -def p_expr_2(p): - '''expr : xor_expr expr_star''' - # 1 2 - p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2) - - -def p_expr_star_1(p): - '''expr_star : VBAR xor_expr''' - # 1 2 - p[0] = [ast.BitOr(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_expr_star_2(p): - '''expr_star : expr_star VBAR xor_expr''' - # 1 2 3 - p[0] = p[1] + [ast.BitOr(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - -# xor_expr: and_expr ('^' and_expr)* - - -def p_xor_expr_1(p): - '''xor_expr : and_expr''' - # 1 - p[0] = p[1] - - -def p_xor_expr_2(p): - '''xor_expr : and_expr xor_expr_star''' - # 1 2 - p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2) - - -def p_xor_expr_star_1(p): - '''xor_expr_star : CIRCUMFLEX and_expr''' - # 1 2 - p[0] = [ast.BitXor(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_xor_expr_star_2(p): - '''xor_expr_star : xor_expr_star CIRCUMFLEX and_expr''' - # 1 2 3 - p[0] = p[1] + [ast.BitXor(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - -# and_expr: shift_expr ('&' shift_expr)* - - -def p_and_expr_1(p): - '''and_expr : shift_expr''' - # 1 - p[0] = p[1] - - -def p_and_expr_2(p): - '''and_expr : shift_expr and_expr_star''' - # 1 2 - p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 0) - - -def p_and_expr_star_1(p): - '''and_expr_star : AMPER shift_expr''' - # 1 2 - p[0] = [ast.BitAnd(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_and_expr_star_2(p): - '''and_expr_star : and_expr_star AMPER shift_expr''' - # 1 2 3 - p[0] = p[1] + [ast.BitAnd(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - -# shift_expr: arith_expr (('<<'|'>>') arith_expr)* - - -def p_shift_expr_1(p): - '''shift_expr : arith_expr''' - # 1 - p[0] = p[1] - - -def p_shift_expr_2(p): - '''shift_expr : arith_expr shift_expr_star''' - # 1 2 - p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2) - - -def p_shift_expr_star_1(p): - '''shift_expr_star : LEFTSHIFT arith_expr''' - # 1 2 - p[0] = [ast.LShift(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_shift_expr_star_2(p): - '''shift_expr_star : RIGHTSHIFT arith_expr''' - # 1 2 - p[0] = [ast.RShift(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_shift_expr_star_3(p): - '''shift_expr_star : shift_expr_star LEFTSHIFT arith_expr''' - # 1 2 3 - p[0] = p[1] + [ast.LShift(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - - -def p_shift_expr_star_4(p): - '''shift_expr_star : shift_expr_star RIGHTSHIFT arith_expr''' - # 1 2 3 - p[0] = p[1] + [ast.RShift(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - -# arith_expr: term (('+'|'-') term)* - - -def p_arith_expr_1(p): - '''arith_expr : term''' - # 1 - p[0] = p[1] - - -def p_arith_expr_2(p): - '''arith_expr : term arith_expr_star''' - # 1 2 - p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2) - - -def p_arith_expr_star_1(p): - '''arith_expr_star : PLUS term''' - # 1 2 - p[0] = [ast.Add(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_arith_expr_star_2(p): - '''arith_expr_star : MINUS term''' - # 1 2 - p[0] = [ast.Sub(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_arith_expr_star_3(p): - '''arith_expr_star : arith_expr_star PLUS term''' - # 1 2 3 - p[0] = p[1] + [ast.Add(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - - -def p_arith_expr_star_4(p): - '''arith_expr_star : arith_expr_star MINUS term''' - # 1 2 3 - p[0] = p[1] + [ast.Sub(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - -# term: factor (('*'|'/'|'%'|'//') factor)* - - -def p_term_1(p): - '''term : factor''' - # 1 - p[0] = p[1] - - -def p_term_2(p): - '''term : factor term_star''' - # 1 2 - p[0] = unwrap_left_associative([p[1]] + p[2], rule=inspect.currentframe().f_code.co_name, alt=len(p[2]) > 2) - - -def p_term_star_1(p): - '''term_star : STAR factor''' - # 1 2 - p[0] = [ast.Mult(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_term_star_2(p): - '''term_star : SLASH factor''' - # 1 2 - p[0] = [ast.Div(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_term_star_3(p): - '''term_star : PERCENT factor''' - # 1 2 - p[0] = [ast.Mod(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_term_star_4(p): - '''term_star : DOUBLESLASH factor''' - # 1 2 - p[0] = [ast.FloorDiv(rule=inspect.currentframe().f_code.co_name, **p[1][1]), p[2]] - - -def p_term_star_5(p): - '''term_star : term_star STAR factor''' - # 1 2 3 - p[0] = p[1] + [ast.Mult(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - - -def p_term_star_6(p): - '''term_star : term_star SLASH factor''' - # 1 2 3 - p[0] = p[1] + [ast.Div(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - - -def p_term_star_7(p): - '''term_star : term_star PERCENT factor''' - # 1 2 3 - p[0] = p[1] + [ast.Mod(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - - -def p_term_star_8(p): - '''term_star : term_star DOUBLESLASH factor''' - # 1 2 3 - p[0] = p[1] + [ast.FloorDiv(rule=inspect.currentframe().f_code.co_name, **p[2][1]), p[3]] - -# factor: ('+'|'-'|'~') factor | power - - -def p_factor_1(p): - '''factor : PLUS factor''' - # 1 2 - op = ast.UAdd(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - p[0] = ast.UnaryOp(op, p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], op) - - -def p_factor_2(p): - '''factor : MINUS factor''' - # 1 2 - if isinstance(p[2], ast.Num) and not hasattr(p[2], "unary"): - p[2].n *= -1 - p[0] = p[2] - p[0].unary = True - inherit_lineno(p[0], p[1][1]) - else: - op = ast.USub(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - p[0] = ast.UnaryOp(op, p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], op) - - -def p_factor_3(p): - '''factor : TILDE factor''' - # 1 2 - op = ast.Invert(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - p[0] = ast.UnaryOp(op, p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], op) - - -def p_factor_4(p): - '''factor : power''' - # 1 - p[0] = p[1] - -# power: atom trailer* ['**' factor] - - -def p_power_1(p): - '''power : atom''' - # 1 - p[0] = p[1] - - -def p_power_2(p): - '''power : atom DOUBLESTAR factor''' - # 1 2 3 - p[0] = ast.BinOp( - p[1], - ast.Pow( - rule=inspect.currentframe().f_code.co_name, - **p[2][1]), - p[3], - rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_power_3(p): - '''power : atom power_star''' - # 1 2 - p[0] = unpack_trailer(p[1], p[2]) - - -def p_power_4(p): - '''power : atom power_star DOUBLESTAR factor''' - # 1 2 3 4 - p[0] = ast.BinOp( - unpack_trailer( - p[1], - p[2]), - ast.Pow( - rule=inspect.currentframe().f_code.co_name, - **p[3][1]), - p[4], - rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_power_star_1(p): - '''power_star : trailer''' - # 1 - p[0] = [p[1]] - - -def p_power_star_2(p): - '''power_star : power_star trailer''' - # 1 2 - p[0] = p[1] + [p[2]] - -# atom: ('(' [yield_expr|testlist_comp] ')' | -# '[' [listmaker] ']' | -# '{' [dictorsetmaker] '}' | -# '`' testlist1 '`' | -# NAME | NUMBER | STRING+ | DOLLARNUMBER) - - -def p_atom_1(p): - '''atom : LPAR RPAR''' - # 1 2 - p[0] = ast.Tuple([], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=True, **p[1][1]) - - -def p_atom_2(p): - '''atom : LPAR yield_expr RPAR''' - # 1 2 3 - p[0] = p[2] - if isinstance(p[0], ast.Tuple): - p[0].paren = True - p[0].alt = p[1][1] - - -def p_atom_3(p): - '''atom : LPAR testlist_comp RPAR''' - # 1 2 3 - p[0] = p[2] - if isinstance(p[0], ast.Tuple): - p[0].paren = True - p[0].alt = p[1][1] - - -def p_atom_4(p): - '''atom : LSQB RSQB''' - # 1 2 - p[0] = ast.List([], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_atom_5(p): - '''atom : LSQB listmaker RSQB''' - # 1 2 3 - if isinstance(p[2], ast.ListComp): - p[0] = p[2] - p[0].alt = p[1][1] - else: - p[0] = ast.List(p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_atom_6(p): - '''atom : LBRACE RBRACE''' - # 1 2 - p[0] = ast.Dict([], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_atom_7(p): - '''atom : LBRACE dictorsetmaker RBRACE''' - # 1 2 3 - if isinstance(p[2], (ast.SetComp, ast.DictComp)): - p[0] = p[2] - p[0].alt = p[1][1] - else: - keys, values = p[2] - if keys is None: - p[0] = ast.Set(values, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - else: - p[0] = ast.Dict(keys, values, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_atom_8(p): - '''atom : BACKQUOTE testlist1 BACKQUOTE''' - # 1 2 3 - p[0] = ast.Repr(p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_atom_9(p): - '''atom : NAME''' - # 1 - p[0] = ast.Name(p[1][0], ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_atom_10(p): - '''atom : NUMBER''' - # 1 - p[0] = ast.Num(p[1][0], rule=inspect.currentframe().f_code.co_name, **p[1][2]) - - -def p_atom_11(p): - '''atom : atom_plus''' - # 1 - p[0] = p[1] - - -def p_atom_12(p): - '''atom : DOLLARNUMBER''' - # 1 - p[0] = DollarNumber(int(p[1][0][1:]), **p[1][1]) - - -def p_atom_plus_1(p): - '''atom_plus : STRING''' - # 1 - p[0] = ast.Str(p[1][0], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_atom_plus_2(p): - '''atom_plus : atom_plus STRING''' - # 1 2 - p[1].s = p[1].s + p[2][0] - p[0] = p[1] - -# listmaker: test ( list_for | (',' test)* [','] ) - - -def p_listmaker_1(p): - '''listmaker : test list_for''' - # 1 2 - p[0] = ast.ListComp(p[1], p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_listmaker_2(p): - '''listmaker : test''' - # 1 - p[0] = [p[1]] - - -def p_listmaker_3(p): - '''listmaker : test COMMA''' - # 1 2 - p[0] = [p[1]] - - -def p_listmaker_4(p): - '''listmaker : test listmaker_star''' - # 1 2 - p[0] = [p[1]] + p[2] - - -def p_listmaker_5(p): - '''listmaker : test listmaker_star COMMA''' - # 1 2 3 - p[0] = [p[1]] + p[2] - - -def p_listmaker_star_1(p): - '''listmaker_star : COMMA test''' - # 1 2 - p[0] = [p[2]] - - -def p_listmaker_star_2(p): - '''listmaker_star : listmaker_star COMMA test''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# testlist_comp: test ( comp_for | (',' test)* [','] ) - - -def p_testlist_comp_1(p): - '''testlist_comp : test comp_for''' - # 1 2 - p[0] = ast.GeneratorExp(p[1], p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_testlist_comp_2(p): - '''testlist_comp : test''' - # 1 - p[0] = p[1] - - -def p_testlist_comp_3(p): - '''testlist_comp : test COMMA''' - # 1 2 - p[0] = ast.Tuple([p[1]], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist_comp_4(p): - '''testlist_comp : test testlist_comp_star''' - # 1 2 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist_comp_5(p): - '''testlist_comp : test testlist_comp_star COMMA''' - # 1 2 3 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist_comp_star_1(p): - '''testlist_comp_star : COMMA test''' - # 1 2 - p[0] = [p[2]] - - -def p_testlist_comp_star_2(p): - '''testlist_comp_star : testlist_comp_star COMMA test''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# lambdef: 'lambda' [varargslist] ':' test - - -def p_lambdef_1(p): - '''lambdef : LAMBDA COLON test''' - # 1 2 3 - p[0] = ast.Lambda( - ast.arguments( - [], - None, - None, - [], - rule=inspect.currentframe().f_code.co_name, - **p[2][1]), - p[3], - rule=inspect.currentframe().f_code.co_name, - **p[1][1]) - - -def p_lambdef_2(p): - '''lambdef : LAMBDA varargslist COLON test''' - # 1 2 3 4 - p[0] = ast.Lambda(p[2], p[4], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME - - -def p_trailer_1(p): - '''trailer : LPAR RPAR''' - # 1 2 - p[0] = ast.Call(None, [], [], None, None, rule=inspect.currentframe().f_code.co_name) - - -def p_trailer_2(p): - '''trailer : LPAR arglist RPAR''' - # 1 2 3 - p[0] = p[2] - - -def p_trailer_3(p): - '''trailer : LSQB subscriptlist RSQB''' - # 1 2 3 - p[0] = ast.Subscript(None, p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name) - - -def p_trailer_4(p): - '''trailer : DOT NAME''' - # 1 2 - p[0] = ast.Attribute(None, p[2][0], ast.Load(), rule=inspect.currentframe().f_code.co_name) - -# subscriptlist: subscript (',' subscript)* [','] - - -def p_subscriptlist_1(p): - '''subscriptlist : subscript''' - # 1 - p[0] = p[1] - - -def p_subscriptlist_2(p): - '''subscriptlist : subscript COMMA''' - # 1 2 - if isinstance(p[1], ast.Index): - tup = ast.Tuple([p[1].value], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(tup, p[1].value) - p[0] = ast.Index(tup, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], tup) - else: - p[0] = ast.ExtSlice([p[1]], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_subscriptlist_3(p): - '''subscriptlist : subscript subscriptlist_star''' - # 1 2 - args = [p[1]] + p[2] - if all(isinstance(x, ast.Index) for x in args): - tup = ast.Tuple([x.value for x in args], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(tup, args[0].value) - p[0] = ast.Index(tup, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], tup) - else: - p[0] = ast.ExtSlice(args, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_subscriptlist_4(p): - '''subscriptlist : subscript subscriptlist_star COMMA''' - # 1 2 3 - args = [p[1]] + p[2] - if all(isinstance(x, ast.Index) for x in args): - tup = ast.Tuple([x.value for x in args], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(tup, args[0].value) - p[0] = ast.Index(tup, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], tup) - else: - p[0] = ast.ExtSlice(args, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_subscriptlist_star_1(p): - '''subscriptlist_star : COMMA subscript''' - # 1 2 - p[0] = [p[2]] - - -def p_subscriptlist_star_2(p): - '''subscriptlist_star : subscriptlist_star COMMA subscript''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] - - -def p_subscript_1(p): - '''subscript : DOT DOT DOT''' - # 1 2 3 - p[0] = ast.Ellipsis(rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_subscript_2(p): - '''subscript : test''' - # 1 - p[0] = ast.Index(p[1], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_subscript_3(p): - '''subscript : COLON''' - # 1 - p[0] = ast.Slice(None, None, None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_subscript_4(p): - '''subscript : COLON sliceop''' - # 1 2 - p[0] = ast.Slice(None, None, p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_subscript_5(p): - '''subscript : COLON test''' - # 1 2 - p[0] = ast.Slice(None, p[2], None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_subscript_6(p): - '''subscript : COLON test sliceop''' - # 1 2 3 - p[0] = ast.Slice(None, p[2], p[3], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_subscript_7(p): - '''subscript : test COLON''' - # 1 2 - p[0] = ast.Slice(p[1], None, None, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_subscript_8(p): - '''subscript : test COLON sliceop''' - # 1 2 3 - p[0] = ast.Slice(p[1], None, p[3], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_subscript_9(p): - '''subscript : test COLON test''' - # 1 2 3 - p[0] = ast.Slice(p[1], p[3], None, rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_subscript_10(p): - '''subscript : test COLON test sliceop''' - # 1 2 3 4 - p[0] = ast.Slice(p[1], p[3], p[4], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - -# sliceop: ':' [test] - - -def p_sliceop_1(p): - '''sliceop : COLON''' - # 1 - p[0] = ast.Name("None", ast.Load(), rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_sliceop_2(p): - '''sliceop : COLON test''' - # 1 2 - p[0] = p[2] - -# exprlist: expr (',' expr)* [','] - - -def p_exprlist_1(p): - '''exprlist : expr''' - # 1 - p[0] = p[1] - - -def p_exprlist_2(p): - '''exprlist : expr COMMA''' - # 1 2 - p[0] = ast.Tuple([p[1]], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_exprlist_3(p): - '''exprlist : expr exprlist_star''' - # 1 2 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_exprlist_4(p): - '''exprlist : expr exprlist_star COMMA''' - # 1 2 3 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_exprlist_star_1(p): - '''exprlist_star : COMMA expr''' - # 1 2 - p[0] = [p[2]] - - -def p_exprlist_star_2(p): - '''exprlist_star : exprlist_star COMMA expr''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# testlist: test (',' test)* [','] - - -def p_testlist_1(p): - '''testlist : test''' - # 1 - p[0] = p[1] - - -def p_testlist_2(p): - '''testlist : test COMMA''' - # 1 2 - p[0] = ast.Tuple([p[1]], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist_3(p): - '''testlist : test testlist_star''' - # 1 2 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist_4(p): - '''testlist : test testlist_star COMMA''' - # 1 2 3 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist_star_1(p): - '''testlist_star : COMMA test''' - # 1 2 - p[0] = [p[2]] - - -def p_testlist_star_2(p): - '''testlist_star : testlist_star COMMA test''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | -# (test (comp_for | (',' test)* [','])) ) - - -def p_dictorsetmaker_1(p): - '''dictorsetmaker : test COLON test comp_for''' - # 1 2 3 4 - p[0] = ast.DictComp(p[1], p[3], p[4], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_dictorsetmaker_2(p): - '''dictorsetmaker : test COLON test''' - # 1 2 3 - p[0] = ([p[1]], [p[3]]) - - -def p_dictorsetmaker_3(p): - '''dictorsetmaker : test COLON test COMMA''' - # 1 2 3 4 - p[0] = ([p[1]], [p[3]]) - - -def p_dictorsetmaker_4(p): - '''dictorsetmaker : test COLON test dictorsetmaker_star''' - # 1 2 3 4 - keys, values = p[4] - p[0] = ([p[1]] + keys, [p[3]] + values) - - -def p_dictorsetmaker_5(p): - '''dictorsetmaker : test COLON test dictorsetmaker_star COMMA''' - # 1 2 3 4 5 - keys, values = p[4] - p[0] = ([p[1]] + keys, [p[3]] + values) - - -def p_dictorsetmaker_6(p): - '''dictorsetmaker : test comp_for''' - # 1 2 - p[0] = ast.SetComp(p[1], p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_dictorsetmaker_7(p): - '''dictorsetmaker : test''' - # 1 - p[0] = (None, [p[1]]) - - -def p_dictorsetmaker_8(p): - '''dictorsetmaker : test COMMA''' - # 1 2 - p[0] = (None, [p[1]]) - - -def p_dictorsetmaker_9(p): - '''dictorsetmaker : test dictorsetmaker_star2''' - # 1 2 - keys, values = p[2] - p[0] = (keys, [p[1]] + values) - - -def p_dictorsetmaker_10(p): - '''dictorsetmaker : test dictorsetmaker_star2 COMMA''' - # 1 2 3 - keys, values = p[2] - p[0] = (keys, [p[1]] + values) - - -def p_dictorsetmaker_star_1(p): - '''dictorsetmaker_star : COMMA test COLON test''' - # 1 2 3 4 - p[0] = ([p[2]], [p[4]]) - - -def p_dictorsetmaker_star_2(p): - '''dictorsetmaker_star : dictorsetmaker_star COMMA test COLON test''' - # 1 2 3 4 5 - keys, values = p[1] - p[0] = (keys + [p[3]], values + [p[5]]) - - -def p_dictorsetmaker_star2_1(p): - '''dictorsetmaker_star2 : COMMA test''' - # 1 2 - p[0] = (None, [p[2]]) - - -def p_dictorsetmaker_star2_2(p): - '''dictorsetmaker_star2 : dictorsetmaker_star2 COMMA test''' - # 1 2 3 - keys, values = p[1] - p[0] = (keys, values + [p[3]]) - -# classdef: 'class' NAME ['(' [testlist] ')'] ':' suite - - -def p_classdef_1(p): - '''classdef : CLASS NAME COLON suite''' - # 1 2 3 4 - p[0] = ast.ClassDef(p[2][0], [], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_classdef_2(p): - '''classdef : CLASS NAME LPAR RPAR COLON suite''' - # 1 2 3 4 5 6 - p[0] = ast.ClassDef(p[2][0], [], p[6], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_classdef_3(p): - '''classdef : CLASS NAME LPAR testlist RPAR COLON suite''' - # 1 2 3 4 5 6 7 - if isinstance(p[4], ast.Tuple): - p[0] = ast.ClassDef(p[2][0], p[4].elts, p[7], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - else: - p[0] = ast.ClassDef(p[2][0], [p[4]], p[7], [], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - -# arglist: (argument ',')* (argument [','] -# |'*' test (',' argument)* [',' '**' test] -# |'**' test) - - -def p_arglist_1(p): - '''arglist : argument''' - # 1 - if notkeyword(p[1]): - p[0] = ast.Call(None, [p[1]], [], None, None, rule=inspect.currentframe().f_code.co_name) - else: - p[0] = ast.Call(None, [], [p[1]], None, None, rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_2(p): - '''arglist : argument COMMA''' - # 1 2 - if notkeyword(p[1]): - p[0] = ast.Call(None, [p[1]], [], None, None, rule=inspect.currentframe().f_code.co_name) - else: - p[0] = ast.Call(None, [], [p[1]], None, None, rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_3(p): - '''arglist : STAR test''' - # 1 2 - p[0] = ast.Call(None, [], [], p[2], None, rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_4(p): - '''arglist : STAR test COMMA DOUBLESTAR test''' - # 1 2 3 4 5 - p[0] = ast.Call(None, [], [], p[2], p[5], rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_5(p): - '''arglist : STAR test arglist_star''' - # 1 2 3 - p[0] = ast.Call(None, filter(notkeyword, p[3]), filter(iskeyword, p[3]), - p[2], None, rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_6(p): - '''arglist : STAR test arglist_star COMMA DOUBLESTAR test''' - # 1 2 3 4 5 6 - p[0] = ast.Call(None, filter(notkeyword, p[3]), filter(iskeyword, p[3]), - p[2], p[6], rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_7(p): - '''arglist : DOUBLESTAR test''' - # 1 2 - p[0] = ast.Call(None, [], [], None, p[2], rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_8(p): - '''arglist : arglist_star2 argument''' - # 1 2 - args = p[1] + [p[2]] - p[0] = ast.Call(None, filter(notkeyword, args), filter(iskeyword, args), - None, None, rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_9(p): - '''arglist : arglist_star2 argument COMMA''' - # 1 2 3 - args = p[1] + [p[2]] - p[0] = ast.Call(None, filter(notkeyword, args), filter(iskeyword, args), - None, None, rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_10(p): - '''arglist : arglist_star2 STAR test''' - # 1 2 3 - p[0] = ast.Call(None, filter(notkeyword, p[1]), filter(iskeyword, p[1]), - p[3], None, rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_11(p): - '''arglist : arglist_star2 STAR test COMMA DOUBLESTAR test''' - # 1 2 3 4 5 6 - p[0] = ast.Call(None, filter(notkeyword, p[1]), filter(iskeyword, p[1]), - p[3], p[6], rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_12(p): - '''arglist : arglist_star2 STAR test arglist_star3''' - # 1 2 3 4 - args = p[1] + p[4] - p[0] = ast.Call(None, filter(notkeyword, args), filter(iskeyword, args), - p[3], None, rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_13(p): - '''arglist : arglist_star2 STAR test arglist_star3 COMMA DOUBLESTAR test''' - # 1 2 3 4 5 6 7 - args = p[1] + p[4] - p[0] = ast.Call(None, filter(notkeyword, args), filter(iskeyword, args), - p[3], p[7], rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_14(p): - '''arglist : arglist_star2 DOUBLESTAR test''' - # 1 2 3 - p[0] = ast.Call(None, filter(notkeyword, p[1]), filter(iskeyword, p[1]), - None, p[3], rule=inspect.currentframe().f_code.co_name) - - -def p_arglist_star_1(p): - '''arglist_star : COMMA argument''' - # 1 2 - p[0] = [p[2]] - - -def p_arglist_star_2(p): - '''arglist_star : arglist_star COMMA argument''' - # 1 2 3 - p[0] = p[1] + [p[3]] - - -def p_arglist_star3_1(p): - '''arglist_star3 : COMMA argument''' - # 1 2 - p[0] = [p[2]] - - -def p_arglist_star3_2(p): - '''arglist_star3 : arglist_star3 COMMA argument''' - # 1 2 3 - p[0] = p[1] + [p[3]] - - -def p_arglist_star2_1(p): - '''arglist_star2 : argument COMMA''' - # 1 2 - p[0] = [p[1]] - - -def p_arglist_star2_2(p): - '''arglist_star2 : arglist_star2 argument COMMA''' - # 1 2 3 - p[0] = p[1] + [p[2]] - -# argument: test [comp_for] | test '=' test - - -def p_argument_1(p): - '''argument : test''' - # 1 - p[0] = p[1] - - -def p_argument_2(p): - '''argument : test comp_for''' - # 1 2 - p[0] = ast.GeneratorExp(p[1], p[2], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - - -def p_argument_3(p): - '''argument : test EQUAL test''' - # 1 2 3 - p[0] = ast.keyword(p[1].id, p[3], rule=inspect.currentframe().f_code.co_name) - inherit_lineno(p[0], p[1]) - -# list_iter: list_for | list_if - - -def p_list_iter_1(p): - '''list_iter : list_for''' - # 1 - p[0] = ([], p[1]) - - -def p_list_iter_2(p): - '''list_iter : list_if''' - # 1 - p[0] = p[1] - -# list_for: 'for' exprlist 'in' testlist_safe [list_iter] - - -def p_list_for_1(p): - '''list_for : FOR exprlist IN testlist_safe''' - # 1 2 3 4 - ctx_to_store(p[2]) - p[0] = [ast.comprehension(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])] - - -def p_list_for_2(p): - '''list_for : FOR exprlist IN testlist_safe list_iter''' - # 1 2 3 4 5 - ctx_to_store(p[2]) - ifs, iters = p[5] - p[0] = [ast.comprehension(p[2], p[4], ifs, rule=inspect.currentframe().f_code.co_name, **p[1][1])] + iters - -# list_if: 'if' old_test [list_iter] - - -def p_list_if_1(p): - '''list_if : IF old_test''' - # 1 2 - p[0] = ([p[2]], []) - - -def p_list_if_2(p): - '''list_if : IF old_test list_iter''' - # 1 2 3 - ifs, iters = p[3] - p[0] = ([p[2]] + ifs, iters) - -# comp_iter: comp_for | comp_if - - -def p_comp_iter_1(p): - '''comp_iter : comp_for''' - # 1 - p[0] = ([], p[1]) - - -def p_comp_iter_2(p): - '''comp_iter : comp_if''' - # 1 - p[0] = p[1] - -# comp_for: 'for' exprlist 'in' or_test [comp_iter] - - -def p_comp_for_1(p): - '''comp_for : FOR exprlist IN or_test''' - # 1 2 3 4 - ctx_to_store(p[2]) - p[0] = [ast.comprehension(p[2], p[4], [], rule=inspect.currentframe().f_code.co_name, **p[1][1])] - - -def p_comp_for_2(p): - '''comp_for : FOR exprlist IN or_test comp_iter''' - # 1 2 3 4 5 - ctx_to_store(p[2]) - ifs, iters = p[5] - p[0] = [ast.comprehension(p[2], p[4], ifs, rule=inspect.currentframe().f_code.co_name, **p[1][1])] + iters - -# comp_if: 'if' old_test [comp_iter] - - -def p_comp_if_1(p): - '''comp_if : IF old_test''' - # 1 2 - p[0] = ([p[2]], []) - - -def p_comp_if_2(p): - '''comp_if : IF old_test comp_iter''' - # 1 2 3 - ifs, iters = p[3] - p[0] = ([p[2]] + ifs, iters) - -# testlist1: test (',' test)* - - -def p_testlist1_1(p): - '''testlist1 : test''' - # 1 - p[0] = p[1] - - -def p_testlist1_2(p): - '''testlist1 : test testlist1_star''' - # 1 2 - p[0] = ast.Tuple([p[1]] + p[2], ast.Load(), rule=inspect.currentframe().f_code.co_name, paren=False) - inherit_lineno(p[0], p[1]) - - -def p_testlist1_star_1(p): - '''testlist1_star : COMMA test''' - # 1 2 - p[0] = [p[2]] - - -def p_testlist1_star_2(p): - '''testlist1_star : testlist1_star COMMA test''' - # 1 2 3 - p[0] = p[1] + [p[3]] - -# encoding_decl: NAME - - -def p_encoding_decl(p): - '''encoding_decl : NAME''' - # 1 - p[0] = p[1] - -# yield_expr: 'yield' [testlist] - - -def p_yield_expr_1(p): - '''yield_expr : YIELD''' - # 1 - p[0] = ast.Yield(None, rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_yield_expr_2(p): - '''yield_expr : YIELD testlist''' - # 1 2 - p[0] = ast.Yield(p[2], rule=inspect.currentframe().f_code.co_name, **p[1][1]) - - -def p_error(p): - lexer = p.lexer - if isinstance(lexer, PythonLexer): - lexer = lexer.lexer - pos = lexer.kwds(lexer.lexpos) - line = re.split("\r?\n", lexer.source)[max(pos["lineno"] - 3, 0):pos["lineno"]] - indicator = "-" * pos["col_offset"] + "^" - raise SyntaxError("invalid syntax\n File " + - lexer.fileName + - ", line " + - str(pos["lineno"]) + - " col " + - str(pos["col_offset"]) + - "\n " + - "\n ".join(line) + - "\n----" + - indicator) - - -def parse(source, fileName=""): - lexer = PythonLexer(fileName=fileName) - parser = yacc.yacc(debug=False, write_tables=True, tabmodule="hgawk_grammar_table", errorlog=yacc.NullLogger()) - return parser.parse(source + "\n", lexer=lexer) diff --git a/histogrammar/hgawk_grammar_table.py b/histogrammar/hgawk_grammar_table.py deleted file mode 100644 index c7d4129..0000000 --- a/histogrammar/hgawk_grammar_table.py +++ /dev/null @@ -1,428 +0,0 @@ -# flake8: noqa - -# hgawk_grammar_table.py -# This file is automatically generated. Do not edit. -_tabversion = '3.8' - -_lr_method = 'LALR' - -_lr_signature = '4D571F86CCD14D6F0A94C69B040BFF0F' - -_lr_action_items = {'DEDENT':([5,6,15,17,19,25,27,40,83,87,92,151,152,209,212,261,328,332,368,404,416,419,431,442,444,445,489,502,503,504,505,508,538,558,559,560,576,585,589,612,613,615,625,626,],[-162,-160,-53,-163,-159,-157,-161,-54,-164,-158,-55,-13,-14,-57,-56,-192,-58,-182,-175,-171,-354,-165,-183,-15,503,-194,-167,-173,-193,-195,-180,-179,-355,-181,-177,-176,-172,-356,-166,-168,-169,-174,-170,-178,]),'LPAR':([0,2,5,6,7,8,9,10,14,15,17,19,25,27,29,30,31,32,34,40,41,42,47,50,52,53,54,56,58,59,62,64,66,67,71,72,81,83,84,87,89,92,94,99,100,101,103,105,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,127,128,129,130,131,132,133,134,135,137,138,140,147,148,149,151,152,154,156,159,161,162,164,165,166,168,174,178,180,181,183,186,189,190,192,195,196,198,203,204,205,208,209,210,212,213,215,216,220,221,222,223,227,231,232,237,238,239,244,245,246,247,249,250,253,255,258,259,261,263,265,267,269,270,271,272,274,277,278,279,281,290,291,292,295,298,299,301,302,304,306,309,310,312,313,315,316,317,319,320,322,326,328,330,331,332,339,349,352,354,355,358,361,364,366,368,370,375,376,378,379,381,383,387,389,390,391,395,404,408,416,419,425,429,431,433,435,440,442,443,444,445,446,448,453,457,459,467,470,476,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,526,532,537,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[2,2,-162,-160,2,2,-3,2,2,-53,-163,-159,-157,-161,2,2,2,2,-288,-54,2,-4,2,161,2,168,2,2,-286,2,2,2,-287,-284,-285,2,2,-164,2,-158,-276,-55,2,2,2,2,2,-144,2,-80,-81,-79,-85,-84,2,2,-88,-83,-78,-89,-82,-87,-86,2,2,2,-227,2,-223,-222,-230,-225,-228,-226,-224,2,255,2,-13,-14,2,2,-281,2,161,2,2,-274,168,2,-289,-5,-6,2,-279,2,2,2,2,315,317,-278,2,2,-277,-57,2,-56,2,2,2,2,2,2,2,-145,2,2,2,2,2,-231,-229,2,2,-283,2,2,168,2,2,-192,2,-282,2,2,2,-306,2,2,2,-275,-309,2,2,168,2,2,2,2,-280,2,2,2,2,2,2,2,2,2,2,2,423,426,2,-58,2,2,-182,-146,2,2,2,2,2,2,2,2,-175,2,-307,2,2,2,-375,2,2,2,-308,168,168,-171,2,-354,-165,496,2,-183,-147,2,2,-15,2,2,-194,2,2,2,2,-376,2,168,168,2,2,2,2,-167,2,-173,-193,-195,-180,2,2,-179,2,2,2,2,2,2,2,2,168,2,168,-355,2,2,2,2,-181,-177,-176,2,2,2,2,-172,2,2,2,-356,2,2,-166,2,2,2,2,2,2,-168,-169,-174,2,-170,-178,]),'ELIF':([92,209,212,261,328,419,489,503,613,625,],[-55,-57,-56,-192,-58,490,541,-193,-169,-170,]),'ENDMARKER':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,151,152,180,181,209,212,261,328,332,368,404,416,419,431,442,489,502,503,505,508,538,558,559,560,576,585,589,612,613,615,625,626,],[1,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,179,-164,-158,-55,-13,-14,-5,-6,-57,-56,-192,-58,-182,-175,-171,-354,-165,-183,-15,-167,-173,-193,-180,-179,-355,-181,-177,-176,-172,-356,-166,-168,-169,-174,-170,-178,]),'RAISE':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[54,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,54,-164,-158,-55,54,-13,-14,54,-5,-6,-57,54,-56,54,-192,54,54,54,-58,54,-182,54,54,-175,-171,-354,-165,-183,-15,54,54,-194,54,54,54,-167,-173,-193,-195,-180,-179,54,54,54,54,-355,54,54,-181,-177,-176,-172,-356,54,54,-166,54,54,-168,-169,-174,54,-170,-178,]),'NAME':([0,2,5,6,7,8,9,10,12,14,15,17,19,25,27,29,30,31,32,33,40,41,42,47,52,53,54,55,56,59,62,64,72,79,81,82,83,84,85,87,92,94,99,100,101,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,161,163,164,165,168,169,173,174,180,181,183,189,190,192,195,201,202,204,205,209,210,212,213,215,216,220,221,222,223,225,226,228,231,232,237,238,239,244,245,246,247,250,253,255,258,259,261,263,267,269,270,272,274,277,281,290,291,292,295,296,298,299,302,304,306,309,310,312,313,315,316,317,319,320,322,323,326,328,330,331,332,340,342,349,352,354,355,358,361,364,366,368,370,376,378,379,381,383,387,389,391,395,397,399,403,404,408,416,419,423,425,426,429,431,435,440,442,443,444,445,446,448,453,457,459,467,470,471,472,474,476,478,480,483,485,489,490,492,494,496,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,526,527,528,532,537,538,539,541,544,546,554,558,559,560,561,564,566,568,571,572,575,576,580,582,584,585,587,588,589,593,596,599,601,607,609,611,612,613,615,616,621,625,626,627,],[67,67,-162,-160,67,67,-3,67,105,67,-53,-163,-159,-157,-161,67,67,67,67,148,-54,67,-4,67,67,170,67,176,67,67,67,67,67,196,67,105,-164,67,105,-158,-55,67,67,67,67,67,67,-80,-81,-79,-85,-84,67,67,-88,-83,-78,-89,-82,-87,-86,67,67,67,-227,67,-223,-222,-230,-225,-228,-226,-224,67,67,-13,-14,67,67,67,279,67,67,170,288,293,67,-5,-6,67,67,67,67,67,105,-128,67,67,-57,67,-56,67,67,67,67,67,67,67,338,339,105,67,67,67,67,67,-231,-229,67,67,67,67,170,67,67,-192,67,67,67,67,67,67,67,67,67,170,67,67,402,67,67,67,67,67,67,67,67,67,67,67,67,67,422,422,-129,67,-58,67,67,-182,433,105,67,67,67,67,67,67,67,67,-175,67,67,67,67,-375,67,67,67,170,170,477,479,481,-171,67,-354,-165,422,422,422,67,-183,67,67,-15,67,67,-194,67,67,67,67,-376,67,170,522,523,525,170,67,67,67,67,-167,67,422,547,422,-173,-193,-195,-180,67,67,-179,67,67,67,67,67,67,67,67,170,573,574,67,170,-355,67,67,67,422,67,-181,-177,-176,67,67,67,67,602,603,605,-172,67,67,67,-356,67,67,-166,67,67,67,619,67,67,67,-168,-169,-174,67,628,-170,-178,629,]),'GLOBAL':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[55,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,55,-164,-158,-55,55,-13,-14,55,-5,-6,-57,55,-56,55,-192,55,55,55,-58,55,-182,55,55,-175,-171,-354,-165,-183,-15,55,55,-194,55,55,55,-167,-173,-193,-195,-180,-179,55,55,55,55,-355,55,55,-181,-177,-176,-172,-356,55,55,-166,55,55,-168,-169,-174,55,-170,-178,]),'NUMBER':([0,2,5,6,7,8,9,10,14,15,17,19,25,27,29,30,31,32,40,41,42,47,52,54,56,59,62,64,72,81,83,84,87,92,94,99,100,101,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,161,164,165,174,180,181,183,189,190,192,195,204,205,209,210,212,213,215,216,220,221,222,223,231,232,237,238,239,244,245,246,247,250,253,258,259,261,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,328,330,331,332,349,352,354,355,358,361,364,366,368,370,376,378,379,381,383,387,389,404,408,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[71,71,-162,-160,71,71,-3,71,71,-53,-163,-159,-157,-161,71,71,71,71,-54,71,-4,71,71,71,71,71,71,71,71,71,-164,71,-158,-55,71,71,71,71,71,71,-80,-81,-79,-85,-84,71,71,-88,-83,-78,-89,-82,-87,-86,71,71,71,-227,71,-223,-222,-230,-225,-228,-226,-224,71,71,-13,-14,71,71,71,71,71,71,-5,-6,71,71,71,71,71,71,71,-57,71,-56,71,71,71,71,71,71,71,71,71,71,71,71,-231,-229,71,71,71,71,71,71,-192,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,71,-58,71,71,-182,71,71,71,71,71,71,71,71,-175,71,71,71,71,-375,71,71,71,-171,71,-354,-165,71,-183,71,71,-15,71,71,-194,71,71,71,71,-376,71,71,71,71,71,-167,71,-173,-193,-195,-180,71,71,-179,71,71,71,71,71,71,71,71,71,-355,71,71,71,71,-181,-177,-176,71,71,71,71,-172,71,71,71,-356,71,71,-166,71,71,71,71,71,71,-168,-169,-174,71,-170,-178,]),'RPAR':([2,11,13,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,81,88,89,90,91,97,98,102,109,126,139,150,155,159,161,162,166,167,170,171,178,182,184,186,191,193,194,195,197,203,205,206,207,208,217,218,219,224,230,240,241,243,249,255,257,264,265,271,273,275,276,278,279,280,286,287,288,289,291,293,294,300,301,307,308,311,313,314,315,317,325,326,334,335,336,337,343,347,348,350,360,362,369,373,374,375,377,380,381,382,390,391,392,393,395,396,398,400,405,410,411,412,413,415,418,420,422,430,436,456,458,459,460,461,469,470,473,475,476,477,479,492,493,495,499,500,515,517,521,522,523,525,526,530,535,536,545,546,547,549,551,552,553,563,565,567,570,573,574,590,592,597,598,600,602,603,605,610,614,618,619,624,628,629,],[89,-256,-236,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-397,203,-276,-298,208,-267,-266,-257,-237,-251,-219,-241,-209,-281,271,-272,-274,-216,-45,-21,-289,-233,-268,-279,-245,-213,-336,-335,-398,-278,-299,-297,-300,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,359,-242,-210,-282,-306,375,-377,-357,-275,-309,-271,-47,393,-38,-26,-22,-40,-304,-234,-280,-246,-247,-214,-337,-338,414,417,-302,-301,-262,-264,-265,-263,-239,-254,-255,-221,441,-243,-211,-359,-363,-307,-364,-378,-358,-273,-308,-48,-49,-46,-27,-31,-41,-305,-235,-248,-249,-215,-339,486,488,-134,-130,-303,-206,-361,-366,-365,-370,-379,-51,-50,-43,-36,-32,-18,-20,-135,-136,548,550,-388,-371,-368,-52,-39,-23,-25,-37,-42,-202,-201,-138,-137,-131,591,-387,-386,-389,-372,-360,-373,-44,-28,-30,-139,-390,-362,-374,-367,-33,-35,-19,-203,-391,-369,-24,-204,-29,-34,]),'ASSERT':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[72,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,72,-164,-158,-55,72,-13,-14,72,-5,-6,-57,72,-56,72,-192,72,72,72,-58,72,-182,72,72,-175,-171,-354,-165,-183,-15,72,72,-194,72,72,72,-167,-173,-193,-195,-180,-179,72,72,72,72,-355,72,72,-181,-177,-176,-172,-356,72,72,-166,72,72,-168,-169,-174,72,-170,-178,]),'LESSEQUAL':([11,13,24,28,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,126,139,150,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,243,249,257,265,271,278,279,280,300,301,307,308,334,335,336,337,343,347,348,350,362,375,382,390,405,410,411,],[-256,-236,-250,138,-288,-269,-240,-270,-286,-232,-287,-284,-285,-244,-276,-267,-266,-257,-237,-251,138,-241,-281,-272,-274,-289,-233,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-221,-243,-307,-273,-308,-235,-248,-249,]),'WHILE':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,151,152,180,181,209,212,261,328,332,364,368,404,416,419,431,442,444,445,489,502,503,504,505,508,538,558,559,560,576,585,589,612,613,615,625,626,],[56,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,56,-164,-158,-55,-13,-14,-5,-6,-57,-56,-192,-58,-182,56,-175,-171,-354,-165,-183,-15,56,-194,-167,-173,-193,-195,-180,-179,-355,-181,-177,-176,-172,-356,-166,-168,-169,-174,-170,-178,]),'COLON':([11,13,20,24,26,28,34,36,37,43,44,46,50,53,58,60,66,67,71,73,74,76,89,95,96,97,98,102,109,126,139,150,155,158,159,162,165,166,167,170,171,172,177,178,182,184,186,191,193,194,195,196,199,203,208,214,217,218,219,224,230,240,241,243,249,256,257,264,265,271,278,279,280,281,283,288,289,291,293,294,300,301,307,308,311,313,314,329,333,334,335,336,337,343,347,348,350,359,362,363,365,366,367,369,375,382,384,387,389,390,393,395,396,398,400,405,410,411,412,413,414,432,436,441,447,449,450,451,465,467,473,475,476,477,479,482,486,491,512,522,523,525,526,530,537,542,543,555,556,557,562,570,573,574,583,586,594,602,603,605,619,628,629,],[-256,-236,-207,-250,-205,-218,-288,-269,-240,154,-208,-217,-270,174,-286,-232,-287,-284,-285,-244,-212,-334,-276,-186,215,-267,-266,-257,-237,-251,-219,-241,-209,269,-281,-272,281,-274,-216,-45,-21,292,298,-289,-233,-268,-279,-245,-213,-336,-335,316,319,-278,-277,330,-258,-260,-261,-259,-238,-252,-253,-220,-283,361,-242,-210,-282,-306,-275,-309,-271,383,387,-38,-26,-22,-40,-304,-234,-280,-246,-247,-214,-337,-338,-187,-184,-262,-264,-265,-263,-239,-254,-255,-221,-16,-243,443,446,-188,448,-211,-307,-273,383,383,281,-308,-46,-27,-31,-41,-305,-235,-248,-249,-215,-339,485,-185,-206,-17,-189,509,510,511,383,281,-43,-36,-32,-18,-20,532,539,544,561,-39,-23,-25,-37,-42,584,587,588,593,-190,-191,596,-44,-28,-30,609,611,616,-33,-35,-19,-24,-29,-34,]),'DEL':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[31,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,31,-164,-158,-55,31,-13,-14,31,-5,-6,-57,31,-56,31,-192,31,31,31,-58,31,-182,31,31,-175,-171,-354,-165,-183,-15,31,31,-194,31,31,31,-167,-173,-193,-195,-180,-179,31,31,31,31,-355,31,31,-181,-177,-176,-172,-356,31,31,-166,31,31,-168,-169,-174,31,-170,-178,]),'VBAR':([11,13,24,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,126,150,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,249,257,265,271,278,279,280,300,301,307,308,334,335,336,337,343,347,348,362,375,382,390,405,410,411,],[-256,-236,-250,-288,-269,-240,-270,-286,183,-287,-284,-285,-244,-276,-267,-266,-257,-237,-251,-241,-281,-272,-274,-289,299,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-243,-307,-273,-308,-235,-248,-249,]),'BACKQUOTE':([0,2,5,6,7,8,9,10,11,13,14,15,17,19,20,24,25,26,27,28,29,30,31,32,34,36,37,40,41,42,44,46,47,50,52,54,56,58,59,60,62,64,66,67,71,72,73,74,81,83,84,87,89,92,94,97,98,99,100,101,102,103,108,109,111,112,113,114,115,116,117,118,119,120,121,123,124,125,126,127,128,129,130,131,132,133,134,135,137,138,139,140,142,143,147,149,150,151,152,154,155,156,159,161,162,164,165,166,167,174,178,180,181,182,183,184,186,189,190,191,192,193,195,203,204,205,208,209,210,212,213,215,216,217,218,219,220,221,222,223,224,230,231,232,237,238,239,240,241,243,244,245,246,247,248,249,250,253,257,258,259,261,263,264,265,267,269,270,271,272,274,277,278,279,280,281,290,292,294,295,298,299,300,301,302,304,306,307,308,309,310,311,312,313,315,316,317,319,326,328,330,331,332,334,335,336,337,343,347,348,349,350,351,352,354,355,358,361,362,364,366,368,369,370,375,376,378,379,381,382,383,387,389,390,400,404,405,408,410,411,412,416,419,429,431,435,436,437,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[30,30,-162,-160,30,30,-3,30,-256,-236,30,-53,-163,-159,-207,-250,-157,-205,-161,-218,30,30,30,30,-288,-269,-240,-54,30,-4,-208,-217,30,-270,30,30,30,-286,30,-232,30,30,-287,-284,-285,30,-244,-212,30,-164,30,-158,-276,-55,30,-267,-266,30,30,30,-257,30,30,-237,-80,-81,-79,-85,-84,30,30,-88,-83,-78,-89,-82,-87,-86,-251,30,30,30,-227,30,-223,-222,-230,-225,-228,-226,-219,-224,-392,249,30,30,-241,-13,-14,30,-209,30,-281,30,-272,30,30,-274,-216,30,-289,-5,-6,-233,30,-268,-279,30,30,-245,30,-213,30,-278,30,30,-277,-57,30,-56,30,30,30,-258,-260,-261,30,30,30,30,-259,-238,30,30,30,30,30,-252,-253,-220,-231,-229,30,30,-393,-283,30,30,-242,30,30,-192,30,-210,-282,30,30,30,-306,30,30,30,-275,-309,-271,30,30,30,-304,30,30,30,-234,-280,30,30,30,-246,-247,30,30,-214,30,30,30,30,30,30,30,-58,30,30,-182,-262,-264,-265,-263,-239,-254,-255,30,-221,-394,30,30,30,30,30,-243,30,30,-175,-211,30,-307,30,30,30,-375,-273,30,30,30,-308,-305,-171,-235,30,-248,-249,-215,-354,-165,30,-183,30,-206,-395,30,-15,30,30,-194,30,30,30,30,-376,30,30,30,30,30,-167,30,-173,-193,-195,-180,30,30,-179,30,30,30,30,30,30,30,30,30,-355,30,30,30,30,-181,-177,-176,30,30,30,30,-172,30,30,30,-356,30,30,-166,30,30,30,30,30,30,-168,-169,-174,30,-170,-178,]),'GREATER':([11,13,24,28,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,126,139,150,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,243,249,257,265,271,278,279,280,300,301,307,308,334,335,336,337,343,347,348,350,362,375,382,390,405,410,411,],[-256,-236,-250,132,-288,-269,-240,-270,-286,-232,-287,-284,-285,-244,-276,-267,-266,-257,-237,-251,132,-241,-281,-272,-274,-289,-233,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-221,-243,-307,-273,-308,-235,-248,-249,]),'PRINT':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[32,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,32,-164,-158,-55,32,-13,-14,32,-5,-6,-57,32,-56,32,-192,32,32,32,-58,32,-182,32,32,-175,-171,-354,-165,-183,-15,32,32,-194,32,32,32,-167,-173,-193,-195,-180,-179,32,32,32,32,-355,32,32,-181,-177,-176,-172,-356,32,32,-166,32,32,-168,-169,-174,32,-170,-178,]),'RETURN':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[29,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,29,-164,-158,-55,29,-13,-14,29,-5,-6,-57,29,-56,29,-192,29,29,29,-58,29,-182,29,29,-175,-171,-354,-165,-183,-15,29,29,-194,29,29,29,-167,-173,-193,-195,-180,-179,29,29,29,29,-355,29,29,-181,-177,-176,-172,-356,29,29,-166,29,29,-168,-169,-174,29,-170,-178,]),'WITH':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,151,152,180,181,209,212,261,328,332,364,368,404,416,419,431,442,444,445,489,502,503,504,505,508,538,558,559,560,576,585,589,612,613,615,625,626,],[7,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,7,-164,-158,-55,-13,-14,-5,-6,-57,-56,-192,-58,-182,7,-175,-171,-354,-165,-183,-15,7,-194,-167,-173,-193,-195,-180,-179,-355,-181,-177,-176,-172,-356,-166,-168,-169,-174,-170,-178,]),'MINUS':([0,2,5,6,7,8,9,10,11,14,15,17,19,24,25,27,29,30,31,32,34,36,40,41,42,47,50,52,54,56,58,59,62,64,66,67,71,72,81,83,84,87,89,92,94,97,98,99,100,101,102,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,126,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,159,161,162,164,165,166,174,178,180,181,183,184,186,189,190,192,195,203,204,205,208,209,210,212,213,215,216,217,218,219,220,221,222,223,224,231,232,237,238,239,240,241,244,245,246,247,249,250,253,258,259,261,263,265,267,269,270,271,272,274,277,278,279,280,281,290,292,295,298,299,301,302,304,306,309,310,312,313,315,316,317,319,326,328,330,331,332,334,335,336,337,347,348,349,352,354,355,358,361,364,366,368,370,375,376,378,379,381,382,383,387,389,390,404,408,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[8,8,-162,-160,8,8,-3,8,-256,8,-53,-163,-159,128,-157,-161,8,8,8,8,-288,-269,-54,8,-4,8,-270,8,8,8,-286,8,8,8,-287,-284,-285,8,8,-164,8,-158,-276,-55,8,-267,-266,8,8,8,-257,8,8,-80,-81,-79,-85,-84,8,8,-88,-83,-78,-89,-82,-87,-86,239,8,8,8,-227,8,-223,-222,-230,-225,-228,-226,-224,8,8,-13,-14,8,8,-281,8,-272,8,8,-274,8,-289,-5,-6,8,-268,-279,8,8,8,8,-278,8,8,-277,-57,8,-56,8,8,8,-258,-260,-261,8,8,8,8,-259,8,8,8,8,8,-252,-253,-231,-229,8,8,-283,8,8,8,8,-192,8,-282,8,8,8,-306,8,8,8,-275,-309,-271,8,8,8,8,8,8,-280,8,8,8,8,8,8,8,8,8,8,8,8,-58,8,8,-182,-262,-264,-265,-263,-254,-255,8,8,8,8,8,8,8,8,-175,8,-307,8,8,8,-375,-273,8,8,8,-308,-171,8,-354,-165,8,-183,8,8,-15,8,8,-194,8,8,8,8,-376,8,8,8,8,8,-167,8,-173,-193,-195,-180,8,8,-179,8,8,8,8,8,8,8,8,8,-355,8,8,8,8,-181,-177,-176,8,8,8,8,-172,8,8,8,-356,8,8,-166,8,8,8,8,8,8,-168,-169,-174,8,-170,-178,]),'DEF':([0,5,6,9,15,17,19,23,25,27,39,40,42,49,59,83,87,92,151,152,160,180,181,209,212,261,318,328,332,364,368,404,416,419,431,442,444,445,487,489,502,503,504,505,508,538,540,558,559,560,576,585,589,612,613,615,625,626,],[33,-162,-160,-3,-53,-163,-159,-11,-157,-161,33,-54,-4,-10,33,-164,-158,-55,-13,-14,-12,-5,-6,-57,-56,-192,-7,-58,-182,33,-175,-171,-354,-165,-183,-15,33,-194,-8,-167,-173,-193,-195,-180,-179,-355,-9,-181,-177,-176,-172,-356,-166,-168,-169,-174,-170,-178,]),'RBRACE':([11,13,20,24,26,28,34,36,37,44,46,47,50,58,60,66,67,71,73,74,89,97,98,102,109,126,139,150,155,157,158,159,162,166,167,178,182,184,186,191,193,203,208,217,218,219,224,230,240,241,243,249,257,264,265,266,267,268,271,278,279,280,294,300,301,307,308,311,334,335,336,337,343,347,348,350,362,369,370,371,372,375,382,390,400,405,410,411,412,436,452,453,454,455,500,513,535,536,551,552,553,592,595,610,614,617,624,],[-256,-236,-207,-250,-205,-218,-288,-269,-240,-208,-217,159,-270,-286,-232,-287,-284,-285,-244,-212,-276,-267,-266,-257,-237,-251,-219,-241,-209,265,-346,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-348,-347,-345,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-349,-352,-341,-307,-273,-308,-305,-235,-248,-249,-215,-206,-353,-342,-340,-343,-388,-344,-202,-201,-387,-386,-389,-390,-350,-203,-391,-351,-204,]),'STAREQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,111,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'LEFTSHIFT':([11,24,34,36,50,58,66,67,71,73,89,97,98,102,126,159,162,166,178,184,186,191,203,208,217,218,219,224,240,241,249,265,271,278,279,280,301,307,308,334,335,336,337,347,348,375,382,390,410,411,],[-256,-250,-288,-269,-270,-286,-287,-284,-285,189,-276,-267,-266,-257,-251,-281,-272,-274,-289,-268,-279,309,-278,-277,-258,-260,-261,-259,-252,-253,-283,-282,-306,-275,-309,-271,-280,-246,-247,-262,-264,-265,-263,-254,-255,-307,-273,-308,-248,-249,]),'GREATEREQUAL':([11,13,24,28,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,126,139,150,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,243,249,257,265,271,278,279,280,300,301,307,308,334,335,336,337,343,347,348,350,362,375,382,390,405,410,411,],[-256,-236,-250,135,-288,-269,-240,-270,-286,-232,-287,-284,-285,-244,-276,-267,-266,-257,-237,-251,135,-241,-281,-272,-274,-289,-233,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-221,-243,-307,-273,-308,-235,-248,-249,]),'MINEQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,113,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'LEFTSHIFTEQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,125,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'SEMI':([3,4,11,13,16,18,20,21,22,24,26,28,29,32,34,35,36,37,38,44,45,46,48,50,51,54,57,58,60,61,63,65,66,67,69,70,71,73,74,75,76,77,78,80,81,86,89,93,97,98,102,104,105,106,107,109,110,122,126,139,141,144,145,146,150,155,159,162,166,167,175,176,178,182,184,186,188,191,193,194,195,197,203,208,211,217,218,219,224,227,229,230,233,234,235,236,240,241,243,249,250,251,252,253,254,257,264,265,271,278,279,280,294,297,300,301,307,308,311,313,314,327,334,335,336,337,338,339,341,343,344,345,346,347,348,350,353,354,355,356,357,362,369,375,382,390,400,401,402,405,409,410,411,412,413,420,421,422,424,427,428,433,434,436,438,439,440,481,492,493,497,498,501,531,545,546,547,548,550,590,591,],[94,-117,-256,-236,-72,-69,-207,-101,-61,-250,-205,-218,-109,-90,-288,-65,-269,-240,-68,-208,-66,-217,-64,-270,-108,-112,-104,-286,-232,-103,-62,-105,-287,-284,-107,-106,-285,-244,-212,-102,-334,-67,-116,-111,-397,-63,-276,210,-267,-266,-257,-132,-144,-118,-140,-237,-152,-73,-251,-219,-110,-100,-328,-91,-241,-209,-281,-272,-274,-216,-113,-148,-289,-233,-268,-279,-155,-245,-213,-336,-335,-398,-278,-277,-59,-258,-260,-261,-259,-145,-141,-238,-70,-71,-74,-75,-252,-253,-220,-283,-329,-330,-93,-92,-95,-242,-210,-282,-306,-275,-309,-271,-304,-149,-234,-280,-246,-247,-214,-337,-338,-60,-262,-264,-265,-263,-133,-146,-142,-239,-153,-76,-77,-254,-255,-221,-332,-331,-94,-98,-96,-243,-211,-307,-273,-308,-305,-114,-150,-235,-156,-248,-249,-215,-339,-134,-119,-130,-121,-125,-127,-147,-143,-206,-333,-99,-97,-151,-135,-136,-122,-124,-154,-115,-138,-137,-131,-120,-126,-139,-123,]),'LESS':([11,13,24,28,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,126,139,150,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,243,249,257,265,271,278,279,280,300,301,307,308,334,335,336,337,343,347,348,350,362,375,382,390,405,410,411,],[-256,-236,-250,133,-288,-269,-240,-270,-286,-232,-287,-284,-285,-244,-276,-267,-266,-257,-237,-251,133,-241,-281,-272,-274,-289,-233,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-221,-243,-307,-273,-308,-235,-248,-249,]),'CIRCUMFLEXEQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,114,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'NEWLINE':([0,3,4,5,6,9,11,13,15,16,17,18,19,20,21,22,24,25,26,27,28,29,32,34,35,36,37,38,40,42,44,45,46,48,50,51,54,57,58,59,60,61,63,65,66,67,69,70,71,73,74,75,76,77,78,80,81,83,86,87,89,92,93,94,97,98,102,104,105,106,107,109,110,122,126,139,141,144,145,146,150,151,152,154,155,159,162,166,167,175,176,178,180,181,182,184,186,188,191,193,194,195,197,198,203,208,209,210,211,212,215,217,218,219,224,227,229,230,233,234,235,236,240,241,243,249,250,251,252,253,254,257,261,264,265,271,278,279,280,294,297,298,300,301,307,308,311,313,314,316,319,327,328,330,332,334,335,336,337,338,339,341,343,344,345,346,347,348,350,353,354,355,356,357,361,362,368,369,375,382,390,400,401,402,404,405,409,410,411,412,413,416,417,419,420,421,422,424,427,428,431,433,434,436,438,439,440,442,443,446,448,481,485,488,489,492,493,497,498,501,502,503,505,508,509,510,511,531,532,538,539,544,545,546,547,548,550,558,559,560,576,585,587,588,589,590,591,593,611,612,613,615,616,625,626,],[9,92,-117,-162,-160,-3,-256,-236,-53,-72,-163,-69,-159,-207,-101,-61,-250,-157,-205,-161,-218,-109,-90,-288,-65,-269,-240,-68,-54,-4,-208,-66,-217,-64,-270,-108,-112,-104,-286,180,-232,-103,-62,-105,-287,-284,-107,-106,-285,-244,-212,-102,-334,-67,-116,-111,-397,-164,-63,-158,-276,-55,209,212,-267,-266,-257,-132,-144,-118,-140,-237,-152,-73,-251,-219,-110,-100,-328,-91,-241,-13,-14,260,-209,-281,-272,-274,-216,-113,-148,-289,-5,-6,-233,-268,-279,-155,-245,-213,-336,-335,-398,318,-278,-277,-57,328,-59,-56,260,-258,-260,-261,-259,-145,-141,-238,-70,-71,-74,-75,-252,-253,-220,-283,-329,-330,-93,-92,-95,-242,-192,-210,-282,-306,-275,-309,-271,-304,-149,260,-234,-280,-246,-247,-214,-337,-338,260,260,-60,-58,260,-182,-262,-264,-265,-263,-133,-146,-142,-239,-153,-76,-77,-254,-255,-221,-332,-331,-94,-98,-96,260,-243,-175,-211,-307,-273,-308,-305,-114,-150,-171,-235,-156,-248,-249,-215,-339,-354,487,-165,-134,-119,-130,-121,-125,-127,-183,-147,-143,-206,-333,-99,-97,-15,260,260,260,-151,260,540,-167,-135,-136,-122,-124,-154,-173,-193,-180,-179,260,260,260,-115,260,-355,260,260,-138,-137,-131,-120,-126,-181,-177,-176,-172,-356,260,260,-166,-139,-123,260,260,-168,-169,-174,260,-170,-178,]),'EXCEPT':([92,209,212,261,262,328,368,503,505,558,],[-55,-57,-56,-192,366,-58,366,-193,-180,-181,]),'DOUBLESLASHEQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,121,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'PLUS':([0,2,5,6,7,8,9,10,11,14,15,17,19,24,25,27,29,30,31,32,34,36,40,41,42,47,50,52,54,56,58,59,62,64,66,67,71,72,81,83,84,87,89,92,94,97,98,99,100,101,102,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,126,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,159,161,162,164,165,166,174,178,180,181,183,184,186,189,190,192,195,203,204,205,208,209,210,212,213,215,216,217,218,219,220,221,222,223,224,231,232,237,238,239,240,241,244,245,246,247,249,250,253,258,259,261,263,265,267,269,270,271,272,274,277,278,279,280,281,290,292,295,298,299,301,302,304,306,309,310,312,313,315,316,317,319,326,328,330,331,332,334,335,336,337,347,348,349,352,354,355,358,361,364,366,368,370,375,376,378,379,381,382,383,387,389,390,404,408,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[10,10,-162,-160,10,10,-3,10,-256,10,-53,-163,-159,127,-157,-161,10,10,10,10,-288,-269,-54,10,-4,10,-270,10,10,10,-286,10,10,10,-287,-284,-285,10,10,-164,10,-158,-276,-55,10,-267,-266,10,10,10,-257,10,10,-80,-81,-79,-85,-84,10,10,-88,-83,-78,-89,-82,-87,-86,238,10,10,10,-227,10,-223,-222,-230,-225,-228,-226,-224,10,10,-13,-14,10,10,-281,10,-272,10,10,-274,10,-289,-5,-6,10,-268,-279,10,10,10,10,-278,10,10,-277,-57,10,-56,10,10,10,-258,-260,-261,10,10,10,10,-259,10,10,10,10,10,-252,-253,-231,-229,10,10,-283,10,10,10,10,-192,10,-282,10,10,10,-306,10,10,10,-275,-309,-271,10,10,10,10,10,10,-280,10,10,10,10,10,10,10,10,10,10,10,10,-58,10,10,-182,-262,-264,-265,-263,-254,-255,10,10,10,10,10,10,10,10,-175,10,-307,10,10,10,-375,-273,10,10,10,-308,-171,10,-354,-165,10,-183,10,10,-15,10,10,-194,10,10,10,10,-376,10,10,10,10,10,-167,10,-173,-193,-195,-180,10,10,-179,10,10,10,10,10,10,10,10,10,-355,10,10,10,10,-181,-177,-176,10,10,10,10,-172,10,10,10,-356,10,10,-166,10,10,10,10,10,10,-168,-169,-174,10,-170,-178,]),'PERCENTEQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,123,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'TILDE':([0,2,5,6,7,8,9,10,14,15,17,19,25,27,29,30,31,32,40,41,42,47,52,54,56,59,62,64,72,81,83,84,87,92,94,99,100,101,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,161,164,165,174,180,181,183,189,190,192,195,204,205,209,210,212,213,215,216,220,221,222,223,231,232,237,238,239,244,245,246,247,250,253,258,259,261,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,328,330,331,332,349,352,354,355,358,361,364,366,368,370,376,378,379,381,383,387,389,404,408,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[62,62,-162,-160,62,62,-3,62,62,-53,-163,-159,-157,-161,62,62,62,62,-54,62,-4,62,62,62,62,62,62,62,62,62,-164,62,-158,-55,62,62,62,62,62,62,-80,-81,-79,-85,-84,62,62,-88,-83,-78,-89,-82,-87,-86,62,62,62,-227,62,-223,-222,-230,-225,-228,-226,-224,62,62,-13,-14,62,62,62,62,62,62,-5,-6,62,62,62,62,62,62,62,-57,62,-56,62,62,62,62,62,62,62,62,62,62,62,62,-231,-229,62,62,62,62,62,62,-192,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,62,-58,62,62,-182,62,62,62,62,62,62,62,62,-175,62,62,62,62,-375,62,62,62,-171,62,-354,-165,62,-183,62,62,-15,62,62,-194,62,62,62,62,-376,62,62,62,62,62,-167,62,-173,-193,-195,-180,62,62,-179,62,62,62,62,62,62,62,62,62,-355,62,62,62,62,-181,-177,-176,62,62,62,62,-172,62,62,62,-356,62,62,-166,62,62,62,62,62,62,-168,-169,-174,62,-170,-178,]),'IMPORT':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,105,151,152,154,180,181,200,201,202,209,210,212,215,227,261,298,316,319,321,323,328,330,332,339,361,364,368,404,416,419,431,433,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[12,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,12,-164,-158,-55,12,-144,-13,-14,12,-5,-6,320,322,-128,-57,12,-56,12,-145,-192,12,12,12,425,-129,-58,12,-182,-146,12,12,-175,-171,-354,-165,-183,-147,-15,12,12,-194,12,12,12,-167,-173,-193,-195,-180,-179,12,12,12,12,-355,12,12,-181,-177,-176,-172,-356,12,12,-166,12,12,-168,-169,-174,12,-170,-178,]),'IN':([11,13,24,28,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,110,126,136,139,145,150,153,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,243,249,250,251,257,265,271,278,279,280,300,301,307,308,324,334,335,336,337,343,347,348,350,353,354,362,375,382,390,405,406,410,411,438,],[-256,-236,-250,137,-288,-269,-240,-270,-286,-232,-287,-284,-285,-244,-276,-267,-266,-257,-237,232,-251,245,137,-328,-241,259,-281,-272,-274,-289,-233,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-329,-330,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,429,-262,-264,-265,-263,-239,-254,-255,-221,-332,-331,-243,-307,-273,-308,-235,483,-248,-249,-333,]),'RSQB':([11,13,20,24,26,28,34,36,37,44,46,50,58,60,64,66,67,71,73,74,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,185,186,187,191,193,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,281,283,284,285,294,300,301,303,304,305,307,308,311,334,335,336,337,343,347,348,350,362,369,375,382,383,384,385,387,388,389,390,400,405,407,408,410,411,412,436,462,463,464,465,466,467,468,484,519,520,533,534,535,536,577,578,579,581,606,607,608,610,622,623,624,],[-256,-236,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,186,-287,-284,-285,-244,-212,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,301,-279,-291,-245,-213,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-318,-317,-310,390,-304,-234,-280,-290,-292,-293,-246,-247,-214,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-326,-320,-319,-322,-312,-311,-308,-305,-235,-295,-294,-248,-249,-215,-206,-327,-321,-316,-324,-323,-313,-314,-296,-325,-315,-382,-196,-202,-201,-383,-381,-380,-197,-384,-198,-199,-203,-385,-200,-204,]),'CLASS':([0,5,6,9,15,17,19,23,25,27,39,40,42,49,59,83,87,92,151,152,160,180,181,209,212,261,318,328,332,364,368,404,416,419,431,442,444,445,487,489,502,503,504,505,508,538,540,558,559,560,576,585,589,612,613,615,625,626,],[79,-162,-160,-3,-53,-163,-159,-11,-157,-161,79,-54,-4,-10,79,-164,-158,-55,-13,-14,-12,-5,-6,-57,-56,-192,-7,-58,-182,79,-175,-171,-354,-165,-183,-15,79,-194,-8,-167,-173,-193,-195,-180,-179,-355,-9,-181,-177,-176,-172,-356,-166,-168,-169,-174,-170,-178,]),'$end':([1,68,179,],[-1,0,-2,]),'SLASHEQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,112,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'RIGHTSHIFT':([11,24,32,34,36,50,58,66,67,71,73,89,97,98,102,126,159,162,166,178,184,186,191,203,208,217,218,219,224,240,241,249,265,271,278,279,280,301,307,308,334,335,336,337,347,348,375,382,390,410,411,],[-256,-250,147,-288,-269,-270,-286,-287,-284,-285,190,-276,-267,-266,-257,-251,-281,-272,-274,-289,-268,-279,310,-278,-277,-258,-260,-261,-259,-252,-253,-283,-282,-306,-275,-309,-271,-280,-246,-247,-262,-264,-265,-263,-254,-255,-307,-273,-308,-248,-249,]),'STRING':([0,2,5,6,7,8,9,10,14,15,17,19,25,27,29,30,31,32,34,40,41,42,47,52,54,56,58,59,62,64,72,81,83,84,87,92,94,99,100,101,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,161,164,165,174,178,180,181,183,189,190,192,195,204,205,209,210,212,213,215,216,220,221,222,223,231,232,237,238,239,244,245,246,247,250,253,258,259,261,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,328,330,331,332,349,352,354,355,358,361,364,366,368,370,376,378,379,381,383,387,389,404,408,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[34,34,-162,-160,34,34,-3,34,34,-53,-163,-159,-157,-161,34,34,34,34,-288,-54,34,-4,34,34,34,34,178,34,34,34,34,34,-164,34,-158,-55,34,34,34,34,34,34,-80,-81,-79,-85,-84,34,34,-88,-83,-78,-89,-82,-87,-86,34,34,34,-227,34,-223,-222,-230,-225,-228,-226,-224,34,34,-13,-14,34,34,34,34,34,34,-289,-5,-6,34,34,34,34,34,34,34,-57,34,-56,34,34,34,34,34,34,34,34,34,34,34,34,-231,-229,34,34,34,34,34,34,-192,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,-58,34,34,-182,34,34,34,34,34,34,34,34,-175,34,34,34,34,-375,34,34,34,-171,34,-354,-165,34,-183,34,34,-15,34,34,-194,34,34,34,34,-376,34,34,34,34,34,-167,34,-173,-193,-195,-180,34,34,-179,34,34,34,34,34,34,34,34,34,-355,34,34,34,34,-181,-177,-176,34,34,34,34,-172,34,34,34,-356,34,34,-166,34,34,34,34,34,34,-168,-169,-174,34,-170,-178,]),'FOR':([0,5,6,9,11,13,15,17,19,20,24,25,26,27,28,34,36,37,40,42,44,46,50,58,59,60,66,67,71,73,74,83,87,89,90,92,97,98,102,109,126,139,150,151,152,155,158,159,162,166,167,178,180,181,182,184,186,187,191,193,203,208,209,212,217,218,219,224,230,240,241,243,249,257,261,264,265,271,275,278,279,280,294,300,301,307,308,311,328,332,334,335,336,337,343,347,348,350,362,364,368,369,372,375,382,390,400,404,405,410,411,412,416,419,431,436,442,444,445,489,500,502,503,504,505,508,533,534,535,536,538,558,559,560,576,581,585,589,592,606,607,608,610,612,613,615,623,624,625,626,],[41,-162,-160,-3,-256,-236,-53,-163,-159,-207,-250,-157,-205,-161,-218,-288,-269,-240,-54,-4,-208,-217,-270,-286,41,-232,-287,-284,-285,-244,-212,-164,-158,-276,204,-55,-267,-266,-257,-237,-251,-219,-241,-13,-14,-209,204,-281,-272,-274,-216,-289,-5,-6,-233,-268,-279,302,-245,-213,-278,-277,-57,-56,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-192,-210,-282,-306,204,-275,-309,-271,-304,-234,-280,-246,-247,-214,-58,-182,-262,-264,-265,-263,-239,-254,-255,-221,-243,41,-175,-211,204,-307,-273,-308,-305,-171,-235,-248,-249,-215,-354,-165,-183,-206,-15,41,-194,-167,204,-173,-193,-195,-180,-179,302,-196,-202,-201,-355,-181,-177,-176,-172,-197,-356,-166,204,302,-198,-199,-203,-168,-169,-174,-200,-204,-170,-178,]),'EXEC':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[14,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,14,-164,-158,-55,14,-13,-14,14,-5,-6,-57,14,-56,14,-192,14,14,14,-58,14,-182,14,14,-175,-171,-354,-165,-183,-15,14,14,-194,14,14,14,-167,-173,-193,-195,-180,-179,14,14,14,14,-355,14,14,-181,-177,-176,-172,-356,14,14,-166,14,14,-168,-169,-174,14,-170,-178,]),'AMPER':([11,24,34,36,37,50,58,66,67,71,73,89,97,98,102,126,150,159,162,166,178,184,186,191,203,208,217,218,219,224,240,241,249,257,265,271,278,279,280,301,307,308,334,335,336,337,347,348,362,375,382,390,410,411,],[-256,-250,-288,-269,149,-270,-286,-287,-284,-285,-244,-276,-267,-266,-257,-251,258,-281,-272,-274,-289,-268,-279,-245,-278,-277,-258,-260,-261,-259,-252,-253,-283,-242,-282,-306,-275,-309,-271,-280,-246,-247,-262,-264,-265,-263,-254,-255,-243,-307,-273,-308,-248,-249,]),'IS':([11,13,24,28,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,126,139,150,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,243,249,257,265,271,278,279,280,300,301,307,308,334,335,336,337,343,347,348,350,362,375,382,390,405,410,411,],[-256,-236,-250,134,-288,-269,-240,-270,-286,-232,-287,-284,-285,-244,-276,-267,-266,-257,-237,-251,134,-241,-281,-272,-274,-289,-233,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-221,-243,-307,-273,-308,-235,-248,-249,]),'PERCENT':([11,34,36,50,58,66,67,71,89,97,98,102,159,162,166,178,184,186,203,208,217,218,219,224,249,265,271,278,279,280,301,334,335,336,337,375,382,390,],[100,-288,-269,-270,-286,-287,-284,-285,-276,-267,-266,221,-281,-272,-274,-289,-268,-279,-278,-277,-258,-260,-261,-259,-283,-282,-306,-275,-309,-271,-280,-262,-264,-265,-263,-307,-273,-308,]),'DOUBLESLASH':([11,34,36,50,58,66,67,71,89,97,98,102,159,162,166,178,184,186,203,208,217,218,219,224,249,265,271,278,279,280,301,334,335,336,337,375,382,390,],[101,-288,-269,-270,-286,-287,-284,-285,-276,-267,-266,222,-281,-272,-274,-289,-268,-279,-278,-277,-258,-260,-261,-259,-283,-282,-306,-275,-309,-271,-280,-262,-264,-265,-263,-307,-273,-308,]),'YIELD':([0,2,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,111,112,113,114,115,116,117,118,119,120,121,123,124,125,151,152,154,180,181,209,210,212,215,237,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[81,81,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,81,-164,-158,-55,81,-80,-81,-79,-85,-84,81,81,-88,-83,-78,-89,-82,-87,-86,-13,-14,81,-5,-6,-57,81,-56,81,81,-192,81,81,81,-58,81,-182,81,81,-175,-171,-354,-165,-183,-15,81,81,-194,81,81,81,-167,-173,-193,-195,-180,-179,81,81,81,81,-355,81,81,-181,-177,-176,-172,-356,81,81,-166,81,81,-168,-169,-174,81,-170,-178,]),'EQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,81,89,97,98,102,109,122,126,139,150,155,159,162,166,167,170,171,178,182,184,186,191,193,194,195,197,203,208,217,218,219,224,230,235,236,240,241,243,249,257,264,265,271,275,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,345,346,347,348,350,362,369,375,382,390,393,398,400,405,410,411,412,413,436,473,],[-256,-236,117,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-397,-276,-267,-266,-257,-237,237,-251,-219,-241,-209,-281,-272,-274,-216,-45,290,-289,-233,-268,-279,-245,-213,-336,-335,-398,-278,-277,-258,-260,-261,-259,-238,-74,-75,-252,-253,-220,-283,-242,-210,-282,-306,379,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-76,-77,-254,-255,-221,-243,-211,-307,-273,-308,-46,478,-305,-235,-248,-249,-215,-339,-206,524,]),'TRY':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,151,152,180,181,209,212,261,328,332,364,368,404,416,419,431,442,444,445,489,502,503,504,505,508,538,558,559,560,576,585,589,612,613,615,625,626,],[43,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,43,-164,-158,-55,-13,-14,-5,-6,-57,-56,-192,-58,-182,43,-175,-171,-354,-165,-183,-15,43,-194,-167,-173,-193,-195,-180,-179,-355,-181,-177,-176,-172,-356,-166,-168,-169,-174,-170,-178,]),'PLUSEQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,120,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'STAR':([11,34,36,50,53,58,66,67,71,89,97,98,102,159,161,162,166,178,184,186,203,208,217,218,219,224,249,255,265,271,274,278,279,280,291,301,317,320,322,334,335,336,337,375,381,382,390,395,425,459,476,526,537,],[99,-288,-269,-270,169,-286,-287,-284,-285,-276,-267,-266,220,-281,270,-272,-274,-289,-268,-279,-278,-277,-258,-260,-261,-259,-283,169,-282,-306,376,-275,-309,-271,397,-280,270,421,427,-262,-264,-265,-263,-307,-375,-273,-308,472,497,-376,527,571,169,]),'AT':([0,5,6,9,15,17,19,23,25,27,40,42,49,59,83,87,92,151,152,160,180,181,209,212,261,318,328,332,364,368,404,416,419,431,442,444,445,487,489,502,503,504,505,508,538,540,558,559,560,576,585,589,612,613,615,625,626,],[82,-162,-160,-3,-53,-163,-159,-11,-157,-161,-54,-4,82,82,-164,-158,-55,-13,-14,-12,-5,-6,-57,-56,-192,-7,-58,-182,82,-175,-171,-354,-165,-183,-15,82,-194,-8,-167,-173,-193,-195,-180,-179,-355,-9,-181,-177,-176,-172,-356,-166,-168,-169,-174,-170,-178,]),'SLASH':([11,34,36,50,58,66,67,71,89,97,98,102,159,162,166,178,184,186,203,208,217,218,219,224,249,265,271,278,279,280,301,334,335,336,337,375,382,390,],[103,-288,-269,-270,-286,-287,-284,-285,-276,-267,-266,223,-281,-272,-274,-289,-268,-279,-278,-277,-258,-260,-261,-259,-283,-282,-306,-275,-309,-271,-280,-262,-264,-265,-263,-307,-273,-308,]),'PASS':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[21,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,21,-164,-158,-55,21,-13,-14,21,-5,-6,-57,21,-56,21,-192,21,21,21,-58,21,-182,21,21,-175,-171,-354,-165,-183,-15,21,21,-194,21,21,21,-167,-173,-193,-195,-180,-179,21,21,21,21,-355,21,21,-181,-177,-176,-172,-356,21,21,-166,21,21,-168,-169,-174,21,-170,-178,]),'NOTEQUAL':([11,13,24,28,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,126,139,150,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,243,249,257,265,271,278,279,280,300,301,307,308,334,335,336,337,343,347,348,350,362,375,382,390,405,410,411,],[-256,-236,-250,130,-288,-269,-240,-270,-286,-232,-287,-284,-285,-244,-276,-267,-266,-257,-237,-251,130,-241,-281,-272,-274,-289,-233,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-221,-243,-307,-273,-308,-235,-248,-249,]),'INDENT':([260,],[364,]),'LSQB':([0,2,5,6,7,8,9,10,14,15,17,19,25,27,29,30,31,32,34,40,41,42,47,50,52,54,56,58,59,62,64,66,67,71,72,81,83,84,87,89,92,94,99,100,101,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,159,161,162,164,165,166,174,178,180,181,183,186,189,190,192,195,203,204,205,208,209,210,212,213,215,216,220,221,222,223,231,232,237,238,239,244,245,246,247,249,250,253,258,259,261,263,265,267,269,270,271,272,274,277,278,279,281,290,292,295,298,299,301,302,304,306,309,310,312,313,315,316,317,319,326,328,330,331,332,349,352,354,355,358,361,364,366,368,370,375,376,378,379,381,383,387,389,390,404,408,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[64,64,-162,-160,64,64,-3,64,64,-53,-163,-159,-157,-161,64,64,64,64,-288,-54,64,-4,64,165,64,64,64,-286,64,64,64,-287,-284,-285,64,64,-164,64,-158,-276,-55,64,64,64,64,64,64,-80,-81,-79,-85,-84,64,64,-88,-83,-78,-89,-82,-87,-86,64,64,64,-227,64,-223,-222,-230,-225,-228,-226,-224,64,64,-13,-14,64,64,-281,64,165,64,64,-274,64,-289,-5,-6,64,-279,64,64,64,64,-278,64,64,-277,-57,64,-56,64,64,64,64,64,64,64,64,64,64,64,64,-231,-229,64,64,-283,64,64,64,64,-192,64,-282,64,64,64,-306,64,64,64,-275,-309,64,64,64,64,64,64,-280,64,64,64,64,64,64,64,64,64,64,64,64,-58,64,64,-182,64,64,64,64,64,64,64,64,-175,64,-307,64,64,64,-375,64,64,64,-308,-171,64,-354,-165,64,-183,64,64,-15,64,64,-194,64,64,64,64,-376,64,64,64,64,64,-167,64,-173,-193,-195,-180,64,64,-179,64,64,64,64,64,64,64,64,64,-355,64,64,64,64,-181,-177,-176,64,64,64,64,-172,64,64,64,-356,64,64,-166,64,64,64,64,64,64,-168,-169,-174,64,-170,-178,]),'ELSE':([11,13,24,28,34,36,37,44,46,50,58,60,66,67,71,73,74,89,92,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,203,208,209,212,217,218,219,224,230,240,241,242,243,249,257,261,264,265,271,278,279,280,300,301,307,308,311,328,334,335,336,337,343,347,348,350,362,368,369,375,382,390,404,405,410,411,412,419,489,502,503,505,558,613,625,],[-256,-236,-250,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-276,-55,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-278,-277,-57,-56,-258,-260,-261,-259,-238,-252,-253,349,-220,-283,-242,-192,-210,-282,-306,-275,-309,-271,-234,-280,-246,-247,-214,-58,-262,-264,-265,-263,-239,-254,-255,-221,-243,450,-211,-307,-273,-308,482,-235,-248,-249,-215,491,542,555,-193,-180,-181,-169,-170,]),'DOT':([34,50,58,66,67,71,85,89,105,159,162,165,166,178,186,201,202,203,208,227,249,265,271,278,279,282,301,323,339,375,386,389,390,433,467,],[-288,163,-286,-287,-284,-285,202,-276,226,-281,163,282,-274,-289,-279,323,-128,-278,-277,340,-283,-282,-306,-275,-309,386,-280,-129,-146,-307,464,282,-308,-147,282,]),'RIGHTSHIFTEQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,124,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'IF':([0,5,6,9,11,13,15,17,19,24,25,26,27,28,34,36,37,40,42,44,46,50,58,59,60,66,67,71,73,74,83,87,89,92,97,98,102,109,126,139,150,151,152,155,159,162,166,167,178,180,181,182,184,186,191,193,203,208,209,212,217,218,219,224,230,240,241,243,249,257,261,264,265,271,278,279,280,300,301,307,308,311,328,332,334,335,336,337,343,347,348,350,362,364,368,369,375,382,390,404,405,410,411,412,416,419,431,442,444,445,489,500,502,503,504,505,508,533,534,535,536,538,558,559,560,576,581,585,589,592,606,607,608,610,612,613,615,623,624,625,626,],[84,-162,-160,-3,-256,-236,-53,-163,-159,-250,-157,129,-161,-218,-288,-269,-240,-54,-4,-208,-217,-270,-286,84,-232,-287,-284,-285,-244,-212,-164,-158,-276,-55,-267,-266,-257,-237,-251,-219,-241,-13,-14,-209,-281,-272,-274,-216,-289,-5,-6,-233,-268,-279,-245,-213,-278,-277,-57,-56,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-192,-210,-282,-306,-275,-309,-271,-234,-280,-246,-247,-214,-58,-182,-262,-264,-265,-263,-239,-254,-255,-221,-243,84,-175,-211,-307,-273,-308,-171,-235,-248,-249,-215,-354,-165,-183,-15,84,-194,-167,554,-173,-193,-195,-180,-179,580,-196,-202,-201,-355,-181,-177,-176,-172,-197,-356,-166,554,580,-198,-199,-203,-168,-169,-174,-200,-204,-170,-178,]),'AND':([11,13,24,28,34,36,37,46,50,58,60,66,67,71,73,74,89,97,98,102,109,126,139,150,159,162,166,167,178,182,184,186,191,193,203,208,217,218,219,224,230,240,241,243,249,257,265,271,278,279,280,300,301,307,308,311,334,335,336,337,343,347,348,350,362,375,382,390,405,410,411,412,],[-256,-236,-250,-218,-288,-269,-240,-217,-270,-286,-232,-287,-284,-285,-244,192,-276,-267,-266,-257,-237,-251,-219,-241,-281,-272,-274,-216,-289,-233,-268,-279,-245,312,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-214,-262,-264,-265,-263,-239,-254,-255,-221,-243,-307,-273,-308,-235,-248,-249,-215,]),'CIRCUMFLEX':([11,13,24,34,36,37,50,58,66,67,71,73,89,97,98,102,109,126,150,159,162,166,178,184,186,191,203,208,217,218,219,224,230,240,241,249,257,265,271,278,279,280,301,307,308,334,335,336,337,343,347,348,362,375,382,390,410,411,],[-256,108,-250,-288,-269,-240,-270,-286,-287,-284,-285,-244,-276,-267,-266,-257,231,-251,-241,-281,-272,-274,-289,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-283,-242,-282,-306,-275,-309,-271,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-243,-307,-273,-308,-248,-249,]),'LBRACE':([0,2,5,6,7,8,9,10,14,15,17,19,25,27,29,30,31,32,40,41,42,47,52,54,56,59,62,64,72,81,83,84,87,92,94,99,100,101,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,161,164,165,174,180,181,183,189,190,192,195,204,205,209,210,212,213,215,216,220,221,222,223,231,232,237,238,239,244,245,246,247,250,253,258,259,261,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,328,330,331,332,349,352,354,355,358,361,364,366,368,370,376,378,379,381,383,387,389,404,408,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[47,47,-162,-160,47,47,-3,47,47,-53,-163,-159,-157,-161,47,47,47,47,-54,47,-4,47,47,47,47,47,47,47,47,47,-164,47,-158,-55,47,47,47,47,47,47,-80,-81,-79,-85,-84,47,47,-88,-83,-78,-89,-82,-87,-86,47,47,47,-227,47,-223,-222,-230,-225,-228,-226,-224,47,47,-13,-14,47,47,47,47,47,47,-5,-6,47,47,47,47,47,47,47,-57,47,-56,47,47,47,47,47,47,47,47,47,47,47,47,-231,-229,47,47,47,47,47,47,-192,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,47,-58,47,47,-182,47,47,47,47,47,47,47,47,-175,47,47,47,47,-375,47,47,47,-171,47,-354,-165,47,-183,47,47,-15,47,47,-194,47,47,47,47,-376,47,47,47,47,47,-167,47,-173,-193,-195,-180,47,47,-179,47,47,47,47,47,47,47,47,47,-355,47,47,47,47,-181,-177,-176,47,47,47,47,-172,47,47,47,-356,47,47,-166,47,47,47,47,47,47,-168,-169,-174,47,-170,-178,]),'FROM':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[85,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,85,-164,-158,-55,85,-13,-14,85,-5,-6,-57,85,-56,85,-192,85,85,85,-58,85,-182,85,85,-175,-171,-354,-165,-183,-15,85,85,-194,85,85,85,-167,-173,-193,-195,-180,-179,85,85,85,85,-355,85,85,-181,-177,-176,-172,-356,85,85,-166,85,85,-168,-169,-174,85,-170,-178,]),'DOLLARNUMBER':([0,2,5,6,7,8,9,10,14,15,17,19,25,27,29,30,31,32,40,41,42,47,52,54,56,59,62,64,72,81,83,84,87,92,94,99,100,101,103,108,111,112,113,114,115,116,117,118,119,120,121,123,124,125,127,128,129,130,131,132,133,134,135,137,138,140,147,149,151,152,154,156,161,164,165,174,180,181,183,189,190,192,195,204,205,209,210,212,213,215,216,220,221,222,223,231,232,237,238,239,244,245,246,247,250,253,258,259,261,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,328,330,331,332,349,352,354,355,358,361,364,366,368,370,376,378,379,381,383,387,389,404,408,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[66,66,-162,-160,66,66,-3,66,66,-53,-163,-159,-157,-161,66,66,66,66,-54,66,-4,66,66,66,66,66,66,66,66,66,-164,66,-158,-55,66,66,66,66,66,66,-80,-81,-79,-85,-84,66,66,-88,-83,-78,-89,-82,-87,-86,66,66,66,-227,66,-223,-222,-230,-225,-228,-226,-224,66,66,-13,-14,66,66,66,66,66,66,-5,-6,66,66,66,66,66,66,66,-57,66,-56,66,66,66,66,66,66,66,66,66,66,66,66,-231,-229,66,66,66,66,66,66,-192,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,66,-58,66,66,-182,66,66,66,66,66,66,66,66,-175,66,66,66,66,-375,66,66,66,-171,66,-354,-165,66,-183,66,66,-15,66,66,-194,66,66,66,66,-376,66,66,66,66,66,-167,66,-173,-193,-195,-180,66,66,-179,66,66,66,66,66,66,66,66,66,-355,66,66,66,66,-181,-177,-176,66,66,66,66,-172,66,66,66,-356,66,66,-166,66,66,66,66,66,66,-168,-169,-174,66,-170,-178,]),'DOUBLESTAREQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,118,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'VBAREQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,115,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'FINALLY':([92,209,212,261,262,328,368,503,505,558,559,],[-55,-57,-56,-192,367,-58,451,-193,-180,-181,594,]),'DOUBLESTAR':([34,50,53,58,66,67,71,89,159,161,162,166,178,186,203,208,249,255,265,271,274,278,279,291,301,317,375,381,390,394,395,457,459,476,514,518,526,529,537,566,569,604,620,],[-288,164,173,-286,-287,-284,-285,-276,-281,272,277,-274,-289,-279,-278,-277,-283,173,-282,-306,378,-275,-309,399,-280,272,-307,-375,-308,471,474,516,-376,528,564,568,572,575,173,599,601,621,627,]),'AS':([11,13,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,89,95,97,98,102,104,105,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,203,208,217,218,219,224,227,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,334,335,336,337,339,343,347,348,350,362,369,375,382,390,400,405,410,411,412,422,433,436,447,],[-256,-236,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-276,213,-267,-266,-257,225,-144,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-278,-277,-258,-260,-261,-259,-145,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-262,-264,-265,-263,-146,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,494,-147,-206,506,]),'EQEQUAL':([11,13,24,28,34,36,37,50,58,60,66,67,71,73,89,97,98,102,109,126,139,150,159,162,166,178,182,184,186,191,203,208,217,218,219,224,230,240,241,243,249,257,265,271,278,279,280,300,301,307,308,334,335,336,337,343,347,348,350,362,375,382,390,405,410,411,],[-256,-236,-250,140,-288,-269,-240,-270,-286,-232,-287,-284,-285,-244,-276,-267,-266,-257,-237,-251,140,-241,-281,-272,-274,-289,-233,-268,-279,-245,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-282,-306,-275,-309,-271,-234,-280,-246,-247,-262,-264,-265,-263,-239,-254,-255,-221,-243,-307,-273,-308,-235,-248,-249,]),'BREAK':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[69,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,69,-164,-158,-55,69,-13,-14,69,-5,-6,-57,69,-56,69,-192,69,69,69,-58,69,-182,69,69,-175,-171,-354,-165,-183,-15,69,69,-194,69,69,69,-167,-173,-193,-195,-180,-179,69,69,69,69,-355,69,69,-181,-177,-176,-172,-356,69,69,-166,69,69,-168,-169,-174,69,-170,-178,]),'CONTINUE':([0,5,6,9,15,17,19,25,27,40,42,59,83,87,92,94,151,152,154,180,181,209,210,212,215,261,298,316,319,328,330,332,361,364,368,404,416,419,431,442,443,444,445,446,448,485,489,502,503,504,505,508,509,510,511,532,538,539,544,558,559,560,576,585,587,588,589,593,611,612,613,615,616,625,626,],[51,-162,-160,-3,-53,-163,-159,-157,-161,-54,-4,51,-164,-158,-55,51,-13,-14,51,-5,-6,-57,51,-56,51,-192,51,51,51,-58,51,-182,51,51,-175,-171,-354,-165,-183,-15,51,51,-194,51,51,51,-167,-173,-193,-195,-180,-179,51,51,51,51,-355,51,51,-181,-177,-176,-172,-356,51,51,-166,51,51,-168,-169,-174,51,-170,-178,]),'NOT':([0,2,5,6,7,9,11,13,15,17,19,24,25,27,28,29,30,32,34,36,37,40,42,47,50,52,54,56,58,59,60,64,66,67,71,72,73,81,83,84,87,89,92,94,97,98,102,109,111,112,113,114,115,116,117,118,119,120,121,123,124,125,126,129,134,139,147,150,151,152,154,156,159,161,162,165,166,174,178,180,181,182,184,186,191,192,195,203,205,208,209,210,212,215,216,217,218,219,224,230,232,237,240,241,243,247,249,253,257,259,261,263,265,267,269,270,271,272,274,278,279,280,281,290,292,295,298,300,301,304,306,307,308,312,313,315,316,317,319,326,328,330,331,332,334,335,336,337,343,347,348,349,350,352,355,358,361,362,364,366,368,370,375,376,378,379,381,382,383,387,389,390,404,405,408,410,411,416,419,429,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[52,52,-162,-160,52,-3,-256,-236,-53,-163,-159,-250,-157,-161,136,52,52,52,-288,-269,-240,-54,-4,52,-270,52,52,52,-286,52,-232,52,-287,-284,-285,52,-244,52,-164,52,-158,-276,-55,52,-267,-266,-257,-237,-80,-81,-79,-85,-84,52,52,-88,-83,-78,-89,-82,-87,-86,-251,52,244,136,52,-241,-13,-14,52,52,-281,52,-272,52,-274,52,-289,-5,-6,-233,-268,-279,-245,52,52,-278,52,-277,-57,52,-56,52,52,-258,-260,-261,-259,-238,52,52,-252,-253,-220,52,-283,52,-242,52,-192,52,-282,52,52,52,-306,52,52,-275,-309,-271,52,52,52,52,52,-234,-280,52,52,-246,-247,52,52,52,52,52,52,52,-58,52,52,-182,-262,-264,-265,-263,-239,-254,-255,52,-221,52,52,52,52,-243,52,52,-175,52,-307,52,52,52,-375,-273,52,52,52,-308,-171,-235,52,-248,-249,-354,-165,52,-183,52,52,-15,52,52,-194,52,52,52,52,-376,52,52,52,52,52,-167,52,-173,-193,-195,-180,52,52,-179,52,52,52,52,52,52,52,52,52,-355,52,52,52,52,-181,-177,-176,52,52,52,52,-172,52,52,52,-356,52,52,-166,52,52,52,52,52,52,-168,-169,-174,52,-170,-178,]),'AMPEREQUAL':([11,13,16,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,194,195,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,294,300,301,307,308,311,313,314,334,335,336,337,343,347,348,350,362,369,375,382,390,400,405,410,411,412,413,436,],[-256,-236,119,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,-334,-276,-267,-266,-257,-237,-251,-219,-241,-209,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-336,-335,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-304,-234,-280,-246,-247,-214,-337,-338,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-305,-235,-248,-249,-215,-339,-206,]),'COMMA':([11,13,20,24,26,28,34,36,37,44,46,50,58,60,66,67,71,73,74,76,89,90,95,96,97,98,102,104,105,107,109,126,139,142,145,146,150,155,158,159,162,166,167,170,171,175,176,178,182,184,186,187,188,191,193,194,203,207,208,214,217,218,219,224,227,229,230,240,241,243,248,249,251,252,254,257,264,265,266,271,275,276,278,279,280,281,283,284,286,288,289,294,297,300,301,305,307,308,311,314,325,329,333,334,335,336,337,338,339,341,343,344,347,348,350,351,353,356,357,362,369,371,372,373,375,377,380,382,383,384,385,387,388,390,392,393,396,398,400,401,402,405,407,410,411,412,413,420,422,430,432,433,434,436,437,438,439,447,452,455,456,458,461,462,463,464,465,466,468,469,473,475,477,481,484,493,500,515,517,519,520,521,523,530,534,535,536,545,547,551,552,553,563,567,570,573,581,590,592,595,598,602,608,610,614,617,623,624,],[-256,-236,-207,-250,-205,-218,-288,-269,-240,-208,-217,-270,-286,-232,-287,-284,-285,-244,-212,195,-276,205,-186,216,-267,-266,-257,-132,-144,228,-237,-251,-219,247,250,253,-241,-209,267,-281,-272,-274,-216,-45,291,295,296,-289,-233,-268,-279,304,306,-245,-213,313,-278,326,-277,331,-258,-260,-261,-259,-145,342,-238,-252,-253,-220,352,-283,354,355,358,-242,-210,-282,370,-306,-377,381,-275,-309,-271,-318,-317,389,391,394,395,-304,403,-234,-280,408,-246,-247,-214,-338,-302,-187,-184,-262,-264,-265,-263,-133,-146,-142,-239,435,-254,-255,-221,-394,-332,-98,440,-243,-211,-352,453,457,-307,459,-378,-273,-326,-320,-319,-322,467,-308,470,-46,476,-41,-305,480,-150,-235,-295,-248,-249,-215,-339,492,-130,-303,-185,-147,-143,-206,-395,-333,-99,507,-353,513,514,518,-379,-327,-321,-316,-324,-323,-314,-51,-43,526,529,-151,-296,546,-388,-371,566,-325,-315,-52,569,-42,582,-202,-201,-138,-131,-387,-386,-389,-372,-373,-44,604,607,-139,-390,-350,-374,620,-199,-203,-391,-351,-200,-204,]),'OR':([11,13,24,28,34,36,37,44,46,50,58,60,66,67,71,73,74,89,97,98,102,109,126,139,150,155,159,162,166,167,178,182,184,186,191,193,203,208,217,218,219,224,230,240,241,243,249,257,264,265,271,278,279,280,300,301,307,308,311,334,335,336,337,343,347,348,350,362,369,375,382,390,405,410,411,412,],[-256,-236,-250,-218,-288,-269,-240,156,-217,-270,-286,-232,-287,-284,-285,-244,-212,-276,-267,-266,-257,-237,-251,-219,-241,263,-281,-272,-274,-216,-289,-233,-268,-279,-245,-213,-278,-277,-258,-260,-261,-259,-238,-252,-253,-220,-283,-242,-210,-282,-306,-275,-309,-271,-234,-280,-246,-247,-214,-262,-264,-265,-263,-239,-254,-255,-221,-243,-211,-307,-273,-308,-235,-248,-249,-215,]),'LAMBDA':([0,2,5,6,7,9,15,17,19,25,27,29,30,32,40,42,47,54,56,59,64,72,81,83,84,87,92,94,111,112,113,114,115,116,117,118,119,120,121,123,124,125,147,151,152,154,161,165,174,180,181,195,205,209,210,212,215,216,232,237,247,253,259,261,267,269,270,272,274,281,290,292,295,298,304,306,313,315,316,317,319,326,328,330,331,332,349,352,355,358,361,364,366,368,370,376,378,379,381,383,387,389,404,408,416,419,431,435,440,442,443,444,445,446,448,453,457,459,467,478,480,483,485,489,490,502,503,504,505,506,507,508,509,510,511,513,514,516,518,524,532,538,539,541,544,554,558,559,560,561,564,566,568,576,580,582,584,585,587,588,589,593,596,599,607,609,611,612,613,615,616,625,626,],[53,53,-162,-160,53,-3,-53,-163,-159,-157,-161,53,53,53,-54,-4,53,53,53,53,53,53,53,-164,53,-158,-55,53,-80,-81,-79,-85,-84,53,53,-88,-83,-78,-89,-82,-87,-86,53,-13,-14,53,53,53,53,-5,-6,53,53,-57,53,-56,53,53,53,53,53,53,53,-192,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,53,-58,53,53,-182,53,53,53,53,53,53,53,-175,53,53,53,53,-375,53,53,53,-171,53,-354,-165,-183,53,53,-15,53,53,-194,53,53,53,53,-376,53,53,53,537,53,-167,53,-173,-193,-195,-180,53,53,-179,53,53,53,53,53,53,53,53,53,-355,53,53,53,537,-181,-177,-176,53,53,53,53,-172,537,537,537,-356,53,53,-166,53,53,53,537,537,53,-168,-169,-174,53,-170,-178,]),} - -_lr_action = {} -for _k, _v in _lr_action_items.items(): - for _x,_y in zip(_v[0],_v[1]): - if not _x in _lr_action: _lr_action[_x] = {} - _lr_action[_x][_k] = _y -del _lr_action_items - -_lr_goto_items = {'if_stmt_star':([419,],[489,]),'import_as_name':([320,322,423,425,426,492,496,546,],[420,420,420,420,420,545,420,590,]),'try_stmt':([0,59,364,444,],[6,6,6,6,]),'small_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[3,3,211,3,327,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,]),'augassign':([16,],[116,]),'import_from':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,]),'import_as_names':([320,322,423,425,426,496,],[424,428,495,498,499,549,]),'try_stmt_plus':([262,],[368,]),'print_stmt_plus':([146,254,],[252,357,]),'import_as_names_star':([420,],[493,]),'comp_op':([28,139,],[131,246,]),'parameters':([148,],[256,]),'listmaker_star':([187,],[305,]),'xor_expr_star':([13,],[109,]),'factor':([0,2,7,8,10,14,29,30,31,32,41,47,52,54,56,59,62,64,72,81,84,94,99,100,101,103,108,116,117,127,128,129,131,147,149,154,156,161,164,165,174,183,189,190,192,195,204,205,210,213,215,216,220,221,222,223,231,232,237,238,239,246,247,250,253,258,259,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[11,11,11,97,98,11,11,11,11,11,11,11,11,11,11,11,184,11,11,11,11,11,217,218,219,224,11,11,11,11,11,11,11,11,11,11,11,11,280,11,11,11,11,11,11,11,11,11,11,11,11,11,334,335,336,337,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,382,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,]),'suite':([154,215,298,316,319,330,361,443,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[262,332,404,416,419,431,442,502,505,508,538,558,559,560,576,585,589,612,613,615,625,626,]),'exec_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,38,]),'testlist_safe_plus':([534,],[581,]),'simple_stmt':([0,59,154,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[15,15,261,261,261,261,261,261,261,15,261,15,261,261,261,261,261,261,261,261,261,261,261,261,261,261,]),'subscriptlist':([165,],[285,]),'testlist':([0,29,59,81,94,116,117,154,210,215,237,259,298,315,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[16,141,16,197,16,234,236,16,16,16,346,363,16,415,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,]),'classdef':([0,39,59,364,444,],[17,151,17,17,17,]),'assert_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,]),'for_stmt':([0,59,364,444,],[19,19,19,19,]),'lambdef':([0,2,7,29,30,32,47,54,56,59,64,72,81,84,94,116,117,147,154,161,165,174,195,205,210,215,216,232,237,247,253,259,267,269,270,272,274,281,290,292,295,298,304,306,313,315,316,317,319,326,330,331,349,352,355,358,361,364,366,370,376,378,379,383,387,389,408,435,440,443,444,446,448,453,457,467,478,480,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,561,564,566,568,587,588,593,596,599,611,616,],[20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,]),'expr_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,]),'decorator':([0,49,59,364,444,],[23,160,23,23,23,]),'term':([0,2,7,14,29,30,31,32,41,47,52,54,56,59,64,72,81,84,94,108,116,117,127,128,129,131,147,149,154,156,161,165,174,183,189,190,192,195,204,205,210,213,215,216,231,232,237,238,239,246,247,250,253,258,259,263,267,269,270,272,274,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,240,241,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,347,348,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,]),'if_stmt':([0,59,364,444,],[25,25,25,25,]),'testlist1_star':([142,],[248,]),'or_test':([0,2,7,29,30,32,47,54,56,59,64,72,81,84,94,116,117,129,147,154,161,165,174,195,205,210,215,216,232,237,247,253,259,267,269,270,272,274,281,290,292,295,298,304,306,313,315,316,317,319,326,330,331,349,352,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,242,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,500,26,26,26,26,26,26,26,26,26,26,26,536,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,536,26,26,26,26,536,536,536,26,26,26,26,26,536,536,26,26,]),'with_stmt':([0,59,364,444,],[27,27,27,27,]),'comp_for':([90,158,275,372,500,592,],[206,268,380,454,552,552,]),'file_input':([0,],[68,]),'trailer':([50,162,],[166,278,]),'fplist_star':([286,],[392,]),'testlist_comp_star':([90,],[207,]),'and_test_star':([74,],[193,]),'listmaker':([64,],[185,]),'arglist':([161,317,],[273,418,]),'flow_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,]),'suite_plus':([364,],[444,]),'testlist_star':([76,],[194,]),'and_expr_star':([37,],[150,]),'shift_expr':([0,2,7,14,29,30,31,32,41,47,52,54,56,59,64,72,81,84,94,108,116,117,129,131,147,149,154,156,161,165,174,183,192,195,204,205,210,213,215,216,231,232,237,246,247,250,253,258,259,263,267,269,270,272,274,281,290,292,295,298,299,302,304,306,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,257,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,362,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,37,]),'list_iter':([533,606,],[577,622,]),'dictorsetmaker':([47,],[157,]),'list_for':([187,533,606,],[303,579,579,]),'or_test_star':([44,],[155,]),'subscript':([165,389,467,],[284,468,520,]),'decorators':([0,59,364,444,],[39,39,39,39,]),'compound_stmt':([0,59,364,444,],[40,40,40,40,]),'dotted_name':([12,82,85,201,228,342,],[104,198,200,321,104,104,]),'power_star':([50,],[162,]),'power':([0,2,7,8,10,14,29,30,31,32,41,47,52,54,56,59,62,64,72,81,84,94,99,100,101,103,108,116,117,127,128,129,131,147,149,154,156,161,164,165,174,183,189,190,192,195,204,205,210,213,215,216,220,221,222,223,231,232,237,238,239,246,247,250,253,258,259,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,]),'subscriptlist_star':([284,],[388,]),'stmt':([0,59,364,444,],[42,181,445,504,]),'import_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,45,]),'sliceop':([281,384,387,465,],[385,463,466,519,]),'comparison':([0,2,7,29,30,32,47,52,54,56,59,64,72,81,84,94,116,117,129,147,154,156,161,165,174,192,195,205,210,215,216,232,237,247,253,259,263,267,269,270,272,274,281,290,292,295,298,304,306,312,313,315,316,317,319,326,330,331,349,352,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,46,]),'pass_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,]),'arith_expr':([0,2,7,14,29,30,31,32,41,47,52,54,56,59,64,72,81,84,94,108,116,117,129,131,147,149,154,156,161,165,174,183,189,190,192,195,204,205,210,213,215,216,231,232,237,246,247,250,253,258,259,263,267,269,270,272,274,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,307,308,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,410,411,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,73,]),'varargslist_star':([171,396,],[289,475,]),'term_star':([11,],[102,]),'comp_iter':([500,592,],[553,614,]),'dotted_as_names':([12,],[106,]),'return_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,57,]),'testlist_comp':([2,],[88,]),'old_test':([483,554,580,582,584,607,609,],[534,592,606,608,610,623,624,]),'atom_plus':([0,2,7,8,10,14,29,30,31,32,41,47,52,54,56,59,62,64,72,81,84,94,99,100,101,103,108,116,117,127,128,129,131,147,149,154,156,161,164,165,174,183,189,190,192,195,204,205,210,213,215,216,220,221,222,223,231,232,237,238,239,246,247,250,253,258,259,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,58,]),'file_input_star':([0,],[59,]),'xor_expr':([0,2,7,14,29,30,31,32,41,47,52,54,56,59,64,72,81,84,94,116,117,129,131,147,154,156,161,165,174,183,192,195,204,205,210,213,215,216,232,237,246,247,250,253,259,263,267,269,270,272,274,281,290,292,295,298,299,302,304,306,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,300,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,405,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,60,]),'expr_star':([60,],[182,]),'continue_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,61,]),'arglist_star2':([161,317,],[274,274,]),'arglist_star3':([458,],[517,]),'global_stmt_star':([176,],[297,]),'dotted_as_name':([12,228,342,],[107,341,434,]),'shift_expr_star':([73,],[191,]),'dotted_name_star':([105,],[227,]),'print_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,]),'dictorsetmaker_star':([372,],[455,]),'atom':([0,2,7,8,10,14,29,30,31,32,41,47,52,54,56,59,62,64,72,81,84,94,99,100,101,103,108,116,117,127,128,129,131,147,149,154,156,161,164,165,174,183,189,190,192,195,204,205,210,213,215,216,220,221,222,223,231,232,237,238,239,246,247,250,253,258,259,263,267,269,270,272,274,277,281,290,292,295,298,299,302,304,306,309,310,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,50,]),'funcdef':([0,39,59,364,444,],[5,152,5,5,5,]),'raise_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,65,]),'old_lambdef':([483,554,580,582,584,607,609,],[535,535,535,535,535,535,535,]),'exprlist':([31,41,204,302,],[144,153,324,406,]),'simple_stmt_star':([3,],[93,]),'decorators_plus':([0,59,364,444,],[49,49,49,49,]),'expr':([0,2,7,14,29,30,31,32,41,47,52,54,56,59,64,72,81,84,94,116,117,129,131,147,154,156,161,165,174,192,195,204,205,210,213,215,216,232,237,246,247,250,253,259,263,267,269,270,272,274,281,290,292,295,298,302,304,306,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[28,28,28,110,28,28,145,28,145,28,28,28,28,28,28,28,28,28,28,28,28,28,243,28,28,28,28,28,28,28,28,145,28,28,329,28,28,28,28,350,28,353,28,28,28,28,28,28,28,28,28,28,28,28,28,145,28,28,28,28,28,28,28,28,28,28,28,28,28,438,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,]),'except_clause':([262,368,],[365,449,]),'dictorsetmaker_star2':([158,],[266,]),'dotted_as_names_star':([107,],[229,]),'arith_expr_star':([24,],[126,]),'and_expr':([0,2,7,14,29,30,31,32,41,47,52,54,56,59,64,72,81,84,94,108,116,117,129,131,147,154,156,161,165,174,183,192,195,204,205,210,213,215,216,231,232,237,246,247,250,253,259,263,267,269,270,272,274,281,290,292,295,298,299,302,304,306,312,313,315,316,317,319,326,330,331,349,352,354,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,230,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,343,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,]),'yield_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,70,]),'import_from_plus':([85,],[201,]),'argument':([161,274,317,457,514,518,566,],[276,377,276,515,563,567,598,]),'fplist':([168,],[287,]),'expr_stmt_star':([16,],[122,]),'testlist_safe':([483,],[533,]),'not_test':([0,2,7,29,30,32,47,52,54,56,59,64,72,81,84,94,116,117,129,147,154,156,161,165,174,192,195,205,210,215,216,232,237,247,253,259,263,267,269,270,272,274,281,290,292,295,298,304,306,312,313,315,316,317,319,326,330,331,349,352,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[74,74,74,74,74,74,74,167,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,311,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,412,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,74,]),'break_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,75,]),'fpdef':([53,168,255,291,391,395,470,476,526,537,],[171,286,171,398,469,473,521,398,473,171,]),'exprlist_star':([145,],[251,]),'list_if':([533,606,],[578,578,]),'test':([0,2,7,29,30,32,47,54,56,59,64,72,81,84,94,116,117,147,154,161,165,174,195,205,210,215,216,232,237,247,253,259,267,269,270,272,274,281,290,292,295,298,304,306,313,315,316,317,319,326,330,331,349,352,355,358,361,364,366,370,376,378,379,383,387,389,408,435,440,443,444,446,448,453,457,467,478,480,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,561,564,566,568,587,588,593,596,599,611,616,],[76,90,95,76,142,146,158,175,177,76,187,188,76,199,76,76,76,254,76,275,283,294,314,325,76,76,95,344,76,351,356,76,371,372,373,374,275,384,396,400,401,76,407,409,413,76,76,275,76,430,76,95,436,437,439,356,76,76,447,452,458,460,461,462,465,283,484,501,439,76,76,76,76,512,275,283,530,531,76,543,556,557,76,76,76,562,275,565,275,570,76,76,586,76,595,597,275,600,76,76,76,617,618,76,76,]),'global_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,77,]),'with_item':([7,216,331,],[96,333,432,]),'import_name':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,78,]),'yield_expr':([0,2,59,94,116,117,154,210,215,237,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[80,91,80,80,233,235,80,80,80,345,80,80,80,80,80,80,80,80,80,80,80,80,80,80,80,80,80,80,80,80,80,80,]),'arglist_star':([373,],[456,]),'comp_if':([500,592,],[551,551,]),'and_test':([0,2,7,29,30,32,47,54,56,59,64,72,81,84,94,116,117,129,147,154,156,161,165,174,195,205,210,215,216,232,237,247,253,259,263,267,269,270,272,274,281,290,292,295,298,304,306,313,315,316,317,319,326,330,331,349,352,355,358,361,364,366,370,376,378,379,383,387,389,408,429,435,440,443,444,446,448,453,457,467,478,480,483,485,490,506,507,509,510,511,513,514,516,518,524,532,539,541,544,554,561,564,566,568,580,582,584,587,588,593,596,599,607,609,611,616,],[44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,264,44,44,44,44,44,44,44,44,44,44,44,44,44,369,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,44,]),'comparison_star':([28,],[139,]),'decorated':([0,59,364,444,],[83,83,83,83,]),'with_stmt_star':([96,],[214,]),'del_stmt':([0,59,94,154,210,215,298,316,319,330,361,364,443,444,446,448,485,509,510,511,532,539,544,587,588,593,611,616,],[86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,86,]),'while_stmt':([0,59,364,444,],[87,87,87,87,]),'varargslist':([53,255,537,],[172,360,583,]),'testlist1':([30,],[143,]),} - -_lr_goto = {} -for _k, _v in _lr_goto_items.items(): - for _x, _y in zip(_v[0], _v[1]): - if not _x in _lr_goto: _lr_goto[_x] = {} - _lr_goto[_x][_k] = _y -del _lr_goto_items -_lr_productions = [ - ("S' -> file_input","S'",1,None,None,None), - ('file_input -> ENDMARKER','file_input',1,'p_file_input_1','hgawk_grammar.py',87), - ('file_input -> file_input_star ENDMARKER','file_input',2,'p_file_input_2','hgawk_grammar.py',91), - ('file_input_star -> NEWLINE','file_input_star',1,'p_file_input_star_1','hgawk_grammar.py',97), - ('file_input_star -> stmt','file_input_star',1,'p_file_input_star_2','hgawk_grammar.py',101), - ('file_input_star -> file_input_star NEWLINE','file_input_star',2,'p_file_input_star_3','hgawk_grammar.py',105), - ('file_input_star -> file_input_star stmt','file_input_star',2,'p_file_input_star_4','hgawk_grammar.py',110), - ('decorator -> AT dotted_name NEWLINE','decorator',3,'p_decorator_1','hgawk_grammar.py',116), - ('decorator -> AT dotted_name LPAR RPAR NEWLINE','decorator',5,'p_decorator_2','hgawk_grammar.py',121), - ('decorator -> AT dotted_name LPAR arglist RPAR NEWLINE','decorator',6,'p_decorator_3','hgawk_grammar.py',126), - ('decorators -> decorators_plus','decorators',1,'p_decorators','hgawk_grammar.py',135), - ('decorators_plus -> decorator','decorators_plus',1,'p_decorators_plus_1','hgawk_grammar.py',140), - ('decorators_plus -> decorators_plus decorator','decorators_plus',2,'p_decorators_plus_2','hgawk_grammar.py',144), - ('decorated -> decorators classdef','decorated',2,'p_decorated_1','hgawk_grammar.py',150), - ('decorated -> decorators funcdef','decorated',2,'p_decorated_2','hgawk_grammar.py',156), - ('funcdef -> DEF NAME parameters COLON suite','funcdef',5,'p_funcdef','hgawk_grammar.py',164), - ('parameters -> LPAR RPAR','parameters',2,'p_parameters_1','hgawk_grammar.py',170), - ('parameters -> LPAR varargslist RPAR','parameters',3,'p_parameters_2','hgawk_grammar.py',174), - ('varargslist -> fpdef COMMA STAR NAME','varargslist',4,'p_varargslist_1','hgawk_grammar.py',183), - ('varargslist -> fpdef COMMA STAR NAME COMMA DOUBLESTAR NAME','varargslist',7,'p_varargslist_2','hgawk_grammar.py',188), - ('varargslist -> fpdef COMMA DOUBLESTAR NAME','varargslist',4,'p_varargslist_3','hgawk_grammar.py',193), - ('varargslist -> fpdef','varargslist',1,'p_varargslist_4','hgawk_grammar.py',198), - ('varargslist -> fpdef COMMA','varargslist',2,'p_varargslist_5','hgawk_grammar.py',203), - ('varargslist -> fpdef varargslist_star COMMA STAR NAME','varargslist',5,'p_varargslist_6','hgawk_grammar.py',208), - ('varargslist -> fpdef varargslist_star COMMA STAR NAME COMMA DOUBLESTAR NAME','varargslist',8,'p_varargslist_7','hgawk_grammar.py',214), - ('varargslist -> fpdef varargslist_star COMMA DOUBLESTAR NAME','varargslist',5,'p_varargslist_8','hgawk_grammar.py',221), - ('varargslist -> fpdef varargslist_star','varargslist',2,'p_varargslist_9','hgawk_grammar.py',227), - ('varargslist -> fpdef varargslist_star COMMA','varargslist',3,'p_varargslist_10','hgawk_grammar.py',232), - ('varargslist -> fpdef EQUAL test COMMA STAR NAME','varargslist',6,'p_varargslist_11','hgawk_grammar.py',237), - ('varargslist -> fpdef EQUAL test COMMA STAR NAME COMMA DOUBLESTAR NAME','varargslist',9,'p_varargslist_12','hgawk_grammar.py',242), - ('varargslist -> fpdef EQUAL test COMMA DOUBLESTAR NAME','varargslist',6,'p_varargslist_13','hgawk_grammar.py',247), - ('varargslist -> fpdef EQUAL test','varargslist',3,'p_varargslist_14','hgawk_grammar.py',252), - ('varargslist -> fpdef EQUAL test COMMA','varargslist',4,'p_varargslist_15','hgawk_grammar.py',257), - ('varargslist -> fpdef EQUAL test varargslist_star COMMA STAR NAME','varargslist',7,'p_varargslist_16','hgawk_grammar.py',262), - ('varargslist -> fpdef EQUAL test varargslist_star COMMA STAR NAME COMMA DOUBLESTAR NAME','varargslist',10,'p_varargslist_17','hgawk_grammar.py',269), - ('varargslist -> fpdef EQUAL test varargslist_star COMMA DOUBLESTAR NAME','varargslist',7,'p_varargslist_18','hgawk_grammar.py',277), - ('varargslist -> fpdef EQUAL test varargslist_star','varargslist',4,'p_varargslist_19','hgawk_grammar.py',284), - ('varargslist -> fpdef EQUAL test varargslist_star COMMA','varargslist',5,'p_varargslist_20','hgawk_grammar.py',290), - ('varargslist -> STAR NAME','varargslist',2,'p_varargslist_21','hgawk_grammar.py',296), - ('varargslist -> STAR NAME COMMA DOUBLESTAR NAME','varargslist',5,'p_varargslist_22','hgawk_grammar.py',301), - ('varargslist -> DOUBLESTAR NAME','varargslist',2,'p_varargslist_23','hgawk_grammar.py',306), - ('varargslist_star -> COMMA fpdef','varargslist_star',2,'p_varargslist_star_1','hgawk_grammar.py',312), - ('varargslist_star -> COMMA fpdef EQUAL test','varargslist_star',4,'p_varargslist_star_2','hgawk_grammar.py',317), - ('varargslist_star -> varargslist_star COMMA fpdef','varargslist_star',3,'p_varargslist_star_3','hgawk_grammar.py',322), - ('varargslist_star -> varargslist_star COMMA fpdef EQUAL test','varargslist_star',5,'p_varargslist_star_4','hgawk_grammar.py',327), - ('fpdef -> NAME','fpdef',1,'p_fpdef_1','hgawk_grammar.py',335), - ('fpdef -> LPAR fplist RPAR','fpdef',3,'p_fpdef_2','hgawk_grammar.py',339), - ('fplist -> fpdef','fplist',1,'p_fplist_1','hgawk_grammar.py',348), - ('fplist -> fpdef COMMA','fplist',2,'p_fplist_2','hgawk_grammar.py',352), - ('fplist -> fpdef fplist_star','fplist',2,'p_fplist_3','hgawk_grammar.py',357), - ('fplist -> fpdef fplist_star COMMA','fplist',3,'p_fplist_4','hgawk_grammar.py',363), - ('fplist_star -> COMMA fpdef','fplist_star',2,'p_fplist_star_1','hgawk_grammar.py',370), - ('fplist_star -> fplist_star COMMA fpdef','fplist_star',3,'p_fplist_star_2','hgawk_grammar.py',374), - ('stmt -> simple_stmt','stmt',1,'p_stmt_1','hgawk_grammar.py',381), - ('stmt -> compound_stmt','stmt',1,'p_stmt_2','hgawk_grammar.py',385), - ('simple_stmt -> small_stmt NEWLINE','simple_stmt',2,'p_simple_stmt_1','hgawk_grammar.py',391), - ('simple_stmt -> small_stmt SEMI NEWLINE','simple_stmt',3,'p_simple_stmt_2','hgawk_grammar.py',395), - ('simple_stmt -> small_stmt simple_stmt_star NEWLINE','simple_stmt',3,'p_simple_stmt_3','hgawk_grammar.py',399), - ('simple_stmt -> small_stmt simple_stmt_star SEMI NEWLINE','simple_stmt',4,'p_simple_stmt_4','hgawk_grammar.py',403), - ('simple_stmt_star -> SEMI small_stmt','simple_stmt_star',2,'p_simple_stmt_star_1','hgawk_grammar.py',408), - ('simple_stmt_star -> simple_stmt_star SEMI small_stmt','simple_stmt_star',3,'p_simple_stmt_star_2','hgawk_grammar.py',412), - ('small_stmt -> expr_stmt','small_stmt',1,'p_small_stmt_1','hgawk_grammar.py',419), - ('small_stmt -> print_stmt','small_stmt',1,'p_small_stmt_2','hgawk_grammar.py',423), - ('small_stmt -> del_stmt','small_stmt',1,'p_small_stmt_3','hgawk_grammar.py',427), - ('small_stmt -> pass_stmt','small_stmt',1,'p_small_stmt_4','hgawk_grammar.py',431), - ('small_stmt -> flow_stmt','small_stmt',1,'p_small_stmt_5','hgawk_grammar.py',435), - ('small_stmt -> import_stmt','small_stmt',1,'p_small_stmt_6','hgawk_grammar.py',439), - ('small_stmt -> global_stmt','small_stmt',1,'p_small_stmt_7','hgawk_grammar.py',443), - ('small_stmt -> exec_stmt','small_stmt',1,'p_small_stmt_8','hgawk_grammar.py',447), - ('small_stmt -> assert_stmt','small_stmt',1,'p_small_stmt_9','hgawk_grammar.py',451), - ('expr_stmt -> testlist augassign yield_expr','expr_stmt',3,'p_expr_stmt_1','hgawk_grammar.py',458), - ('expr_stmt -> testlist augassign testlist','expr_stmt',3,'p_expr_stmt_2','hgawk_grammar.py',464), - ('expr_stmt -> testlist','expr_stmt',1,'p_expr_stmt_3','hgawk_grammar.py',470), - ('expr_stmt -> testlist expr_stmt_star','expr_stmt',2,'p_expr_stmt_4','hgawk_grammar.py',475), - ('expr_stmt_star -> EQUAL yield_expr','expr_stmt_star',2,'p_expr_stmt_star_1','hgawk_grammar.py',484), - ('expr_stmt_star -> EQUAL testlist','expr_stmt_star',2,'p_expr_stmt_star_2','hgawk_grammar.py',488), - ('expr_stmt_star -> expr_stmt_star EQUAL yield_expr','expr_stmt_star',3,'p_expr_stmt_star_3','hgawk_grammar.py',492), - ('expr_stmt_star -> expr_stmt_star EQUAL testlist','expr_stmt_star',3,'p_expr_stmt_star_4','hgawk_grammar.py',496), - ('augassign -> PLUSEQUAL','augassign',1,'p_augassign_1','hgawk_grammar.py',503), - ('augassign -> MINEQUAL','augassign',1,'p_augassign_2','hgawk_grammar.py',507), - ('augassign -> STAREQUAL','augassign',1,'p_augassign_3','hgawk_grammar.py',511), - ('augassign -> SLASHEQUAL','augassign',1,'p_augassign_4','hgawk_grammar.py',515), - ('augassign -> PERCENTEQUAL','augassign',1,'p_augassign_5','hgawk_grammar.py',519), - ('augassign -> AMPEREQUAL','augassign',1,'p_augassign_6','hgawk_grammar.py',523), - ('augassign -> VBAREQUAL','augassign',1,'p_augassign_7','hgawk_grammar.py',527), - ('augassign -> CIRCUMFLEXEQUAL','augassign',1,'p_augassign_8','hgawk_grammar.py',531), - ('augassign -> LEFTSHIFTEQUAL','augassign',1,'p_augassign_9','hgawk_grammar.py',535), - ('augassign -> RIGHTSHIFTEQUAL','augassign',1,'p_augassign_10','hgawk_grammar.py',539), - ('augassign -> DOUBLESTAREQUAL','augassign',1,'p_augassign_11','hgawk_grammar.py',543), - ('augassign -> DOUBLESLASHEQUAL','augassign',1,'p_augassign_12','hgawk_grammar.py',547), - ('print_stmt -> PRINT','print_stmt',1,'p_print_stmt_1','hgawk_grammar.py',554), - ('print_stmt -> PRINT test','print_stmt',2,'p_print_stmt_2','hgawk_grammar.py',558), - ('print_stmt -> PRINT test COMMA','print_stmt',3,'p_print_stmt_3','hgawk_grammar.py',562), - ('print_stmt -> PRINT test print_stmt_plus','print_stmt',3,'p_print_stmt_4','hgawk_grammar.py',566), - ('print_stmt -> PRINT test print_stmt_plus COMMA','print_stmt',4,'p_print_stmt_5','hgawk_grammar.py',570), - ('print_stmt -> PRINT RIGHTSHIFT test','print_stmt',3,'p_print_stmt_6','hgawk_grammar.py',574), - ('print_stmt -> PRINT RIGHTSHIFT test print_stmt_plus','print_stmt',4,'p_print_stmt_7','hgawk_grammar.py',578), - ('print_stmt -> PRINT RIGHTSHIFT test print_stmt_plus COMMA','print_stmt',5,'p_print_stmt_8','hgawk_grammar.py',582), - ('print_stmt_plus -> COMMA test','print_stmt_plus',2,'p_print_stmt_plus_1','hgawk_grammar.py',587), - ('print_stmt_plus -> print_stmt_plus COMMA test','print_stmt_plus',3,'p_print_stmt_plus_2','hgawk_grammar.py',591), - ('del_stmt -> DEL exprlist','del_stmt',2,'p_del_stmt','hgawk_grammar.py',597), - ('pass_stmt -> PASS','pass_stmt',1,'p_pass_stmt','hgawk_grammar.py',607), - ('flow_stmt -> break_stmt','flow_stmt',1,'p_flow_stmt_1','hgawk_grammar.py',613), - ('flow_stmt -> continue_stmt','flow_stmt',1,'p_flow_stmt_2','hgawk_grammar.py',617), - ('flow_stmt -> return_stmt','flow_stmt',1,'p_flow_stmt_3','hgawk_grammar.py',621), - ('flow_stmt -> raise_stmt','flow_stmt',1,'p_flow_stmt_4','hgawk_grammar.py',625), - ('flow_stmt -> yield_stmt','flow_stmt',1,'p_flow_stmt_5','hgawk_grammar.py',629), - ('break_stmt -> BREAK','break_stmt',1,'p_break_stmt','hgawk_grammar.py',636), - ('continue_stmt -> CONTINUE','continue_stmt',1,'p_continue_stmt','hgawk_grammar.py',642), - ('return_stmt -> RETURN','return_stmt',1,'p_return_stmt_1','hgawk_grammar.py',648), - ('return_stmt -> RETURN testlist','return_stmt',2,'p_return_stmt_2','hgawk_grammar.py',652), - ('yield_stmt -> yield_expr','yield_stmt',1,'p_yield_stmt','hgawk_grammar.py',658), - ('raise_stmt -> RAISE','raise_stmt',1,'p_raise_stmt_1','hgawk_grammar.py',664), - ('raise_stmt -> RAISE test','raise_stmt',2,'p_raise_stmt_2','hgawk_grammar.py',668), - ('raise_stmt -> RAISE test COMMA test','raise_stmt',4,'p_raise_stmt_3','hgawk_grammar.py',672), - ('raise_stmt -> RAISE test COMMA test COMMA test','raise_stmt',6,'p_raise_stmt_4','hgawk_grammar.py',676), - ('import_stmt -> import_name','import_stmt',1,'p_import_stmt_1','hgawk_grammar.py',682), - ('import_stmt -> import_from','import_stmt',1,'p_import_stmt_2','hgawk_grammar.py',686), - ('import_name -> IMPORT dotted_as_names','import_name',2,'p_import_name','hgawk_grammar.py',692), - ('import_from -> FROM dotted_name IMPORT STAR','import_from',4,'p_import_from_1','hgawk_grammar.py',699), - ('import_from -> FROM dotted_name IMPORT LPAR import_as_names RPAR','import_from',6,'p_import_from_2','hgawk_grammar.py',709), - ('import_from -> FROM dotted_name IMPORT import_as_names','import_from',4,'p_import_from_3','hgawk_grammar.py',719), - ('import_from -> FROM import_from_plus dotted_name IMPORT STAR','import_from',5,'p_import_from_4','hgawk_grammar.py',729), - ('import_from -> FROM import_from_plus dotted_name IMPORT LPAR import_as_names RPAR','import_from',7,'p_import_from_5','hgawk_grammar.py',739), - ('import_from -> FROM import_from_plus dotted_name IMPORT import_as_names','import_from',5,'p_import_from_6','hgawk_grammar.py',749), - ('import_from -> FROM import_from_plus IMPORT STAR','import_from',4,'p_import_from_7','hgawk_grammar.py',759), - ('import_from -> FROM import_from_plus IMPORT LPAR import_as_names RPAR','import_from',6,'p_import_from_8','hgawk_grammar.py',763), - ('import_from -> FROM import_from_plus IMPORT import_as_names','import_from',4,'p_import_from_9','hgawk_grammar.py',767), - ('import_from_plus -> DOT','import_from_plus',1,'p_import_from_plus_1','hgawk_grammar.py',772), - ('import_from_plus -> import_from_plus DOT','import_from_plus',2,'p_import_from_plus_2','hgawk_grammar.py',776), - ('import_as_name -> NAME','import_as_name',1,'p_import_as_name_1','hgawk_grammar.py',782), - ('import_as_name -> NAME AS NAME','import_as_name',3,'p_import_as_name_2','hgawk_grammar.py',786), - ('dotted_as_name -> dotted_name','dotted_as_name',1,'p_dotted_as_name_1','hgawk_grammar.py',792), - ('dotted_as_name -> dotted_name AS NAME','dotted_as_name',3,'p_dotted_as_name_2','hgawk_grammar.py',803), - ('import_as_names -> import_as_name','import_as_names',1,'p_import_as_names_1','hgawk_grammar.py',816), - ('import_as_names -> import_as_name COMMA','import_as_names',2,'p_import_as_names_2','hgawk_grammar.py',820), - ('import_as_names -> import_as_name import_as_names_star','import_as_names',2,'p_import_as_names_3','hgawk_grammar.py',824), - ('import_as_names -> import_as_name import_as_names_star COMMA','import_as_names',3,'p_import_as_names_4','hgawk_grammar.py',828), - ('import_as_names_star -> COMMA import_as_name','import_as_names_star',2,'p_import_as_names_star_1','hgawk_grammar.py',833), - ('import_as_names_star -> import_as_names_star COMMA import_as_name','import_as_names_star',3,'p_import_as_names_star_2','hgawk_grammar.py',837), - ('dotted_as_names -> dotted_as_name','dotted_as_names',1,'p_dotted_as_names_1','hgawk_grammar.py',843), - ('dotted_as_names -> dotted_as_name dotted_as_names_star','dotted_as_names',2,'p_dotted_as_names_2','hgawk_grammar.py',847), - ('dotted_as_names_star -> COMMA dotted_as_name','dotted_as_names_star',2,'p_dotted_as_names_star_1','hgawk_grammar.py',852), - ('dotted_as_names_star -> dotted_as_names_star COMMA dotted_as_name','dotted_as_names_star',3,'p_dotted_as_names_star_2','hgawk_grammar.py',856), - ('dotted_name -> NAME','dotted_name',1,'p_dotted_name_1','hgawk_grammar.py',862), - ('dotted_name -> NAME dotted_name_star','dotted_name',2,'p_dotted_name_2','hgawk_grammar.py',866), - ('dotted_name_star -> DOT NAME','dotted_name_star',2,'p_dotted_name_star_1','hgawk_grammar.py',880), - ('dotted_name_star -> dotted_name_star DOT NAME','dotted_name_star',3,'p_dotted_name_star_2','hgawk_grammar.py',884), - ('global_stmt -> GLOBAL NAME','global_stmt',2,'p_global_stmt_1','hgawk_grammar.py',890), - ('global_stmt -> GLOBAL NAME global_stmt_star','global_stmt',3,'p_global_stmt_2','hgawk_grammar.py',894), - ('global_stmt_star -> COMMA NAME','global_stmt_star',2,'p_global_stmt_star_1','hgawk_grammar.py',899), - ('global_stmt_star -> global_stmt_star COMMA NAME','global_stmt_star',3,'p_global_stmt_star_2','hgawk_grammar.py',903), - ('exec_stmt -> EXEC expr','exec_stmt',2,'p_exec_stmt_1','hgawk_grammar.py',909), - ('exec_stmt -> EXEC expr IN test','exec_stmt',4,'p_exec_stmt_2','hgawk_grammar.py',913), - ('exec_stmt -> EXEC expr IN test COMMA test','exec_stmt',6,'p_exec_stmt_3','hgawk_grammar.py',917), - ('assert_stmt -> ASSERT test','assert_stmt',2,'p_assert_stmt_1','hgawk_grammar.py',923), - ('assert_stmt -> ASSERT test COMMA test','assert_stmt',4,'p_assert_stmt_2','hgawk_grammar.py',927), - ('compound_stmt -> if_stmt','compound_stmt',1,'p_compound_stmt_1','hgawk_grammar.py',933), - ('compound_stmt -> while_stmt','compound_stmt',1,'p_compound_stmt_2','hgawk_grammar.py',937), - ('compound_stmt -> for_stmt','compound_stmt',1,'p_compound_stmt_3','hgawk_grammar.py',941), - ('compound_stmt -> try_stmt','compound_stmt',1,'p_compound_stmt_4','hgawk_grammar.py',945), - ('compound_stmt -> with_stmt','compound_stmt',1,'p_compound_stmt_5','hgawk_grammar.py',949), - ('compound_stmt -> funcdef','compound_stmt',1,'p_compound_stmt_6','hgawk_grammar.py',953), - ('compound_stmt -> classdef','compound_stmt',1,'p_compound_stmt_7','hgawk_grammar.py',957), - ('compound_stmt -> decorated','compound_stmt',1,'p_compound_stmt_8','hgawk_grammar.py',961), - ('if_stmt -> IF test COLON suite','if_stmt',4,'p_if_stmt_1','hgawk_grammar.py',967), - ('if_stmt -> IF test COLON suite ELSE COLON suite','if_stmt',7,'p_if_stmt_2','hgawk_grammar.py',971), - ('if_stmt -> IF test COLON suite if_stmt_star','if_stmt',5,'p_if_stmt_3','hgawk_grammar.py',975), - ('if_stmt -> IF test COLON suite if_stmt_star ELSE COLON suite','if_stmt',8,'p_if_stmt_4','hgawk_grammar.py',979), - ('if_stmt_star -> ELIF test COLON suite','if_stmt_star',4,'p_if_stmt_star_1','hgawk_grammar.py',988), - ('if_stmt_star -> if_stmt_star ELIF test COLON suite','if_stmt_star',5,'p_if_stmt_star_2','hgawk_grammar.py',993), - ('while_stmt -> WHILE test COLON suite','while_stmt',4,'p_while_stmt_1','hgawk_grammar.py',1004), - ('while_stmt -> WHILE test COLON suite ELSE COLON suite','while_stmt',7,'p_while_stmt_2','hgawk_grammar.py',1008), - ('for_stmt -> FOR exprlist IN testlist COLON suite','for_stmt',6,'p_for_stmt_1','hgawk_grammar.py',1014), - ('for_stmt -> FOR exprlist IN testlist COLON suite ELSE COLON suite','for_stmt',9,'p_for_stmt_2','hgawk_grammar.py',1019), - ('try_stmt -> TRY COLON suite try_stmt_plus','try_stmt',4,'p_try_stmt_1','hgawk_grammar.py',1030), - ('try_stmt -> TRY COLON suite try_stmt_plus FINALLY COLON suite','try_stmt',7,'p_try_stmt_2','hgawk_grammar.py',1034), - ('try_stmt -> TRY COLON suite try_stmt_plus ELSE COLON suite','try_stmt',7,'p_try_stmt_3','hgawk_grammar.py',1038), - ('try_stmt -> TRY COLON suite try_stmt_plus ELSE COLON suite FINALLY COLON suite','try_stmt',10,'p_try_stmt_4','hgawk_grammar.py',1042), - ('try_stmt -> TRY COLON suite FINALLY COLON suite','try_stmt',6,'p_try_stmt_5','hgawk_grammar.py',1046), - ('try_stmt_plus -> except_clause COLON suite','try_stmt_plus',3,'p_try_stmt_plus_1','hgawk_grammar.py',1051), - ('try_stmt_plus -> try_stmt_plus except_clause COLON suite','try_stmt_plus',4,'p_try_stmt_plus_2','hgawk_grammar.py',1056), - ('with_stmt -> WITH with_item COLON suite','with_stmt',4,'p_with_stmt_1','hgawk_grammar.py',1063), - ('with_stmt -> WITH with_item with_stmt_star COLON suite','with_stmt',5,'p_with_stmt_2','hgawk_grammar.py',1068), - ('with_stmt_star -> COMMA with_item','with_stmt_star',2,'p_with_stmt_star_1','hgawk_grammar.py',1078), - ('with_stmt_star -> with_stmt_star COMMA with_item','with_stmt_star',3,'p_with_stmt_star_2','hgawk_grammar.py',1082), - ('with_item -> test','with_item',1,'p_with_item_1','hgawk_grammar.py',1092), - ('with_item -> test AS expr','with_item',3,'p_with_item_2','hgawk_grammar.py',1097), - ('except_clause -> EXCEPT','except_clause',1,'p_except_clause_1','hgawk_grammar.py',1105), - ('except_clause -> EXCEPT test','except_clause',2,'p_except_clause_2','hgawk_grammar.py',1109), - ('except_clause -> EXCEPT test AS test','except_clause',4,'p_except_clause_3','hgawk_grammar.py',1113), - ('except_clause -> EXCEPT test COMMA test','except_clause',4,'p_except_clause_4','hgawk_grammar.py',1118), - ('suite -> simple_stmt','suite',1,'p_suite_1','hgawk_grammar.py',1125), - ('suite -> NEWLINE INDENT suite_plus DEDENT','suite',4,'p_suite_2','hgawk_grammar.py',1129), - ('suite_plus -> stmt','suite_plus',1,'p_suite_plus_1','hgawk_grammar.py',1134), - ('suite_plus -> suite_plus stmt','suite_plus',2,'p_suite_plus_2','hgawk_grammar.py',1138), - ('testlist_safe -> old_test','testlist_safe',1,'p_testlist_safe_1','hgawk_grammar.py',1144), - ('testlist_safe -> old_test testlist_safe_plus','testlist_safe',2,'p_testlist_safe_2','hgawk_grammar.py',1148), - ('testlist_safe -> old_test testlist_safe_plus COMMA','testlist_safe',3,'p_testlist_safe_3','hgawk_grammar.py',1153), - ('testlist_safe_plus -> COMMA old_test','testlist_safe_plus',2,'p_testlist_safe_plus_1','hgawk_grammar.py',1159), - ('testlist_safe_plus -> testlist_safe_plus COMMA old_test','testlist_safe_plus',3,'p_testlist_safe_plus_2','hgawk_grammar.py',1163), - ('old_test -> or_test','old_test',1,'p_old_test_1','hgawk_grammar.py',1169), - ('old_test -> old_lambdef','old_test',1,'p_old_test_2','hgawk_grammar.py',1173), - ('old_lambdef -> LAMBDA COLON old_test','old_lambdef',3,'p_old_lambdef_1','hgawk_grammar.py',1179), - ('old_lambdef -> LAMBDA varargslist COLON old_test','old_lambdef',4,'p_old_lambdef_2','hgawk_grammar.py',1183), - ('test -> or_test','test',1,'p_test_1','hgawk_grammar.py',1189), - ('test -> or_test IF or_test ELSE test','test',5,'p_test_2','hgawk_grammar.py',1193), - ('test -> lambdef','test',1,'p_test_3','hgawk_grammar.py',1198), - ('or_test -> and_test','or_test',1,'p_or_test_1','hgawk_grammar.py',1204), - ('or_test -> and_test or_test_star','or_test',2,'p_or_test_2','hgawk_grammar.py',1208), - ('or_test_star -> OR and_test','or_test_star',2,'p_or_test_star_1','hgawk_grammar.py',1216), - ('or_test_star -> or_test_star OR and_test','or_test_star',3,'p_or_test_star_2','hgawk_grammar.py',1220), - ('and_test -> not_test','and_test',1,'p_and_test_1','hgawk_grammar.py',1226), - ('and_test -> not_test and_test_star','and_test',2,'p_and_test_2','hgawk_grammar.py',1230), - ('and_test_star -> AND not_test','and_test_star',2,'p_and_test_star_1','hgawk_grammar.py',1238), - ('and_test_star -> and_test_star AND not_test','and_test_star',3,'p_and_test_star_2','hgawk_grammar.py',1242), - ('not_test -> NOT not_test','not_test',2,'p_not_test_1','hgawk_grammar.py',1248), - ('not_test -> comparison','not_test',1,'p_not_test_2','hgawk_grammar.py',1254), - ('comparison -> expr','comparison',1,'p_comparison_1','hgawk_grammar.py',1260), - ('comparison -> expr comparison_star','comparison',2,'p_comparison_2','hgawk_grammar.py',1264), - ('comparison_star -> comp_op expr','comparison_star',2,'p_comparison_star_1','hgawk_grammar.py',1271), - ('comparison_star -> comparison_star comp_op expr','comparison_star',3,'p_comparison_star_2','hgawk_grammar.py',1276), - ('comp_op -> LESS','comp_op',1,'p_comp_op_1','hgawk_grammar.py',1284), - ('comp_op -> GREATER','comp_op',1,'p_comp_op_2','hgawk_grammar.py',1288), - ('comp_op -> EQEQUAL','comp_op',1,'p_comp_op_3','hgawk_grammar.py',1292), - ('comp_op -> GREATEREQUAL','comp_op',1,'p_comp_op_4','hgawk_grammar.py',1296), - ('comp_op -> LESSEQUAL','comp_op',1,'p_comp_op_5','hgawk_grammar.py',1300), - ('comp_op -> NOTEQUAL','comp_op',1,'p_comp_op_6','hgawk_grammar.py',1304), - ('comp_op -> IN','comp_op',1,'p_comp_op_7','hgawk_grammar.py',1308), - ('comp_op -> NOT IN','comp_op',2,'p_comp_op_8','hgawk_grammar.py',1312), - ('comp_op -> IS','comp_op',1,'p_comp_op_9','hgawk_grammar.py',1316), - ('comp_op -> IS NOT','comp_op',2,'p_comp_op_10','hgawk_grammar.py',1320), - ('expr -> xor_expr','expr',1,'p_expr_1','hgawk_grammar.py',1326), - ('expr -> xor_expr expr_star','expr',2,'p_expr_2','hgawk_grammar.py',1330), - ('expr_star -> VBAR xor_expr','expr_star',2,'p_expr_star_1','hgawk_grammar.py',1335), - ('expr_star -> expr_star VBAR xor_expr','expr_star',3,'p_expr_star_2','hgawk_grammar.py',1339), - ('xor_expr -> and_expr','xor_expr',1,'p_xor_expr_1','hgawk_grammar.py',1345), - ('xor_expr -> and_expr xor_expr_star','xor_expr',2,'p_xor_expr_2','hgawk_grammar.py',1349), - ('xor_expr_star -> CIRCUMFLEX and_expr','xor_expr_star',2,'p_xor_expr_star_1','hgawk_grammar.py',1354), - ('xor_expr_star -> xor_expr_star CIRCUMFLEX and_expr','xor_expr_star',3,'p_xor_expr_star_2','hgawk_grammar.py',1358), - ('and_expr -> shift_expr','and_expr',1,'p_and_expr_1','hgawk_grammar.py',1364), - ('and_expr -> shift_expr and_expr_star','and_expr',2,'p_and_expr_2','hgawk_grammar.py',1368), - ('and_expr_star -> AMPER shift_expr','and_expr_star',2,'p_and_expr_star_1','hgawk_grammar.py',1373), - ('and_expr_star -> and_expr_star AMPER shift_expr','and_expr_star',3,'p_and_expr_star_2','hgawk_grammar.py',1377), - ('shift_expr -> arith_expr','shift_expr',1,'p_shift_expr_1','hgawk_grammar.py',1383), - ('shift_expr -> arith_expr shift_expr_star','shift_expr',2,'p_shift_expr_2','hgawk_grammar.py',1387), - ('shift_expr_star -> LEFTSHIFT arith_expr','shift_expr_star',2,'p_shift_expr_star_1','hgawk_grammar.py',1392), - ('shift_expr_star -> RIGHTSHIFT arith_expr','shift_expr_star',2,'p_shift_expr_star_2','hgawk_grammar.py',1396), - ('shift_expr_star -> shift_expr_star LEFTSHIFT arith_expr','shift_expr_star',3,'p_shift_expr_star_3','hgawk_grammar.py',1400), - ('shift_expr_star -> shift_expr_star RIGHTSHIFT arith_expr','shift_expr_star',3,'p_shift_expr_star_4','hgawk_grammar.py',1404), - ('arith_expr -> term','arith_expr',1,'p_arith_expr_1','hgawk_grammar.py',1410), - ('arith_expr -> term arith_expr_star','arith_expr',2,'p_arith_expr_2','hgawk_grammar.py',1414), - ('arith_expr_star -> PLUS term','arith_expr_star',2,'p_arith_expr_star_1','hgawk_grammar.py',1419), - ('arith_expr_star -> MINUS term','arith_expr_star',2,'p_arith_expr_star_2','hgawk_grammar.py',1423), - ('arith_expr_star -> arith_expr_star PLUS term','arith_expr_star',3,'p_arith_expr_star_3','hgawk_grammar.py',1427), - ('arith_expr_star -> arith_expr_star MINUS term','arith_expr_star',3,'p_arith_expr_star_4','hgawk_grammar.py',1431), - ('term -> factor','term',1,'p_term_1','hgawk_grammar.py',1437), - ('term -> factor term_star','term',2,'p_term_2','hgawk_grammar.py',1441), - ('term_star -> STAR factor','term_star',2,'p_term_star_1','hgawk_grammar.py',1446), - ('term_star -> SLASH factor','term_star',2,'p_term_star_2','hgawk_grammar.py',1450), - ('term_star -> PERCENT factor','term_star',2,'p_term_star_3','hgawk_grammar.py',1454), - ('term_star -> DOUBLESLASH factor','term_star',2,'p_term_star_4','hgawk_grammar.py',1458), - ('term_star -> term_star STAR factor','term_star',3,'p_term_star_5','hgawk_grammar.py',1462), - ('term_star -> term_star SLASH factor','term_star',3,'p_term_star_6','hgawk_grammar.py',1466), - ('term_star -> term_star PERCENT factor','term_star',3,'p_term_star_7','hgawk_grammar.py',1470), - ('term_star -> term_star DOUBLESLASH factor','term_star',3,'p_term_star_8','hgawk_grammar.py',1474), - ('factor -> PLUS factor','factor',2,'p_factor_1','hgawk_grammar.py',1480), - ('factor -> MINUS factor','factor',2,'p_factor_2','hgawk_grammar.py',1486), - ('factor -> TILDE factor','factor',2,'p_factor_3','hgawk_grammar.py',1498), - ('factor -> power','factor',1,'p_factor_4','hgawk_grammar.py',1504), - ('power -> atom','power',1,'p_power_1','hgawk_grammar.py',1510), - ('power -> atom DOUBLESTAR factor','power',3,'p_power_2','hgawk_grammar.py',1514), - ('power -> atom power_star','power',2,'p_power_3','hgawk_grammar.py',1519), - ('power -> atom power_star DOUBLESTAR factor','power',4,'p_power_4','hgawk_grammar.py',1523), - ('power_star -> trailer','power_star',1,'p_power_star_1','hgawk_grammar.py',1529), - ('power_star -> power_star trailer','power_star',2,'p_power_star_2','hgawk_grammar.py',1533), - ('atom -> LPAR RPAR','atom',2,'p_atom_1','hgawk_grammar.py',1543), - ('atom -> LPAR yield_expr RPAR','atom',3,'p_atom_2','hgawk_grammar.py',1547), - ('atom -> LPAR testlist_comp RPAR','atom',3,'p_atom_3','hgawk_grammar.py',1554), - ('atom -> LSQB RSQB','atom',2,'p_atom_4','hgawk_grammar.py',1561), - ('atom -> LSQB listmaker RSQB','atom',3,'p_atom_5','hgawk_grammar.py',1565), - ('atom -> LBRACE RBRACE','atom',2,'p_atom_6','hgawk_grammar.py',1573), - ('atom -> LBRACE dictorsetmaker RBRACE','atom',3,'p_atom_7','hgawk_grammar.py',1577), - ('atom -> BACKQUOTE testlist1 BACKQUOTE','atom',3,'p_atom_8','hgawk_grammar.py',1589), - ('atom -> NAME','atom',1,'p_atom_9','hgawk_grammar.py',1593), - ('atom -> NUMBER','atom',1,'p_atom_10','hgawk_grammar.py',1597), - ('atom -> atom_plus','atom',1,'p_atom_11','hgawk_grammar.py',1601), - ('atom -> DOLLARNUMBER','atom',1,'p_atom_12','hgawk_grammar.py',1605), - ('atom_plus -> STRING','atom_plus',1,'p_atom_plus_1','hgawk_grammar.py',1610), - ('atom_plus -> atom_plus STRING','atom_plus',2,'p_atom_plus_2','hgawk_grammar.py',1614), - ('listmaker -> test list_for','listmaker',2,'p_listmaker_1','hgawk_grammar.py',1621), - ('listmaker -> test','listmaker',1,'p_listmaker_2','hgawk_grammar.py',1626), - ('listmaker -> test COMMA','listmaker',2,'p_listmaker_3','hgawk_grammar.py',1630), - ('listmaker -> test listmaker_star','listmaker',2,'p_listmaker_4','hgawk_grammar.py',1634), - ('listmaker -> test listmaker_star COMMA','listmaker',3,'p_listmaker_5','hgawk_grammar.py',1638), - ('listmaker_star -> COMMA test','listmaker_star',2,'p_listmaker_star_1','hgawk_grammar.py',1643), - ('listmaker_star -> listmaker_star COMMA test','listmaker_star',3,'p_listmaker_star_2','hgawk_grammar.py',1647), - ('testlist_comp -> test comp_for','testlist_comp',2,'p_testlist_comp_1','hgawk_grammar.py',1653), - ('testlist_comp -> test','testlist_comp',1,'p_testlist_comp_2','hgawk_grammar.py',1658), - ('testlist_comp -> test COMMA','testlist_comp',2,'p_testlist_comp_3','hgawk_grammar.py',1662), - ('testlist_comp -> test testlist_comp_star','testlist_comp',2,'p_testlist_comp_4','hgawk_grammar.py',1667), - ('testlist_comp -> test testlist_comp_star COMMA','testlist_comp',3,'p_testlist_comp_5','hgawk_grammar.py',1672), - ('testlist_comp_star -> COMMA test','testlist_comp_star',2,'p_testlist_comp_star_1','hgawk_grammar.py',1678), - ('testlist_comp_star -> testlist_comp_star COMMA test','testlist_comp_star',3,'p_testlist_comp_star_2','hgawk_grammar.py',1682), - ('lambdef -> LAMBDA COLON test','lambdef',3,'p_lambdef_1','hgawk_grammar.py',1688), - ('lambdef -> LAMBDA varargslist COLON test','lambdef',4,'p_lambdef_2','hgawk_grammar.py',1692), - ('trailer -> LPAR RPAR','trailer',2,'p_trailer_1','hgawk_grammar.py',1698), - ('trailer -> LPAR arglist RPAR','trailer',3,'p_trailer_2','hgawk_grammar.py',1702), - ('trailer -> LSQB subscriptlist RSQB','trailer',3,'p_trailer_3','hgawk_grammar.py',1706), - ('trailer -> DOT NAME','trailer',2,'p_trailer_4','hgawk_grammar.py',1710), - ('subscriptlist -> subscript','subscriptlist',1,'p_subscriptlist_1','hgawk_grammar.py',1716), - ('subscriptlist -> subscript COMMA','subscriptlist',2,'p_subscriptlist_2','hgawk_grammar.py',1720), - ('subscriptlist -> subscript subscriptlist_star','subscriptlist',2,'p_subscriptlist_3','hgawk_grammar.py',1731), - ('subscriptlist -> subscript subscriptlist_star COMMA','subscriptlist',3,'p_subscriptlist_4','hgawk_grammar.py',1743), - ('subscriptlist_star -> COMMA subscript','subscriptlist_star',2,'p_subscriptlist_star_1','hgawk_grammar.py',1756), - ('subscriptlist_star -> subscriptlist_star COMMA subscript','subscriptlist_star',3,'p_subscriptlist_star_2','hgawk_grammar.py',1760), - ('subscript -> DOT DOT DOT','subscript',3,'p_subscript_1','hgawk_grammar.py',1766), - ('subscript -> test','subscript',1,'p_subscript_2','hgawk_grammar.py',1770), - ('subscript -> COLON','subscript',1,'p_subscript_3','hgawk_grammar.py',1775), - ('subscript -> COLON sliceop','subscript',2,'p_subscript_4','hgawk_grammar.py',1779), - ('subscript -> COLON test','subscript',2,'p_subscript_5','hgawk_grammar.py',1783), - ('subscript -> COLON test sliceop','subscript',3,'p_subscript_6','hgawk_grammar.py',1787), - ('subscript -> test COLON','subscript',2,'p_subscript_7','hgawk_grammar.py',1791), - ('subscript -> test COLON sliceop','subscript',3,'p_subscript_8','hgawk_grammar.py',1796), - ('subscript -> test COLON test','subscript',3,'p_subscript_9','hgawk_grammar.py',1801), - ('subscript -> test COLON test sliceop','subscript',4,'p_subscript_10','hgawk_grammar.py',1806), - ('sliceop -> COLON','sliceop',1,'p_sliceop_1','hgawk_grammar.py',1813), - ('sliceop -> COLON test','sliceop',2,'p_sliceop_2','hgawk_grammar.py',1817), - ('exprlist -> expr','exprlist',1,'p_exprlist_1','hgawk_grammar.py',1823), - ('exprlist -> expr COMMA','exprlist',2,'p_exprlist_2','hgawk_grammar.py',1827), - ('exprlist -> expr exprlist_star','exprlist',2,'p_exprlist_3','hgawk_grammar.py',1832), - ('exprlist -> expr exprlist_star COMMA','exprlist',3,'p_exprlist_4','hgawk_grammar.py',1837), - ('exprlist_star -> COMMA expr','exprlist_star',2,'p_exprlist_star_1','hgawk_grammar.py',1843), - ('exprlist_star -> exprlist_star COMMA expr','exprlist_star',3,'p_exprlist_star_2','hgawk_grammar.py',1847), - ('testlist -> test','testlist',1,'p_testlist_1','hgawk_grammar.py',1853), - ('testlist -> test COMMA','testlist',2,'p_testlist_2','hgawk_grammar.py',1857), - ('testlist -> test testlist_star','testlist',2,'p_testlist_3','hgawk_grammar.py',1862), - ('testlist -> test testlist_star COMMA','testlist',3,'p_testlist_4','hgawk_grammar.py',1867), - ('testlist_star -> COMMA test','testlist_star',2,'p_testlist_star_1','hgawk_grammar.py',1873), - ('testlist_star -> testlist_star COMMA test','testlist_star',3,'p_testlist_star_2','hgawk_grammar.py',1877), - ('dictorsetmaker -> test COLON test comp_for','dictorsetmaker',4,'p_dictorsetmaker_1','hgawk_grammar.py',1884), - ('dictorsetmaker -> test COLON test','dictorsetmaker',3,'p_dictorsetmaker_2','hgawk_grammar.py',1889), - ('dictorsetmaker -> test COLON test COMMA','dictorsetmaker',4,'p_dictorsetmaker_3','hgawk_grammar.py',1893), - ('dictorsetmaker -> test COLON test dictorsetmaker_star','dictorsetmaker',4,'p_dictorsetmaker_4','hgawk_grammar.py',1897), - ('dictorsetmaker -> test COLON test dictorsetmaker_star COMMA','dictorsetmaker',5,'p_dictorsetmaker_5','hgawk_grammar.py',1902), - ('dictorsetmaker -> test comp_for','dictorsetmaker',2,'p_dictorsetmaker_6','hgawk_grammar.py',1907), - ('dictorsetmaker -> test','dictorsetmaker',1,'p_dictorsetmaker_7','hgawk_grammar.py',1912), - ('dictorsetmaker -> test COMMA','dictorsetmaker',2,'p_dictorsetmaker_8','hgawk_grammar.py',1916), - ('dictorsetmaker -> test dictorsetmaker_star2','dictorsetmaker',2,'p_dictorsetmaker_9','hgawk_grammar.py',1920), - ('dictorsetmaker -> test dictorsetmaker_star2 COMMA','dictorsetmaker',3,'p_dictorsetmaker_10','hgawk_grammar.py',1925), - ('dictorsetmaker_star -> COMMA test COLON test','dictorsetmaker_star',4,'p_dictorsetmaker_star_1','hgawk_grammar.py',1931), - ('dictorsetmaker_star -> dictorsetmaker_star COMMA test COLON test','dictorsetmaker_star',5,'p_dictorsetmaker_star_2','hgawk_grammar.py',1935), - ('dictorsetmaker_star2 -> COMMA test','dictorsetmaker_star2',2,'p_dictorsetmaker_star2_1','hgawk_grammar.py',1941), - ('dictorsetmaker_star2 -> dictorsetmaker_star2 COMMA test','dictorsetmaker_star2',3,'p_dictorsetmaker_star2_2','hgawk_grammar.py',1945), - ('classdef -> CLASS NAME COLON suite','classdef',4,'p_classdef_1','hgawk_grammar.py',1952), - ('classdef -> CLASS NAME LPAR RPAR COLON suite','classdef',6,'p_classdef_2','hgawk_grammar.py',1956), - ('classdef -> CLASS NAME LPAR testlist RPAR COLON suite','classdef',7,'p_classdef_3','hgawk_grammar.py',1960), - ('arglist -> argument','arglist',1,'p_arglist_1','hgawk_grammar.py',1971), - ('arglist -> argument COMMA','arglist',2,'p_arglist_2','hgawk_grammar.py',1978), - ('arglist -> STAR test','arglist',2,'p_arglist_3','hgawk_grammar.py',1985), - ('arglist -> STAR test COMMA DOUBLESTAR test','arglist',5,'p_arglist_4','hgawk_grammar.py',1989), - ('arglist -> STAR test arglist_star','arglist',3,'p_arglist_5','hgawk_grammar.py',1993), - ('arglist -> STAR test arglist_star COMMA DOUBLESTAR test','arglist',6,'p_arglist_6','hgawk_grammar.py',1997), - ('arglist -> DOUBLESTAR test','arglist',2,'p_arglist_7','hgawk_grammar.py',2001), - ('arglist -> arglist_star2 argument','arglist',2,'p_arglist_8','hgawk_grammar.py',2005), - ('arglist -> arglist_star2 argument COMMA','arglist',3,'p_arglist_9','hgawk_grammar.py',2010), - ('arglist -> arglist_star2 STAR test','arglist',3,'p_arglist_10','hgawk_grammar.py',2015), - ('arglist -> arglist_star2 STAR test COMMA DOUBLESTAR test','arglist',6,'p_arglist_11','hgawk_grammar.py',2019), - ('arglist -> arglist_star2 STAR test arglist_star3','arglist',4,'p_arglist_12','hgawk_grammar.py',2023), - ('arglist -> arglist_star2 STAR test arglist_star3 COMMA DOUBLESTAR test','arglist',7,'p_arglist_13','hgawk_grammar.py',2028), - ('arglist -> arglist_star2 DOUBLESTAR test','arglist',3,'p_arglist_14','hgawk_grammar.py',2033), - ('arglist_star -> COMMA argument','arglist_star',2,'p_arglist_star_1','hgawk_grammar.py',2038), - ('arglist_star -> arglist_star COMMA argument','arglist_star',3,'p_arglist_star_2','hgawk_grammar.py',2042), - ('arglist_star3 -> COMMA argument','arglist_star3',2,'p_arglist_star3_1','hgawk_grammar.py',2047), - ('arglist_star3 -> arglist_star3 COMMA argument','arglist_star3',3,'p_arglist_star3_2','hgawk_grammar.py',2051), - ('arglist_star2 -> argument COMMA','arglist_star2',2,'p_arglist_star2_1','hgawk_grammar.py',2056), - ('arglist_star2 -> arglist_star2 argument COMMA','arglist_star2',3,'p_arglist_star2_2','hgawk_grammar.py',2060), - ('argument -> test','argument',1,'p_argument_1','hgawk_grammar.py',2066), - ('argument -> test comp_for','argument',2,'p_argument_2','hgawk_grammar.py',2070), - ('argument -> test EQUAL test','argument',3,'p_argument_3','hgawk_grammar.py',2075), - ('list_iter -> list_for','list_iter',1,'p_list_iter_1','hgawk_grammar.py',2082), - ('list_iter -> list_if','list_iter',1,'p_list_iter_2','hgawk_grammar.py',2086), - ('list_for -> FOR exprlist IN testlist_safe','list_for',4,'p_list_for_1','hgawk_grammar.py',2092), - ('list_for -> FOR exprlist IN testlist_safe list_iter','list_for',5,'p_list_for_2','hgawk_grammar.py',2097), - ('list_if -> IF old_test','list_if',2,'p_list_if_1','hgawk_grammar.py',2105), - ('list_if -> IF old_test list_iter','list_if',3,'p_list_if_2','hgawk_grammar.py',2109), - ('comp_iter -> comp_for','comp_iter',1,'p_comp_iter_1','hgawk_grammar.py',2116), - ('comp_iter -> comp_if','comp_iter',1,'p_comp_iter_2','hgawk_grammar.py',2120), - ('comp_for -> FOR exprlist IN or_test','comp_for',4,'p_comp_for_1','hgawk_grammar.py',2126), - ('comp_for -> FOR exprlist IN or_test comp_iter','comp_for',5,'p_comp_for_2','hgawk_grammar.py',2131), - ('comp_if -> IF old_test','comp_if',2,'p_comp_if_1','hgawk_grammar.py',2139), - ('comp_if -> IF old_test comp_iter','comp_if',3,'p_comp_if_2','hgawk_grammar.py',2143), - ('testlist1 -> test','testlist1',1,'p_testlist1_1','hgawk_grammar.py',2150), - ('testlist1 -> test testlist1_star','testlist1',2,'p_testlist1_2','hgawk_grammar.py',2154), - ('testlist1_star -> COMMA test','testlist1_star',2,'p_testlist1_star_1','hgawk_grammar.py',2160), - ('testlist1_star -> testlist1_star COMMA test','testlist1_star',3,'p_testlist1_star_2','hgawk_grammar.py',2164), - ('encoding_decl -> NAME','encoding_decl',1,'p_encoding_decl','hgawk_grammar.py',2170), - ('yield_expr -> YIELD','yield_expr',1,'p_yield_expr_1','hgawk_grammar.py',2176), - ('yield_expr -> YIELD testlist','yield_expr',2,'p_yield_expr_2','hgawk_grammar.py',2180), -] diff --git a/histogrammar/notebooks/histogrammar_tutorial_advanced.ipynb b/histogrammar/notebooks/histogrammar_tutorial_advanced.ipynb index 62b9bb7..7e99bde 100644 --- a/histogrammar/notebooks/histogrammar_tutorial_advanced.ipynb +++ b/histogrammar/notebooks/histogrammar_tutorial_advanced.ipynb @@ -31,7 +31,6 @@ "source": [ "%%capture\n", "# install histogrammar (if not installed yet)\n", - "import sys\n", "\n", "!\"{sys.executable}\" -m pip install histogrammar" ] @@ -51,9 +50,7 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib" + "import pandas as pd" ] }, { @@ -72,6 +69,7 @@ "source": [ "# open a pandas dataframe for use below\n", "from histogrammar import resources\n", + "\n", "df = pd.read_csv(resources.data(\"test.csv.gz\"), parse_dates=[\"date\"])" ] }, @@ -101,9 +99,9 @@ "source": [ "# download histogrammar jar files if not already installed, used for histogramming of spark dataframe\n", "try:\n", + " from pyspark import __version__ as pyspark_version\n", " from pyspark.sql import SparkSession\n", " from pyspark.sql.functions import col\n", - " from pyspark import __version__ as pyspark_version\n", " pyspark_installed = True\n", "except ImportError:\n", " print(\"pyspark needs to be installed for this example\")\n", @@ -123,7 +121,7 @@ " scala = '2.12' if int(pyspark_version[0]) >= 3 else '2.11'\n", " hist_jar = f'io.github.histogrammar:histogrammar_{scala}:1.0.20'\n", " hist_spark_jar = f'io.github.histogrammar:histogrammar-sparksql_{scala}:1.0.20'\n", - " \n", + "\n", " spark = SparkSession.builder.config(\n", " \"spark.jars.packages\", f'{hist_spark_jar},{hist_jar}'\n", " ).getOrCreate()\n", @@ -182,7 +180,7 @@ "if pyspark_installed:\n", " hist = hg.SparselyHistogram(binWidth=100, quantity=col('transaction'))\n", " hist.fill.sparksql(sdf)\n", - " hist.plot.matplotlib();" + " hist.plot.matplotlib()" ] }, { @@ -202,7 +200,7 @@ "source": [ "if pyspark_installed:\n", " h2 = sdf.hg_SparselyProfileErr(25, col('longitude'), col('age'))\n", - " h2.plot.matplotlib();" + " h2.plot.matplotlib()" ] }, { @@ -213,7 +211,7 @@ "source": [ "if pyspark_installed:\n", " h3 = sdf.hg_TwoDimensionallySparselyHistogram(25, col('longitude'), 10, col('latitude'))\n", - " h3.plot.matplotlib();" + " h3.plot.matplotlib()" ] }, { @@ -456,7 +454,7 @@ " 'z': {'deviate': True}, # Deviate histogram\n", " 'f': {'average': True}, # Average histogram\n", " 'a:f': [{'edges': [0, 10, 101]}, {'average': True}], # IrregularlyBin vs Average histograms\n", - " 'g': {'thresholds': [0, 2, 10, 11, 21, 101]}, # Stack histogram \n", + " 'g': {'thresholds': [0, 2, 10, 11, 21, 101]}, # Stack histogram\n", " 'h': {'bag': True}, # Bag histogram\n", " }\n", "\n", @@ -471,7 +469,7 @@ "metadata": {}, "outputs": [], "source": [ - "# For example \n", + "# For example\n", "features = ['latitude:age', 'longitude:age', 'age', 'longitude']\n", "\n", "bin_specs = {\n", diff --git a/histogrammar/notebooks/histogrammar_tutorial_basic.ipynb b/histogrammar/notebooks/histogrammar_tutorial_basic.ipynb index 1adc044..588fc2d 100644 --- a/histogrammar/notebooks/histogrammar_tutorial_basic.ipynb +++ b/histogrammar/notebooks/histogrammar_tutorial_basic.ipynb @@ -27,7 +27,6 @@ "source": [ "%%capture\n", "# install histogrammar (if not installed yet)\n", - "import sys\n", "\n", "!\"{sys.executable}\" -m pip install histogrammar" ] @@ -47,9 +46,8 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", "import numpy as np\n", - "import matplotlib" + "import pandas as pd" ] }, { @@ -68,6 +66,7 @@ "source": [ "# open a pandas dataframe for use below\n", "from histogrammar import resources\n", + "\n", "df = pd.read_csv(resources.data(\"test.csv.gz\"), parse_dates=[\"date\"])" ] }, @@ -252,7 +251,7 @@ "# Alternatively, do:\n", "hist3 = df.hg_SparselyBin(binWidth=10, origin=0, quantity='age')\n", "\n", - "# ... where hist3 automatically picks up column age from the dataframe, \n", + "# ... where hist3 automatically picks up column age from the dataframe,\n", "# ... and does not need to be filled by calling fill.numpy() explicitly." ] }, @@ -455,6 +454,7 @@ "outputs": [], "source": [ "from histogrammar.plot.hist_numpy import get_2dgrid\n", + "\n", "x_labels, y_labels, grid = get_2dgrid(hist2)" ] }, @@ -670,7 +670,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Stack counts how often data points are greater or equal to the provided thresholds \n", + "# Stack counts how often data points are greater or equal to the provided thresholds\n", "thresholds = [-200, -150, -100, -50, 0, 50, 100, 150, 200]" ] }, @@ -914,6 +914,7 @@ "source": [ "# we can store the histograms if we want to\n", "import json\n", + "\n", "from histogrammar.util import dumper\n", "\n", "# store\n", diff --git a/histogrammar/notebooks/histogrammar_tutorial_exercises.ipynb b/histogrammar/notebooks/histogrammar_tutorial_exercises.ipynb index 5867f75..63d34c6 100644 --- a/histogrammar/notebooks/histogrammar_tutorial_exercises.ipynb +++ b/histogrammar/notebooks/histogrammar_tutorial_exercises.ipynb @@ -23,7 +23,6 @@ "source": [ "%%capture\n", "# install histogrammar (if not installed yet)\n", - "import sys\n", "\n", "!\"{sys.executable}\" -m pip install histogrammar" ] @@ -43,9 +42,7 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib" + "import pandas as pd" ] }, { @@ -64,6 +61,7 @@ "source": [ "# open a pandas dataframe for use below\n", "from histogrammar import resources\n", + "\n", "df = pd.read_csv(resources.data(\"test.csv.gz\"), parse_dates=[\"date\"])" ] }, diff --git a/histogrammar/parsing.py b/histogrammar/parsing.py deleted file mode 100644 index 9770112..0000000 --- a/histogrammar/parsing.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 DIANA-HEP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import histogrammar.pycparser.c_parser -import histogrammar.pycparser.c_generator -import histogrammar.pycparser.c_ast - - -class C99SourceToAst(object): - def __init__(self, wholeFile=False): - self.wholeFile = wholeFile - self.parser = histogrammar.pycparser.c_parser.CParser( - lextab="histogrammar.pycparser.lextab", yacctab="histogrammar.pycparser.yacctab") - - def __call__(self, src): - if self.wholeFile: - return self.parser.parse(src) - else: - src = "void wrappedAsFcn() {" + src + ";}" - ast = self.parser.parse(src).ext[0].body.block_items - if len(ast) < 1: - raise SyntaxError("empty expression") - else: - return [x for x in ast if not isinstance(x, histogrammar.pycparser.c_ast.EmptyStatement)] - - -class C99AstToSource(object): - def __init__(self): - self.generator = histogrammar.pycparser.c_generator.CGenerator() - - def __call__(self, ast): - if isinstance(ast, (list, tuple)): - return "; ".join(self.generator.visit(x).strip() for x in ast) - return self.generator.visit(ast).strip() diff --git a/histogrammar/plot/bokeh.py b/histogrammar/plot/bokeh.py index 59dc419..b288f2c 100644 --- a/histogrammar/plot/bokeh.py +++ b/histogrammar/plot/bokeh.py @@ -15,7 +15,6 @@ # limitations under the License. # "Public" methods; what we want to attach to the Histogram as a mix-in. -from __future__ import absolute_import import math @@ -23,491 +22,547 @@ # from histogrammar.util import * -class HistogramMethods(object): - def plotbokeh(self, glyphType="line", glyphSize=1, fillColor="red", - lineColor="black", lineAlpha=1, fillAlpha=0.1, lineDash='solid'): - +class HistogramMethods: + def plotbokeh( + self, + glyphType="line", + glyphSize=1, + fillColor="red", + lineColor="black", + lineAlpha=1, + fillAlpha=0.1, + lineDash="solid", + ): # glyphs - from bokeh.models.glyphs import Rect, Line - from bokeh.models.renderers import GlyphRenderer - from bokeh.models.markers import (Circle, Cross, - Diamond, Square, - Triangle) - # data from bokeh.models import ColumnDataSource + from bokeh.models.glyphs import Line, Rect + from bokeh.models.markers import Circle, Cross, Diamond, Square, Triangle + from bokeh.models.renderers import GlyphRenderer # Parameters of the histogram lo = self.low hi = self.high num = self.num - bin_width = (hi-lo)/num - x = list() + bin_width = (hi - lo) / num + x = [] center = lo for _ in range(num): - x.append(center+bin_width/2) + x.append(center + bin_width / 2) center += bin_width y = self.numericalValues - ci = [2.*v for v in self.confidenceIntervalValues()] + ci = [2.0 * v for v in self.confidenceIntervalValues()] - source = ColumnDataSource(data=dict(x=x, y=y, ci=ci)) + source = ColumnDataSource(data={"x": x, "y": y, "ci": ci}) glyph = None if glyphType == "square": glyph = Square( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "diamond": glyph = Diamond( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "cross": glyph = Cross( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "triangle": glyph = Triangle( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "circle": glyph = Circle( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "rect": glyph = Rect( - x='x', - y='y', + x="x", + y="y", width=bin_width, height=0.1, fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) elif glyphType == "errors": glyph = Rect( - x='x', - y='y', + x="x", + y="y", width=bin_width, - height='ci', + height="ci", fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) elif glyphType == "histogram": h = y - y = [yy/2 for yy in y] - source = ColumnDataSource(dict(x=x, y=y, h=h)) + y = [yy / 2 for yy in y] + source = ColumnDataSource({"x": x, "y": y, "h": h}) glyph = Rect( - x='x', - y='y', + x="x", + y="y", width=bin_width, - height='h', + height="h", fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) else: glyph = Line( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, line_alpha=lineAlpha, line_width=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) return GlyphRenderer(glyph=glyph, data_source=source) -class SparselyHistogramMethods(object): - def plotbokeh(self, glyphType="line", glyphSize=1, fillColor="red", - lineColor="black", lineAlpha=1, fillAlpha=0.1, lineDash='solid'): - +class SparselyHistogramMethods: + def plotbokeh( + self, + glyphType="line", + glyphSize=1, + fillColor="red", + lineColor="black", + lineAlpha=1, + fillAlpha=0.1, + lineDash="solid", + ): # glyphs - from bokeh.models.glyphs import Rect, Line - from bokeh.models.renderers import GlyphRenderer - from bokeh.models.markers import (Circle, Cross, - Diamond, Square, - Triangle) - # data from bokeh.models import ColumnDataSource + from bokeh.models.glyphs import Line, Rect + from bokeh.models.markers import Circle, Cross, Diamond, Square, Triangle + from bokeh.models.renderers import GlyphRenderer # Parameters of the histogram lo = self.low hi = self.high num = self.numFilled - bin_width = (hi-lo)/num - x = list() + bin_width = (hi - lo) / num + x = [] center = lo for _ in range(num): - x.append(center+bin_width/2) + x.append(center + bin_width / 2) center += bin_width y = [v.entries for _, v in sorted(self.bins.items())] - source = ColumnDataSource(data=dict(x=x, y=y)) + source = ColumnDataSource(data={"x": x, "y": y}) glyph = None if glyphType == "square": glyph = Square( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "diamond": glyph = Diamond( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "cross": glyph = Cross( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "triangle": glyph = Triangle( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "circle": glyph = Circle( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "rect": glyph = Rect( - x='x', - y='y', + x="x", + y="y", width=bin_width, height=0.1, fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) elif glyphType == "errors": - ci = [2.*v for v in self.confidenceIntervalValues()] - source = ColumnDataSource(data=dict(x=x, y=y, ci=ci)) + ci = [2.0 * v for v in self.confidenceIntervalValues()] + source = ColumnDataSource(data={"x": x, "y": y, "ci": ci}) glyph = Rect( - x='x', - y='y', + x="x", + y="y", width=bin_width, - height='ci', + height="ci", fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) elif glyphType == "histogram": h = y - y = [yy/2 for yy in y] - source = ColumnDataSource(dict(x=x, y=y, h=h)) + y = [yy / 2 for yy in y] + source = ColumnDataSource({"x": x, "y": y, "h": h}) glyph = Rect( - x='x', - y='y', + x="x", + y="y", width=bin_width, - height='h', + height="h", fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) else: glyph = Line( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, line_alpha=lineAlpha, line_width=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) return GlyphRenderer(glyph=glyph, data_source=source) -class CategorizeHistogramMethods(object): +class CategorizeHistogramMethods: pass -class IrregularlyHistogramMethods(object): +class IrregularlyHistogramMethods: pass -class CentrallyHistogramMethods(object): +class CentrallyHistogramMethods: pass -class ProfileMethods(object): - def plotbokeh(self, glyphType="line", glyphSize=1, fillColor="red", - lineColor="black", lineAlpha=1, fillAlpha=0.1, lineDash='solid'): - +class ProfileMethods: + def plotbokeh( + self, + glyphType="line", + glyphSize=1, + fillColor="red", + lineColor="black", + lineAlpha=1, + fillAlpha=0.1, + lineDash="solid", + ): # glyphs - from bokeh.models.glyphs import Rect, Line - from bokeh.models.renderers import GlyphRenderer - from bokeh.models.markers import (Circle, Cross, - Diamond, Square, - Triangle) - # data from bokeh.models import ColumnDataSource + from bokeh.models.glyphs import Line, Rect + from bokeh.models.markers import Circle, Cross, Diamond, Square, Triangle + from bokeh.models.renderers import GlyphRenderer # Parameters of the histogram lo = self.low hi = self.high num = self.num - bin_width = (hi-lo)/num - x = list() - y = list() + bin_width = (hi - lo) / num + x = [] + y = [] center = lo for v in self.values: if not math.isnan(v.mean): y.append(v.mean) - x.append(center+bin_width/2) + x.append(center + bin_width / 2) center += bin_width - source = ColumnDataSource(data=dict(x=x, y=y)) + source = ColumnDataSource(data={"x": x, "y": y}) glyph = None if glyphType == "square": glyph = Square( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "diamond": glyph = Diamond( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "cross": glyph = Cross( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "triangle": glyph = Triangle( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "circle": glyph = Circle( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "histogram": w = [bin_width for _ in x] h = y - y = [yy/2 for yy in y] - source = ColumnDataSource(dict(x=x, y=y, w=w, h=h)) + y = [yy / 2 for yy in y] + source = ColumnDataSource({"x": x, "y": y, "w": w, "h": h}) glyph = Rect( - x='x', - y='y', - width='w', - height='h', + x="x", + y="y", + width="w", + height="h", fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) else: glyph = Line( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, line_alpha=lineAlpha, line_width=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) return GlyphRenderer(glyph=glyph, data_source=source) -class SparselyProfileMethods(object): +class SparselyProfileMethods: pass -class ProfileErrMethods(object): - def plotbokeh(self, glyphType="line", glyphSize=1, fillColor="red", - lineColor="black", lineAlpha=1, fillAlpha=0.1, lineDash='solid'): - +class ProfileErrMethods: + def plotbokeh( + self, + glyphType="line", + glyphSize=1, + fillColor="red", + lineColor="black", + lineAlpha=1, + fillAlpha=0.1, + lineDash="solid", + ): # glyphs - from bokeh.models.glyphs import Rect, Line - from bokeh.models.renderers import GlyphRenderer - from bokeh.models.markers import (Circle, Cross, - Diamond, Square, - Triangle) + from math import sqrt # data from bokeh.models import ColumnDataSource - - from math import sqrt + from bokeh.models.glyphs import Line, Rect + from bokeh.models.markers import Circle, Cross, Diamond, Square, Triangle + from bokeh.models.renderers import GlyphRenderer # Parameters of the histogram lo = self.low hi = self.high num = self.num - bin_width = (hi-lo)/num - x = list() - y = list() + bin_width = (hi - lo) / num + x = [] + y = [] center = lo for v in self.values: if not math.isnan(v.mean): y.append(v.mean) - x.append(center+bin_width/2) + x.append(center + bin_width / 2) center += bin_width - source = ColumnDataSource(data=dict(x=x, y=y)) + source = ColumnDataSource(data={"x": x, "y": y}) glyph = None if glyphType == "square": glyph = Square( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "diamond": glyph = Diamond( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "cross": glyph = Cross( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "triangle": glyph = Triangle( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "circle": glyph = Circle( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, fill_color=fillColor, line_alpha=lineAlpha, size=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) elif glyphType == "errors": w = [bin_width for _ in x] - h = [sqrt(v.variance/v.entries) if v.entries > 0 else 0.0 for v in self.values] - source = ColumnDataSource(dict(x=x, y=y, w=w, h=h)) + h = [sqrt(v.variance / v.entries) if v.entries > 0 else 0.0 for v in self.values] + source = ColumnDataSource({"x": x, "y": y, "w": w, "h": h}) glyph = Rect( - x='x', - y='y', - width='w', - height='h', + x="x", + y="y", + width="w", + height="h", fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) elif glyphType == "histogram": w = [bin_width for _ in x] h = y - y = [yy/2 for yy in y] - source = ColumnDataSource(dict(x=x, y=y, w=w, h=h)) + y = [yy / 2 for yy in y] + source = ColumnDataSource({"x": x, "y": y, "w": w, "h": h}) glyph = Rect( - x='x', - y='y', - width='w', - height='h', + x="x", + y="y", + width="w", + height="h", fill_alpha=fillAlpha, line_color=lineColor, - fill_color=fillColor) + fill_color=fillColor, + ) else: glyph = Line( - x='x', - y='y', + x="x", + y="y", line_color=lineColor, line_alpha=lineAlpha, line_width=glyphSize, - line_dash=lineDash) + line_dash=lineDash, + ) return GlyphRenderer(glyph=glyph, data_source=source) -class SparselyProfileErrMethods(object): +class SparselyProfileErrMethods: pass -class StackedHistogramMethods(object): +class StackedHistogramMethods: nMaxStacked = 10 - glyphTypeDefaults = ["circle"]*nMaxStacked - glyphSizeDefaults = [1]*nMaxStacked - fillColorDefaults = ["red"]*nMaxStacked - lineColorDefaults = ["red"]*nMaxStacked - lineAlphaDefaults = [1]*nMaxStacked - fillAlphaDefaults = [0.1]*nMaxStacked - lineDashDefaults = ["solid"]*nMaxStacked - - def plotbokeh(self, glyphTypes=glyphTypeDefaults, glyphSizes=glyphSizeDefaults, fillColors=fillColorDefaults, - lineColors=lineColorDefaults, lineAlphas=lineAlphaDefaults, fillAlphas=fillAlphaDefaults, - lineDashes=lineDashDefaults): - nChildren = len(self.children)-1 + glyphTypeDefaults = ["circle"] * nMaxStacked + glyphSizeDefaults = [1] * nMaxStacked + fillColorDefaults = ["red"] * nMaxStacked + lineColorDefaults = ["red"] * nMaxStacked + lineAlphaDefaults = [1] * nMaxStacked + fillAlphaDefaults = [0.1] * nMaxStacked + lineDashDefaults = ["solid"] * nMaxStacked + + def plotbokeh( + self, + glyphTypes=glyphTypeDefaults, + glyphSizes=glyphSizeDefaults, + fillColors=fillColorDefaults, + lineColors=lineColorDefaults, + lineAlphas=lineAlphaDefaults, + fillAlphas=fillAlphaDefaults, + lineDashes=lineDashDefaults, + ): + nChildren = len(self.children) - 1 assert len(glyphSizes) >= nChildren assert len(glyphTypes) >= nChildren @@ -517,68 +572,70 @@ def plotbokeh(self, glyphTypes=glyphTypeDefaults, glyphSizes=glyphSizeDefaults, assert len(fillAlphas) >= nChildren assert len(lineDashes) >= nChildren - stackedGlyphs = list() + stackedGlyphs = [] # for ichild, p in enumerate(self.children,start=1): for ichild in range(nChildren): - stackedGlyphs.append(self.children[ichild+1].plotbokeh(glyphTypes[ichild], - glyphSizes[ichild], - fillColors[ichild], - lineColors[ichild], - lineAlphas[ichild], - fillAlphas[ichild], - lineDashes[ichild])) + stackedGlyphs.append( + self.children[ichild + 1].plotbokeh( + glyphTypes[ichild], + glyphSizes[ichild], + fillColors[ichild], + lineColors[ichild], + lineAlphas[ichild], + fillAlphas[ichild], + lineDashes[ichild], + ) + ) return stackedGlyphs -class PartitionedHistogramMethods(object): +class PartitionedHistogramMethods: pass -class FractionedHistogramMethods(object): +class FractionedHistogramMethods: pass -class TwoDimensionallyHistogramMethods(object): +class TwoDimensionallyHistogramMethods: pass -class SparselyTwoDimensionallyHistogramMethods(object): +class SparselyTwoDimensionallyHistogramMethods: pass -class IrregularlyTwoDimensionallyHistogramMethods(object): +class IrregularlyTwoDimensionallyHistogramMethods: pass -class CentrallyTwoDimensionallyHistogramMethods(object): +class CentrallyTwoDimensionallyHistogramMethods: pass -def plot(xLabel='x', yLabel='y', *args): - - from bokeh.models import DataRange1d, Plot, LinearAxis - from bokeh.models import PanTool, WheelZoomTool +def plot(xLabel="x", yLabel="y", *args): + from bokeh.models import DataRange1d, LinearAxis, PanTool, Plot, WheelZoomTool xdr = DataRange1d() ydr = DataRange1d() plot = Plot(x_range=xdr, y_range=ydr, min_border=80) - extra = list() + extra = [] if not isinstance(xLabel, str) and not isinstance(yLabel, str): extra.append(xLabel) extra.append(yLabel) - xLabel = 'x' - yLabel = 'y' + xLabel = "x" + yLabel = "y" elif not isinstance(xLabel, str): extra.append(xLabel) - xLabel = 'x' + xLabel = "x" elif not isinstance(yLabel, str): extra.append(yLabel) - yLabel = 'y' + yLabel = "y" - args = extra+list(args) + args = extra + list(args) for renderer in args: if not isinstance(renderer, list): plot.renderers.append(renderer) @@ -587,9 +644,9 @@ def plot(xLabel='x', yLabel='y', *args): # axes xaxis = LinearAxis(axis_label=xLabel) - plot.add_layout(xaxis, 'below') + plot.add_layout(xaxis, "below") yaxis = LinearAxis(axis_label=yLabel) - plot.add_layout(yaxis, 'left') + plot.add_layout(yaxis, "left") # add grid to the plot # plot.add_layout(Grid(dimension=0, ticker=xaxis.ticker)) # plot.add_layout(Grid(dimension=1, ticker=yaxis.ticker)) @@ -602,14 +659,16 @@ def plot(xLabel='x', yLabel='y', *args): def save(plot, fname): # SaveTool https://github.com/bokeh/bokeh/blob/118b6a765ee79232b1fef0e82ed968a9dbb0e17f/examples/models/line.py - from bokeh.io import save, output_file + from bokeh.io import output_file, save + output_file(fname) save(plot) def view(plot, show=False): - from bokeh.plotting import curdoc from bokeh.client import push_session + from bokeh.plotting import curdoc + if show: session = push_session(curdoc()) session.show(plot) diff --git a/histogrammar/plot/hist_numpy.py b/histogrammar/plot/hist_numpy.py index 7a90982..379b5aa 100644 --- a/histogrammar/plot/hist_numpy.py +++ b/histogrammar/plot/hist_numpy.py @@ -19,6 +19,7 @@ import warnings + import numpy as np @@ -31,11 +32,7 @@ def prepare_2dgrid(hist): :return: two comma-separated lists of unique x and y keys """ if hist.n_dim < 2: - warnings.warn( - "Input histogram only has {n} dimensions (<2). Returning empty lists.".format( - n=hist.n_dim - ) - ) + warnings.warn(f"Input histogram only has {hist.n_dim} dimensions (<2). Returning empty lists.") return [], [] xkeys = set() @@ -75,11 +72,7 @@ def set_2dgrid(hist, xkeys, ykeys): grid = np.zeros((len(ykeys), len(xkeys))) if hist.n_dim < 2: - warnings.warn( - "Input histogram only has {n} dimensions (<2). Returning original grid.".format( - n=hist.n_dim - ) - ) + warnings.warn(f"Input histogram only has {hist.n_dim} dimensions (<2). Returning original grid.") return grid # SparselyBin, Categorize, IrregularlyBin, CentrallyBin @@ -91,10 +84,10 @@ def set_2dgrid(hist, xkeys, ykeys): i = xkeys.index(k) if hasattr(h, "bins"): h_bins = dict(h.bins) - for l, g in h_bins.items(): - if l not in ykeys: + for li, g in h_bins.items(): + if li not in ykeys: continue - j = ykeys.index(l) + j = ykeys.index(li) grid[j, i] = g.entries elif hasattr(h, "values"): for j, g in enumerate(h.values): @@ -104,10 +97,10 @@ def set_2dgrid(hist, xkeys, ykeys): for i, h in enumerate(hist.values): if hasattr(h, "bins"): h_bins = dict(h.bins) - for l, g in h_bins.items(): - if l not in ykeys: + for lj, g in h_bins.items(): + if lj not in ykeys: continue - j = ykeys.index(l) + j = ykeys.index(lj) grid[j, i] = g.entries elif hasattr(h, "values"): for j, g in enumerate(h.values): @@ -121,14 +114,8 @@ def get_2dgrid(hist): :param hist: input histogrammar histogram :return: x,y,grid of first two dimenstions of input histogram """ - import numpy as np - if hist.n_dim < 2: - warnings.warn( - "Input histogram only has {n} dimensions (<2). Returning empty grid.".format( - n=hist.n_dim - ) - ) + warnings.warn(f"Input histogram only has {hist.n_dim} dimensions (<2). Returning empty grid.") return np.zeros((0, 0)) xkeys, ykeys = prepare_2dgrid(hist) @@ -141,8 +128,7 @@ def get_2dgrid(hist): def get_x_labels(hist, xkeys): - xlabels = [str(hist._center_from_key(key)) for key in xkeys] - return xlabels + return [str(hist._center_from_key(key)) for key in xkeys] def get_y_labels(hist, ykeys): @@ -153,8 +139,7 @@ def get_y_labels(hist, ykeys): # Bin elif hasattr(hist, "values"): h = hist.values[0] - ylabels = [str(h._center_from_key(key)) for key in ykeys] - return ylabels + return [str(h._center_from_key(key)) for key in ykeys] def prepare2Dsparse(sparse): diff --git a/histogrammar/plot/matplotlib.py b/histogrammar/plot/matplotlib.py index 096898e..f4725ef 100644 --- a/histogrammar/plot/matplotlib.py +++ b/histogrammar/plot/matplotlib.py @@ -14,22 +14,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import -# python 2/3 compatibility fixes -from histogrammar.util import xrange +import numpy as np + from histogrammar.plot.hist_numpy import get_2dgrid, prepare2Dsparse, set2Dsparse +# python 2/3 compatibility fixes +from histogrammar.util import xrange # 1d plotting of counts + generic 2d plotting of counts -class HistogramMethods(object): + +class HistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ - name : title of the plot. - kwargs : `matplotlib.patches.Rectangle` properties. + """name : title of the plot. + kwargs : `matplotlib.patches.Rectangle` properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ # catch generic 2d plotting of counts if self.n_dim >= 2: @@ -37,13 +38,14 @@ def plotmatplotlib(self, name=None, **kwargs): # specialized 1d plotting of counts from here on import matplotlib.pyplot as plt + ax = plt.gca() edges = self.bin_edges() entries = self.bin_entries() width = self.bin_width() - ax.bar(edges[:-1], entries, width=width, align='edge', **kwargs) + ax.bar(edges[:-1], entries, width=width, align="edge", **kwargs) if name is not None: ax.set_title(name) @@ -53,13 +55,12 @@ def plotmatplotlib(self, name=None, **kwargs): return ax -class SparselyHistogramMethods(object): +class SparselyHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ - name : title of the plot. - kwargs : `matplotlib.patches.Rectangle` properties. + """name : title of the plot. + kwargs : `matplotlib.patches.Rectangle` properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ # catch generic 2d plotting of counts if self.n_dim >= 2: @@ -67,13 +68,14 @@ def plotmatplotlib(self, name=None, **kwargs): # specialized 1d plotting of counts from here on import matplotlib.pyplot as plt + ax = plt.gca() edges = self.bin_edges() entries = self.bin_entries() width = self.bin_width() - ax.bar(edges[:-1], entries, width=width, align='edge', **kwargs) + ax.bar(edges[:-1], entries, width=width, align="edge", **kwargs) ax.set_xlim(self.low, self.high) if name is not None: @@ -84,13 +86,12 @@ def plotmatplotlib(self, name=None, **kwargs): return ax -class IrregularlyHistogramMethods(object): +class IrregularlyHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ - name : title of the plot. - kwargs : `matplotlib.patches.Rectangle` properties. + """name : title of the plot. + kwargs : `matplotlib.patches.Rectangle` properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ # catch generic 2d plotting of counts if self.n_dim >= 2: @@ -98,13 +99,14 @@ def plotmatplotlib(self, name=None, **kwargs): # specialized 1d plotting of counts from here on import matplotlib.pyplot as plt + ax = plt.gca() edges = self.edges[1:-1] entries = self.bin_entries()[1:-1] width = self.bin_width() - ax.bar(edges, entries, width=width, align='edge', **kwargs) + ax.bar(edges, entries, width=width, align="edge", **kwargs) if name is not None: ax.set_title(name) @@ -114,13 +116,12 @@ def plotmatplotlib(self, name=None, **kwargs): return ax -class CentrallyHistogramMethods(object): +class CentrallyHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ - name : title of the plot. - kwargs : `matplotlib.patches.Rectangle` properties. + """name : title of the plot. + kwargs : `matplotlib.patches.Rectangle` properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ # catch generic 2d plotting of counts if self.n_dim >= 2: @@ -128,16 +129,17 @@ def plotmatplotlib(self, name=None, **kwargs): # specialized 1d plotting of counts from here on import matplotlib.pyplot as plt - import numpy as np + ax = plt.gca() - width = kwargs.pop('width', 0.8) + width = kwargs.pop("width", 0.8) labels = self.bin_centers() values = self.bin_entries() - assert len(labels) == len(values), \ - 'labels and values have different array lengths: %d vs %d.' % \ - (len(labels), len(values)) + assert len(labels) == len(values), "labels and values have different array lengths: %d vs %d." % ( + len(labels), + len(values), + ) # plot histogram tick_pos = np.arange(len(labels)) + 0.5 @@ -147,9 +149,10 @@ def plotmatplotlib(self, name=None, **kwargs): def xtick(lab): lab = str(lab) if len(lab) > 20: - lab = lab[:17] + '...' + lab = lab[:17] + "..." return lab - ax.set_xlim((0., float(len(labels)))) + + ax.set_xlim((0.0, float(len(labels)))) ax.set_xticks(tick_pos) ax.set_xticklabels([xtick(lab) for lab in labels], fontsize=12, rotation=90) @@ -162,13 +165,12 @@ def xtick(lab): return ax -class CategorizeHistogramMethods(object): +class CategorizeHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ - name : title of the plot. - kwargs : `matplotlib.patches.Rectangle` properties. + """name : title of the plot. + kwargs : `matplotlib.patches.Rectangle` properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ # catch generic 2d plotting of counts if self.n_dim >= 2: @@ -176,16 +178,17 @@ def plotmatplotlib(self, name=None, **kwargs): # specialized 1d plotting of counts from here on import matplotlib.pyplot as plt - import numpy as np + ax = plt.gca() - width = kwargs.pop('width', 0.8) + width = kwargs.pop("width", 0.8) labels = self.bin_labels() values = self.bin_entries() - assert len(labels) == len(values), \ - 'labels and values have different array lengths: %d vs %d.' % \ - (len(labels), len(values)) + assert len(labels) == len(values), "labels and values have different array lengths: %d vs %d." % ( + len(labels), + len(values), + ) # sort labels alphabetically idx = np.argsort(labels) @@ -200,9 +203,10 @@ def plotmatplotlib(self, name=None, **kwargs): def xtick(lab): lab = str(lab) if len(lab) > 20: - lab = lab[:17] + '...' + lab = lab[:17] + "..." return lab - ax.set_xlim((0., float(len(labels)))) + + ax.set_xlim((0.0, float(len(labels)))) ax.set_xticks(tick_pos) ax.set_xticklabels([xtick(lab) for lab in labels], fontsize=12, rotation=90) @@ -217,15 +221,17 @@ def xtick(lab): # 1d plotting of profiles -class ProfileMethods(object): + +class ProfileMethods: def plotmatplotlib(self, name=None, **kwargs): - """ Plotting method for Bin of Average - name : title of the plot. - kwargs : matplotlib.collections.LineCollection properties. + """Plotting method for Bin of Average + name : title of the plot. + kwargs : matplotlib.collections.LineCollection properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ import matplotlib.pyplot as plt + ax = plt.gca() xranges = [self.range(x) for x in self.indexes] @@ -241,22 +247,22 @@ def plotmatplotlib(self, name=None, **kwargs): return ax -class SparselyProfileMethods(object): +class SparselyProfileMethods: def plotmatplotlib(self, name=None, **kwargs): - """ Plotting method for SparselyBin of Average - name : title of the plot. - kwargs : matplotlib.collections.LineCollection properties. + """Plotting method for SparselyBin of Average + name : title of the plot. + kwargs : matplotlib.collections.LineCollection properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ import matplotlib.pyplot as plt - import numpy as np + ax = plt.gca() xmins = np.arange(self.low, self.high, self.binWidth) xmaxs = np.arange(self.low + self.binWidth, self.high + self.binWidth, self.binWidth) - means = np.nan*np.ones(xmaxs.shape) + means = np.nan * np.ones(xmaxs.shape) for i in xrange(self.minBin, self.maxBin + 1): if i in self.bins: @@ -272,20 +278,20 @@ def plotmatplotlib(self, name=None, **kwargs): return ax -class ProfileErrMethods(object): +class ProfileErrMethods: def plotmatplotlib(self, name=None, aspect=True, **kwargs): - """ Plotting method for Bin of Deviate - name : title of the plot. - aspect : - kwargs : `matplotlib.collections.LineCollection` properties. + """Plotting method for Bin of Deviate + name : title of the plot. + aspect : + kwargs : `matplotlib.collections.LineCollection` properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ import matplotlib.pyplot as plt - import numpy as np + ax = plt.gca() - bin_centers = [sum(self.range(x))/2.0 for x in self.indexes] + bin_centers = [sum(self.range(x)) / 2.0 for x in self.indexes] xranges = [self.range(x) for x in self.indexes] means = self.meanValues variances = self.varianceValues @@ -297,8 +303,8 @@ def plotmatplotlib(self, name=None, aspect=True, **kwargs): if aspect is True: counts = [p.entries for p in self.values] - ymins = [means[i] - np.sqrt(variances[i])/np.sqrt(counts[i]) for i in range(num_bins)] - ymaxs = [means[i] + np.sqrt(variances[i])/np.sqrt(counts[i]) for i in range(num_bins)] + ymins = [means[i] - np.sqrt(variances[i]) / np.sqrt(counts[i]) for i in range(num_bins)] + ymaxs = [means[i] + np.sqrt(variances[i]) / np.sqrt(counts[i]) for i in range(num_bins)] else: ymins = [means[i] - np.sqrt(variances[i]) for i in range(num_bins)] ymaxs = [means[i] + np.sqrt(variances[i]) for i in range(num_bins)] @@ -312,20 +318,19 @@ def plotmatplotlib(self, name=None, aspect=True, **kwargs): return ax -class SparselyProfileErrMethods(object): +class SparselyProfileErrMethods: def plotmatplotlib(self, name=None, aspect=True, **kwargs): - """ Plotting method for - """ + """Plotting method for""" import matplotlib.pyplot as plt - import numpy as np + ax = plt.gca() xmins = np.arange(self.low, self.high, self.binWidth) xmaxs = np.arange(self.low + self.binWidth, self.high + self.binWidth, self.binWidth) - means = np.nan*np.ones(xmaxs.shape) - variances = np.nan*np.ones(xmaxs.shape) - counts = np.nan*np.ones(xmaxs.shape) + means = np.nan * np.ones(xmaxs.shape) + variances = np.nan * np.ones(xmaxs.shape) + counts = np.nan * np.ones(xmaxs.shape) for i in xrange(self.minBin, self.maxBin + 1): if i in self.bins: @@ -342,10 +347,10 @@ def plotmatplotlib(self, name=None, aspect=True, **kwargs): ax.hlines(means, xmins, xmaxs, **kwargs) - bin_centers = (self.binWidth/2.0) + xmins + bin_centers = (self.binWidth / 2.0) + xmins if aspect is True: - ymins = [means[i] - np.sqrt(variances[i])/np.sqrt(counts[i]) for i in xrange(len(means))] - ymaxs = [means[i] + np.sqrt(variances[i])/np.sqrt(counts[i]) for i in xrange(len(means))] + ymins = [means[i] - np.sqrt(variances[i]) / np.sqrt(counts[i]) for i in xrange(len(means))] + ymaxs = [means[i] + np.sqrt(variances[i]) / np.sqrt(counts[i]) for i in xrange(len(means))] else: ymins = [means[i] - np.sqrt(variances[i]) for i in xrange(len(means))] ymaxs = [means[i] + np.sqrt(variances[i]) for i in xrange(len(means))] @@ -361,13 +366,14 @@ def plotmatplotlib(self, name=None, aspect=True, **kwargs): # other 1d/2d plotting -class StackedHistogramMethods(object): + +class StackedHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ Plotting method for - """ + """Plotting method for""" import matplotlib.pyplot as plt + ax = plt.gca() - color_cycle = plt.rcParams['axes.color_cycle'] + color_cycle = plt.rcParams["axes.color_cycle"] if "color" in kwargs: kwargs.pop("color") @@ -383,13 +389,13 @@ def plotmatplotlib(self, name=None, **kwargs): return ax -class PartitionedHistogramMethods(object): +class PartitionedHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ Plotting method for - """ + """Plotting method for""" import matplotlib.pyplot as plt + ax = plt.gca() - color_cycle = plt.rcParams['axes.color_cycle'] + color_cycle = plt.rcParams["axes.color_cycle"] if "color" in kwargs: kwargs.pop("color") @@ -405,12 +411,11 @@ def plotmatplotlib(self, name=None, **kwargs): return ax -class FractionedHistogramMethods(object): +class FractionedHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ Plotting method for - """ + """Plotting method for""" import matplotlib.pyplot as plt - import numpy as np + ax = plt.gca() if isinstance(self.numerator, HistogramMethods): @@ -422,19 +427,19 @@ def plotmatplotlib(self, name=None, **kwargs): ax.hlines(fracs, xmins, xmaxs, **kwargs) elif isinstance(self.numerator, SparselyHistogramMethods): - assert self.numerator.binWidth == self.denominator.binWidth,\ - "Fraction numerator and denominator histograms must have same binWidth." + assert ( + self.numerator.binWidth == self.denominator.binWidth + ), "Fraction numerator and denominator histograms must have same binWidth." numerator = self.numerator denominator = self.denominator xmins = np.arange(numerator.low, numerator.high, numerator.binWidth) xmaxs = np.arange( - numerator.low + + numerator.low + numerator.binWidth, + numerator.high + numerator.binWidth, numerator.binWidth, - numerator.high + - numerator.binWidth, - numerator.binWidth) + ) - fracs = np.nan*np.zeros(xmaxs.shape) + fracs = np.nan * np.zeros(xmaxs.shape) for i in xrange(denominator.minBin, denominator.maxBin + 1): if i in self.numerator.bins and i in self.denominator.bins: @@ -452,15 +457,17 @@ def plotmatplotlib(self, name=None, **kwargs): # specialized 2d plotting of counts -class TwoDimensionallyHistogramMethods(object): + +class TwoDimensionallyHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ Plotting method for Bin of Bin of Count - name : title of the plot. - kwargs: matplotlib.collections.QuadMesh properties. + """Plotting method for Bin of Bin of Count + name : title of the plot. + kwargs: matplotlib.collections.QuadMesh properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ import matplotlib.pyplot as plt + fig, ax = plt.subplots(nrows=1) x_ranges, y_ranges, grid = self.xy_ranges_grid() @@ -478,10 +485,7 @@ def plotmatplotlib(self, name=None, **kwargs): return ax def xy_ranges_grid(self): - """ Return x and y ranges and x,y grid - """ - import numpy as np - + """Return x and y ranges and x,y grid""" samp = self.values[0] x_ranges = np.unique(np.array([self.range(i) for i in self.indexes]).flatten()) y_ranges = np.unique(np.array([samp.range(i) for i in samp.indexes]).flatten()) @@ -495,32 +499,36 @@ def xy_ranges_grid(self): return x_ranges, y_ranges, grid def x_lim(self): - """ return x low high tuble - """ + """return x low high tuble""" return (self.low, self.high) def y_lim(self): - """ return y low high tuble - """ + """return y low high tuble""" samp = self.values[0] return (samp.low, samp.high) def project_on_x(self): - """ project 2d histogram onto x-axis + """project 2d histogram onto x-axis :returns: on x-axis projected histogram (1d) :rtype: histogrammar.Bin """ from histogrammar import Bin, Count - h_x = Bin(num=self.num, low=self.low, high=self.high, quantity=self.quantity, value=Count()) + h_x = Bin( + num=self.num, + low=self.low, + high=self.high, + quantity=self.quantity, + value=Count(), + ) # loop over all counters and integrate over y (=j) for i, bi in enumerate(self.values): h_x.values[i].entries += sum(bj.entries for bj in bi.values) return h_x def project_on_y(self): - """ project 2d histogram onto y-axis + """project 2d histogram onto y-axis :returns: on y-axis projected histogram (1d) :rtype: histogrammar.Bin @@ -528,7 +536,13 @@ def project_on_y(self): from histogrammar import Bin, Count ybin = self.values[0] - h_y = Bin(num=ybin.num, low=ybin.low, high=ybin.high, quantity=ybin.quantity, value=Count()) + h_y = Bin( + num=ybin.num, + low=ybin.low, + high=ybin.high, + quantity=ybin.quantity, + value=Count(), + ) # loop over all counters and integrate over x (=i) for bi in self.values: for j, bj in enumerate(bi.values): @@ -536,15 +550,16 @@ def project_on_y(self): return h_y -class SparselyTwoDimensionallyHistogramMethods(object): +class SparselyTwoDimensionallyHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ Plotting method for SparselyBin of SparselyBin of Count - name : title of the plot. - kwargs: matplotlib.collections.QuadMesh properties. + """Plotting method for SparselyBin of SparselyBin of Count + name : title of the plot. + kwargs: matplotlib.collections.QuadMesh properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ import matplotlib.pyplot as plt + fig, ax = plt.subplots(nrows=1) x_ranges, y_ranges, grid = self.xy_ranges_grid() @@ -562,17 +577,14 @@ def plotmatplotlib(self, name=None, **kwargs): return ax def xy_ranges_grid(self): - """ Return x and y ranges and x,y grid - """ - import numpy as np - + """Return x and y ranges and x,y grid""" yminBin, ymaxBin, ynum, ylow, yhigh = prepare2Dsparse(self) xbinWidth = self.binWidth try: ykey = list(self.bins.keys())[0] except BaseException: - raise KeyError('SparselyBin 2d hist is not filled.') + raise KeyError("SparselyBin 2d hist is not filled.") ybinWidth = self.bins[ykey].binWidth xmaxBin = max(self.bins.keys()) @@ -589,8 +601,7 @@ def xy_ranges_grid(self): return x_ranges, y_ranges, grid def x_lim(self): - """ return x low high tuble - """ + """return x low high tuble""" xmaxBin = max(self.bins.keys()) xminBin = min(self.bins.keys()) xlow = xminBin * self.binWidth + self.origin @@ -598,21 +609,24 @@ def x_lim(self): return (xlow, xhigh) def y_lim(self): - """ return y low high tuble - """ + """return y low high tuble""" yminBin, ymaxBin, ynum, ylow, yhigh = prepare2Dsparse(self) return (ylow, yhigh) def project_on_x(self): - """ project 2d sparselybin histogram onto x-axis + """project 2d sparselybin histogram onto x-axis :returns: on x-axis projected histogram (1d) :rtype: histogrammar.SparselyBin """ - from histogrammar import SparselyBin, Count + from histogrammar import Count, SparselyBin - h_x = SparselyBin(binWidth=self.binWidth, origin=self.origin, - quantity=self.quantity, value=Count()) + h_x = SparselyBin( + binWidth=self.binWidth, + origin=self.origin, + quantity=self.quantity, + value=Count(), + ) # loop over all counters and integrate over y (=j) for i in self.bins: bi = self.bins[i] @@ -620,20 +634,24 @@ def project_on_x(self): return h_x def project_on_y(self): - """ project 2d sparselybin histogram onto y-axis + """project 2d sparselybin histogram onto y-axis :returns: on y-axis projected histogram (1d) :rtype: histogrammar.SparselyBin """ - from histogrammar import SparselyBin, Count + from histogrammar import Count, SparselyBin try: ykey = list(self.bins.keys())[0] except BaseException: - raise KeyError('SparselyBin 2d hist is not filled. Cannot project on y-axis.') + raise KeyError("SparselyBin 2d hist is not filled. Cannot project on y-axis.") ybin = self.bins[ykey] - h_y = SparselyBin(binWidth=ybin.binWidth, origin=ybin.origin, - quantity=ybin.quantity, value=Count()) + h_y = SparselyBin( + binWidth=ybin.binWidth, + origin=ybin.origin, + quantity=ybin.quantity, + value=Count(), + ) # loop over all counters and integrate over x (=i) for i in self.bins: bi = self.bins[i] @@ -644,15 +662,16 @@ def project_on_y(self): return h_y -class IrregularlyTwoDimensionallyHistogramMethods(object): +class IrregularlyTwoDimensionallyHistogramMethods: def plotmatplotlib(self, name=None, **kwargs): - """ Plotting method for Bin of Bin of Count - name : title of the plot. - kwargs: matplotlib.collections.QuadMesh properties. + """Plotting method for Bin of Bin of Count + name : title of the plot. + kwargs: matplotlib.collections.QuadMesh properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ import matplotlib.pyplot as plt + fig, ax = plt.subplots(nrows=1) x_ranges, y_ranges, grid = self.xy_ranges_grid() @@ -670,10 +689,7 @@ def plotmatplotlib(self, name=None, **kwargs): return ax def xy_ranges_grid(self): - """ Return x and y ranges and x,y grid - """ - import numpy as np - + """Return x and y ranges and x,y grid""" samp = self.bins[0][1] x_ranges = self.bin_edges()[1:-1] # cut underflow and overflow bins y_ranges = samp.bin_edges()[1:-1] @@ -682,32 +698,30 @@ def xy_ranges_grid(self): for j in range(1, self.n_bins - 1): for i in range(1, samp.n_bins - 1): - grid[i-1, j-1] = (self.bins[j][1]).bins[i][1].entries + grid[i - 1, j - 1] = (self.bins[j][1]).bins[i][1].entries return x_ranges, y_ranges, grid def x_lim(self): - """ return x low high tuble - """ + """return x low high tuble""" # cut underflow and overflow bins x_ranges = self.bin_edges()[1:-1] return (x_ranges[0], x_ranges[-1]) def y_lim(self): - """ return y low high tuble - """ + """return y low high tuble""" # cut underflow and overflow bins samp = self.bins[0][1] y_ranges = samp.bin_edges()[1:-1] return (y_ranges[0], y_ranges[-1]) def project_on_x(self): - """ project 2d histogram onto x-axis + """project 2d histogram onto x-axis :returns: on x-axis projected histogram (1d) :rtype: histogrammar.Bin """ - from histogrammar import IrregularlyBin, Count + from histogrammar import Count, IrregularlyBin h_x = IrregularlyBin(edges=self.edges[1:], quantity=self.quantity, value=Count()) # loop over all counters and integrate over y (=j) @@ -716,12 +730,12 @@ def project_on_x(self): return h_x def project_on_y(self): - """ project 2d histogram onto y-axis + """project 2d histogram onto y-axis :returns: on y-axis projected histogram (1d) :rtype: histogrammar.Bin """ - from histogrammar import IrregularlyBin, Count + from histogrammar import Count, IrregularlyBin ybin = self.bins[0][1] h_y = IrregularlyBin(edges=ybin.edges[1:], quantity=ybin.quantity, value=Count()) @@ -734,15 +748,16 @@ def project_on_y(self): # generic 2d plotting function of counts + def plot2dmatplotlib(self, name=None, **kwargs): - """ General plotting method for 2d Bin/SparselyBin/Categorize/CentrallyBin/IrregularlyBin of Count - name : title of the plot. - kwargs: matplotlib.collections.QuadMesh properties. + """General plotting method for 2d Bin/SparselyBin/Categorize/CentrallyBin/IrregularlyBin of Count + name : title of the plot. + kwargs: matplotlib.collections.QuadMesh properties. - Returns a matplotlib.axes instance + Returns a matplotlib.axes instance """ import matplotlib.pyplot as plt - import numpy as np + fig, ax = plt.subplots(nrows=1) x_labels, y_labels, grid = get_2dgrid(self) @@ -751,8 +766,9 @@ def plot2dmatplotlib(self, name=None, **kwargs): def tick(lab): lab = str(lab) if len(lab) > 20: - lab = lab[:17] + '...' + lab = lab[:17] + "..." return lab + x_labels = [tick(lab) for lab in x_labels] y_labels = [tick(lab) for lab in y_labels] @@ -764,13 +780,13 @@ def tick(lab): ytick_pos = np.arange(len(y_labels)) + 0.5 # always needs egdes, so one more than grid length - im = ax.pcolormesh(xedges, yedges, grid, shading='auto', **kwargs) + im = ax.pcolormesh(xedges, yedges, grid, shading="auto", **kwargs) fig.colorbar(im, ax=ax) ax.set_xticks(xtick_pos) ax.set_yticks(ytick_pos) - ax.set_xlim((0., float(len(x_labels)))) - ax.set_ylim((0., float(len(y_labels)))) + ax.set_xlim((0.0, float(len(x_labels)))) + ax.set_ylim((0.0, float(len(y_labels)))) ax.set_xticklabels(x_labels, rotation=90) ax.set_yticklabels(y_labels) diff --git a/histogrammar/plot/root.py b/histogrammar/plot/root.py deleted file mode 100644 index 3fd1574..0000000 --- a/histogrammar/plot/root.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 DIANA-HEP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# "Private" methods; not attached to the histogram because not a member of the class, -# but within scope because it's a closure. - -import math -import types - -# python 2/3 compatibility fixes -from histogrammar.util import xrange - -try: - from collections import OrderedDict -except ImportError: - class OrderedDict(dict): - def __init__(self): - self._data = [] - - def __setattr__(self, key, value): - self._data.append((key, value)) - - def __getattr__(self, key): - return dict(self._data)[key] - - def items(self): - return self._data - - def keys(self): - return [k for k, v in self._data] - - def values(self): - return [v for k, v in self._data] - - -def setTH1(entries, values, underflow, overflow, th1): - th1.SetBinContent(0, underflow) - for i, v in enumerate(values): - th1.SetBinContent(i + 1, v) - th1.SetBinContent(len(values) + 1, overflow) - th1.SetEntries(entries) - - -def prepareTH2sparse(sparse): - sample = list(sparse.bins.values())[0] - yminBins = [v.minBin for v in sparse.bins.values() if v.minBin is not None] - ymaxBins = [v.maxBin for v in sparse.bins.values() if v.maxBin is not None] - if len(yminBins) > 0 and len(ymaxBins) > 0: - yminBin = min(yminBins) - ymaxBin = max(ymaxBins) - else: - yminBin = 0 - ymaxBin = 0 - ynum = 1 + ymaxBin - yminBin - ylow = yminBin * sample.binWidth + sample.origin - yhigh = (ymaxBin + 1) * sample.binWidth + sample.origin - return yminBin, ymaxBin, ynum, ylow, yhigh - - -def setTH2sparse(sparse, yminBin, ymaxBin, th2): - for i, iindex in enumerate(xrange(sparse.minBin, sparse.maxBin + 1)): - for j, jindex in enumerate(xrange(yminBin, ymaxBin + 1)): - if iindex in sparse.bins and jindex in sparse.bins[iindex].bins: - th2.SetBinContent(i + 1, j + 1, sparse.bins[iindex].bins[jindex].entries) - -# "Public" methods; what we want to attach to the Histogram as a mix-in. - - -class HistogramMethods(object): - def plotroot(self, name, title="", binType="D"): - import ROOT - constructor = getattr(ROOT, "TH1" + binType) - th1 = constructor(name, title, len(self.values), self.low, self.high) - setTH1(self.entries, [x.entries for x in self.values], self.underflow.entries, self.overflow.entries, th1) - return th1 - - -class SparselyHistogramMethods(object): - def plotroot(self, name, title="", binType="D"): - import ROOT - constructor = getattr(ROOT, "TH1" + binType) - if self.minBin is None or self.maxBin is None: - th1 = constructor(name, title, 1, self.origin, self.origin + 1.0) - else: - size = 1 + self.maxBin - self.minBin - th1 = constructor(name, title, size, self.low, self.high) - setTH1( - self.entries, [ - self.bins[i].entries if i in self.bins else 0.0 for i in xrange( - self.minBin, self.maxBin + 1)], 0.0, 0.0, th1) - return th1 - - -class IrregularlyHistogramMethods(object): - pass - - -class CentrallyHistogramMethods(object): - pass - - -class CategorizeHistogramMethods(object): - def plotroot(self, name, title="", binType="C"): - """ Construct a ROOT histogram - - :param str name: name of the histogram - :param str title: title of the histogram (optional) - :param str binType: histogram bin type. Default is "C" (char). - :returns: ROOT histgram - """ - import ROOT - constructor = getattr(ROOT, "TH1" + binType) - th1 = constructor(name, title, len(self.bins), 0, 1) - th1.SetMinimum(0) - for i, key in enumerate(self.bins.keys()): - b = self.bins[key] - try: - label = str(key) - except BaseException: - label = 'bin_%d' % i - th1.Fill(label, b.entries) - return th1 - - -class ProfileMethods(object): - def plotroot(self, name, title=""): - import ROOT - tprofile = ROOT.TProfile(name, title, len(self.values), self.low, self.high) - tprofile.SetBinContent(0, self.underflow.entries*self.underflow.entries) - tprofile.SetBinEntries(0, self.underflow.entries) - for i, v in enumerate(self.values): - if not math.isnan(v.mean): - tprofile.SetBinError(i + 1, math.sqrt(v.entries) * v.mean) - tprofile.SetBinContent(i + 1, v.entries * v.mean) - tprofile.SetBinEntries(i + 1, v.entries) - tprofile.SetBinContent(len(self.values) + 1, self.overflow.entries*self.overflow.entries) - tprofile.SetBinEntries(len(self.values) + 1, self.overflow.entries) - tprofile.SetEntries(self.entries) - return tprofile - - -class SparselyProfileMethods(object): - def plotroot(self, name, title=""): - import ROOT - if self.minBin is None or self.maxBin is None: - tprofile = ROOT.TProfile(name, title, 1, self.origin, self.origin + 1.0) - else: - tprofile = ROOT.TProfile(name, title, 1 + self.maxBin - self.minBin, self.low, self.high) - for i, index in enumerate(xrange(self.minBin, self.maxBin + 1)): - if index in self.bins: - v = self.bins[index] - if not math.isnan(v.mean): - tprofile.SetBinError(i + 1, math.sqrt(v.entries) * v.mean) - tprofile.SetBinContent(i + 1, v.entries * v.mean) - tprofile.SetBinEntries(i + 1, v.entries) - tprofile.SetBinContent(0, 0.0) - tprofile.SetBinEntries(0, 0.0) - tprofile.SetBinContent(1 + self.maxBin - self.minBin, 0.0) - tprofile.SetBinEntries(1 + self.maxBin - self.minBin, 0.0) - tprofile.SetEntries(self.entries) - return tprofile - - -class ProfileErrMethods(object): - def plotroot(self, name, title=""): - import ROOT - tprofile = ROOT.TProfile(name, title, len(self.values), self.low, self.high) - tprofile.SetBinContent(0, self.underflow.entries*self.underflow.entries) - tprofile.SetBinEntries(0, self.underflow.entries) - for i, v in enumerate(self.values): - if not math.isnan(v.mean): - tprofile.SetBinError(i + 1, math.sqrt(v.entries*(v.variance + v.mean*v.mean))) - tprofile.SetBinContent(i + 1, v.entries * v.mean) - tprofile.SetBinEntries(i + 1, v.entries) - tprofile.SetBinContent(len(self.values) + 1, self.overflow.entries*self.overflow.entries) - tprofile.SetBinEntries(len(self.values) + 1, self.overflow.entries) - tprofile.SetEntries(self.entries) - return tprofile - - -class SparselyProfileErrMethods(object): - def plotroot(self, name, title=""): - import ROOT - if self.minBin is None or self.maxBin is None: - tprofile = ROOT.TProfile(name, title, 1, self.origin, self.origin + 1.0) - else: - tprofile = ROOT.TProfile(name, title, 1 + self.maxBin - self.minBin, self.low, self.high) - for i, index in enumerate(xrange(self.minBin, self.maxBin + 1)): - if index in self.bins: - v = self.bins[index] - if not math.isnan(v.mean): - tprofile.SetBinError(i + 1, math.sqrt(v.entries*(v.variance + v.mean*v.mean))) - tprofile.SetBinContent(i + 1, v.entries * v.mean) - tprofile.SetBinEntries(i + 1, v.entries) - tprofile.SetBinContent(0, 0.0) - tprofile.SetBinEntries(0, 0.0) - tprofile.SetBinContent(1 + self.maxBin - self.minBin, 0.0) - tprofile.SetBinEntries(1 + self.maxBin - self.minBin, 0.0) - tprofile.SetEntries(self.entries) - return tprofile - - -class StackedHistogramMethods(object): - def plotroot(self, *names): - out = OrderedDict() - for n, (c, v) in zip(names, self.thresholds): - if isinstance(n, (list, tuple)) and len(n) == 2: - name, title = n - else: - name, title = n, "" - out[c] = v.plotroot(name, title) - - def Draw(self, options=""): - first = True - for v in self.values(): - v.Draw(options) - if first: - options = options + "same" - first = False - - out.Draw = types.MethodType(Draw, out) - return out - - -class PartitionedHistogramMethods(object): - def plotroot(self, *names): - out = OrderedDict() - for n, (c, v) in zip(names, self.thresholds): - if isinstance(n, (list, tuple)) and len(n) == 2: - name, title = n - else: - name, title = n, "" - out[c] = v.plotroot(name, title) - - def Draw(self, options=""): - first = True - for v in self.values(): - v.Draw(options) - if first: - options = options + "same" - first = False - - out.Draw = types.MethodType(Draw, out) - return out - - -class FractionedHistogramMethods(object): - def plotroot(self, numeratorName, denominatorName): - import ROOT - denominator = self.denominator.plotroot(denominatorName) - num = denominator.GetNbinsX() - low = denominator.GetBinLowEdge(1) - high = denominator.GetBinLowEdge(num) + denominator.GetBinWidth(num) - - numerator = ROOT.TH1D(numeratorName, "", num, low, high) - if isinstance(self.numerator, HistogramMethods): - setTH1(self.numerator.entries, [x.entries for x in self.numerator.values], - self.numerator.underflow.entries, self.numerator.overflow.entries, numerator) - elif isinstance(self.numerator, SparselyHistogramMethods): - setTH1(self.numerator.entries, - [self.numerator.bins[i].entries if i in self.numerator.bins else 0.0 for i in xrange( - self.denominator.minBin, self.denominator.maxBin + 1)], - 0.0, - 0.0, - numerator) - - return ROOT.TEfficiency(numerator, denominator) - - -class TwoDimensionallyHistogramMethods(object): - def plotroot(self, name, title="", binType="D"): - import ROOT - constructor = getattr(ROOT, "TH2" + binType) - sample = self.values[0] - th2 = constructor(name, title, int(self.num), float(self.low), float(self.high), - int(sample.num), float(sample.low), float(sample.high)) - for i in xrange(self.num): - for j in xrange(sample.num): - th2.SetBinContent(i + 1, j + 1, self.values[i].values[j].entries) - return th2 - - -class SparselyTwoDimensionallyHistogramMethods(object): - def plotroot(self, name, title="", binType="D"): - import ROOT - constructor = getattr(ROOT, "TH2" + binType) - yminBin, ymaxBin, ynum, ylow, yhigh = prepareTH2sparse(self) - th2 = constructor(name, title, int(self.num), float(self.low), float(self.high), - int(ynum), float(ylow), float(yhigh)) - setTH2sparse(self, yminBin, ymaxBin, th2) - return th2 - - -class IrregularlyTwoDimensionallyHistogramMethods(object): - pass - - -class CentrallyTwoDimensionallyHistogramMethods(object): - pass diff --git a/histogrammar/plot/vega/__init__.py b/histogrammar/plot/vega/__init__.py deleted file mode 100644 index a571a5c..0000000 --- a/histogrammar/plot/vega/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 DIANA-HEP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/histogrammar/plot/vega/jsontrans.py b/histogrammar/plot/vega/jsontrans.py deleted file mode 100644 index 91000e4..0000000 --- a/histogrammar/plot/vega/jsontrans.py +++ /dev/null @@ -1,363 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 DIANA-HEP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import math -import sys - -# Definitions for python 2/3 compatability -if sys.version_info[0] > 2: - basestring = str - long = int - -MAX_REPR = 50 - - -class JsonObject(dict): - def __init__(self, *pairs, **kwarg): - if isinstance(pairs, dict): - self._pairs = tuple(pairs.items()) - else: - self._pairs = pairs - if len(kwarg) > 0: - self._pairs = self._pairs + tuple(kwarg.items()) - - if any(not isinstance(kv, tuple) or len(kv) != 2 for kv in self._pairs): - raise TypeError("JsonObject pairs must all be two-element tuples") - - if any(not isinstance(k, basestring) or not (v is None or isinstance( - v, (basestring, bool, int, long, float, JsonObject, JsonArray))) for k, v in self._pairs): - raise TypeError( - "JsonObject keys must be strings and values must be (string, bool, int, float, JsonObject, JsonArray)") - - def toJsonString(self, prefix="", indent=2): - out = ["{\n", prefix, " "] - first = True - for k, v in self._pairs: - if first: - first = False - else: - out.append(",\n") - out.append(prefix) - out.append(" ") - out.append(json.dumps(k)) - out.append(": ") - if isinstance(v, float) and (math.isnan(v) or math.isinf(v)): - raise ValueError("cannot JSON-serialize NaN or Infinity") - elif isinstance(v, (basestring, bool, int, long, float)): - v = json.dumps(v) - else: - v = v.toJsonString(prefix + (" " * indent), indent) - out.append(v) - out.append("\n") - out.append(prefix) - out.append("}") - return "".join(out) - - def _index(self, key): - for i, (k, v) in enumerate(self._pairs): - if k == key: - return i - return -1 - - def set(self, *path, **kwds): - if "to" not in kwds: - raise TypeError("missing keyword argument 'to' in set(path, to=value)") - elif len(kwds) != 1: - raise TypeError("unrecognized keyword arguments in set(path, to=value)") - value = kwds["to"] - - if len(path) < 1: - raise TypeError("missing path in set(path, to=value)") - key = path[0] - index = self._index(key) - if len(path) == 1: - if index == -1: - return JsonObject(*(self._pairs + ((key, value),))) - else: - return JsonObject(*[(key, value) if k == key else (k, v) for k, v in self._pairs]) - else: - if index == -1: - raise ValueError("JsonObject field {0} does not contain path ({1})".format( - repr(key), ", ".join(map(repr, path[1:])))) - elif not isinstance(self._pairs[index][1], (JsonObject, JsonArray)): - raise ValueError("JsonObject field {0} does not contain path ({1})".format( - repr(key), ", ".join(map(repr, path[1:])))) - else: - return JsonObject(*[(k, v.set(*path[1:], **kwds)) if k == key else (k, v) for k, v in self._pairs]) - - def without(self, *path): - if len(path) < 1: - raise TypeError("missing path in without(path)") - key = path[0] - index = self._index(key) - if len(path) == 1: - if index == -1: - return self - else: - return JsonObject(*[(k, v) for k, v in self._pairs if k != key]) - else: - if index == -1: - raise ValueError("JsonObject field {0} does not contain path ({1})".format( - repr(key), ", ".join(map(repr, path[1:])))) - elif not isinstance(self._pairs[index][1], (JsonObject, JsonArray)): - raise ValueError("JsonObject field {0} does not contain path ({1})".format( - repr(key), ", ".join(map(repr, path[1:])))) - else: - return JsonObject(*[(k, v.without(*path[1:])) if k == key else (k, v) for k, v in self._pairs]) - - def overlay(self, other): - out = self - for k, v in other.items(): - out = out.set(k, to=v) - return out - - # override built-in dict methods - - def __cmp__(self, other): - def cmp(a, b): - return (a > b) - (a < b) - return cmp(dict(self._pairs), dict(other._pairs)) - - def __contains__(self, key): - return any(k == key for k, v in self._pairs) - - def __delattr_(self, key): - raise TypeError("JsonObject cannot be changed in-place; no immutable equivalent") - - def __delitem__(self, key): - raise TypeError("JsonObject cannot be changed in-place; use .without(key)") - - def __eq__(self, other): - return isinstance(other, JsonObject) and self._pairs == other._pairs - - def __format__(self, format_spec): - return str(self) - - def __getitem__(self, key): - index = self._index(key) - if index == -1: - raise KeyError(key) - else: - return self._pairs[index][1] - - def __hash__(self): - return hash(("JsonObject", self._pairs)) - - def __iter__(self): - return self.keys() - - def __len__(self): - return len(self._pairs) - - def __reduce__(self): - return self.__reduce_ex__(0) - - def __reduce_ex__(self, protocol): - return (self.__class__, self._pairs) - - def __repr__(self): - out = "{" - first = True - for k, v in self._pairs: - if first: - first = False - else: - out += ", " - if len(out) > MAX_REPR - 1: - break - if isinstance(v, (basestring, bool, int, long, float)): - v = json.dumps(v) - else: - v = repr(v) - out += json.dumps(k) + ": " + v - if len(out) > MAX_REPR - 1: - out = out[:(MAX_REPR - 4)] + "..." - return out + "}" - - def __setitem__(self, key, value): - raise TypeError("JsonObject cannot be changed in-place; use .set(path, to=value)") - - def __sizeof__(self): - return super(dict, self).__sizeof__() - - def __str__(self): - out = ["{"] - first = True - for k, v in self._pairs: - if first: - first = False - else: - out.append(",") - out.append(json.dumps(k)) - out.append(":") - if isinstance(v, (basestring, bool, int, long, float)): - v = json.dumps(v) - else: - v = str(v) - out.append(v) - out.append("}") - return "".join(out) - - def clear(self): - raise TypeError("JsonObject cannot be changed in-place; use JsonObject() constructor to make a new one") - - def copy(self): - return self # because we're immutable - - def __copy__(self): - return self # because we're immutable - - def __deepcopy__(self, memo): - return self # because we're immutable - - def get(self, key, default=None): - index = self._index(key) - if index == -1: - return default - else: - return self._pairs[index][1] - - def has_key(self, key): - return key in self - - def items(self): - for k, v in self._pairs: - yield k, v - - def iteritems(self): - return self.items() - - def iterkeys(self): - return self.keys() - - def itervalues(self): - return self.values() - - def keys(self): - for k, v in self._pairs: - yield k - - def pop(self, key, default=None): - raise TypeError("JsonObject cannot be changed in-place; no immutable equivalent") - - def popitem(self, key, default=None): - raise TypeError("JsonObject cannot be changed in-place; no immutable equivalent") - - def setdefault(self, key, default=None): - raise TypeError("JsonObject cannot be changed in-place; no immutable equivalent") - - def update(self, other): - raise TypeError("JsonObject cannot be changed in-place; use .overlay(other)") - - def values(self): - for k, v in self._pairs: - yield v - - def viewitems(self): - return self.items() - - def viewkeys(self): - return self.keys() - - def viewvalues(self): - return self.values() - - -class JsonArray(tuple): - def __init__(self, *values): - self._values = values - if any(not (v is None or isinstance(v, (basestring, bool, int, long, float, JsonObject, JsonArray))) - for v in self._values): - raise TypeError("JsonArray values must be (string, bool, int, float, JsonObject, JsonArray)") - - def toJsonString(self, prefix="", indent=2): - out = [prefix, "[\n", prefix, " "] - first = True - for v in self._values: - if first: - first = False - else: - out.append(",\n") - out.append(prefix) - out.append(" ") - if isinstance(v, float) and (math.isnan(v) or math.isinf(v)): - raise ValueError("cannot JSON-serialize NaN or Infinity") - elif isinstance(v, (basestring, bool, int, long, float)): - v = json.dumps(v) - else: - v = v.toJsonString(prefix + (" " * indent), indent) - out.append(v) - out.append("\n") - out.append(prefix) - out.append("]") - return "".join(out) - - # override built-in tuple methods - -# __add__ -# __cmp__ -# __contains__ -# __delattr__ -# __delitem__ -# __eq__ -# __format__ -# __getitem__ -# __getnewargs__ -# __getslice__ -# __hash__ -# __iter__ -# __len__ -# __mul__ -# __reduce__ -# __reduce_ex__ - - def __repr__(self): - out = "[" - first = True - for v in self._values: - if first: - first = False - else: - out += ", " - if len(out) > MAX_REPR - 1: - break - out += repr(v) - if len(out) > MAX_REPR - 1: - out = out[:(MAX_REPR - 4)] + "..." - return out + "]" - -# __rmul__ -# __sizeof__ - - def __str__(self): - out = ["["] - first = False - for v in self._values: - if first: - first = False - else: - out.append(",") - if isinstance(v, (basestring, bool, int, long, float)): - v = json.dumps(v) - else: - v = str(v) - out.append(v) - out.append("]") - return "".join(out) - -# count -# index diff --git a/histogrammar/plot/vega/nodejs.py b/histogrammar/plot/vega/nodejs.py deleted file mode 100644 index 8ff469a..0000000 --- a/histogrammar/plot/vega/nodejs.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 DIANA-HEP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import subprocess -import tempfile -import os - - -def write(vegaSpec, outputFile, format=None): - """Use the 'vega' package in Nodejs to write to SVG or PNG files. - - Unlike interactive plotting, this does not require a round trip through a web browser, but it does require a - Nodejs installation on your computer (to evaluate the Javascript). - - To install the prerequisites on an Ubuntu system, do - - # Cairo dependencies for generating PNG: - sudo apt-get install install libcairo2-dev libjpeg-dev libgif-dev libpango1.0-dev build-essential g++ - # Nodejs and its package manager, npm: - sudo apt-get install npm - - # Get the 'vega' package with npm; user-install, not global (no sudo)! - npm install vega - - Parameters: - vegaSpec (string or dict): JSON string or its dict-of-dicts equivalent - outputFile (string or None): output file name or None to return output as a string - format ('svg', 'png', or None): None (default) guesses format from outputFile extension - """ - - if format is None and outputFile is None: - format = "svg" - elif format is None and outputFile.endswith(".svg"): - format = "svg" - elif format is None and outputFile.endswith(".png"): - format = "png" - else: - raise IOError("Could not infer format from outputFile") - - if format == "png": - cmd = "vg2png" - elif format == "svg": - cmd = "vg2svg" - else: - raise IOError("Only 'png' and 'svg' output is supported.") - - npmbin = subprocess.Popen(["npm", "bin"], stdout=subprocess.PIPE) - if npmbin.wait() == 0: - npmbin = npmbin.stdout.read().strip() - else: - raise IOError("Nodejs Package Manager 'npm' must be installed to use nodejs.write function.") - - tmp = tempfile.NamedTemporaryFile(delete=False) - - if isinstance(vegaSpec, dict): - vegaSpec = json.dump(tmp, vegaSpec) - else: - tmp.write(vegaSpec) - - tmp.close() - - if outputFile is None: - vg2x = subprocess.Popen([cmd, tmp.name], stdout=subprocess.PIPE, env=dict( - os.environ, PATH=npmbin + ":" + os.environ.get("PATH", ""))) - if vg2x.wait() == 0: - return vg2x.stdout.read() - else: - os.unlink(tmp.name) - raise IOError("Command '{0}' failed; if it's not installed, install it with 'npm install vega'".format(cmd)) - - else: - vg2x = subprocess.Popen([cmd, tmp.name, outputFile], stdout=subprocess.PIPE, - env=dict(os.environ, PATH=npmbin + ":" + os.environ.get("PATH", ""))) - if vg2x.wait() != 0: - os.unlink(tmp.name) - raise IOError("Command '{0}' failed; if it's not installed, install it with 'npm install vega'".format(cmd)) diff --git a/histogrammar/primitives/average.py b/histogrammar/primitives/average.py index 756af54..a1019d2 100644 --- a/histogrammar/primitives/average.py +++ b/histogrammar/primitives/average.py @@ -14,14 +14,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import math import numbers -import struct -from histogrammar.defs import Container, Factory, identity, JsonFormatException, ContainerException -from histogrammar.util import n_dim, datatype, serializable, inheritdoc, maybeAdd, floatToJson, hasKeys, numeq, \ - basestring +from histogrammar.defs import ( + Container, + ContainerException, + Factory, + JsonFormatException, + identity, +) +from histogrammar.util import ( + basestring, + datatype, + floatToJson, + hasKeys, + inheritdoc, + maybeAdd, + n_dim, + numeq, + serializable, +) class Average(Factory, Container): @@ -41,12 +54,16 @@ def ed(entries, mean): mean (float): the mean. """ - if not isinstance(entries, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("entries ({0}) must be a number".format(entries)) + if not isinstance(entries, numbers.Real) and entries not in ( + "nan", + "inf", + "-inf", + ): + raise TypeError(f"entries ({entries}) must be a number") if not isinstance(mean, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("mean ({0}) must be a number".format(mean)) + raise TypeError(f"mean ({mean}) must be a number") if entries < 0.0: - raise ValueError("entries ({0}) cannot be negative".format(entries)) + raise ValueError(f"entries ({entries}) cannot be negative") out = Average(None) out.entries = float(entries) out.mean = float(mean) @@ -70,7 +87,7 @@ def __init__(self, quantity=identity): self.quantity = serializable(identity(quantity) if isinstance(quantity, str) else quantity) self.entries = 0.0 self.mean = float("nan") - super(Average, self).__init__() + super().__init__() self.specialize() @inheritdoc(Container) @@ -87,10 +104,9 @@ def __add__(self, other): elif other.entries == 0.0: out.mean = self.mean else: - out.mean = (self.entries*self.mean + other.entries*other.mean)/(self.entries + other.entries) + out.mean = (self.entries * self.mean + other.entries * other.mean) / (self.entries + other.entries) return out.specialize() - else: - raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) + raise ContainerException(f"cannot add {self.name} and {other.name}") @inheritdoc(Container) def __iadd__(self, other): @@ -103,11 +119,10 @@ def __iadd__(self, other): def __mul__(self, factor): if math.isnan(factor) or factor <= 0.0: return self.zero() - else: - out = self.zero() - out.entries = factor * self.entries - out.mean = self.mean - return out.specialize() + out = self.zero() + out.entries = factor * self.entries + out.mean = self.mean + return out.specialize() @inheritdoc(Container) def __rmul__(self, factor): @@ -120,7 +135,7 @@ def fill(self, datum, weight=1.0): if weight > 0.0: q = self.quantity(datum) if not isinstance(q, numbers.Real): - raise TypeError("function return value ({0}) must be boolean or number".format(q)) + raise TypeError(f"function return value ({q}) must be boolean or number") # no possibility of exception from here on out (for rollback) if self.entries == 0.0: @@ -132,192 +147,19 @@ def fill(self, datum, weight=1.0): elif math.isinf(self.mean) or math.isinf(q): if math.isinf(self.mean) and math.isinf(q) and self.mean * q < 0.0: - self.mean = float("nan") # opposite-sign infinities is bad + self.mean = float("nan") # opposite-sign infinities is bad elif math.isinf(q): - self.mean = q # mean becomes infinite with sign of q + self.mean = q # mean becomes infinite with sign of q else: - pass # mean is already infinite + pass # mean is already infinite if math.isinf(self.entries) or math.isnan(self.entries): - self.mean = float("nan") # non-finite denominator is bad + self.mean = float("nan") # non-finite denominator is bad - else: # handle finite case + else: # handle finite case delta = q - self.mean shift = delta * weight / self.entries self.mean += shift - def _cppGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - return self._c99GenerateCode(parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, - fillPrefix, fillIndent, weightVars, weightVarStack, tmpVarTypes) - - def _c99GenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0;") - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".mean = 0.0;") - - normexpr = self._c99QuantityExpr( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - None) - fillCode.append(" " * fillIndent + self._c99ExpandPrefix(*fillPrefix) + ".entries += " + - weightVarStack[-1] + ";") - - delta = "delta_" + str(len(tmpVarTypes)) - tmpVarTypes[delta] = "double" - shift = "shift_" + str(len(tmpVarTypes)) - tmpVarTypes[shift] = "double" - - fillCode.append("""{indent}if (std::isnan({mean}) || std::isnan({q})) {{ -{indent} {mean} = NAN; -{indent}}} -{indent}else if (std::isinf({mean}) || std::isinf({q})) {{ -{indent} if (std::isinf({mean}) && std::isinf({q}) && {mean} * {q} < 0.0) -{indent} {mean} = NAN; -{indent} else if (std::isinf({q})) -{indent} {mean} = {q}; -{indent} else -{indent} {{ }} -{indent} if (std::isinf({entries}) || std::isnan({entries})) -{indent} {mean} = NAN; -{indent}}} -{indent}else {{ -{indent} {delta} = {q} - {mean}; -{indent} {shift} = {delta} * {weight} / {entries}; -{indent} {mean} += {shift}; -{indent}}}""".format(indent=" " * fillIndent, - entries=self._c99ExpandPrefix(*fillPrefix) + ".entries", - mean=self._c99ExpandPrefix(*fillPrefix) + ".mean", - q=normexpr, - delta=delta, - shift=shift, - weight=weightVarStack[-1])) - - storageStructs[self._c99StructName()] = """ - typedef struct {{ - double entries; - double mean; - }} {0}; -""".format(self._c99StructName()) - - def _clingUpdate(self, filler, *extractorPrefix): - obj = self._clingExpandPrefix(filler, *extractorPrefix) - - entries = self.entries + obj.entries - if self.entries == 0.0: - mean = obj.mean - elif obj.entries == 0.0: - mean = self.mean - else: - mean = (self.entries*self.mean + obj.entries*obj.mean)/(self.entries + obj.entries) - - self.entries = entries - self.mean = mean - - def _c99StructName(self): - return "Av" - - def _cudaGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, - fillPrefix, fillIndent, combineCode, totalPrefix, itemPrefix, combineIndent, jsonCode, - jsonPrefix, jsonIndent, weightVars, weightVarStack, tmpVarTypes, suppressName): - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0f;") - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".sum = 0.0f;") - - normexpr = self._cudaQuantityExpr( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - None) - fillCode.append(" " * fillIndent + "atomicAdd(&" + - self._c99ExpandPrefix(*fillPrefix) + ".entries, " + weightVarStack[-1] + ");") - fillCode.append(" " * fillIndent + "atomicAdd(&" + self._c99ExpandPrefix(*fillPrefix) + - ".sum, " + weightVarStack[-1] + " * " + normexpr + ");") - - combineCode.append( - " " * - combineIndent + - "atomicAdd(&" + - self._c99ExpandPrefix( - * - totalPrefix) + - ".entries, " + - self._c99ExpandPrefix( - * - itemPrefix) + - ".entries);") - combineCode.append( - " " * - combineIndent + - "atomicAdd(&" + - self._c99ExpandPrefix( - * - totalPrefix) + - ".sum, " + - self._c99ExpandPrefix( - * - itemPrefix) + - ".sum);") - - jsonCode.append(" " * jsonIndent + "fprintf(out, \"{\\\"entries\\\": \");") - jsonCode.append(" " * jsonIndent + "floatToJson(out, " + self._c99ExpandPrefix(*jsonPrefix) + ".entries);") - jsonCode.append(" " * jsonIndent + "fprintf(out, \", \\\"mean\\\": \");") - jsonCode.append(" " * jsonIndent + "if (" + self._c99ExpandPrefix(*jsonPrefix) + ".entries == 0.0f)") - jsonCode.append(" " * jsonIndent + " fprintf(out, \"\\\"nan\\\"\");") - jsonCode.append(" " * jsonIndent + "else") - jsonCode.append( - " " * - jsonIndent + - " floatToJson(out, " + - self._c99ExpandPrefix( - * - jsonPrefix) + - ".sum / " + - self._c99ExpandPrefix( - * - jsonPrefix) + - ".entries);") - if suppressName or self.quantity.name is None: - jsonCode.append(" " * jsonIndent + "fprintf(out, \"}\");") - else: - jsonCode.append(" " * jsonIndent + "fprintf(out, \", \\\"name\\\": " + - json.dumps(json.dumps(self.quantity.name))[1:-1] + "}\");") - - storageStructs[self._c99StructName()] = """ - typedef struct {{ - float entries; - float sum; - }} {0}; -""".format(self._c99StructName()) - - def _cudaUnpackAndFill(self, data, bigendian, alignment): - format = " 0.0 q = q[selection] weights = weights[selection] @@ -341,7 +184,7 @@ def _numpy(self, data, weights, shape): self.mean = float("nan") elif ca_plus_cb > 0.0: mb = numpy.average(q, weights=weights) - self.mean = float((ca*ma + (ca_plus_cb - ca)*mb) / ca_plus_cb) + self.mean = float((ca * ma + (ca_plus_cb - ca) * mb) / ca_plus_cb) def _sparksql(self, jvm, converter): return converter.Average(self.quantity.asSparkSQL()) @@ -353,8 +196,10 @@ def children(self): @inheritdoc(Container) def toJsonFragment(self, suppressName): - return maybeAdd({"entries": floatToJson(self.entries), "mean": floatToJson(self.mean)}, - name=(None if suppressName else self.quantity.name)) + return maybeAdd( + {"entries": floatToJson(self.entries), "mean": floatToJson(self.mean)}, + name=(None if suppressName else self.quantity.name), + ) @staticmethod @inheritdoc(Factory) @@ -381,15 +226,18 @@ def fromJsonFragment(json, nameFromParent): out.quantity.name = nameFromParent if name is None else name return out.specialize() - else: - raise JsonFormatException(json, "Average") + raise JsonFormatException(json, "Average") def __repr__(self): - return "".format(self.mean) + return f"" def __eq__(self, other): - return isinstance(other, Average) and self.quantity == other.quantity and numeq( - self.entries, other.entries) and numeq(self.mean, other.mean) + return ( + isinstance(other, Average) + and self.quantity == other.quantity + and numeq(self.entries, other.entries) + and numeq(self.mean, other.mean) + ) def __ne__(self, other): return not self == other diff --git a/histogrammar/primitives/bag.py b/histogrammar/primitives/bag.py index a0b2424..b1623ea 100644 --- a/histogrammar/primitives/bag.py +++ b/histogrammar/primitives/bag.py @@ -17,9 +17,41 @@ import math import numbers -from histogrammar.defs import Container, Factory, identity, JsonFormatException, ContainerException -from histogrammar.util import n_dim, datatype, serializable, inheritdoc, maybeAdd, floatToJson, hasKeys, numeq, \ - floatOrNan, rangeToJson, basestring, xrange +from histogrammar.defs import ( + Container, + ContainerException, + Factory, + JsonFormatException, + identity, +) +from histogrammar.util import ( + basestring, + datatype, + floatOrNan, + floatToJson, + hasKeys, + inheritdoc, + maybeAdd, + n_dim, + numeq, + rangeToJson, + serializable, +) + + +class Sorter: + def __init__(self, x): + self.x = x + + def __lt__(self, other): + for xi, yi in zip(self.x, other.x): + if isinstance(xi, str) and isinstance(yi, float): + return False + if isinstance(xi, float) and isinstance(yi, str) or xi < yi: + return True + if xi > yi: + return False + return False class Bag(Factory, Container): @@ -46,12 +78,16 @@ def ed(entries, values, range): or string. """ - if not isinstance(entries, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("entries ({0}) must be a number".format(entries)) + if not isinstance(entries, numbers.Real) and entries not in ( + "nan", + "inf", + "-inf", + ): + raise TypeError(f"entries ({entries}) must be a number") if not isinstance(values, dict) and not all(isinstance(k, numbers.Real) for k, v in values.items()): - raise TypeError("values ({0}) must be a dict from numbers to range type".format(values)) + raise TypeError(f"values ({values}) must be a dict from numbers to range type") if float(entries) < 0.0: - raise ValueError("entries ({0}) cannot be negative".format(entries)) + raise ValueError(f"entries ({entries}) cannot be negative") out = Bag(None, range) out.entries = float(entries) out.values = values @@ -83,7 +119,7 @@ def __init__(self, quantity=identity, range="S"): self.dimension = int(range[1:]) except BaseException: self.dimension = 0 - super(Bag, self).__init__() + super().__init__() self.specialize() @inheritdoc(Container) @@ -94,9 +130,7 @@ def zero(self): def __add__(self, other): if isinstance(other, Bag): if self.range != other.range: - raise ContainerException( - "cannot add Bag because range differs ({0} vs {1})".format( - self.range, other.range)) + raise ContainerException(f"cannot add Bag because range differs ({self.range} vs {other.range})") out = Bag(self.quantity, self.range) @@ -111,8 +145,7 @@ def __add__(self, other): return out.specialize() - else: - raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) + raise ContainerException(f"cannot add {self.name} and {other.name}") @inheritdoc(Container) def __iadd__(self, other): @@ -124,12 +157,12 @@ def __iadd__(self, other): def __mul__(self, factor): if math.isnan(factor) or factor <= 0.0: return self.zero() - else: - out = self.zero() - out.entries = factor * self.entries - for value, count in self.values.items(): - out.values[value] = factor * count - return out.specialize() + + out = self.zero() + out.entries = factor * self.entries + for value, count in self.values.items(): + out.values[value] = factor * count + return out.specialize() @inheritdoc(Container) def __rmul__(self, factor): @@ -146,22 +179,24 @@ def fill(self, datum, weight=1.0): def _update(self, q, weight): if self.range == "S": if not isinstance(q, basestring): - raise TypeError("function return value ({0}) must be a string for range {1}".format(q, self.range)) + raise TypeError(f"function return value ({q}) must be a string for range {self.range}") elif self.range == "N": try: q = floatOrNan(q) except BaseException: - raise TypeError("function return value ({0}) must be a number for range {1}".format(q, self.range)) + raise TypeError(f"function return value ({q}) must be a number for range {self.range}") else: try: q = tuple(floatOrNan(qi) for qi in q) assert len(q) == self.dimension except BaseException: - raise TypeError( - "function return value ({0}) must be a list/tuple of numbers with length {1} for range {2}".format( - q, self.dimension, self.range)) + msg = ( + f"function return value ({q}) must be a list/tuple of numbers with length {self.dimension}" + f" for range {self.range}" + ) + raise TypeError(msg) # no possibility of exception from here on out (for rollback) self.entries += weight @@ -170,91 +205,9 @@ def _update(self, q, weight): else: self.values[q] = weight - def _cppGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - normexpr = self._c99QuantityExpr( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - None) - - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0;") - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".values.clear();") - fillCode.append(" " * fillIndent + self._c99ExpandPrefix(*fillPrefix) + - ".entries += " + weightVarStack[-1] + ";") - - fillCode.append("""{indent}if ({values}.find({q}) == {values}.end()) -{indent} {values}[{q}] = 0.0; -{indent}{values}[{q}] += {weight};""".format( - indent=" " * fillIndent, - values=self._c99ExpandPrefix(*fillPrefix) + ".values", - q=normexpr, - weight=weightVarStack[-1] - )) - - if self.range[0] == "N" and len(self.range) > 1: - storageStructs[self.range] = """ - class {0} {{ - public: - double {1}; - {0}({2}): {3} {{ }} - {0}(const {0}& other): {4} {{ }} - {0}(): {5} {{ }} - Bool_t operator<(const {0}& other) const {{ - {6} - else return false; - }} - }}; -""".format(self.range, - ", ".join("v" + str(i) for i in xrange(self.dimension)), - ", ".join("double v" + str(i) for i in xrange(self.dimension)), - ", ".join("v" + str(i) + "(v" + str(i) + ")" for i in xrange(self.dimension)), - ", ".join("v" + str(i) + "(other.v" + str(i) + ")" for i in xrange(self.dimension)), - ", ".join("v" + str(i) + "(0.0)" for i in xrange(self.dimension)), - "\n ".join( - ("else " if i != 0 else "") + - "if (v" + - str(i) + - " < other.v" + - str(i) + - ") return true;" for i in xrange( - self.dimension)) - ) - - storageStructs[self._c99StructName()] = """ - typedef struct {{ - double entries; - std::map<{1}, double> values; - double getValues({1} i) {{ return values[i]; }} - }} {0}; -""".format(self._c99StructName(), "double" if self.range == "N" else "std::string" if self.range == "S" else self.range) - - def _c99GenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - raise NotImplementedError("no C99-compliant implementation of Bag (only C++)") - - def _clingUpdate(self, filler, *extractorPrefix): - obj = self._clingExpandPrefix(filler, *extractorPrefix) - self.entries += obj.entries - - for i in obj.values: - key = i.first - if self.range[0] == "N" and len(self.range) > 1: - key = tuple(getattr(key, "v" + str(x)) for x in xrange(self.dimension)) - if key not in self.values: - self.values[key] = 0.0 - self.values[key] += i.second - - def _c99StructName(self): - return "Bg" + self.range + "_" - def _numpy(self, data, weights, shape): import numpy + q = self.quantity(data) assert isinstance(q, numpy.ndarray) if shape[0] is None: @@ -267,9 +220,7 @@ def _numpy(self, data, weights, shape): for x, w in zip(q, weights): if w > 0.0: - if isinstance(x, numpy.ndarray): - x = x.tolist() - self._update(x, float(w)) + self._update(x.tolist() if isinstance(x, numpy.ndarray) else x, float(w)) def _sparksql(self, jvm, converter): return converter.Bag(self.quantity.asSparkSQL(), self.range) @@ -287,31 +238,22 @@ def toJsonFragment(self, suppressName): aslist.append(("nan", self.values["nan"])) elif self.range[0] == "N": - class Sorter(object): - def __init__(self, x): - self.x = x - - def __lt__(self, other): - for xi, yi in zip(self.x, other.x): - if isinstance(xi, str) and isinstance(yi, float): - return False - elif isinstance(xi, float) and isinstance(yi, str): - return True - elif xi < yi: - return True - elif xi > yi: - return False - return False - aslist = sorted((x for x in self.values.items()), key=lambda y: tuple(Sorter(z) for z in y)) + aslist = sorted( + (x for x in self.values.items()), + key=lambda y: tuple(Sorter(z) for z in y), + ) else: aslist = sorted(x for x in self.values.items()) - return maybeAdd({ - "entries": floatToJson(self.entries), - "values": [{"w": floatToJson(n), "v": rangeToJson(v)} for v, n in aslist], - "range": self.range, - }, name=(None if suppressName else self.quantity.name)) + return maybeAdd( + { + "entries": floatToJson(self.entries), + "values": [{"w": floatToJson(n), "v": rangeToJson(v)} for v, n in aslist], + "range": self.range, + }, + name=(None if suppressName else self.quantity.name), + ) @staticmethod @inheritdoc(Factory) @@ -339,7 +281,7 @@ def fromJsonFragment(json, nameFromParent): if nv["w"] in ("nan", "inf", "-inf") or isinstance(nv["w"], numbers.Real): n = float(nv["w"]) else: - raise JsonFormatException(nv["w"], "Bag.values {0} n".format(i)) + raise JsonFormatException(nv["w"], f"Bag.values {i} n") if nv["v"] in ("nan", "inf", "-inf") or isinstance(nv["v"], numbers.Real): v = floatOrNan(nv["v"]) @@ -348,15 +290,15 @@ def fromJsonFragment(json, nameFromParent): elif isinstance(nv["v"], (list, tuple)): for j, d in enumerate(nv["v"]): if d not in ("nan", "inf", "-inf") and not isinstance(d, numbers.Real): - raise JsonFormatException(d, "Bag.values {0} v {1}".format(i, j)) + raise JsonFormatException(d, f"Bag.values {i} v {j}") v = tuple(map(floatOrNan, nv["v"])) else: - raise JsonFormatException(nv["v"], "Bag.values {0} v".format(i)) + raise JsonFormatException(nv["v"], f"Bag.values {i} v") values[v] = n else: - raise JsonFormatException(nv, "Bag.values {0}".format(i)) + raise JsonFormatException(nv, f"Bag.values {i}") elif json["values"] is None: values = None @@ -373,11 +315,10 @@ def fromJsonFragment(json, nameFromParent): out.quantity.name = nameFromParent if name is None else name return out.specialize() - else: - raise JsonFormatException(json, "Bag") + raise JsonFormatException(json, "Bag") def __repr__(self): - return "".format(len(self.values), self.range) + return f"" def __eq__(self, other): if len(self.values) != len(other.values): @@ -391,55 +332,57 @@ def __eq__(self, other): two = sorted(x for x in other.values.items() if x[0] != "nan") + [("nan", other.values.get("nan"))] elif self.range[0] == "N": - class Sorter(object): - def __init__(self, x): - self.x = x - - def __lt__(self, other): - for xi, yi in zip(self.x, other.x): - if isinstance(xi, str) and isinstance(yi, float): - return False - elif isinstance(xi, float) and isinstance(yi, str): - return True - elif xi < yi: - return True - elif xi > yi: - return False - return False - one = sorted((x for x in self.values.items()), key=lambda y: tuple(Sorter(z) for z in y)) - two = sorted((x for x in other.values.items()), key=lambda y: tuple(Sorter(z) for z in y)) + one = sorted( + (x for x in self.values.items()), + key=lambda y: tuple(Sorter(z) for z in y), + ) + two = sorted( + (x for x in other.values.items()), + key=lambda y: tuple(Sorter(z) for z in y), + ) else: one = sorted(x for x in self.values.items()) two = sorted(x for x in other.values.items()) + return_false = False for (v1, w1), (v2, w2) in zip(one, two): if isinstance(v1, basestring) and isinstance(v2, basestring): if v1 != v2: - return False + return_false = True + break elif isinstance(v1, numbers.Real) and isinstance(v2, numbers.Real): if not numeq(v1, v2): - return False + return_false = True + break elif isinstance(v1, tuple) and isinstance(v2, tuple) and len(v1) == len(v2): for v1i, v2i in zip(v1, v2): if isinstance(v1i, numbers.Real) and isinstance(v2i, numbers.Real): if not numeq(v1i, v2i): - return False + return_false = True + break elif isinstance(v1i, basestring) and isinstance(v2i, basestring): if v1i != v2i: - return False + return_false = True + break else: - return False + return_false = True + break else: - return False + return_false = True + break if v1 == "nan" and v2 == "nan" and w1 is None and w2 is None: pass elif isinstance(w1, numbers.Real) and isinstance(w2, numbers.Real): if not numeq(w1, w2): - return False + return_false = True + break else: - return False + return_false = True + break + if return_false: + return False return isinstance(other, Bag) and self.quantity == other.quantity and numeq(self.entries, other.entries) diff --git a/histogrammar/primitives/bin.py b/histogrammar/primitives/bin.py index c0431c3..443bcdd 100644 --- a/histogrammar/primitives/bin.py +++ b/histogrammar/primitives/bin.py @@ -14,16 +14,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import math import numbers -import struct -from histogrammar.defs import Container, Factory, identity, JsonFormatException, ContainerException -from histogrammar.util import n_dim, datatype, serializable, inheritdoc, maybeAdd, floatToJson, hasKeys, numeq, \ - xrange, long, basestring +import numpy as np +from histogrammar.defs import ( + Container, + ContainerException, + Factory, + JsonFormatException, + identity, +) from histogrammar.primitives.count import Count +from histogrammar.util import ( + basestring, + datatype, + floatToJson, + hasKeys, + inheritdoc, + long, + maybeAdd, + n_dim, + numeq, + serializable, +) class Bin(Factory, Container): @@ -66,39 +81,70 @@ def ed(low, high, entries, values, underflow, overflow, nanflow): nanflow (:doc:`Container `): is the filled nanflow bin. """ if not isinstance(low, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("low ({0}) must be a number".format(low)) + raise TypeError(f"low ({low}) must be a number") if not isinstance(high, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("high ({0}) must be a number".format(high)) - if not isinstance(entries, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("entries ({0}) must be a number".format(entries)) + raise TypeError(f"high ({high}) must be a number") + if not isinstance(entries, numbers.Real) and entries not in ( + "nan", + "inf", + "-inf", + ): + raise TypeError(f"entries ({entries}) must be a number") if not isinstance(values, (list, tuple)) and not all(isinstance(v, Container) for v in values): - raise TypeError("values ({0}) must be a list of Containers".format(values)) + raise TypeError(f"values ({values}) must be a list of Containers") if not isinstance(underflow, Container): - raise TypeError("underflow ({0}) must be a Container".format(underflow)) + raise TypeError(f"underflow ({underflow}) must be a Container") if not isinstance(overflow, Container): - raise TypeError("overflow ({0}) must be a Container".format(overflow)) + raise TypeError(f"overflow ({overflow}) must be a Container") if not isinstance(nanflow, Container): - raise TypeError("nanflow ({0}) must be a Container".format(nanflow)) + raise TypeError(f"nanflow ({nanflow}) must be a Container") if low >= high: - raise ValueError("low ({0}) must be less than high ({1})".format(low, high)) + raise ValueError(f"low ({low}) must be less than high ({high})") if entries < 0.0: - raise ValueError("entries ({0}) cannot be negative".format(entries)) + raise ValueError(f"entries ({entries}) cannot be negative") if len(values) < 1: - raise ValueError("values ({0}) must have at least one element".format(values)) - - out = Bin(len(values), float(low), float(high), None, None, underflow, overflow, nanflow) + raise ValueError(f"values ({values}) must have at least one element") + + out = Bin( + len(values), + float(low), + float(high), + None, + None, + underflow, + overflow, + nanflow, + ) out.entries = float(entries) out.values = values out.contentType = values[0].name return out.specialize() @staticmethod - def ing(num, low, high, quantity, value=Count(), underflow=Count(), overflow=Count(), nanflow=Count()): + def ing( + num, + low, + high, + quantity, + value=Count(), + underflow=Count(), + overflow=Count(), + nanflow=Count(), + ): """Synonym for ``__init__``.""" return Bin(num, low, high, quantity, value, underflow, overflow, nanflow) - def __init__(self, num, low, high, quantity=identity, value=Count(), - underflow=Count(), overflow=Count(), nanflow=Count()): + def __init__( + self, + num, + low, + high, + quantity=identity, + value=Count(), + underflow=Count(), + overflow=Count(), + nanflow=Count(), + ): """Create a Bin that is capable of being filled and added. Parameters: @@ -122,23 +168,23 @@ def __init__(self, num, low, high, quantity=identity, value=Count(), """ if not isinstance(num, (int, long)): - raise TypeError("num ({0}) must be an integer".format(num)) + raise TypeError(f"num ({num}) must be an integer") if not isinstance(low, numbers.Real): - raise TypeError("low ({0}) must be a number".format(low)) + raise TypeError(f"low ({low}) must be a number") if not isinstance(high, numbers.Real): - raise TypeError("high ({0}) must be a number".format(high)) + raise TypeError(f"high ({high}) must be a number") if value is not None and not isinstance(value, Container): - raise TypeError("value ({0}) must be a Container".format(value)) + raise TypeError(f"value ({value}) must be a Container") if not isinstance(underflow, Container): - raise TypeError("underflow ({0}) must be a Container".format(underflow)) + raise TypeError(f"underflow ({underflow}) must be a Container") if not isinstance(overflow, Container): - raise TypeError("overflow ({0}) must be a Container".format(overflow)) + raise TypeError(f"overflow ({overflow}) must be a Container") if not isinstance(nanflow, Container): - raise TypeError("nanflow ({0}) must be a Container".format(nanflow)) + raise TypeError(f"nanflow ({nanflow}) must be a Container") if num < 1: - raise ValueError("num ({0}) must be least one".format(num)) + raise ValueError(f"num ({num}) must be least one") if low >= high: - raise ValueError("low ({0}) must be less than high ({1})".format(low, high)) + raise ValueError(f"low ({low}) must be less than high ({high})") self.entries = 0.0 self.low = float(low) @@ -153,7 +199,7 @@ def __init__(self, num, low, high, quantity=identity, value=Count(), self.underflow = underflow.copy() self.overflow = overflow.copy() self.nanflow = nanflow.copy() - super(Bin, self).__init__() + super().__init__() self.specialize() def ascii(self): @@ -176,7 +222,7 @@ def ascii(self): dots = [None] * length i = 0 while i < length: - dots[i] = int(round((values[i] - minimum)*prop)) + dots[i] = int(round((values[i] - minimum) * prop)) i += 1 # Get range of values corresponding to each bin @@ -186,24 +232,38 @@ def ascii(self): ranges[i] = "[" + str(self.range(i))[1:] i += 1 - printedValues = ["{0:<.4g}".format(v) for v in values] + printedValues = [f"{v:<.4g}" for v in values] printedValuesWidth = max(len(x) for x in printedValues) - formatter = "{0:<14} {1:<%s} {2:<65}" % printedValuesWidth + formatter = f"{{0:<14}} {{1:<{printedValuesWidth}}} {{2:<65}}" - print(" " * printedValuesWidth + "{0:>16}{1:>65}".format(minimum, maximum)) + print(" " * printedValuesWidth + f"{minimum:>16}{maximum:>65}") print(" " * (16 + printedValuesWidth) + "+" + "-" * 62 + "+") i = 0 while i < length: - print(formatter.format(ranges[i], printedValues[i], "|" + "*" * dots[i] + " " * (62 - dots[i]) + "|")) + print( + formatter.format( + ranges[i], + printedValues[i], + "|" + "*" * dots[i] + " " * (62 - dots[i]) + "|", + ) + ) i += 1 print(" " * (16 + printedValuesWidth) + "+" + "-" * 62 + "+") def histogram(self): """Return a plain histogram by converting all sub-aggregator values into Counts""" - out = Bin(len(self.values), self.low, self.high, self.quantity, None, - self.underflow.copy(), self.overflow.copy(), self.nanflow.copy()) + out = Bin( + len(self.values), + self.low, + self.high, + self.quantity, + None, + self.underflow.copy(), + self.overflow.copy(), + self.nanflow.copy(), + ) out.entries = float(self.entries) for i, v in enumerate(self.values): out.values[i] = Count.ed(v.entries) @@ -211,76 +271,81 @@ def histogram(self): @inheritdoc(Container) def zero(self): - return Bin(len(self.values), self.low, self.high, self.quantity, self.values[0].zero(), self.underflow.zero(), - self.overflow.zero(), self.nanflow.zero()) + return Bin( + len(self.values), + self.low, + self.high, + self.quantity, + self.values[0].zero(), + self.underflow.zero(), + self.overflow.zero(), + self.nanflow.zero(), + ) @inheritdoc(Container) def __add__(self, other): if isinstance(other, Bin): if self.low != other.low: - raise ContainerException("cannot add Bins because low differs ({0} vs {1})".format(self.low, other.low)) + raise ContainerException(f"cannot add Bins because low differs ({self.low} vs {other.low})") if self.high != other.high: - raise ContainerException( - "cannot add Bins because high differs ({0} vs {1})".format( - self.high, other.high)) + raise ContainerException(f"cannot add Bins because high differs ({self.high} vs {other.high})") if len(self.values) != len(other.values): - raise ContainerException("cannot add Bins because nubmer of values differs ({0} vs {1})".format( - len(self.values), len(other.values))) + raise ContainerException( + f"cannot add Bins because nubmer of values differs ({len(self.values)} vs {len(other.values)})" + ) if len(self.values) == 0: raise ContainerException("cannot add Bins because number of values is zero") - out = Bin(len(self.values), - self.low, - self.high, - self.quantity, - self.values[0], - self.underflow + other.underflow, - self.overflow + other.overflow, - self.nanflow + other.nanflow) + out = Bin( + len(self.values), + self.low, + self.high, + self.quantity, + self.values[0], + self.underflow + other.underflow, + self.overflow + other.overflow, + self.nanflow + other.nanflow, + ) out.entries = self.entries + other.entries out.values = [x + y for x, y in zip(self.values, other.values)] return out.specialize() - else: - raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) + raise ContainerException(f"cannot add {self.name} and {other.name}") @inheritdoc(Container) def __iadd__(self, other): if isinstance(other, Bin): if self.low != other.low: - raise ContainerException("cannot add Bins because low differs ({0} vs {1})".format(self.low, other.low)) + raise ContainerException(f"cannot add Bins because low differs ({self.low} vs {other.low})") if self.high != other.high: - raise ContainerException( - "cannot add Bins because high differs ({0} vs {1})".format( - self.high, other.high)) + raise ContainerException(f"cannot add Bins because high differs ({self.high} vs {other.high})") if len(self.values) != len(other.values): - raise ContainerException("cannot add Bins because nubmer of values differs ({0} vs {1})".format( - len(self.values), len(other.values))) + raise ContainerException( + f"cannot add Bins because nubmer of values differs ({len(self.values)} vs {len(other.values)})" + ) if len(self.values) == 0: raise ContainerException("cannot add Bins because number of values is zero") self.entries += other.entries - for x, y in zip(self.values, other.values): - x += y + for i in range(len(self.values)): + self.values[i] += other.values[i] self.underflow += other.underflow self.overflow += other.overflow self.nanflow += other.nanflow return self - else: - raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) + raise ContainerException(f"cannot add {self.name} and {other.name}") @inheritdoc(Container) def __mul__(self, factor): if math.isnan(factor) or factor <= 0.0: return self.zero() - else: - out = self.zero() - out.entries = factor * self.entries - for i, v in enumerate(self.values): - out.values[i] = v * factor - out.overflow = self.overflow * factor - out.underflow = self.underflow * factor - out.nanflow = self.nanflow * factor - return out.specialize() + out = self.zero() + out.entries = factor * self.entries + for i, v in enumerate(self.values): + out.values[i] = v * factor + out.overflow = self.overflow * factor + out.underflow = self.underflow * factor + out.nanflow = self.nanflow * factor + return out.specialize() @inheritdoc(Container) def __rmul__(self, factor): @@ -298,8 +363,7 @@ def bin(self, x): """ if self.under(x) or self.over(x) or self.nan(x): return -1 - else: - return int(math.floor(self.num * (x - self.low) / (self.high - self.low))) + return int(math.floor(self.num * (x - self.low) / (self.high - self.low))) def under(self, x): """Return ``true`` iff ``x`` is in the underflow region (less than ``low``).""" @@ -320,8 +384,10 @@ def indexes(self): def range(self, index): """Get the low and high edge of a bin (given by index number).""" - return ((self.high - self.low) * index / self.num + self.low, - (self.high - self.low) * (index + 1) / self.num + self.low) + return ( + (self.high - self.low) * index / self.num + self.low, + (self.high - self.low) * (index + 1) / self.num + self.low, + ) @inheritdoc(Container) def fill(self, datum, weight=1.0): @@ -330,7 +396,7 @@ def fill(self, datum, weight=1.0): if weight > 0.0: q = self.quantity(datum) if not isinstance(q, numbers.Real): - raise TypeError("function return value ({0}) must be boolean or number".format(q)) + raise TypeError(f"function return value ({q}) must be boolean or number") if self.under(q): self.underflow.fill(datum, weight) @@ -344,438 +410,6 @@ def fill(self, datum, weight=1.0): # no possibility of exception from here on out (for rollback) self.entries += weight - def _cppGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - return self._c99GenerateCode(parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, - fillPrefix, fillIndent, weightVars, weightVarStack, tmpVarTypes) - - def _c99GenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - normexpr = self._c99QuantityExpr( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - None) - - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0;") - fillCode.append(" " * fillIndent + self._c99ExpandPrefix(*fillPrefix) + - ".entries += " + weightVarStack[-1] + ";") - - fillCode.append(" " * fillIndent + "if (std::isnan({0})) {{".format(normexpr)) - self.nanflow._c99GenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "nanflow"), - ), - initIndent, - fillCode, - fillPrefix + (("var", - "nanflow"), - ), - fillIndent + 2, - weightVars, - weightVarStack, - tmpVarTypes) - fillCode.append(" " * fillIndent + "}") - - fillCode.append(" " * fillIndent + "else if ({0} < {1}) {{".format(normexpr, self.low)) - self.underflow._c99GenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "underflow"), - ), - initIndent, - fillCode, - fillPrefix + (("var", - "underflow"), - ), - fillIndent + 2, - weightVars, - weightVarStack, - tmpVarTypes) - fillCode.append(" " * fillIndent + "}") - - fillCode.append(" " * fillIndent + "else if ({0} >= {1}) {{".format(normexpr, self.high)) - self.overflow._c99GenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "overflow"), - ), - initIndent, - fillCode, - fillPrefix + (("var", - "overflow"), - ), - fillIndent + 2, - weightVars, - weightVarStack, - tmpVarTypes) - fillCode.append(" " * fillIndent + "}") - - fillCode.append(" " * fillIndent + "else {") - - bin = "bin_" + str(len(tmpVarTypes)) - tmpVarTypes[bin] = "int" - initCode.append(" " * initIndent + "for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.values))) - - fillCode.append(" " * (fillIndent + 2) + - "{0} = floor(({1} - {2}) * {3});".format(bin, normexpr, self.low, - len(self.values)/(self.high - self.low))) - - self.values[0]._c99GenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "values"), - ("index", - bin)), - initIndent + 2, - fillCode, - fillPrefix + (("var", - "values"), - ("index", - bin)), - fillIndent + 2, - weightVars, - weightVarStack, - tmpVarTypes) - - initCode.append(" " * initIndent + "}") - fillCode.append(" " * fillIndent + "}") - - storageStructs[self._c99StructName()] = """ - typedef struct {{ - double entries; - {3} underflow; - {4} overflow; - {5} nanflow; - {1} values[{2}]; - {1}& getValues(int i) {{ return values[i]; }} - }} {0}; -""".format(self._c99StructName(), self.values[0]._c99StorageType(), len(self.values), self.underflow._c99StorageType(), - self.overflow._c99StorageType(), self.nanflow._c99StorageType()) - - def _clingUpdate(self, filler, *extractorPrefix): - obj = self._clingExpandPrefix(filler, *extractorPrefix) - self.entries += obj.entries - for i in xrange(len(self.values)): - self.values[i]._clingUpdate(obj, ("func", ["getValues", i])) - self.underflow._clingUpdate(obj, ("var", "underflow")) - self.overflow._clingUpdate(obj, ("var", "overflow")) - self.nanflow._clingUpdate(obj, ("var", "nanflow")) - - def _c99StructName(self): - return "Bn" + str(len(self.values)) + self.values[0]._c99StructName( - ) + self.underflow._c99StructName() + self.overflow._c99StructName() + self.nanflow._c99StructName() - - def _cudaGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, combineCode, totalPrefix, itemPrefix, combineIndent, jsonCode, jsonPrefix, - jsonIndent, weightVars, weightVarStack, tmpVarTypes, suppressName): - normexpr = self._cudaQuantityExpr( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - None) - - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0f;") - fillCode.append(" " * fillIndent + - "atomicAdd(&" + self._c99ExpandPrefix(*fillPrefix) + ".entries, " + weightVarStack[-1] + ");") - combineCode.append( - " " * - combineIndent + - "atomicAdd(&" + - self._c99ExpandPrefix( - * - totalPrefix) + - ".entries, " + - self._c99ExpandPrefix( - * - itemPrefix) + - ".entries);") - jsonCode.append(" " * - jsonIndent + - "fprintf(out, \"{\\\"low\\\": " + - str(self.low) + - ", \\\"high\\\": " + - str(self.high) + - ", \\\"entries\\\": \");") - jsonCode.append(" " * jsonIndent + "floatToJson(out, " + self._c99ExpandPrefix(*jsonPrefix) + ".entries);") - - fillCode.append(" " * fillIndent + "if (isnan({0})) {{".format(normexpr)) - jsonCode.append( - " " * - jsonIndent + - "fprintf(out, \", \\\"nanflow:type\\\": \\\"" + - self.nanflow.name + - "\\\"\");") - jsonCode.append(" " * jsonIndent + "fprintf(out, \", \\\"nanflow\\\": \");") - self.nanflow._cudaGenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "nanflow"), - ), - initIndent + 2, - fillCode, - fillPrefix + (("var", - "nanflow"), - ), - fillIndent + 2, - combineCode, - totalPrefix + (("var", - "nanflow"), - ), - itemPrefix + (("var", - "nanflow"), - ), - combineIndent, - jsonCode, - jsonPrefix + (("var", - "nanflow"), - ), - jsonIndent, - weightVars, - weightVarStack, - tmpVarTypes, - False) - fillCode.append(" " * fillIndent + "}") - - fillCode.append(" " * fillIndent + "else if ({0} < {1}) {{".format(normexpr, self.low)) - jsonCode.append( - " " * - jsonIndent + - "fprintf(out, \", \\\"underflow:type\\\": \\\"" + - self.underflow.name + - "\\\"\");") - jsonCode.append(" " * jsonIndent + "fprintf(out, \", \\\"underflow\\\": \");") - self.underflow._cudaGenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "underflow"), - ), - initIndent + 2, - fillCode, - fillPrefix + (("var", - "underflow"), - ), - fillIndent + 2, - combineCode, - totalPrefix + (("var", - "underflow"), - ), - itemPrefix + (("var", - "underflow"), - ), - combineIndent, - jsonCode, - jsonPrefix + (("var", - "underflow"), - ), - jsonIndent, - weightVars, - weightVarStack, - tmpVarTypes, - False) - fillCode.append(" " * fillIndent + "}") - - fillCode.append(" " * fillIndent + "else if ({0} >= {1}) {{".format(normexpr, self.high)) - jsonCode.append( - " " * - jsonIndent + - "fprintf(out, \", \\\"overflow:type\\\": \\\"" + - self.overflow.name + - "\\\"\");") - jsonCode.append(" " * jsonIndent + "fprintf(out, \", \\\"overflow\\\": \");") - self.overflow._cudaGenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "overflow"), - ), - initIndent + 2, - fillCode, - fillPrefix + (("var", - "overflow"), - ), - fillIndent + 2, - combineCode, - totalPrefix + (("var", - "overflow"), - ), - itemPrefix + (("var", - "overflow"), - ), - combineIndent, - jsonCode, - jsonPrefix + (("var", - "overflow"), - ), - jsonIndent, - weightVars, - weightVarStack, - tmpVarTypes, - False) - fillCode.append(" " * fillIndent + "}") - - fillCode.append(" " * fillIndent + "else {") - - bin = "bin_" + str(len(tmpVarTypes)) - tmpVarTypes[bin] = "int" - - initCode.append(" " * initIndent + "for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.values))) - - fillCode.append(" " * (fillIndent + 2) + - "{0} = floor(({1} - {2}) * {3});".format(bin, normexpr, self.low, - len(self.values)/(self.high - self.low))) - - combineCode.append(" " * combineIndent + "for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.values))) - - jsonCode.append( - " " * - jsonIndent + - "fprintf(out, \", \\\"values:type\\\": \\\"" + - self.values[0].name + - "\\\"\");") - if hasattr(self.values[0], "quantity") and self.values[0].quantity.name is not None: - jsonCode.append( - " " * - jsonIndent + - "fprintf(out, \", \\\"values:name\\\": \\\"" + - self.values[0].quantity.name + - "\\\"\");") - jsonCode.append(" " * jsonIndent + "fprintf(out, \", \\\"values\\\": [\");") - jsonCode.append(" " * jsonIndent + "for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.values))) - self.values[0]._cudaGenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "values"), - ("index", - bin)), - initIndent + 2, - fillCode, - fillPrefix + (("var", - "values"), - ("index", - bin)), - fillIndent + 2, - combineCode, - totalPrefix + (("var", - "values"), - ("index", - bin)), - itemPrefix + (("var", - "values"), - ("index", - bin)), - combineIndent + 2, - jsonCode, - jsonPrefix + (("var", - "values"), - ("index", - bin)), - jsonIndent + 2, - weightVars, - weightVarStack, - tmpVarTypes, - True) - - initCode.append(" " * initIndent + "}") - - fillCode.append(" " * fillIndent + "}") - - combineCode.append(" " * combineIndent + "}") - - jsonCode.append(" " * jsonIndent + " if ({0} != {1})".format(bin, len(self.values) - 1)) - jsonCode.append(" " * jsonIndent + " fprintf(out, \", \");") - jsonCode.append(" " * jsonIndent + "}") - - if suppressName or self.quantity.name is None: - jsonCode.append(" " * jsonIndent + "fprintf(out, \"]}\");") - else: - jsonCode.append(" " * jsonIndent + "fprintf(out, \"], \\\"name\\\": " + - json.dumps(json.dumps(self.quantity.name))[1:-1] + "}\");") - - storageStructs[self._c99StructName()] = """ - typedef struct {{ - float entries; - {3} underflow; - {4} overflow; - {5} nanflow; - {1} values[{2}]; - }} {0}; -""".format(self._c99StructName(), self.values[0]._cudaStorageType(), len(self.values), - self.underflow._cudaStorageType(), self.overflow._cudaStorageType(), self.nanflow._cudaStorageType()) - - def _cudaUnpackAndFill(self, data, bigendian, alignment): - format = "".format( - len(self.values), self.low, self.high, self.values[0].name, self.underflow.name, self.overflow.name, - self.nanflow.name) + return ( + f"" + ) def __eq__(self, other): - return isinstance(other, Bin) and numeq(self.low, other.low) and numeq(self.high, other.high) and \ - self.quantity == other.quantity and numeq(self.entries, other.entries) and \ - self.values == other.values and self.underflow == other.underflow and \ - self.overflow == other.overflow and self.nanflow == other.nanflow + return ( + isinstance(other, Bin) + and numeq(self.low, other.low) + and numeq(self.high, other.high) + and self.quantity == other.quantity + and numeq(self.entries, other.entries) + and self.values == other.values + and self.underflow == other.underflow + and self.overflow == other.overflow + and self.nanflow == other.nanflow + ) def __ne__(self, other): return not self == other def __hash__(self): - return hash((self.low, self.high, self.quantity, self.entries, tuple( - self.values), self.underflow, self.overflow, self.nanflow)) + return hash( + ( + self.low, + self.high, + self.quantity, + self.entries, + tuple(self.values), + self.underflow, + self.overflow, + self.nanflow, + ) + ) @property def size(self): - """Get number of bins, consistent with SparselyBin and Categorize """ + """Get number of bins, consistent with SparselyBin and Categorize""" return self.num @property def n_bins(self): - """Get number of bins, consistent with SparselyBin and Categorize """ + """Get number of bins, consistent with SparselyBin and Categorize""" return self.num def num_bins(self, low=None, high=None): - """ - Returns number of bins of a given (sub-)range + """Returns number of bins of a given (sub-)range Possible to set range with low and high params @@ -976,14 +653,13 @@ def num_bins(self, low=None, high=None): :returns: number of bins in range :rtype: int """ - import numpy as np # trivial cases first if low is None and high is None: return len(self.values) # catch weird cases - elif low is not None and high is not None: + if low is not None and high is not None: if low > high: - raise RuntimeError('low {low} greater than high {high}'.format(low=low, high=high)) + raise RuntimeError(f"low {low} greater than high {high}") if low < self.low and high < self.low: # note: all these data end up in the underflow bin, with no real index return 0 @@ -1005,18 +681,14 @@ def num_bins(self, low=None, high=None): maxBin -= 1 high = self.low + self.bin_width() * (maxBin + 1) # number of bins. use np.round to correct for machine level rounding errors - num_bins = int(np.round((high - low) / self.bin_width())) - return num_bins + return int(np.round((high - low) / self.bin_width())) def bin_width(self): - """ - Returns bin width - """ + """Returns bin width""" return (self.high - self.low) / len(self.values) def bin_entries(self, low=None, high=None, xvalues=[]): - """ - Returns bin values + """Returns bin values Possible to set range with low and high params, and list of selected x-values @@ -1026,14 +698,13 @@ def bin_entries(self, low=None, high=None, xvalues=[]): :returns: numpy array with numbers of entries for selected bins :rtype: numpy.array """ - import numpy as np # trivial case if low is None and high is None and len(xvalues) == 0: return np.array([x.entries for x in self.values]) # catch weird cases - elif low is not None and high is not None and len(xvalues) == 0: + if low is not None and high is not None and len(xvalues) == 0: if low > high: - raise RuntimeError('low {low} greater than high {high}'.format(low=low, high=high)) + raise RuntimeError(f"low {low} greater than high {high}") if low < self.low and high < self.low: # note: all these data end up in the underflow bin return np.array([]) @@ -1045,10 +716,8 @@ def bin_entries(self, low=None, high=None, xvalues=[]): entries = [self.values[self.bin(x)].entries if self.bin(x) in self.indexes else 0.0 for x in xvalues] return np.array(entries) # lowest edge - if low is None or low < self.low: - minBin = 0 - else: # low >= self.low and low < self.high - minBin = self.bin(low) + # low >= self.low and low < self.high + minBin = 0 if low is None or low < self.low else self.bin(low) # highest edge if high is None or high >= self.high: maxBin = len(self.values) - 1 @@ -1059,23 +728,21 @@ def bin_entries(self, low=None, high=None, xvalues=[]): return np.array([self.values[i].entries for i in range(minBin, maxBin + 1)]) def bin_edges(self, low=None, high=None): - """ - Returns bin edges + """Returns bin edges :param low: lower edge of range, default is None :param high: higher edge of range, default is None :returns: numpy array with bin edges for selected range :rtype: numpy.array """ - import numpy as np num_bins = self.num_bins(low, high) # trivial cases first if low is None and high is None: return np.linspace(self.low, self.high, num_bins + 1) # catch weird cases - elif low is not None and high is not None: + if low is not None and high is not None: if low > high: - raise RuntimeError('low {low} greater than high {high}'.format(low=low, high=high)) + raise RuntimeError(f"low {low} greater than high {high}") if low < self.low and high < self.low: # note: all these data end up in the underflow bin return np.linspace(self.low, self.low, num_bins + 1) @@ -1098,27 +765,24 @@ def bin_edges(self, low=None, high=None): high = self.low + self.bin_width() * (maxBin + 1) # new low and high values reset, so redo num_bins num_bins = self.num_bins(low + np.finfo(float).eps, high - np.finfo(float).eps) - edges = np.linspace(low, high, num_bins + 1) - return edges + return np.linspace(low, high, num_bins + 1) def bin_centers(self, low=None, high=None): - """ - Returns bin centers + """Returns bin centers :param low: lower edge of range, default is None :param high: higher edge of range, default is None :returns: numpy array with bin centers for selected range :rtype: numpy.array """ - import numpy as np # trivial case if low is None and high is None: bw = self.bin_width() - return np.arange(self.low + bw / 2., self.high + bw / 2., bw) + return np.arange(self.low + bw / 2.0, self.high + bw / 2.0, bw) # catch weird cases - elif low is not None and high is not None: + if low is not None and high is not None: if low > high: - raise RuntimeError('low {low} greater than high {high}'.format(low=low, high=high)) + raise RuntimeError(f"low {low} greater than high {high}") if low < self.low and high < self.low: # note: all these data end up in the underflow bin return np.array([]) @@ -1126,10 +790,8 @@ def bin_centers(self, low=None, high=None): # note: all these data end up in the overflow bin return np.array([]) # lowest edge - if low is None or low < self.low: - minBin = 0 - else: # low >= self.low and low < self.high - minBin = self.bin(low) + # low >= self.low and low < self.high + minBin = 0 if low is None or low < self.low else self.bin(low) # highest edge if high is None or high >= self.high: maxBin = len(self.values) - 1 @@ -1141,20 +803,17 @@ def bin_centers(self, low=None, high=None): return self.low + (np.linspace(minBin, maxBin, maxBin - minBin + 1) + 0.5) * self.bin_width() def _center_from_key(self, idx): - xc = (idx + 0.5) * self.bin_width() + self.low - return xc + return (idx + 0.5) * self.bin_width() + self.low @property def mpv(self): - """Return bin-center of most probable value - """ + """Return bin-center of most probable value""" bin_entries = self.bin_entries() bin_centers = self.bin_centers() # if two max elements are equal, this will return the element with the lowest index. max_idx = max(enumerate(bin_entries), key=lambda x: x[1])[0] - bc = bin_centers[max_idx] - return bc + return bin_centers[max_idx] # extra properties: number of dimensions and datatypes of sub-hists diff --git a/histogrammar/primitives/categorize.py b/histogrammar/primitives/categorize.py index 987e9bd..364f33a 100644 --- a/histogrammar/primitives/categorize.py +++ b/histogrammar/primitives/categorize.py @@ -16,12 +16,28 @@ import math import numbers + import numpy as np -from histogrammar.defs import Container, Factory, identity, JsonFormatException, ContainerException -from histogrammar.util import n_dim, datatype, serializable, inheritdoc, maybeAdd, floatToJson, hasKeys, numeq, \ - basestring +from histogrammar.defs import ( + Container, + ContainerException, + Factory, + JsonFormatException, + identity, +) from histogrammar.primitives.count import Count +from histogrammar.util import ( + basestring, + datatype, + floatToJson, + hasKeys, + inheritdoc, + maybeAdd, + n_dim, + numeq, + serializable, +) class Categorize(Factory, Container): @@ -45,14 +61,18 @@ def ed(entries, contentType, binsAsDict=None, **bins): `bins` is empty). bins (dict from str to :doc:`Container `): the non-empty bin categories and their values. """ - if not isinstance(entries, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("entries ({0}) must be a number".format(entries)) + if not isinstance(entries, numbers.Real) and entries not in ( + "nan", + "inf", + "-inf", + ): + raise TypeError(f"entries ({entries}) must be a number") if not isinstance(contentType, basestring): - raise TypeError("contentType ({0}) must be a string".format(contentType)) + raise TypeError(f"contentType ({contentType}) must be a string") if not all(isinstance(k, basestring) and isinstance(v, Container) for k, v in bins.items()): - raise TypeError("bins ({0}) must be a dict from strings to Containers".format(bins)) + raise TypeError(f"bins ({bins}) must be a dict from strings to Containers") if entries < 0.0: - raise ValueError("entries ({0}) cannot be negative".format(entries)) + raise ValueError(f"entries ({entries}) cannot be negative") out = Categorize(None, None) out.entries = float(entries) @@ -82,7 +102,7 @@ def __init__(self, quantity=identity, value=Count()): fill with values when their `entries` become non-zero. """ if value is not None and not isinstance(value, Container): - raise TypeError("value ({0}) must be None or a Container".format(value)) + raise TypeError(f"value ({value}) must be None or a Container") self.entries = 0.0 self.quantity = serializable(identity(quantity) if isinstance(quantity, str) else quantity) self.value = value @@ -91,7 +111,7 @@ def __init__(self, quantity=identity, value=Count()): self.contentType = value.name else: self.contentType = "Count" - super(Categorize, self).__init__() + super().__init__() self.specialize() @property @@ -150,8 +170,7 @@ def __add__(self, other): out.bins[k] = other.bins[k].copy() return out.specialize() - else: - raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) + raise ContainerException(f"cannot add {self.name} and {other.name}") @inheritdoc(Container) def __iadd__(self, other): @@ -163,19 +182,17 @@ def __iadd__(self, other): elif k not in self.bins and k in other.bins: self.bins[k] = other.bins[k].copy() return self - else: - raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) + raise ContainerException(f"cannot add {self.name} and {other.name}") @inheritdoc(Container) def __mul__(self, factor): if math.isnan(factor) or factor <= 0.0: return self.zero() - else: - out = self.zero() - out.entries = factor * self.entries - for k, v in self.bins.items(): - out.bins[k] = v * factor - return out.specialize() + out = self.zero() + out.entries = factor * self.entries + for k, v in self.bins.items(): + out.bins[k] = v * factor + return out.specialize() @inheritdoc(Container) def __rmul__(self, factor): @@ -190,9 +207,9 @@ def fill(self, datum, weight=1.0): if isinstance(q, (basestring, bool)): pass elif q is None or np.isnan(q): - q = 'NaN' + q = "NaN" if not isinstance(q, (basestring, bool)): - raise TypeError("function return value ({0}) must be a string or bool".format(q)) + raise TypeError(f"function return value ({q}) must be a string or bool") if q not in self.bins: self.bins[q] = self.value.zero() @@ -201,92 +218,18 @@ def fill(self, datum, weight=1.0): # no possibility of exception from here on out (for rollback) self.entries += weight - def _cppGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - normexpr = self._c99QuantityExpr( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - None) - - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0;") - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".bins.clear();") - fillCode.append(" " * fillIndent + self._c99ExpandPrefix(*fillPrefix) + - ".entries += " + weightVarStack[-1] + ";") - - value = "value_" + str(len(tmpVarTypes)) - tmpVarTypes[value] = self.value._c99StorageType() + "*" - - fillCode.append("""{indent}if ({bins}.find({q}) == {bins}.end()) -{indent} {bins}[{q}] = {prototype}; // copy -{indent}{value} = &({bins}[{q}]); // reference""".format( - indent=" " * fillIndent, - q=normexpr, - value=value, - prototype=self._c99ExpandPrefix(*fillPrefix) + ".value", - bins=self._c99ExpandPrefix(*fillPrefix) + ".bins")) - - self.value._c99GenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "value"), - ), - initIndent, - fillCode, - (("var", - "(*" + value + ")"), - ), - fillIndent, - weightVars, - weightVarStack, - tmpVarTypes) - - storageStructs[self._c99StructName()] = """ - typedef struct {{ - double entries; - {1} value; - std::unordered_map bins; - {1}& getValues(std::string i) {{ return bins[i]; }} - }} {0}; -""".format(self._c99StructName(), self.value._c99StorageType()) - - def _c99GenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - raise NotImplementedError("no C99-compliant implementation of Categorize (only C++)") - - def _clingUpdate(self, filler, *extractorPrefix): - obj = self._clingExpandPrefix(filler, *extractorPrefix) - self.entries += obj.entries - - for i in obj.bins: - key = i.first - if key not in self.bins: - self.bins[key] = self.value.copy() - self.bins[key]._clingUpdate(obj, ("func", ["getValues", key])) - - def _c99StructName(self): - return "Cz" + self.value._c99StructName() - def _numpy(self, data, weights, shape): q = self.quantity(data) if isinstance(q, (list, tuple)): q = np.array(q) self._checkNPQuantity(q, shape) - if isinstance(weights, (float, int)) and weights == 1: - all_weights_one = True - elif isinstance(weights, np.ndarray) and np.all(weights == 1): + if ( + isinstance(weights, (float, int)) + and weights == 1 + or isinstance(weights, np.ndarray) + and np.all(weights == 1) + ): all_weights_one = True else: all_weights_one = False @@ -302,13 +245,14 @@ def _numpy(self, data, weights, shape): uniques, counts = np.unique(q, return_counts=True) for c, x in zip(counts, uniques): - if isinstance(x, (basestring, bool)): + xval = x + if isinstance(xval, (basestring, bool)): pass - elif x is None or np.isnan(x): - x = 'NaN' - if x not in self.bins: - self.bins[x] = self.value.zero() - self.bins[x]._numpy(None, c, [None]) + elif xval is None or np.isnan(xval): + xval = "NaN" + if xval not in self.bins: + self.bins[xval] = self.value.zero() + self.bins[xval]._numpy(None, c, [None]) else: # all other cases ... selection = np.empty(q.shape, dtype=bool) @@ -316,18 +260,19 @@ def _numpy(self, data, weights, shape): # no possibility of exception from here on out (for rollback) for i, x in enumerate(uniques): - if isinstance(x, (basestring, bool)): + xval = x + if isinstance(xval, (basestring, bool)): pass - elif x is None or np.isnan(x): - x = 'NaN' - if x not in self.bins: - self.bins[x] = self.value.zero() + elif xval is None or np.isnan(xval): + xval = "NaN" + if xval not in self.bins: + self.bins[xval] = self.value.zero() # passing on the full array seems faster for one- AND multi-dim histograms np.not_equal(inverse, i, selection) subweights[:] = weights subweights[selection] = 0.0 - self.bins[x]._numpy(data, subweights, shape) + self.bins[xval]._numpy(data, subweights, shape) self.entries += float(newentries) @@ -365,14 +310,19 @@ def toJsonFragment(self, suppressName): else: bins_type = self.contentType - return maybeAdd({ - # for json serialization all keys need to be strings, else json libs throws TypeError - # e.g. boolean keys get converted to strings here - "entries": floatToJson(self.entries), - "bins:type": bins_type, - "bins": dict((str(k), v.toJsonFragment(True)) for k, v in self.bins.items()), - }, **{"name": None if suppressName else self.quantity.name, - "bins:name": binsName}) + return maybeAdd( + { + # for json serialization all keys need to be strings, else json libs throws TypeError + # e.g. boolean keys get converted to strings here + "entries": floatToJson(self.entries), + "bins:type": bins_type, + "bins": {str(k): v.toJsonFragment(True) for k, v in self.bins.items()}, + }, + **{ + "name": None if suppressName else self.quantity.name, + "bins:name": binsName, + }, + ) @staticmethod @inheritdoc(Factory) @@ -404,7 +354,7 @@ def fromJsonFragment(json, nameFromParent): raise JsonFormatException(json["bins:name"], "Categorize.bins:name") if isinstance(json["bins"], dict): - bins = dict((k, factory.fromJsonFragment(v, dataName)) for k, v in json["bins"].items()) + bins = {k: factory.fromJsonFragment(v, dataName) for k, v in json["bins"].items()} else: raise JsonFormatException(json, "Categorize.bins") @@ -412,17 +362,19 @@ def fromJsonFragment(json, nameFromParent): out.quantity.name = nameFromParent if name is None else name return out.specialize() - else: - raise JsonFormatException(json, "Categorize") + raise JsonFormatException(json, "Categorize") def __repr__(self): - return " 0 else self.value.name if self.value is not None else self.contentType, - self.size) + vals = self.values[0].name if self.size > 0 else self.value.name if self.value is not None else self.contentType + return f" 0: label = label[:max_length] except BaseException: - label = 'bin_%d' % i + label = "bin_%d" % i labels.append(label) return np.array(labels) def bin_centers(self, max_length=-1): - """ - Returns bin labels + """Returns bin labels Compatible function call with Bin and SparselyBin @@ -487,15 +436,13 @@ def _center_from_key(self, bin_key): @property def mpv(self): - """Return bin-label of most probable value - """ + """Return bin-label of most probable value""" bin_entries = self.bin_entries() bin_labels = self.bin_labels() # if two max elements are equal, this will return the element with the lowest index. max_idx = max(enumerate(bin_entries), key=lambda x: x[1])[0] - bl = bin_labels[max_idx] - return bl + return bin_labels[max_idx] # extra properties: number of dimensions and datatypes of sub-hists diff --git a/histogrammar/primitives/centrallybin.py b/histogrammar/primitives/centrallybin.py index 9b0a92b..5206540 100644 --- a/histogrammar/primitives/centrallybin.py +++ b/histogrammar/primitives/centrallybin.py @@ -14,15 +14,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import math import numbers -import struct -from histogrammar.defs import Container, Factory, identity, JsonFormatException, ContainerException -from histogrammar.util import n_dim, datatype, serializable, inheritdoc, maybeAdd, floatToJson, hasKeys, numeq, \ - floatToC99, basestring, xrange +import numpy as np + +from histogrammar.defs import ( + Container, + ContainerException, + Factory, + JsonFormatException, + identity, +) from histogrammar.primitives.count import Count +from histogrammar.util import ( + basestring, + datatype, + floatToJson, + hasKeys, + inheritdoc, + maybeAdd, + n_dim, + numeq, + serializable, + xrange, +) class CentrallyBin(Factory, Container): @@ -42,15 +58,24 @@ def ed(entries, bins, nanflow): their accumulated data. nanflow (:doc:`Container `): the filled nanflow bin. """ - if not isinstance(entries, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("entries ({0}) must be a number".format(entries)) - if not isinstance(bins, (list, tuple)) and not all(isinstance(v, (list, tuple)) and len( - v) == 2 and isinstance(v[0], numbers.Real) and isinstance(v[1], Container) for v in bins): - raise TypeError("bins ({0}) must be a list of number, Container pairs".format(bins)) + if not isinstance(entries, numbers.Real) and entries not in ( + "nan", + "inf", + "-inf", + ): + raise TypeError(f"entries ({entries}) must be a number") + if not isinstance(bins, (list, tuple)) and not all( + isinstance(v, (list, tuple)) + and len(v) == 2 + and isinstance(v[0], numbers.Real) + and isinstance(v[1], Container) + for v in bins + ): + raise TypeError(f"bins ({bins}) must be a list of number, Container pairs") if not isinstance(nanflow, Container): - raise TypeError("nanflow ({0}) must be a Container".format(nanflow)) + raise TypeError(f"nanflow ({nanflow}) must be a Container") if entries < 0.0: - raise ValueError("entries ({0}) cannot be negative".format(entries)) + raise ValueError(f"entries ({entries}) cannot be negative") out = CentrallyBin(bins, None, None, nanflow) out.entries = float(entries) out.bins = bins @@ -77,15 +102,20 @@ def __init__(self, centers, quantity=identity, value=Count(), nanflow=Count()): sub-aggregators in each bin. """ - if not isinstance(centers, (list, tuple)) and not all(isinstance(v, (list, tuple)) and len( - v) == 2 and isinstance(v[0], numbers.Real) and isinstance(v[1], Container) for v in centers): - raise TypeError("centers ({0}) must be a list of number, Container pairs".format(centers)) + if not isinstance(centers, (list, tuple)) and not all( + isinstance(v, (list, tuple)) + and len(v) == 2 + and isinstance(v[0], numbers.Real) + and isinstance(v[1], Container) + for v in centers + ): + raise TypeError(f"centers ({centers}) must be a list of number, Container pairs") if value is not None and not isinstance(value, Container): - raise TypeError("value ({0}) must be None or a Container".format(value)) + raise TypeError(f"value ({value}) must be None or a Container") if not isinstance(nanflow, Container): - raise TypeError("nanflow ({0}) must be a Container".format(nanflow)) + raise TypeError(f"nanflow ({nanflow}) must be a Container") if len(centers) < 2: - raise ValueError("number of centers ({0}) must be at least two".format(len(centers))) + raise ValueError(f"number of centers ({len(centers)}) must be at least two") self.entries = 0.0 if value is None: @@ -97,7 +127,7 @@ def __init__(self, centers, quantity=identity, value=Count(), nanflow=Count()): self.value = value self.nanflow = nanflow.copy() - super(CentrallyBin, self).__init__() + super().__init__() self.specialize() def histogram(self): @@ -131,11 +161,11 @@ def index(self, x, greater=True): thisCenter = self.bins[index][0] nextCenter = self.bins[index + 1][0] if greater: - if x < (thisCenter + nextCenter)/2.0: - return index - else: - if x <= (thisCenter + nextCenter)/2.0: + if x < (thisCenter + nextCenter) / 2.0: return index + elif x <= (thisCenter + nextCenter) / 2.0: + return index + return None def _lower_index(self, x): return self.index(x) @@ -159,32 +189,29 @@ def neighbors(self, center): """Find the lower and upper neighbors of a bin (given by exact bin center).""" closestIndex = self.index(center) if self.bins[closestIndex][0] != center: - raise TypeError("position {0} is not the exact center of a bin".format(center)) - elif closestIndex == 0: + raise TypeError(f"position {center} is not the exact center of a bin") + if closestIndex == 0: return None, self.bins[closestIndex + 1][0] - elif closestIndex == len(self.bins) - 1: + if closestIndex == len(self.bins) - 1: return self.bins[closestIndex - 1][0], None - else: - return self.bins[closestIndex - 1][0], self.bins[closestIndex + 1][0] + return self.bins[closestIndex - 1][0], self.bins[closestIndex + 1][0] def range(self, center): """Get the low and high edge of a bin (given by exact bin center).""" - below, above = self.neighbors(center) # is never None, None + below, above = self.neighbors(center) # is never None, None if below is None: - return float("-inf"), (center + above)/2.0 - elif above is None: - return (below + center)/2.0, float("inf") - else: - return (below + center)/2.0, (above + center)/2.0 + return float("-inf"), (center + above) / 2.0 + if above is None: + return (below + center) / 2.0, float("inf") + return (below + center) / 2.0, (above + center) / 2.0 @property def n_bins(self): - """Get number of bins, consistent with SparselyBin and Categorize """ + """Get number of bins, consistent with SparselyBin and Categorize""" return len(self.bins) def num_bins(self, low=None, high=None): - """ - Returns number of bins of a given (sub-)range + """Returns number of bins of a given (sub-)range Possible to set range with low and high params @@ -197,9 +224,8 @@ def num_bins(self, low=None, high=None): if low is None and high is None: return len(self.bins) # catch weird cases - elif low is not None and high is not None: - if low > high: - raise RuntimeError('low {low} greater than high {high}'.format(low=low, high=high)) + if low is not None and high is not None and low > high: + raise RuntimeError(f"low {low} greater than high {high}") # lowest, highest edge reset if low is None: low = float("-inf") @@ -211,8 +237,7 @@ def num_bins(self, low=None, high=None): return hidx - lidx + 1 def bin_entries(self, low=None, high=None, xvalues=[]): - """ - Returns bin values + """Returns bin values Possible to set range with low and high params, and list of selected x-values @@ -222,14 +247,13 @@ def bin_entries(self, low=None, high=None, xvalues=[]): :returns: numpy array with numbers of entries for selected bins :rtype: numpy.array """ - import numpy as np # trivial case if low is None and high is None and len(xvalues) == 0: return np.array([b[1].entries for b in self.bins]) # catch weird cases - elif low is not None and high is not None and len(xvalues) == 0: + if low is not None and high is not None and len(xvalues) == 0: if low > high: - raise RuntimeError('low {low} greater than high {high}'.format(low=low, high=high)) + raise RuntimeError(f"low {low} greater than high {high}") # entries at request list of x-values elif len(xvalues) > 0: return np.array([(self.bins[self.index(x)])[1].entries for x in xvalues]) @@ -244,44 +268,39 @@ def bin_entries(self, low=None, high=None, xvalues=[]): return np.array([(self.bins[i])[1].entries for i in xrange(lidx, hidx + 1)]) def bin_edges(self, low=None, high=None): - """ - Returns bin edges + """Returns bin edges :param low: lower edge of range, default is None :param high: higher edge of range, default is None :returns: numpy array with bin edges for selected range :rtype: numpy.array """ - import numpy as np # catch weird cases if low is not None and high is not None and low > high: - raise RuntimeError('low {low} greater than high {high}'.format(low=low, high=high)) + raise RuntimeError(f"low {low} greater than high {high}") # lowest, highest edge reset if low is None: - low = float('-inf') + low = float("-inf") if high is None: - high = float('inf') + high = float("inf") # return bin edges lidx = self._lower_index(low) hidx = self._upper_index(high) lowest_edge = self.range(self.center(low))[0] - upper_edges = np.array([self.range(c)[1] for c in self.centers[lidx: hidx + 1]]) - bin_edges = np.concatenate([[lowest_edge], upper_edges]) - return bin_edges + upper_edges = np.array([self.range(c)[1] for c in self.centers[lidx : hidx + 1]]) + return np.concatenate([[lowest_edge], upper_edges]) def bin_centers(self, low=None, high=None): - """ - Returns bin centers + """Returns bin centers :param low: lower edge of range, default is None :param high: higher edge of range, default is None :returns: numpy array with bin centers for selected range :rtype: numpy.array """ - import numpy as np # catch weird cases if low is not None and high is not None and low > high: - raise RuntimeError('low {low} greater than high {high}'.format(low=low, high=high)) + raise RuntimeError(f"low {low} greater than high {high}") # lowest, highest edge reset if low is None: low = -np.inf @@ -290,20 +309,18 @@ def bin_centers(self, low=None, high=None): # return bin entries lidx = self._lower_index(low) hidx = self._upper_index(high) - return np.array(self.centers[lidx: hidx + 1]) + return np.array(self.centers[lidx : hidx + 1]) def _center_from_key(self, center): return center @property def mpv(self): - """Return bin-center of most probable value - """ + """Return bin-center of most probable value""" bin_entries = self.bin_entries() # if two max elements are equal, this will return the element with the lowest index. max_idx = max(enumerate(bin_entries), key=lambda x: x[1])[0] - bc = self.centers[max_idx] - return bc + return self.centers[max_idx] @inheritdoc(Container) def zero(self): @@ -313,12 +330,17 @@ def zero(self): def __add__(self, other): if self.centers != other.centers: raise ContainerException( - "cannot add CentrallyBin because centers are different:\n {0}\nvs\n {1}".format( - self.centers, other.centers)) + f"cannot add CentrallyBin because centers are different:\n {self.centers}\nvs\n {other.centers}" + ) newbins = [(c1, v1 + v2) for (c1, v1), (_, v2) in zip(self.bins, other.bins)] - out = CentrallyBin([c for c, v in self.bins], self.quantity, self.value, self.nanflow + other.nanflow) + out = CentrallyBin( + [c for c, v in self.bins], + self.quantity, + self.value, + self.nanflow + other.nanflow, + ) out.entries = self.entries + other.entries out.bins = newbins return out.specialize() @@ -327,11 +349,11 @@ def __add__(self, other): def __iadd__(self, other): if self.centers != other.centers: raise ContainerException( - "cannot add CentrallyBin because centers are different:\n {0}\nvs\n {1}".format( - self.centers, other.centers)) + f"cannot add CentrallyBin because centers are different:\n {self.centers}\nvs\n {other.centers}" + ) self.entries += other.entries for (c1, v1), (_, v2) in zip(self.bins, other.bins): - v1 += v2 + v1 += v2 # noqa: PLW2901 self.nanflow += other.nanflow return self @@ -339,12 +361,11 @@ def __iadd__(self, other): def __mul__(self, factor): if math.isnan(factor) or factor <= 0.0: return self.zero() - else: - out = self.zero() - out.entries = factor * self.entries - out.bins = [(c, v * factor) for (c, v) in self.bins] - out.nanflow = self.nanflow * factor - return out.specialize() + out = self.zero() + out.entries = factor * self.entries + out.bins = [(c, v * factor) for (c, v) in self.bins] + out.nanflow = self.nanflow * factor + return out.specialize() @inheritdoc(Container) def __rmul__(self, factor): @@ -357,7 +378,7 @@ def fill(self, datum, weight=1.0): if weight > 0.0: q = self.quantity(datum) if not isinstance(q, numbers.Real): - raise TypeError("function return value ({0}) must be boolean or number".format(q)) + raise TypeError(f"function return value ({q}) must be boolean or number") if self.nan(q): self.nanflow.fill(datum, weight) @@ -367,309 +388,6 @@ def fill(self, datum, weight=1.0): # no possibility of exception from here on out (for rollback) self.entries += weight - def _cppGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - return self._c99GenerateCode(parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, - fillPrefix, fillIndent, weightVars, weightVarStack, tmpVarTypes) - - def _c99GenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - normexpr = self._c99QuantityExpr( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - None) - - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0;") - fillCode.append(" " * fillIndent + self._c99ExpandPrefix(*fillPrefix) + - ".entries += " + weightVarStack[-1] + ";") - - fillCode.append(" " * fillIndent + "if (std::isnan({0})) {{".format(normexpr)) - self.nanflow._c99GenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "nanflow"), - ), - initIndent, - fillCode, - fillPrefix + (("var", - "nanflow"), - ), - fillIndent + 2, - weightVars, - weightVarStack, - tmpVarTypes) - fillCode.append(" " * fillIndent + "}") - fillCode.append(" " * fillIndent + "else {") - - bin = "bin_" + str(len(tmpVarTypes)) - tmpVarTypes[bin] = "int" - - initCode.append(" " * initIndent + "for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.bins))) - - fillCode.append(" " * fillIndent + - " const double edges[{0}] = {{{1}}};".format(len(self.values) - 1, - ", ".join(floatToC99((self.bins[idx - 1][0] + - self.bins[idx][0])/2.0) - for idx in xrange(1, len(self.bins))))) - - fillCode.append(" " * fillIndent + " for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.bins) - 1)) - fillCode.append(" " * fillIndent + " if ({0} < edges[{1}])".format(normexpr, bin)) - fillCode.append(" " * fillIndent + " break;") - fillCode.append(" " * fillIndent + " }") - - self.bins[0][1]._c99GenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "values"), - ("index", - bin)), - initIndent + 2, - fillCode, - fillPrefix + (("var", - "values"), - ("index", - bin)), - fillIndent + 2, - weightVars, - weightVarStack, - tmpVarTypes) - - initCode.append(" " * initIndent + "}") - fillCode.append(" " * fillIndent + "}") - - storageStructs[self._c99StructName()] = """ - typedef struct {{ - double entries; - {3} nanflow; - {1} values[{2}]; - {1}& getValues(int i) {{ return values[i]; }} - }} {0}; -""".format(self._c99StructName(), - self.bins[0][1]._c99StorageType(), - len(self.values), - self.nanflow._c99StorageType()) - - def _clingUpdate(self, filler, *extractorPrefix): - obj = self._clingExpandPrefix(filler, *extractorPrefix) - self.entries += obj.entries - for i in xrange(len(self.values)): - self.bins[i][1]._clingUpdate(obj, ("func", ["getValues", i])) - self.nanflow._clingUpdate(obj, ("var", "nanflow")) - - def _c99StructName(self): - return "Cb" + str(len(self.bins)) + self.bins[0][1]._c99StructName() + self.nanflow._c99StructName() - - def _cudaGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, combineCode, totalPrefix, itemPrefix, combineIndent, jsonCode, jsonPrefix, - jsonIndent, weightVars, weightVarStack, tmpVarTypes, suppressName): - normexpr = self._cudaQuantityExpr( - parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - None) - - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0f;") - fillCode.append(" " * fillIndent + "atomicAdd(&" + self._c99ExpandPrefix(*fillPrefix) + ".entries, " + - weightVarStack[-1] + ");") - combineCode.append( - " " * - combineIndent + - "atomicAdd(&" + - self._c99ExpandPrefix( - * - totalPrefix) + - ".entries, " + - self._c99ExpandPrefix( - * - itemPrefix) + - ".entries);") - jsonCode.append(" " * jsonIndent + "fprintf(out, \"{\\\"entries\\\": \");") - jsonCode.append(" " * jsonIndent + "floatToJson(out, " + self._c99ExpandPrefix(*jsonPrefix) + ".entries);") - - fillCode.append(" " * fillIndent + "if (isnan({0})) {{".format(normexpr)) - jsonCode.append( - " " * - jsonIndent + - "fprintf(out, \", \\\"nanflow:type\\\": \\\"" + - self.nanflow.name + - "\\\"\");") - jsonCode.append(" " * jsonIndent + "fprintf(out, \", \\\"nanflow\\\": \");") - self.nanflow._cudaGenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "nanflow"), - ), - initIndent + 2, - fillCode, - fillPrefix + (("var", - "nanflow"), - ), - fillIndent + 2, - combineCode, - totalPrefix + (("var", - "nanflow"), - ), - itemPrefix + (("var", - "nanflow"), - ), - combineIndent, - jsonCode, - jsonPrefix + (("var", - "nanflow"), - ), - jsonIndent, - weightVars, - weightVarStack, - tmpVarTypes, - False) - fillCode.append(" " * fillIndent + "}") - fillCode.append(" " * fillIndent + "else {") - - bin = "bin_" + str(len(tmpVarTypes)) - tmpVarTypes[bin] = "int" - - initCode.append(" " * initIndent + "for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.bins))) - - fillCode.append(" " * fillIndent + " const float edges[{0}] = {{{1}}};".format( - len(self.values) - 1, ", ".join(floatToC99((self.bins[index - 1][0] + self.bins[index][0])/2.0) - for index in xrange(1, len(self.bins))))) - fillCode.append(" " * fillIndent + " for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.bins) - 1)) - fillCode.append(" " * fillIndent + " if ({0} < edges[{1}])".format(normexpr, bin)) - fillCode.append(" " * fillIndent + " break;") - fillCode.append(" " * fillIndent + " }") - - combineCode.append(" " * combineIndent + "for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.bins))) - - jsonCode.append( - " " * - jsonIndent + - "fprintf(out, \", \\\"bins:type\\\": \\\"" + - self.bins[0][1].name + - "\\\"\");") - if hasattr(self.bins[0][1], "quantity") and self.bins[0][1].quantity.name is not None: - jsonCode.append( - " " * - jsonIndent + - "fprintf(out, \", \\\"bins:name\\\": \\\"" + - self.bins[0][1].quantity.name + - "\\\"\");") - jsonCode.append(" " * jsonIndent + "{") - jsonCode.append(" " * jsonIndent + " const float centers[{0}] = {{{1}}};".format( - len(self.values), ", ".join(floatToC99(center) for center, value in self.bins))) - jsonCode.append(" " * jsonIndent + " fprintf(out, \", \\\"bins\\\": [\");") - jsonCode.append(" " * jsonIndent + " for ({0} = 0; {0} < {1}; ++{0}) {{".format(bin, len(self.values))) - jsonCode.append(" " * jsonIndent + " fprintf(out, \"{\\\"center\\\": \");") - jsonCode.append(" " * jsonIndent + " floatToJson(out, centers[" + bin + "]);") - jsonCode.append(" " * jsonIndent + " fprintf(out, \", \\\"data\\\": \");") - - self.bins[0][1]._cudaGenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "values"), - ("index", - bin)), - initIndent + 2, - fillCode, - fillPrefix + (("var", - "values"), - ("index", - bin)), - fillIndent + 2, - combineCode, - totalPrefix + (("var", - "values"), - ("index", - bin)), - itemPrefix + (("var", - "values"), - ("index", - bin)), - combineIndent + 2, - jsonCode, - jsonPrefix + (("var", - "values"), - ("index", - bin)), - jsonIndent + 4, - weightVars, - weightVarStack, - tmpVarTypes, - True) - - initCode.append(" " * initIndent + "}") - fillCode.append(" " * fillIndent + "}") - combineCode.append(" " * combineIndent + "}") - jsonCode.append(" " * jsonIndent + " fprintf(out, \"}\");") - jsonCode.append(" " * jsonIndent + " if ({0} != {1})".format(bin, len(self.values) - 1)) - jsonCode.append(" " * jsonIndent + " fprintf(out, \", \");") - jsonCode.append(" " * jsonIndent + " }") - jsonCode.append(" " * jsonIndent + "}") - - if suppressName or self.quantity.name is None: - jsonCode.append(" " * jsonIndent + "fprintf(out, \"]}\");") - else: - jsonCode.append(" " * jsonIndent + "fprintf(out, \"], \\\"name\\\": " + - json.dumps(json.dumps(self.quantity.name))[1:-1] + "}\");") - - storageStructs[self._c99StructName()] = """ - typedef struct {{ - float entries; - {3} nanflow; - {1} values[{2}]; - }} {0}; -""".format(self._c99StructName(), - self.bins[0][1]._cudaStorageType(), - len(self.values), - self.nanflow._cudaStorageType()) - - def _cudaUnpackAndFill(self, data, bigendian, alignment): - format = "".format( - self.bins[0][1].name, len(self.bins), self.nanflow.name) + return f"" def __eq__(self, other): - return isinstance(other, CentrallyBin) and self.quantity == other.quantity and numeq( - self.entries, other.entries) and self.bins == other.bins and self.nanflow == other.nanflow + return ( + isinstance(other, CentrallyBin) + and self.quantity == other.quantity + and numeq(self.entries, other.entries) + and self.bins == other.bins + and self.nanflow == other.nanflow + ) def __ne__(self, other): return not self == other diff --git a/histogrammar/primitives/collection.py b/histogrammar/primitives/collection.py index 0aecc81..2aa78d9 100644 --- a/histogrammar/primitives/collection.py +++ b/histogrammar/primitives/collection.py @@ -14,293 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import math import numbers -import struct - -from histogrammar.defs import Container, Factory, JsonFormatException, ContainerException -from histogrammar.util import inheritdoc, floatToJson, hasKeys, numeq, basestring, xrange - - -class Collection(object): - def _c99CanonicalOrder(self, items): - return sorted((v._c99StructName(), k, v) for k, v in items) - - def _c99StructName(self): - letter = self.name[0] - out = [letter, "_"] - last = None - count = 0 - for s, k, v in self._c99CanonicalOrder(self.pairs.items()): - if s != last: - if last is not None: - out.append(str(count)) - out.append("_") - out.append(s) - count = 0 - count += 1 - last = s - out.extend([str(count), "_", letter.lower()]) - return "".join(out) - - def _c99Struct(self): - out = [""" - typedef struct { - """] - last = None - count = 0 - n = 0 - lastN = 0 - lastType = 0 - for s, k, v in self._c99CanonicalOrder(self.pairs.items()): - if s != last: - if last is not None: - out.append("[{0}];\n ".format(count)) - out.append("{0}& getSub{1}(int i) {{ return sub{1}[i]; }}\n ".format(lastType, lastN)) - out.append("{0} sub{1}".format(s, n)) - lastType = s - lastN = n - n += 1 - count = 0 - count += 1 - last = s - out.append( - """[{0}];\n {1}& getSub{2}(int i) {{ return sub{2}[i]; }}\n double entries;\n }} {3};""".format( - count, - lastType, - lastN, - self._c99StructName())) - return "".join(out) - - def _cppGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - return self._c99GenerateCode(parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, - fillPrefix, fillIndent, weightVars, weightVarStack, tmpVarTypes) - - def _c99GenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, weightVars, weightVarStack, tmpVarTypes): - last = None - n = 0 - i = 0 - for s, k, v in self._c99CanonicalOrder(self.pairs.items()): - if last is not None and s != last: - n += 1 - i = 0 - v._c99GenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "sub" + str(n)), - ("index", - i)), - initIndent, - fillCode, - fillPrefix + (("var", - "sub" + str(n)), - ("index", - i)), - fillIndent, - weightVars, - weightVarStack, - tmpVarTypes) - i += 1 - last = s - - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0;") - fillCode.append(" " * fillIndent + self._c99ExpandPrefix(*fillPrefix) + - ".entries += " + weightVarStack[-1] + ";") - - storageStructs[self._c99StructName()] = self._c99Struct() - - def _clingUpdate(self, filler, *extractorPrefix): - obj = self._clingExpandPrefix(filler, *extractorPrefix) - self.entries += obj.entries - last = None - n = 0 - i = 0 - for s, k, v in self._c99CanonicalOrder(self.pairs.items()): - if last is not None and s != last: - n += 1 - i = 0 - v._clingUpdate(obj, ("func", ["getSub" + str(n), i])) - i += 1 - last = s - - def _cudaStruct(self): - out = [""" - typedef struct { - """] - last = None - n = 0 - count = 0 - for s, k, v in self._c99CanonicalOrder(self.pairs.items()): - if s != last: - if last is not None: - out.append("[{0}];\n ".format(count)) - out.append("{0} sub{1}".format(s, n)) - n += 1 - count = 0 - count += 1 - last = s - out.append("""[{0}];\n float entries;\n }} {1};""".format(count, self._c99StructName())) - return "".join(out) - - def _cudaGenerateCode(self, parser, generator, inputFieldNames, inputFieldTypes, derivedFieldTypes, - derivedFieldExprs, storageStructs, initCode, initPrefix, initIndent, fillCode, fillPrefix, - fillIndent, combineCode, totalPrefix, itemPrefix, combineIndent, jsonCode, jsonPrefix, - jsonIndent, weightVars, weightVarStack, tmpVarTypes, suppressName): - tmpJsonCode = [] - - if isinstance(self, (Label, Index)): - tmpJsonCode.append( - " " * - jsonIndent + - "fprintf(out, \"{\\\"sub:type\\\": \\\"" + - self.values[0].name + - "\\\", \\\"data\\\": \");") - else: - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"{\\\"data\\\": \");") - if isinstance(self, (Label, UntypedLabel)): - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"{\");") - else: - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"[\");") - - if isinstance(self, Branch): - rightOrder = {} - lastJsonCode = len(tmpJsonCode) - - last = None - n = 0 - i = 0 - for s, k, v in self._c99CanonicalOrder(self.pairs.items()): - if not isinstance(self, Branch) and last is not None: - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \", \");") - if isinstance(self, (Label, UntypedLabel)): - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"" + json.dumps(json.dumps(k))[1:-1] + ": \");") - if isinstance(self, (UntypedLabel, Branch)): - tmpJsonCode.append( - " " * - jsonIndent + - "fprintf(out, \"{\\\"type\\\": \\\"" + - v.name + - "\\\", \\\"data\\\": \");") - - if last is not None and s != last: - n += 1 - i = 0 - v._cudaGenerateCode(parser, - generator, - inputFieldNames, - inputFieldTypes, - derivedFieldTypes, - derivedFieldExprs, - storageStructs, - initCode, - initPrefix + (("var", - "sub" + str(n)), - ("index", - i)), - initIndent, - fillCode, - fillPrefix + (("var", - "sub" + str(n)), - ("index", - i)), - fillIndent, - combineCode, - totalPrefix + (("var", - "sub" + str(n)), - ("index", - i)), - itemPrefix + (("var", - "sub" + str(n)), - ("index", - i)), - combineIndent, - tmpJsonCode, - jsonPrefix + (("var", - "sub" + str(n)), - ("index", - i)), - jsonIndent, - weightVars, - weightVarStack, - tmpVarTypes, - suppressName) - - i += 1 - last = s - - if isinstance(self, (UntypedLabel, Branch)): - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"}\");") - - if isinstance(self, Branch): - rightOrder[k] = tmpJsonCode[lastJsonCode:] - tmpJsonCode = tmpJsonCode[:lastJsonCode] - - if isinstance(self, Branch): - for k in xrange(len(rightOrder)): - if k != 0: - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \", \");") - tmpJsonCode.extend(rightOrder[k]) - - if isinstance(self, (Label, UntypedLabel)): - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"}\");") - else: - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"]\");") - - initCode.append(" " * initIndent + self._c99ExpandPrefix(*initPrefix) + ".entries = 0.0f;") - fillCode.append(" " * fillIndent + "atomicAdd(&" + self._c99ExpandPrefix(*fillPrefix) + ".entries, " + - weightVarStack[-1] + ");") - combineCode.append( - " " * - combineIndent + - "atomicAdd(&" + - self._c99ExpandPrefix( - * - totalPrefix) + - ".entries, " + - self._c99ExpandPrefix( - * - itemPrefix) + - ".entries);") - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \", \\\"entries\\\": \");") - tmpJsonCode.append(" " * jsonIndent + "floatToJson(out, " + self._c99ExpandPrefix(*jsonPrefix) + ".entries);") - tmpJsonCode.append(" " * jsonIndent + "fprintf(out, \"}\");") - - jsonCode.extend(tmpJsonCode) - - storageStructs[self._c99StructName()] = self._cudaStruct() - - def _cudaUnpackAndFill(self, data, bigendian, alignment): - last = None - n = 0 - i = 0 - for s, k, v in self._c99CanonicalOrder(self.pairs.items()): - if last is not None and s != last: - n += 1 - i = 0 - data = v._cudaUnpackAndFill(data, bigendian, alignment) - i += 1 - last = s - - format = "` pairs): the collection of filled aggregators. """ - if not isinstance(entries, numbers.Real) and entries not in ("nan", "inf", "-inf"): - raise TypeError("entries ({0}) must be a number".format(entries)) + if not isinstance(entries, numbers.Real) and entries not in ( + "nan", + "inf", + "-inf", + ): + raise TypeError(f"entries ({entries}) must be a number") if not all(isinstance(k, basestring) and isinstance(v, Container) for k, v in pairs.items()): - raise TypeError("pairs ({0}) must be a dict from strings to Containers".format(pairs)) + raise TypeError(f"pairs ({pairs}) must be a dict from strings to Containers") if entries < 0.0: - raise ValueError("entries ({0}) cannot be negative".format(entries)) + raise ValueError(f"entries ({entries}) cannot be negative") if pairsAsDict is None: pairsAsDict = {} @@ -359,8 +93,8 @@ def __init__(self, **pairs): entries (float): the number of entries, initially 0.0. """ if not all(isinstance(k, basestring) and isinstance(v, Container) for k, v in pairs.items()): - raise TypeError("pairs ({0}) must be a dict from strings to Containers".format(pairs)) - if any(not isinstance(x, basestring) for x in pairs.keys()): + raise TypeError(f"pairs ({pairs}) must be a dict from strings to Containers") + if any(not isinstance(x, basestring) for x in pairs): raise ValueError("all Label keys must be strings") if len(pairs) < 1: raise ValueError("at least one pair required") @@ -376,7 +110,7 @@ def __init__(self, **pairs): self.entries = 0.0 self.pairs = pairs - super(Label, self).__init__() + super().__init__() self.specialize() @property @@ -403,8 +137,7 @@ def __call__(self, x, *rest): """Attempt to get key ``index``, throwing an exception if it does not exist.""" if len(rest) == 0: return self.pairs[x] - else: - return self.pairs[x](*rest) + return self.pairs[x](*rest) def get(self, x): """Attempt to get key ``x``, returning ``None`` if it does not exist.""" @@ -416,56 +149,49 @@ def getOrElse(self, x, default): @inheritdoc(Container) def zero(self): - return Label(**dict((k, v.zero()) for k, v in self.pairs.items())) + return Label(**{k: v.zero() for k, v in self.pairs.items()}) @inheritdoc(Container) def __add__(self, other): if isinstance(other, Label): if self.keySet != other.keySet: raise ContainerException( - "cannot add Labels because keys differ:\n {0}\n {1}".format( - ", ".join( - sorted( - self.keys)), ", ".join( - sorted( - other.keys)))) - - out = Label(**dict((k, self(k) + other(k)) for k in self.keys)) + "cannot add Labels because keys differ:\n {}\n {}".format( + ", ".join(sorted(self.keys)), ", ".join(sorted(other.keys)) + ) + ) + + out = Label(**{k: self(k) + other(k) for k in self.keys}) out.entries = self.entries + other.entries return out.specialize() - else: - raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) + raise ContainerException(f"cannot add {self.name} and {other.name}") @inheritdoc(Container) def __iadd__(self, other): if isinstance(other, Label): if self.keySet != other.keySet: raise ContainerException( - "cannot add Labels because keys differ:\n {0}\n {1}".format( - ", ".join( - sorted( - self.keys)), ", ".join( - sorted( - other.keys)))) + "cannot add Labels because keys differ:\n {}\n {}".format( + ", ".join(sorted(self.keys)), ", ".join(sorted(other.keys)) + ) + ) self.entries += other.entries for k in self.keys: v = self(k) v += other(k) return self - else: - raise ContainerException("cannot add {0} and {1}".format(self.name, other.name)) + raise ContainerException(f"cannot add {self.name} and {other.name}") @inheritdoc(Container) def __mul__(self, factor): if math.isnan(factor) or factor <= 0.0: return self.zero() - else: - out = self.zero() - out.entries = factor * self.entries - for k, v in self.pairs.items(): - out.pairs[k] = v * factor - return out.specialize() + out = self.zero() + out.entries = factor * self.entries + for k, v in self.pairs.items(): + out.pairs[k] = v * factor + return out.specialize() @inheritdoc(Container) def __rmul__(self, factor): @@ -491,7 +217,6 @@ def _numpy(self, data, weights, shape): x._numpy(data, weights, shape) # no possibility of exception from here on out (for rollback) - import numpy if isinstance(weights, numpy.ndarray): self.entries += float(weights.sum()) else: @@ -507,10 +232,11 @@ def children(self): @inheritdoc(Container) def toJsonFragment(self, suppressName): - return {"entries": floatToJson(self.entries), - "sub:type": self.values[0].name, - "data": dict((k, v.toJsonFragment(False)) for k, v in self.pairs.items()) - } + return { + "entries": floatToJson(self.entries), + "sub:type": self.values[0].name, + "data": {k: v.toJsonFragment(False) for k, v in self.pairs.items()}, + } @staticmethod @inheritdoc(Factory) @@ -527,17 +253,16 @@ def fromJsonFragment(json, nameFromParent): raise JsonFormatException(json, "Label.sub:type") if isinstance(json["data"], dict): - pairs = dict((k, factory.fromJsonFragment(v, None)) for k, v in json["data"].items()) + pairs = {k: factory.fromJsonFragment(v, None) for k, v in json["data"].items()} else: raise JsonFormatException(json, "Label.data") return Label.ed(entries, **pairs) - else: - raise JsonFormatException(json, "Label") + raise JsonFormatException(json, "Label") def __repr__(self): - return "