From 7b60a9ee3e5cd5167816cdcd62688325abaa56fb Mon Sep 17 00:00:00 2001 From: Pablo Perez Adrian Date: Thu, 20 May 2010 17:12:45 +0200 Subject: [PATCH 001/100] correccion error en el fichero /test7flscriptparse --- test/flscriptparser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/flscriptparser b/test/flscriptparser index 74fd89b..76736c5 100755 --- a/test/flscriptparser +++ b/test/flscriptparser @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash PARSE_FILES="$*" if [ "$1" = "--openwith" ]; then OPENWITH="$2" From 841e26d4a601a8dbc4219083bd83813c48b7dc1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mart=C3=ADnez=20Mart=C3=AD?= Date: Tue, 29 Jun 2010 18:15:55 +0200 Subject: [PATCH 002/100] Add instalation script --- INSTALL-ES | 3 +++ Makefile | 7 +++++++ dependencies.debian | 3 +++ install.sh | 18 ++++++++++++++++++ 4 files changed, 31 insertions(+) create mode 100644 Makefile create mode 100644 dependencies.debian create mode 100644 install.sh diff --git a/INSTALL-ES b/INSTALL-ES index d6298b0..88e6e43 100644 --- a/INSTALL-ES +++ b/INSTALL-ES @@ -2,6 +2,9 @@ Parseador de Código fuente tipo Javascript para AbanQ ------------------------------------------------------- DEPENDENCIAS: +- python-ply +- git-core +- realpath flscriptparser depende únicamente del proyecto PLY de Python. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c9cf0d1 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ + +all: + @echo "> run sudo make install to install." + +install: + bash install.sh + diff --git a/dependencies.debian b/dependencies.debian new file mode 100644 index 0000000..8fb8480 --- /dev/null +++ b/dependencies.debian @@ -0,0 +1,3 @@ +python-ply +git-core +realpath \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..46786fe --- /dev/null +++ b/install.sh @@ -0,0 +1,18 @@ +#!/bin/bash +[[ $(basename $(pwd)) != "flscriptparser" ]] && { + echo "This program MUST be run from its folder!!!" + exit 1 +} +DEPS=$(cat dependencies.debian) +aptitude install $DEPS -y +while read line; do + test -e /usr/local/bin/fldesigner && unlink /usr/local/bin/fldesigner + ln -s "$line" /usr/local/bin/fldesigner + break +done < <(find /usr/local/ -type f -name designer) + +while read line; do + bname=$(basename "$line") + test -e /usr/local/bin/$bname && unlink /usr/local/bin/$bname + ln -s "$line" /usr/local/bin/$bname +done < <(find $(pwd) -executable -type f \! -path "*/.*") \ No newline at end of file From 17575b6c66124d06d4e6e751e2f9158b572ba97e Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 30 Jun 2010 10:14:07 +0200 Subject: [PATCH 003/100] Bugfix --- flpatcher | 2 +- xmlparse.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 xmlparse.py diff --git a/flpatcher b/flpatcher index 726b5c8..8b84dd3 100755 --- a/flpatcher +++ b/flpatcher @@ -269,7 +269,7 @@ for filename in "${FILE_LIST[@]}"; do else echo "El fichero $MERGE no existe en este proyecto pero si fue modificado en el parche" echo "(para recordar esta respuesta las proximas veces escriba: b* i*)" - $question="Elija una opcion [(b)orrar|(i)ncluir]: " + question="Elija una opcion [(b)orrar|(i)ncluir]: " read -p "$question" answer type="${answer:0:1}" [[ ${answer:1:1} == "*" ]] && TYPE_BL=$type diff --git a/xmlparse.py b/xmlparse.py new file mode 100644 index 0000000..06c981e --- /dev/null +++ b/xmlparse.py @@ -0,0 +1,87 @@ +import xml.parsers.expat +import sys +from optparse import OptionParser +import re +elements = [] +show_end = True +lstelements = [] + +# 3 handler functions +def start_element(name, attrs): + global elements, show_end, lstelements + lstattrs=list(sorted([ "%s=%s" % (k,v) for k,v in attrs.iteritems() ])) + completename=name + if len(lstattrs): + completename+="?"+"&".join(lstattrs) + + show_end = False + elements.append(completename) + #print 'Start element:', name, attrs + lstelements.append("/".join(elements)) + #print "/".join(elements) + +def end_element(name): + global elements, show_end, lstelements + if show_end: + #print "/".join(elements) + "/" + lstelements.append("/".join(elements) + "/") + + show_end = True + elements.pop() + #print 'End element:', name + +def char_data(data): + global elements, show_end, lstelements + #data = data.strip() + if data.strip(): + #show_end = True + txt = lstelements.pop() + lstelements.append(txt+"(%s)" % repr(data)) + + + #print "/".join(elements)+ "(%s)" % repr(data) + + + + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + (options, args) = parser.parse_args() + if options.optdebug: + print options, args + + p = xml.parsers.expat.ParserCreate() + + p.StartElementHandler = start_element + p.EndElementHandler = end_element + p.CharacterDataHandler = char_data + global lstelements + r1 = re.compile("/widget") + for fname in args: + fhandler = open(fname) + p.ParseFile(fhandler) + for t in lstelements: + gl = r1.split(t) + if len(gl)>1: + print "(..%d..)%s" % (len(gl)-1,gl[-1]) + else: + print t + lstelements = [] + fhandler.close() + + +if __name__ == "__main__": main() \ No newline at end of file From ca15998776b4ef127131cb6c84beb0de23c45ea9 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 1 Jul 2010 08:57:04 +0200 Subject: [PATCH 004/100] First tests with xmlparsing --- xmlparse.py | 162 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 136 insertions(+), 26 deletions(-) diff --git a/xmlparse.py b/xmlparse.py index 06c981e..0b7e000 100644 --- a/xmlparse.py +++ b/xmlparse.py @@ -4,45 +4,122 @@ import re elements = [] show_end = True +lasttextdata = "" lstelements = [] +def reset(): + global elements, show_end, lstelements, lasttextdata + elements = [] + show_end = True + lasttextdata = "" + lstelements = [] + + # 3 handler functions def start_element(name, attrs): - global elements, show_end, lstelements + global elements, show_end, lstelements, lasttextdata lstattrs=list(sorted([ "%s=%s" % (k,v) for k,v in attrs.iteritems() ])) completename=name if len(lstattrs): - completename+="?"+"&".join(lstattrs) + completename+="&"+"&".join(lstattrs) show_end = False elements.append(completename) #print 'Start element:', name, attrs lstelements.append("/".join(elements)) #print "/".join(elements) + lasttextdata = "" def end_element(name): - global elements, show_end, lstelements + global elements, show_end, lstelements, lasttextdata + lasttextdata = "" if show_end: #print "/".join(elements) + "/" - lstelements.append("/".join(elements) + "/") + lstelements.append("/".join(elements) + ";") show_end = True elements.pop() #print 'End element:', name def char_data(data): - global elements, show_end, lstelements + global elements, show_end, lstelements, lasttextdata #data = data.strip() - if data.strip(): + lasttextdata+=data + if lasttextdata.strip(): #show_end = True - txt = lstelements.pop() - lstelements.append(txt+"(%s)" % repr(data)) + lstelements.pop() + lstelements.append("/".join(elements)+"(%s)" % repr(lasttextdata.strip())) #print "/".join(elements)+ "(%s)" % repr(data) +def unmap(lines): + + runmap = re.compile(r"^(?P/*)(?P\w+)(?P&[^\(]+)*(?P\(.+\))?$") + # depthlevel + # tagname + elementpool = [] + text = [] + for line in lines: + line = line.strip() + if line[-1] == ";": continue + rg1 = runmap.match(line) + if not rg1: + print "error:" + print line + break + + depth = len(rg1.group('depth')) + tagname = str(rg1.group('tagname')) + t_attrs = rg1.group('attrs') + attrs = [] + if t_attrs: + lattrs = t_attrs[1:].split("&") + for attr in lattrs: + key, val = attr.split("=") + attrs.append( (key,val) ) + + t_txt = rg1.group('txt') + txt = "" + if t_txt: + txt = eval(t_txt[1:-1]) + + while depth < len(elementpool): + toclose = elementpool.pop() + text.append("" % toclose) + text.append("\n" + " " * len(elementpool)) + + if depth == len(elementpool): + #print depth, tagname, attrs, txt + txtattrs = "" + if attrs: + for k,v in attrs: + txtattrs+=" %s=\"%s\"" % (k,v) + + if txt: + txt = txt.encode("utf-8") + txt = txt.replace("&","&") + txt = txt.replace("<","<") + else: + txt = "" + text.append("<%s%s>%s" % (tagname, txtattrs,txt)) + elementpool.append(tagname) + + + else: + print "error:" + print depth, len(elementpool) + break + + while len(elementpool): + toclose = elementpool.pop() + text.append("" % toclose) + text.append("\n" + " " * len(elementpool)) + + return text + def main(): parser = OptionParser() @@ -59,29 +136,62 @@ def main(): parser.add_option("--debug", action="store_true", dest="debug", default=False, help="prints lots of useless messages") + (options, args) = parser.parse_args() if options.optdebug: print options, args - - p = xml.parsers.expat.ParserCreate() + if len(args) < 2: + print "Se necesita al menos una accion y un argumento extra." + print "xmlparse (map|unmap) file1 [file2] [file3]" + return + action = args.pop(0) + + if action == "map": + global lstelements + separators = [ + "hbox", + "vbox", + "grid", + ] + r1 = re.compile("/widget") + for fname in args: + p = xml.parsers.expat.ParserCreate() - p.StartElementHandler = start_element - p.EndElementHandler = end_element - p.CharacterDataHandler = char_data - global lstelements - r1 = re.compile("/widget") - for fname in args: - fhandler = open(fname) - p.ParseFile(fhandler) - for t in lstelements: - gl = r1.split(t) - if len(gl)>1: - print "(..%d..)%s" % (len(gl)-1,gl[-1]) - else: - print t - lstelements = [] - fhandler.close() + p.StartElementHandler = start_element + p.EndElementHandler = end_element + p.CharacterDataHandler = char_data + fhandler = open(fname) + fw = open(fname+".map","w") + reset() + p.ParseFile(fhandler) + for t in lstelements: + elems = t.split("/") + lbox = [] + for n,e in enumerate(elems): + if e in separators: lbox.append(n) + if len(lbox)>1: + nlbox = lbox[-2] + while len(elems[nlbox:]) < 2: + nlbox -= 1 + else: + nlbox = 0 + fw.write("/"*(len(elems)-1) + "/".join(elems[-1:])+ "\n") + #print "/"*(len(elems)-1) + "/".join(elems[-1:]) + lstelements = [] + fhandler.close() + fw.close() + elif action == "unmap": + for fname in args: + fhandler = open(fname) + fw = open(fname+".ui","w") + for line in unmap(fhandler): + fw.write(line) + fw.close() + fhandler.close() + + else: + print "Unkown action '%s'" % action if __name__ == "__main__": main() \ No newline at end of file From a118a84faf444937ea20c061a5789f2bb3ebb053 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 1 Jul 2010 09:27:11 +0200 Subject: [PATCH 005/100] Delete temporal files every call of mergetool or patcher --- flmergetool | 14 +++++++++++++- flpatcher | 12 ++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/flmergetool b/flmergetool index 7e10cec..6ad0187 100755 --- a/flmergetool +++ b/flmergetool @@ -63,6 +63,7 @@ else exit 1 fi +( for file1 in "$BASE" "$LOCAL" "$REMOTE"; do is_done=1 until [[ $is_done == 0 ]] @@ -134,4 +135,15 @@ then else echo "KDIFF3 FAILED" exit 1 -fi \ No newline at end of file +fi +) + +for file1 in "$BASE" "$LOCAL" "$REMOTE"; do + unlink "$file1.xml" + unlink "$file1.blocks" + unlink "$file1.aligned" + unlink "$file1.hash" +fi + + +exit $?; \ No newline at end of file diff --git a/flpatcher b/flpatcher index 8b84dd3..02a7d86 100755 --- a/flpatcher +++ b/flpatcher @@ -154,13 +154,18 @@ while read -a line; do done < <(git diff --numstat "$SRC_START_COMMIT" "$SRC_END_COMMIT") unset IFS +TMP_FILES=() + git_show() { local commit="$1" local src="$2" local dst="$3" - git show "$commit":"$src" >"$dst" 2>/dev/null || { + if git show "$commit":"$src" >"$dst" 2>/dev/null + then + TMP_FILES+=("$dst") + else unlink "$dst" - } + fi } # 2.- Para cada fichero en la diferencia, extraer las versiones BASE y REMOTE @@ -286,6 +291,9 @@ for filename in "${FILE_LIST[@]}"; do fi done +for f in ${TMP_FILES[@]}; do + unlink "$f" +done # 5.- Para cada mezcla finalizada, añadirla al proyecto DST (git add) From 3ac08cfeb11cbfda9148bd341a666dc5d0c0ca2d Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 1 Jul 2010 10:24:15 +0200 Subject: [PATCH 006/100] Automatically add commit message for GIT GUI --- flpatcher | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/flpatcher b/flpatcher index 02a7d86..f7f2fd2 100755 --- a/flpatcher +++ b/flpatcher @@ -111,9 +111,13 @@ is_valid_git_commit() { local gitfolder="$1" local commit="$2" ( + local COMMIT_LINE="" cd "$gitfolder" || exit 1 [[ "$commit" ]] || exit 2 - git log "$commit" --pretty=oneline --abbrev-commit -1 >/dev/null 2>&1 || exit 3 + COMMIT_LINE=$(git log "$commit" --pretty=oneline --abbrev=8 --abbrev-commit -1 2>/dev/null) + [[ $? != 0 ]] || exit 3 + LAST_VALID_COMMIT_ID=${COMMIT_LINE:0:8} + LAST_VALID_COMMIT_MSG=${COMMIT_LINE:9} ) return $? } @@ -133,14 +137,22 @@ is_valid_git_commit "$SRC_PROJECT" "$SRC_START_COMMIT" || { echo "El commit inicial <$SRC_START_COMMIT> no es valido." >&2; exit 1; } +SRC_START_COMMIT=$LAST_VALID_COMMIT_ID +SRC_START_COMMIT_MSG=$LAST_VALID_COMMIT_MSG + is_valid_git_commit "$SRC_PROJECT" "$SRC_END_COMMIT" || { echo "El commit final <$SRC_END_COMMIT> no es valido." >&2; exit 1; } +SRC_END_COMMIT=$LAST_VALID_COMMIT_ID +SRC_END_COMMIT_MSG=$LAST_VALID_COMMIT_MSG + is_valid_git_commit "$DST_PROJECT" "$DST_APPLY_COMMIT" || { echo "El commit a aplicar <$DST_APPLY_COMMIT> no es valido." >&2; exit 1; } +DST_APPLY_COMMIT=$LAST_VALID_COMMIT_ID +DST_APPLY_COMMIT_MSG=$LAST_VALID_COMMIT_MSG # 1.- Calcular lista de ficheros que componen la diferencia en SRC @@ -283,7 +295,7 @@ for filename in "${FILE_LIST[@]}"; do case "${type}" in b) rm "$BASE" "$REMOTE" && git rm "$MERGE" ;; i) cp "$REMOTE" "$MERGE" && git add "$MERGE";; - *) echo "Unexpected type" ;; + *) echo "Unexpected type" ;; esac else @@ -301,6 +313,21 @@ done # 6.- Si todos los ficheros se mezclaron sin error: # hacer un commit, proponer un mensaje de commit. +GIT_MSGFILE="$DST_PROJECT/.git/GITGUI_MSG" +echo "patch: $SRC_START_COMMIT..$SRC_END_COMMIT > $DST_APPLY_COMMIT -- $SRC_PROJECT" > $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +echo "project: $SRC_PROJECT" >> $GIT_MSGFILE +echo "start: $SRC_START_COMMIT $SRC_START_COMMIT_MSG" >> $GIT_MSGFILE +echo "end: $SRC_END_COMMIT $SRC_END_COMMIT_MSG" >> $GIT_MSGFILE +echo "applied: $DST_APPLY_COMMIT $DST_APPLY_COMMIT_MSG" >> $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +echo " -- files patched: " >> $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +for filename in "${FILE_LIST[@]}"; do + echo " $filename" >> $GIT_MSGFILE +done +echo "" >> $GIT_MSGFILE + popd >/dev/null From 9a5ee2873419f9761c077e5169d3bd7ea05ca865 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 1 Jul 2010 13:32:55 +0200 Subject: [PATCH 007/100] Added xml2json , at the moment is only a test app. --- xml2json.py | 280 ++++++++++++++++++++++++++++++++++++++++++++++++++++ xmlparse.py | 1 + 2 files changed, 281 insertions(+) create mode 100644 xml2json.py diff --git a/xml2json.py b/xml2json.py new file mode 100644 index 0000000..6850994 --- /dev/null +++ b/xml2json.py @@ -0,0 +1,280 @@ +import json +import xml.parsers.expat +from optparse import OptionParser + +# json_string = json.dumps(python_variable) +# python_var = json.loads("string_encoded_jsonvar") + +def printr(*args): + print args[0], + for arg in args[1:]: + if type(arg) is unicode: + arg=arg.encode("utf-8") + print repr(arg), + print + + +class JSON_Base: + def __init__(self, finput, foutput): + self.finput = finput + self.foutput = foutput + + self.init_vars() + + def process(self): + print "Please define a process function." + + def init_vars(self): + pass + +""" + Possible Format: + + List-per-tag: + [ depth, tagname, attrs, textdata ] + + tagname: \w+ -> ElementTag + tagname: !\w+ -> DoctypeTag + tagname: ?\w+ -> XmlDeclTag + + Problems: + * Handling C-DATA + + Example 1: AbanQ UI (UTF-8) + StartDoctypeDeclHandler: 'UI' None None 0 + EndDoctypeDeclHandler: + StartElementHandler: 'UI' {u'version': u'3.3', u'stdsetdef': u'1'} + CharacterDataHandler: '\n' + StartElementHandler: 'class' {} + CharacterDataHandler: 'formLineasAlbaranesCli' + EndElementHandler: 'class' + + Example 2: JasperReports JRXML (UTF-8) + XmlDeclHandler: '1.0' 'UTF-8' -1 + Unhandled data: '\n' + StartElementHandler: 'jasperReport' {u'xmlns': u'http://jasperreports.sourceforge.net/jasperreports', u'name': u'report1', u'language': u'groovy', u'pageWidth': u'842', u'columnWidth': u'802', u'topMargin': u'20', u'rightMargin': u'20', u'bottomMargin': u'20', u'xmlns:xsi': u'http://www.w3.org/2001/XMLSchema-instance', u'leftMargin': u'20', u'xsi:schemaLocation': u'http://jasperreports.sourceforge.net/jasperreports http://jasperreports.sourceforge.net/xsd/jasperreport.xsd', u'pageHeight': u'595', u'orientation': u'Landscape'} + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + StartElementHandler: 'style' {u'fontName': u'Times New Roman', u'name': u'Title', u'isBold': u'false', u'forecolor': u'#FFFFFF', u'fontSize': u'50', u'isDefault': u'false', u'pdfFontName': u'Times-Bold'} + EndElementHandler: 'style' + + Example 3: AbanQ Actions XML (ISO-8859-1) + StartElementHandler: 'ACTIONS' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'action' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'name' {} + CharacterDataHandler: 'albaranescli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'description' {} + CharacterDataHandler: 'QT_TRANSLATE_NOOP("MetaData","Son los documentos que justifican la entrega de una mercancia a un ciente")' + EndElementHandler: 'description' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + Example 4: AbanQ Tables MTD (ISO-8859-1) + StartDoctypeDeclHandler: 'TMD' None None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'TMD' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'name' {} + CharacterDataHandler: 'facturascli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CommentHandler: 'Facturas de cliente' + CharacterDataHandler: ' ' + StartElementHandler: 'alias' {} + CharacterDataHandler: 'QT_TRANSLATE_NOOP("MetaData","Facturas de Clientes")' + EndElementHandler: 'alias' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + Example 5: AbanQ Report QRY (ISO-8859-1) + StartDoctypeDeclHandler: 'QRY' None None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'QRY' {} + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + StartElementHandler: 'name' {} + CharacterDataHandler: 'presupuestoscli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + + Example 6: AbanQ Report KUT (ISO-8859-1) + XmlDeclHandler: '1.0' 'UTF-8' -1 + Unhandled data: '\n' + StartDoctypeDeclHandler: 'KugarTemplate' 'kugartemplate.dtd' None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'KugarTemplate' {u'TopMargin': u'50', u'PageSize': u'0', u'RightMargin': u'30', u'PageOrientation': u'0', u'BottomMargin': u'50', u'LeftMargin': u'30'} + CharacterDataHandler: '\n' + StartElementHandler: 'Detail' {u'Level': u'0', u'Height': u'0'} + EndElementHandler: 'Detail' + CharacterDataHandler: '\n' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + + + +""" + +class JSON_Converter(JSON_Base): + + def init_vars(self): + self.p = xml.parsers.expat.ParserCreate() + + self.p.StartElementHandler = self.StartElementHandler + self.p.EndElementHandler = self.EndElementHandler + self.p.CharacterDataHandler = self.CharacterDataHandler + self.p.XmlDeclHandler = self.XmlDeclHandler + self.p.StartDoctypeDeclHandler = self.StartDoctypeDeclHandler + self.p.EndDoctypeDeclHandler = self.EndDoctypeDeclHandler + self.p.ElementDeclHandler = self.ElementDeclHandler + self.p.AttlistDeclHandler = self.AttlistDeclHandler + self.p.ProcessingInstructionHandler = self.ProcessingInstructionHandler + self.p.CharacterDataHandler = self.CharacterDataHandler + self.p.EntityDeclHandler = self.EntityDeclHandler + self.p.NotationDeclHandler = self.NotationDeclHandler + self.p.StartNamespaceDeclHandler = self.StartNamespaceDeclHandler + self.p.EndNamespaceDeclHandler = self.EndNamespaceDeclHandler + self.p.CommentHandler = self.CommentHandler + self.p.StartCdataSectionHandler = self.StartCdataSectionHandler + self.p.EndCdataSectionHandler = self.EndCdataSectionHandler + self.p.DefaultHandler = self.DefaultHandler + + def process(self): + self.p.ParseFile(self.finput) + + + def StartElementHandler(self, name, attributes): + printr( "StartElementHandler:", name, attributes) + + def EndElementHandler(self, name): + printr( "EndElementHandler:", name) + + def XmlDeclHandler(self, version, encoding, standalone): + printr( "XmlDeclHandler:", version, encoding, standalone) + + def StartDoctypeDeclHandler(self, doctypeName, systemId, publicId, has_internal_subset): + printr( "StartDoctypeDeclHandler:", doctypeName, systemId, publicId, has_internal_subset) + + def EndDoctypeDeclHandler(self): + printr( "EndDoctypeDeclHandler:") + + def ElementDeclHandler(self, name, model): + printr( "ElementDeclHandler:", name, model) + + def AttlistDeclHandler(self, elname, attname, type, default, required): + printr( "AttlistDeclHandler:", elname, attname, type, default, required) + + def ProcessingInstructionHandler(self, target, data): + printr( "ProcessingInstructionHandler:", target, data) + + def CharacterDataHandler(self, data): + printr( "CharacterDataHandler:", data) + + def EntityDeclHandler(self, entityName, is_parameter_entity, value, base, systemId, publicId, notationName): + printr( "EntityDeclHandler:" , entityName, is_parameter_entity, value, base, systemId, publicId, notationName) + + def NotationDeclHandler(self, notationName, base, systemId, publicId): + printr( "NotationDeclHandler:", notationName, base, systemId, publicId) + + def StartNamespaceDeclHandler(self, prefix, uri): + printr( "StartNamespaceDeclHandler:", prefix, uri) + + def EndNamespaceDeclHandler(self, prefix): + printr( "EndNamespaceDeclHandler:", prefix) + + def CommentHandler(self, data): + printr( "CommentHandler:", data) + + def StartCdataSectionHandler(self): + printr( "StartCdataSectionHandler:") + + def EndCdataSectionHandler(self): + printr( "EndCdataSectionHandler:") + + def DefaultHandler(self,data): + printr( "Unhandled data:", data) + +class JSON_Reverter(JSON_Base): + pass + + + + + + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + + (options, args) = parser.parse_args() + if options.optdebug: + print options, args + if len(args) < 2: + print "xml2json needs at least an action and a file." + print "xml2json (revert|convert) file1 [file2] [file3]" + return + action = args.pop(0) + + if action == "convert": + for fname in args: + + fhandler = open(fname) + fw = open(fname+".json","w") + jconv = JSON_Converter(fhandler, fw) + jconv.process() + + fhandler.close() + fw.close() + elif action == "revert": + for fname in args: + lExt = fname.split(".") + ext = "xml" + if lExt[-1]=="json" and len(lExt[-2])>=1 and len(lExt[-2])<=6: + ext = lExt[-2] + fhandler = open(fname) + + fw = open(fname+"."+ext,"w") + jrev = JSON_Reverter(fhandler, fw) + jrev.process() + + fw.close() + fhandler.close() + + else: + print "Unkown action '%s'" % action + + + + + + + + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/xmlparse.py b/xmlparse.py index 0b7e000..706176b 100644 --- a/xmlparse.py +++ b/xmlparse.py @@ -2,6 +2,7 @@ import sys from optparse import OptionParser import re + elements = [] show_end = True lasttextdata = "" From c7375bc2df1d1bf73ab643124cf6036c08cde661 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 1 Jul 2010 14:01:31 +0200 Subject: [PATCH 008/100] More documentation for xml2json --- xml2json.py | 62 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/xml2json.py b/xml2json.py index 6850994..e0266f3 100644 --- a/xml2json.py +++ b/xml2json.py @@ -31,14 +31,48 @@ def init_vars(self): Possible Format: List-per-tag: - [ depth, tagname, attrs, textdata ] + [ depth, tagname, attrs, ttype, tdata ] + + depth: 0,1,2,3,4...N tagname: \w+ -> ElementTag tagname: !\w+ -> DoctypeTag - tagname: ?\w+ -> XmlDeclTag + tagname: ?\w+ -> XmlDeclTag (always: xml) (attrs = version, encoding?, standalone?) + tagname: #\w+ -> CommentTag (always: comment) (attrs = []) (tdata = comment) + + attrs: dict { attr : val , attr2 : val2 } + + ttype: text|cdata|mixed -multiline? + + tdata: raw text + cdata combined. + Problems: * Handling C-DATA + * Handling multiline texts + * Handling tabs and spaces at the start of each line of multiline text + * Handling comments + + Non-treated: + * NameSpaces + 32.18 + + * Entity Declarations + + + * Element Declarations + + + * Notation Declarations + + + * Attribute List Delcarations + + + + Example 1: AbanQ UI (UTF-8) StartDoctypeDeclHandler: 'UI' None None 0 @@ -159,11 +193,23 @@ def process(self): def StartElementHandler(self, name, attributes): printr( "StartElementHandler:", name, attributes) + def CharacterDataHandler(self, data): + printr( "CharacterDataHandler:", data) + def EndElementHandler(self, name): printr( "EndElementHandler:", name) def XmlDeclHandler(self, version, encoding, standalone): printr( "XmlDeclHandler:", version, encoding, standalone) + + def CommentHandler(self, data): + printr( "CommentHandler:", data) + + def StartCdataSectionHandler(self): + printr( "StartCdataSectionHandler:") + + def EndCdataSectionHandler(self): + printr( "EndCdataSectionHandler:") def StartDoctypeDeclHandler(self, doctypeName, systemId, publicId, has_internal_subset): printr( "StartDoctypeDeclHandler:", doctypeName, systemId, publicId, has_internal_subset) @@ -180,9 +226,6 @@ def AttlistDeclHandler(self, elname, attname, type, default, required): def ProcessingInstructionHandler(self, target, data): printr( "ProcessingInstructionHandler:", target, data) - def CharacterDataHandler(self, data): - printr( "CharacterDataHandler:", data) - def EntityDeclHandler(self, entityName, is_parameter_entity, value, base, systemId, publicId, notationName): printr( "EntityDeclHandler:" , entityName, is_parameter_entity, value, base, systemId, publicId, notationName) @@ -195,15 +238,6 @@ def StartNamespaceDeclHandler(self, prefix, uri): def EndNamespaceDeclHandler(self, prefix): printr( "EndNamespaceDeclHandler:", prefix) - def CommentHandler(self, data): - printr( "CommentHandler:", data) - - def StartCdataSectionHandler(self): - printr( "StartCdataSectionHandler:") - - def EndCdataSectionHandler(self): - printr( "EndCdataSectionHandler:") - def DefaultHandler(self,data): printr( "Unhandled data:", data) From c2b03fd2dc1f7af95b5982db8c92c90e62d27e31 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 12 Jul 2010 13:32:58 +0200 Subject: [PATCH 009/100] Partial: Json xml encoder --- xml2json.py | 174 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 169 insertions(+), 5 deletions(-) diff --git a/xml2json.py b/xml2json.py index e0266f3..d18ec00 100644 --- a/xml2json.py +++ b/xml2json.py @@ -1,4 +1,11 @@ -import json +try: + from json import dumps as json_dumps + from json import loads as json_loads +except: + from json import write as json_dumps + from json import read as json_loads + + import xml.parsers.expat from optparse import OptionParser @@ -6,6 +13,7 @@ # python_var = json.loads("string_encoded_jsonvar") def printr(*args): + return print args[0], for arg in args[1:]: if type(arg) is unicode: @@ -14,10 +22,52 @@ def printr(*args): print +class xmlElement: + def __init__(self, parent, tagname, attrs = {}, ttype = "text", tdata = ""): + self.parent = parent + + self.tagname = tagname + self.attrs = attrs + self.ttype = ttype + self.tdata = tdata + + if self.parent: + self.depth = parent.depth + 1 + self.path = self.parent.path + [self.tagname] + else: + self.depth = 0 + self.path = [self.tagname] + + def append(self,text): + self.tdata += text + + def export(self,encoding): + depth = self.depth + tagname = self.tagname + attrs = [ [k,v] for k,v in self.attrs.iteritems() ] + attrs.sort() + tdata = self.tdata.strip() + ttype = self.ttype + + if len(tdata) == 0: + tdata = "" + ttype = "" + + #v = [depth,tagname,attrs,ttype,tdata] + #vt1 = json_dumps(v) + vt2 = "%d)%s" % (depth,tagname) + if attrs: vt2 +="\tattrs:" + json_dumps(attrs) + if ttype: vt2 +="\t%s:%s" % ( ttype, json_dumps(tdata)) + + return vt2.encode(encoding) + + + class JSON_Base: - def __init__(self, finput, foutput): + def __init__(self, finput, foutput, encoding): self.finput = finput self.foutput = foutput + self.encoding = encoding self.init_vars() @@ -165,7 +215,10 @@ def init_vars(self): class JSON_Converter(JSON_Base): def init_vars(self): - self.p = xml.parsers.expat.ParserCreate() + self.real_encoding = self.getRealEncoding() + self.xmltag = None + self.taglist = [] + self.p = xml.parsers.expat.ParserCreate(self.real_encoding) self.p.StartElementHandler = self.StartElementHandler self.p.EndElementHandler = self.EndElementHandler @@ -185,37 +238,106 @@ def init_vars(self): self.p.StartCdataSectionHandler = self.StartCdataSectionHandler self.p.EndCdataSectionHandler = self.EndCdataSectionHandler self.p.DefaultHandler = self.DefaultHandler + + def getRealEncoding(self): + validEncodings = ["UTF-8", "UTF-16", "ISO-8859-1"] + self.encoding = self.encoding.upper() + + if self.encoding in validEncodings: return self.encoding + if self.encoding.find("UTF")>=0: + if self.encoding.find("8"): + return "UTF-8" + if self.encoding.find("16"): + return "UTF-16" + return "UTF-8" + + if self.encoding.find("ISO")>=0: + return "ISO-8859-1" + + if self.encoding.find("1252")>=0: + return "ISO-8859-1" + + if self.encoding.find("CP")==0: + return "ISO-8859-1" + + if self.encoding.find("WIN")==0: + return "ISO-8859-1" + + return "UTF-8" + def process(self): self.p.ParseFile(self.finput) + for tag in self.taglist: + self.foutput.write(tag.export(self.real_encoding)+"\n") + + def startTag(self,*args): + newtag = xmlElement(self.xmltag, *args) + self.xmltag = newtag + self.taglist.append(newtag) + return newtag + + def endTag(self): + self.xmltag = self.xmltag.parent + return self.xmltag + + - + def StartElementHandler(self, name, attributes): printr( "StartElementHandler:", name, attributes) + # tagname: \w+ -> ElementTag + self.startTag(name, attributes) def CharacterDataHandler(self, data): printr( "CharacterDataHandler:", data) + self.xmltag.append(data) def EndElementHandler(self, name): printr( "EndElementHandler:", name) + self.endTag() def XmlDeclHandler(self, version, encoding, standalone): printr( "XmlDeclHandler:", version, encoding, standalone) + # tagname: ?\w+ -> XmlDeclTag (always: xml) (attrs = version, encoding?, standalone?) + attrs = { 'version' : version } + if encoding: attrs['encoding'] = encoding + if standalone: attrs['standalone'] = standalone + + self.startTag("?xml", attrs) + + self.endTag() def CommentHandler(self, data): printr( "CommentHandler:", data) + # tagname: #\w+ -> CommentTag (always: comment) (attrs = []) (tdata = comment) + self.startTag("#comment") + self.xmltag.append(data) + self.endTag() + def StartCdataSectionHandler(self): printr( "StartCdataSectionHandler:") + self.xmltag.ttype = "cdata" def EndCdataSectionHandler(self): printr( "EndCdataSectionHandler:") + # se descarta el cierre... def StartDoctypeDeclHandler(self, doctypeName, systemId, publicId, has_internal_subset): printr( "StartDoctypeDeclHandler:", doctypeName, systemId, publicId, has_internal_subset) + # tagname: !\w+ -> DoctypeTag + attrs = {} + if systemId: attrs['systemId'] = systemId + if publicId: attrs['publicId'] = publicId + if has_internal_subset: attrs['has_internal_subset'] = has_internal_subset + + self.startTag("!"+doctypeName,attrs) + def EndDoctypeDeclHandler(self): printr( "EndDoctypeDeclHandler:") + self.endTag() def ElementDeclHandler(self, name, model): printr( "ElementDeclHandler:", name, model) @@ -246,6 +368,45 @@ class JSON_Reverter(JSON_Base): +def autodetectXmlEncoding(rawtext): + lines = [ line.strip() for line in rawtext.split("\n") if line.strip() ] + if lines[0].find("")>=0: + # File is QtDesigner UI + return "UTF-8" + + if lines[0].find("UTF-8")>=0: + # Unkown, standard xml (like jrxml) + return "UTF-8" + + if lines[0].find("")>=0: + # AbanQ actions XML + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ table MTD + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ report Query + return "ISO-8859-15" + + if lines[0].find('')>=0: + # AbanQ report Kut + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ translations + return "ISO-8859-15" + + try: + import chardet + dictEncoding = chardet.detect(rawtext) + encoding = dictEncoding["encoding"] + return encoding + except ImportError: + print "python-chardet library is not installed. Assuming input file is UTF-8." + #encoding= + #UTF-8, UTF-16, ISO-8859-1 @@ -280,7 +441,10 @@ def main(): fhandler = open(fname) fw = open(fname+".json","w") - jconv = JSON_Converter(fhandler, fw) + rawtext = fhandler.read() + fhandler.seek(0) + encoding = autodetectXmlEncoding(rawtext) + jconv = JSON_Converter(fhandler, fw, encoding) jconv.process() fhandler.close() From 8e3dd0b0b24fc9ce715b7adb3899d248498d7b57 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 13 Jul 2010 09:18:45 +0200 Subject: [PATCH 010/100] Bugfixes in mergetool --- flmergetool | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/flmergetool b/flmergetool index 6ad0187..eb6a4d2 100755 --- a/flmergetool +++ b/flmergetool @@ -3,6 +3,11 @@ # FLMERGETOOL. # To add this tool to git, do the following: # flmergetool INSTALL +editors=(medit kate gedit kwrite nano vim vi) +EDITOR= +for editor in "${editor[@]}"; do + EDITOR=$(command -v $editor) && break +done thisprg=`realpath $0` thisdir=`dirname $thisprg` @@ -80,22 +85,26 @@ for file1 in "$BASE" "$LOCAL" "$REMOTE"; do patched=1 } done - if [[ $patched == 0 ]]; then - read -p "Fallo en el parseo del fichero, desea abrir un editor para corregirlo manualmente? [Y/n]:" editor - [[ $editor == "n" ]] && { - echo "FLScriptParse FAILED" - exit 1 - } - bname=$(basename "$file1") - [[ $editor ]] || editor="medit" - [[ $editor == "Y" ]] || editor="medit" - [[ $editor == "y" ]] || editor="medit" - cp $file1 /tmp/$bname.orig - "$editor" "$file1" - read -p "Pulse Intro si ha terminado." - diff -U6 -p --minimal -d /tmp/$bname.orig $file1 >/tmp/knownpatch.$bname.patch.tmp && \ - mv /tmp/knownpatch.$bname.patch.tmp /tmp/knownpatch.$bname.patch - fi + + read -p "Fallo en el parseo del fichero, desea abrir un editor para corregirlo manualmente? [Y/n]:" editor + [[ $editor == "n" ]] && { + echo "FLScriptParse FAILED" + exit 1 + } + bname=$(basename "$file1") + [[ $editor ]] || editor=$EDITOR + [[ $editor == "Y" ]] || editor=$EDITOR + [[ $editor == "y" ]] || editor=$EDITOR + cp $file1 /tmp/$bname.orig + "$editor" "$file1" + read -p "Pulse Intro si ha terminado." + diff -U6 -p --minimal -d /tmp/$bname.orig $file1 >/tmp/knownpatch.$bname.patch.tmp + if [[ $? == 1 ]] && [[ wc -l /tmp/knownpatch.$bname.patch.tmp > 0 ]] ; then + mv /tmp/knownpatch.$bname.patch.tmp /tmp/knownpatch.$bname.patch + else + unlink /tmp/knownpatch.$bname.patch.tmp + fi + fi done done @@ -143,7 +152,7 @@ for file1 in "$BASE" "$LOCAL" "$REMOTE"; do unlink "$file1.blocks" unlink "$file1.aligned" unlink "$file1.hash" -fi +done exit $?; \ No newline at end of file From 67513f15f6f7f934535684879b0912007cac668c Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 13 Jul 2010 14:01:45 +0200 Subject: [PATCH 011/100] Finished xml merging with xml2json --- flmergetool | 206 +++++++++++++++++++++++++++++++--------------------- xml2json.py | 160 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 280 insertions(+), 86 deletions(-) diff --git a/flmergetool b/flmergetool index eb6a4d2..457fdbd 100755 --- a/flmergetool +++ b/flmergetool @@ -60,99 +60,141 @@ else fi fi -if echo $MERGE | grep -E '\.qs' -then - OK=1 -else - echo "File $MERGE does not have .qs" - exit 1 -fi -( -for file1 in "$BASE" "$LOCAL" "$REMOTE"; do - is_done=1 - until [[ $is_done == 0 ]] - do - if python $thisdir/flscriptparse.py -O file "$file1" && test -f "$file1.xml" - then - #echo "FLScriptParse OK" - is_done=0 - else - patched=0 - for kpatch in /tmp/knownpatch.*; do - [[ $kpatch == "/tmp/knownpatch.*" ]] && break - patch "$file1" "$kpatch" -f -F4 -l && { - patched=1 +merge_other() { + kdiff3 "$BASE" "$LOCAL" "$REMOTE" -o "$MERGE" --auto >/dev/null 2>&1 + return $? +} + +merge_xml() { + python $thisdir/xml2json.py convert "$BASE" "$LOCAL" "$REMOTE" && \ + kdiff3 "$BASE.json" "$LOCAL.json" "$REMOTE.json" -o "$MERGE.json" --auto >/dev/null 2>&1 && \ + python $thisdir/xml2json.py revert "$MERGE.json" && \ + cp $MERGE.json.* $MERGE + for f in "$BASE" "$LOCAL" "$REMOTE" "$MERGE"; do + rm "$f".json "$f".json.* + done + return $? +} + +merge_qs() { + if echo $MERGE | grep -E '\.qs' + then + OK=1 + else + echo "File $MERGE does not have .qs" + exit 1 + fi + + ( + for file1 in "$BASE" "$LOCAL" "$REMOTE"; do + is_done=1 + until [[ $is_done == 0 ]] + do + if python $thisdir/flscriptparse.py -O file "$file1" && test -f "$file1.xml" + then + #echo "FLScriptParse OK" + is_done=0 + else + patched=0 + for kpatch in /tmp/knownpatch.*; do + [[ $kpatch == "/tmp/knownpatch.*" ]] && break + patch "$file1" "$kpatch" -f -F4 -l && { + patched=1 + } + done + + read -p "Fallo en el parseo del fichero, desea abrir un editor para corregirlo manualmente? [Y/n]:" editor + [[ $editor == "n" ]] && { + echo "FLScriptParse FAILED" + exit 1 } - done + bname=$(basename "$file1") + [[ $editor ]] || editor=$EDITOR + [[ $editor == "Y" ]] || editor=$EDITOR + [[ $editor == "y" ]] || editor=$EDITOR + cp $file1 /tmp/$bname.orig + "$editor" "$file1" + read -p "Pulse Intro si ha terminado." + diff -U6 -p --minimal -d /tmp/$bname.orig $file1 >/tmp/knownpatch.$bname.patch.tmp + if [[ $? == 1 ]] && [[ $(wc -l /tmp/knownpatch.$bname.patch.tmp) > 0 ]] ; then + mv /tmp/knownpatch.$bname.patch.tmp /tmp/knownpatch.$bname.patch + else + unlink /tmp/knownpatch.$bname.patch.tmp + fi - read -p "Fallo en el parseo del fichero, desea abrir un editor para corregirlo manualmente? [Y/n]:" editor - [[ $editor == "n" ]] && { - echo "FLScriptParse FAILED" - exit 1 - } - bname=$(basename "$file1") - [[ $editor ]] || editor=$EDITOR - [[ $editor == "Y" ]] || editor=$EDITOR - [[ $editor == "y" ]] || editor=$EDITOR - cp $file1 /tmp/$bname.orig - "$editor" "$file1" - read -p "Pulse Intro si ha terminado." - diff -U6 -p --minimal -d /tmp/$bname.orig $file1 >/tmp/knownpatch.$bname.patch.tmp - if [[ $? == 1 ]] && [[ wc -l /tmp/knownpatch.$bname.patch.tmp > 0 ]] ; then - mv /tmp/knownpatch.$bname.patch.tmp /tmp/knownpatch.$bname.patch - else - unlink /tmp/knownpatch.$bname.patch.tmp fi - - fi + done done -done -if python $thisdir/flpremerge.py $DEBUG $BASE $LOCAL $REMOTE -then - #echo "FLPreMerge OK" - OK=1 -else - echo "FLPreMerge FAILED" - exit 1 -fi + if python $thisdir/flpremerge.py $DEBUG $BASE $LOCAL $REMOTE + then + #echo "FLPreMerge OK" + OK=1 + else + echo "FLPreMerge FAILED" + exit 1 + fi -if python $thisdir/flalign.py $DEBUG $BASE $LOCAL $REMOTE -then - #echo "FLAlign OK" - OK=1 -else - echo "FLAlign FAILED" - exit 1 -fi + if python $thisdir/flalign.py $DEBUG $BASE $LOCAL $REMOTE + then + #echo "FLAlign OK" + OK=1 + else + echo "FLAlign FAILED" + exit 1 + fi -if test -f $BASE.aligned && test -f $LOCAL.aligned && test -f $REMOTE.aligned -then - OK=1 -else - echo "FLAlign FAILED" - exit 1 + if test -f $BASE.aligned && test -f $LOCAL.aligned && test -f $REMOTE.aligned + then + OK=1 + else + echo "FLAlign FAILED" + exit 1 + fi + unlink $MERGE + #echo "kdiff3 $BASE.aligned $LOCAL.aligned $REMOTE.aligned -o $MERGE --auto" + if kdiff3 --auto $BASE.aligned $LOCAL.aligned $REMOTE.aligned -o $MERGE >/dev/null 2>&1 + then + #echo "KDIFF3 OK" + OK=1 + else + echo "KDIFF3 FAILED" + exit 1 + fi + ) + + for file1 in "$BASE" "$LOCAL" "$REMOTE"; do + unlink "$file1.xml" + unlink "$file1.blocks" + unlink "$file1.aligned" + unlink "$file1.hash" + done + + + return $?; +} + + + + + + +if echo $MERGE | grep -E '\.qs' ; then + merge_qs + exit $? fi -unlink $MERGE -#echo "kdiff3 $BASE.aligned $LOCAL.aligned $REMOTE.aligned -o $MERGE --auto" -if kdiff3 --auto $BASE.aligned $LOCAL.aligned $REMOTE.aligned -o $MERGE >/dev/null 2>&1 -then - #echo "KDIFF3 OK" - OK=1 -else - echo "KDIFF3 FAILED" - exit 1 +if echo $MERGE | grep -E '\.(xml|mtd|ui)' ; then + merge_xml + exit $? +fi +if echo $MERGE | grep -E '\.(kut|qry)' ; then + merge_other + exit $? fi -) -for file1 in "$BASE" "$LOCAL" "$REMOTE"; do - unlink "$file1.xml" - unlink "$file1.blocks" - unlink "$file1.aligned" - unlink "$file1.hash" -done +print "Unknown extension." +exit 3 -exit $?; \ No newline at end of file diff --git a/xml2json.py b/xml2json.py index d18ec00..c23a747 100644 --- a/xml2json.py +++ b/xml2json.py @@ -21,6 +21,24 @@ def printr(*args): print repr(arg), print +def entity_rep(txt,entities=""): + entity_list = list("&'\"<>") + entity_dict = { + '"' : """, + "'" : "'", + '<' : "<", + '>' : ">", + "&" : "&", + } + if entities == "": entities = entity_list + entities = list(entities) + if "&" not in entities: entities.append("&") + + for entity in entity_list: + if entity not in entities: continue + #print entity,entity_dict[entity] + txt = txt.replace(entity,entity_dict[entity]) + return txt class xmlElement: def __init__(self, parent, tagname, attrs = {}, ttype = "text", tdata = ""): @@ -31,9 +49,12 @@ def __init__(self, parent, tagname, attrs = {}, ttype = "text", tdata = ""): self.ttype = ttype self.tdata = tdata + self.children = [] + if self.parent: self.depth = parent.depth + 1 self.path = self.parent.path + [self.tagname] + self.parent.children.append(self) else: self.depth = 0 self.path = [self.tagname] @@ -60,6 +81,50 @@ def export(self,encoding): if ttype: vt2 +="\t%s:%s" % ( ttype, json_dumps(tdata)) return vt2.encode(encoding) + + def exportXML(self): + if type(self.attrs) is dict: + attrs = [ [k,v] for k,v in self.attrs.iteritems() ] + attrs.sort() + else: + attrs = self.attrs + + txtattrs = "" + + if attrs: + txtattrs=" " + for key,value in attrs: + txtattrs += '%s="%s" ' % (key,entity_rep(value,'&<"')) + + + depthpad = u" " * self.depth + output = u"" + if self.tagname == "#comment": + output += u"%s\n" % (depthpad, self.tdata) + elif self.tagname[0] == "!": + output += u"%s\n" % (depthpad, self.tagname[1:],txtattrs) + elif self.children: + output += u"%s<%s%s>\n" % (depthpad, self.tagname,txtattrs) + for child in self.children: + output += child.exportXML() + + output += u"%s\n" % (depthpad, self.tagname) + else: + if self.tdata == "": + if txtattrs=="": txtattrs = " " + output += u"%s<%s%s/>\n" % (depthpad, self.tagname,txtattrs) + else: + tdata = self.tdata + if tdata.find("\n")>-1: tdata = "\n%s\n" % tdata + if self.ttype=="cdata": tdata = "" % self.tdata + else: tdata = entity_rep(self.tdata) + + output += u"%s<%s%s>%s\n" % (depthpad, self.tagname,txtattrs, tdata, self.tagname) + + + return output + + @@ -76,6 +141,85 @@ def process(self): def init_vars(self): pass + + + +class JSON_Reverter(JSON_Base): + def init_vars(self): + self.cElement = None + self.rootXML = [] + + def processCmd(self,key,val): + if key == "encoding": + if self.encoding != "auto": + self.encoding = self.encoding.upper() + if val.upper() != self.encoding: + print " ignoring %s=%s , using specified value '%s' instead" % (key,val,self.encoding) + return + self.encoding = val.upper() + return + + print "ERROR: unknown key %s='%s'" % (key,val) + + def newElement(self,depth,tagname,text,ttype,attrs): + parent = self.cElement + if parent: parentdepth = parent.depth + else: parentdepth = -1 + while parent and parentdepth > depth - 1: + parent = parent.parent + if parent: parentdepth = parent.depth + else: parentdepth = -1 + + self.cElement = xmlElement(parent, tagname, attrs, ttype , text) + if parent is None: self.rootXML.append(self.cElement) + + + def process(self): + for line in self.finput: + line = line.strip() + if len(line) == 0: continue + if line[0]=="!": + lstkeys = line[1:].split(":") + key, val = lstkeys + self.processCmd(key.strip(),val.strip()) + continue + + fields = line.split("\t") + + depth, tag = fields[0].split(")") + depth = int(depth) + text = "" + ttype = "text" + attrs = {} + + #self.foutput.write("%d\t%s\n" % (depth, tag)) + for field in fields[1:]: + tpos = field.find(":") + if tpos == -1: + print "unexpected character:", line + return + ftype = field[:tpos] + try: + fvalue = json_loads(field[tpos+1:]) + except ValueError: + print "ValueError:", field[tpos+1:] + + #if type(fvalue) is unicode: + # self.foutput.write("%s = %s\n" % (ftype, fvalue.encode(self.encoding))) + #else: + # self.foutput.write("%s = %s\n" % (ftype, repr(fvalue))) + if ftype == "text": + text = fvalue + ttype = "text" + if ftype == "cdata": + text = fvalue + ttype = "cdata" + if ftype == "attrs": attrs = fvalue + + self.newElement(depth,tag,text,ttype,attrs) + + for element in self.rootXML: + self.foutput.write(element.exportXML().encode(self.encoding)) """ Possible Format: @@ -268,6 +412,7 @@ def getRealEncoding(self): def process(self): self.p.ParseFile(self.finput) + self.foutput.write("!encoding: "+ self.real_encoding+"\n") for tag in self.taglist: self.foutput.write(tag.export(self.real_encoding)+"\n") @@ -363,8 +508,9 @@ def EndNamespaceDeclHandler(self, prefix): def DefaultHandler(self,data): printr( "Unhandled data:", data) -class JSON_Reverter(JSON_Base): - pass + + + @@ -426,6 +572,9 @@ def main(): action="store_true", dest="debug", default=False, help="prints lots of useless messages") + parser.add_option("-E", "--encoding", dest="encoding", default = "auto", + help="Set encoding=ENC: auto,utf-8,iso-8859-15", metavar="ENC") + (options, args) = parser.parse_args() if options.optdebug: @@ -443,7 +592,10 @@ def main(): fw = open(fname+".json","w") rawtext = fhandler.read() fhandler.seek(0) - encoding = autodetectXmlEncoding(rawtext) + if options.encoding == "auto": + encoding = autodetectXmlEncoding(rawtext) + else: + encoding = options.encoding jconv = JSON_Converter(fhandler, fw, encoding) jconv.process() @@ -458,7 +610,7 @@ def main(): fhandler = open(fname) fw = open(fname+"."+ext,"w") - jrev = JSON_Reverter(fhandler, fw) + jrev = JSON_Reverter(fhandler, fw, options.encoding) jrev.process() fw.close() From b1f389de8687ffaf51fa03afb4bc85dd0b9a7795 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maria=20Yulieth=20Pe=C3=B1uela=20Carvajal?= Date: Mon, 19 Jul 2010 13:39:27 +0200 Subject: [PATCH 012/100] bugfix: --porcielain option not avaliable in all git versions --- flpatcher | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flpatcher b/flpatcher index f7f2fd2..ada25c8 100755 --- a/flpatcher +++ b/flpatcher @@ -102,7 +102,7 @@ is_valid_git_folder() { *) echo "Modified file in working directory '$filename' ($status)"; exit 1;; esac # echo "*$status* '$filename'" - done < <(git status --porcelain) + done < <(git status --porcelain 2>/dev/null) ) || return 1 From 2e1c154e92fbe594a3208c94236267cb6b87a2af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Verd=C3=BA=20Mateu?= Date: Tue, 20 Jul 2010 10:26:59 +0200 Subject: [PATCH 013/100] various bugfixes --- flmergetool | 13 ++++++------- flpatcher | 29 ++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/flmergetool b/flmergetool index 457fdbd..4d2845a 100755 --- a/flmergetool +++ b/flmergetool @@ -4,10 +4,12 @@ # To add this tool to git, do the following: # flmergetool INSTALL editors=(medit kate gedit kwrite nano vim vi) -EDITOR= -for editor in "${editor[@]}"; do - EDITOR=$(command -v $editor) && break +EDITOR_FILEPATH= +for editor in "${editors[@]}"; do + EDITOR_FILEPATH=$(command -v $editor) + [[ $EDITOR_FILEPATH ]] && break done +echo $EDITOR_FILEPATH thisprg=`realpath $0` thisdir=`dirname $thisprg` @@ -110,11 +112,8 @@ merge_qs() { exit 1 } bname=$(basename "$file1") - [[ $editor ]] || editor=$EDITOR - [[ $editor == "Y" ]] || editor=$EDITOR - [[ $editor == "y" ]] || editor=$EDITOR cp $file1 /tmp/$bname.orig - "$editor" "$file1" + "$EDITOR_FILEPATH" "$file1" read -p "Pulse Intro si ha terminado." diff -U6 -p --minimal -d /tmp/$bname.orig $file1 >/tmp/knownpatch.$bname.patch.tmp if [[ $? == 1 ]] && [[ $(wc -l /tmp/knownpatch.$bname.patch.tmp) > 0 ]] ; then diff --git a/flpatcher b/flpatcher index ada25c8..290df58 100755 --- a/flpatcher +++ b/flpatcher @@ -107,18 +107,22 @@ is_valid_git_folder() { ) || return 1 } +LAST_VALID_COMMIT_ID="" +LAST_VALID_COMMIT_MSG="" + is_valid_git_commit() { local gitfolder="$1" local commit="$2" - ( - local COMMIT_LINE="" - cd "$gitfolder" || exit 1 - [[ "$commit" ]] || exit 2 - COMMIT_LINE=$(git log "$commit" --pretty=oneline --abbrev=8 --abbrev-commit -1 2>/dev/null) - [[ $? != 0 ]] || exit 3 - LAST_VALID_COMMIT_ID=${COMMIT_LINE:0:8} - LAST_VALID_COMMIT_MSG=${COMMIT_LINE:9} - ) + + local COMMIT_LINE="" + pushd "$gitfolder" || exit 1 + [[ "$commit" ]] || exit 2 + COMMIT_LINE=$(git log "$commit" --pretty=oneline --abbrev=8 --abbrev-commit -1 ) + [[ $? != 0 ]] && { echo "$(pwd) git log salio con estado $? " >&2; exit 3; } + LAST_VALID_COMMIT_ID=${COMMIT_LINE:0:8} + LAST_VALID_COMMIT_MSG=${COMMIT_LINE:9} + popd + return $? } @@ -134,13 +138,17 @@ is_valid_git_folder "$DST_PROJECT" || { } is_valid_git_commit "$SRC_PROJECT" "$SRC_START_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; echo "El commit inicial <$SRC_START_COMMIT> no es valido." >&2; exit 1; } +echo "<$LAST_VALID_COMMIT_ID>" + SRC_START_COMMIT=$LAST_VALID_COMMIT_ID SRC_START_COMMIT_MSG=$LAST_VALID_COMMIT_MSG is_valid_git_commit "$SRC_PROJECT" "$SRC_END_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; echo "El commit final <$SRC_END_COMMIT> no es valido." >&2; exit 1; } @@ -148,6 +156,7 @@ SRC_END_COMMIT=$LAST_VALID_COMMIT_ID SRC_END_COMMIT_MSG=$LAST_VALID_COMMIT_MSG is_valid_git_commit "$DST_PROJECT" "$DST_APPLY_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; echo "El commit a aplicar <$DST_APPLY_COMMIT> no es valido." >&2; exit 1; } @@ -155,6 +164,8 @@ DST_APPLY_COMMIT=$LAST_VALID_COMMIT_ID DST_APPLY_COMMIT_MSG=$LAST_VALID_COMMIT_MSG +echo " [$SRC_START_COMMIT] - [$SRC_END_COMMIT] -> [$DST_APPLY_COMMIT]" + # 1.- Calcular lista de ficheros que componen la diferencia en SRC FILE_LIST=() From db28a0f791b7f5be1ab4b1efa844004da24fd8ee Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 30 Jul 2010 10:04:52 +0200 Subject: [PATCH 014/100] Bugfixes: Now 3 equal files are merged exactly equal. --- flalign.py | 2 +- flpremerge.py | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/flalign.py b/flalign.py index acc6e2b..1f8a939 100644 --- a/flalign.py +++ b/flalign.py @@ -399,7 +399,7 @@ def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = text = "".join( F[Fwhich].lines[int(linebegin):int(lineend)] ) - text = text.replace("\t"," ") + #text = text.replace("\t"," ") sline = line.split(":") if sline[0]=="classdeclaration": if len(classlist): diff --git a/flpremerge.py b/flpremerge.py index 1aa8725..67dfeb9 100644 --- a/flpremerge.py +++ b/flpremerge.py @@ -74,7 +74,8 @@ def computeSortedBlocks(self, fout=None): self.linePosChar = linePosChar for pk, bl_name in list(sorted(self.sortedNames))+[((None,None),None)]: desde, hasta = pk - if desde is None or desde > anthasta + 1: + if desde is None or desde >= anthasta +1: + bdesde = anthasta + 1 fB.seek(bdesde) if desde: @@ -87,8 +88,16 @@ def computeSortedBlocks(self, fout=None): while linenum < len(linePosChar) and linePosChar[linenum+1]<=bdesde: linenum += 1 startline = linenum - while linenum < len(linePosChar) and linePosChar[linenum+1] 1 and linePosChar[startline-1]-bdesde >=-2: startline-=1 + + #print linePosChar[startline-1]-bdesde, linePosChar[startline]-bdesde, linePosChar[startline+1]-bdesde #print (startline, endline), "BLOCK", (linePosChar[startline], linePosChar[endline]), (bdesde , bhasta), (bhasta-bdesde)+1 @@ -99,6 +108,8 @@ def computeSortedBlocks(self, fout=None): bblocks = [] blockdesc = [] if len(sB.splitlines(1)) != endline-startline+1: + print startline, endline, repr(sB) + print linePosChar[endline-1]-bhasta, linePosChar[endline]-bhasta, linePosChar[endline+1]-bhasta print "Block lines doesnt match:", len(sB.splitlines(1)), endline-startline, startline, endline print linePosChar[startline-2:startline+3],bdesde @@ -174,16 +185,19 @@ def computeSortedBlocks(self, fout=None): while linenum < len(linePosChar) and linePosChar[linenum] 0 or linePosChar[endline] - hasta < 1: - print linePosChar[startline] - desde, linePosChar[endline] - hasta, bl_name + if linePosChar[startline] - desde != 0 or linePosChar[endline] - hasta < 1 or linePosChar[endline] - hasta > 2: + print linePosChar[startline] - desde, linePosChar[endline-1] - hasta, linePosChar[endline] - hasta, bl_name name = bl_name #print (startline, endline), name, (linePosChar[startline], linePosChar[endline]), pk, (hasta-desde)+1 self.computedBlocks.append((startline, endline,name)) fout.write("%d\t%d\t%s\n" % (startline, endline,name)) #print "%s" % name antdesde, anthasta = pk - if anthasta < linePosChar[linenum] + 1: + if anthasta <= linePosChar[linenum-1] + 1: + anthasta = linePosChar[linenum-1] + 1 + elif anthasta <= linePosChar[linenum] + 1: anthasta = linePosChar[linenum] + 1 + fB.close() fout.close() From 55054120da0af86a92ba39d73606c13ebb69e9f3 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 25 Jul 2011 10:29:33 +0200 Subject: [PATCH 015/100] Add support for [], {} and for-in --- flex.py | 2 +- flscriptparse.py | 11 +++++++++++ xml2json.py | 4 +++- 3 files changed, 15 insertions(+), 2 deletions(-) mode change 100644 => 100755 xml2json.py diff --git a/flex.py b/flex.py index ee93427..8b9bc6a 100644 --- a/flex.py +++ b/flex.py @@ -12,7 +12,7 @@ # Reserved words reserved = [ 'BREAK', 'CASE', 'CONST', 'CONTINUE', 'DEFAULT', 'DO', - 'ELSE', 'FOR', 'IF', + 'ELSE', 'FOR', 'IF', 'IN', 'RETURN', #'STRUCT', 'SWITCH', diff --git a/flscriptparse.py b/flscriptparse.py index c564d9d..30c90b6 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -47,6 +47,10 @@ def p_parse(token): | funccall | error + array_value : LBRACKET RBRACKET + + dictobject_value : LBRACE RBRACE + base_expression : exprval | base_expression mathoperator base_expression | base_expression cmp_symbol base_expression @@ -58,6 +62,8 @@ def p_parse(token): | NEW funccall_1 | NEW ID | base_expression CONDITIONAL1 base_expression COLON base_expression + | array_value + | dictobject_value expression : base_expression | error @@ -93,6 +99,7 @@ def p_parse(token): | whilestatement | withstatement | forstatement + | forinstatement | switch | trycatch @@ -267,6 +274,10 @@ def p_parse(token): | FOR LPAREN SEMI base_expression SEMI storeinstruction RPAREN statement_block | error + forinstatement : FOR LPAREN VAR vardecl IN base_expression RPAREN statement_block + | FOR LPAREN variable IN base_expression RPAREN statement_block + | error + switch : SWITCH LPAREN expression RPAREN LBRACE case_block_list RBRACE optid : ID diff --git a/xml2json.py b/xml2json.py old mode 100644 new mode 100755 index c23a747..e6b4e24 --- a/xml2json.py +++ b/xml2json.py @@ -1,3 +1,5 @@ +#!/usr/bin/python + try: from json import dumps as json_dumps from json import loads as json_loads @@ -627,4 +629,4 @@ def main(): -if __name__ == "__main__": main() \ No newline at end of file +if __name__ == "__main__": main() From c3f46398dc4c79d95460edd276779d9b7e616a9c Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 1 Dec 2011 08:42:23 +0100 Subject: [PATCH 016/100] =?UTF-8?q?Agregar=20soporte=20para=20definici?= =?UTF-8?q?=C3=B3n=20de=20diccionarios=20y=20eliminar=20la=20impresi=C3=B3?= =?UTF-8?q?n=20en=20XML.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flscriptparse.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 30c90b6..1e6e1dd 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -50,7 +50,13 @@ def p_parse(token): array_value : LBRACKET RBRACKET dictobject_value : LBRACE RBRACE - + | LBRACE dictobject_value_elemlist RBRACE + + dictobject_value_elemlist : dictobject_value_elem + | dictobject_value_elemlist COMMA dictobject_value_elem + + dictobject_value_elem : constant COLON exprval + base_expression : exprval | base_expression mathoperator base_expression | base_expression cmp_symbol base_expression @@ -425,7 +431,10 @@ def calctree(obj, depth = 0, num = [], otype = "source"): if type(value) is dict: #print "*" - tree_obj = calctree(value,depth+1,num+[str(n)], ctype) + if depth < 60: + tree_obj = calctree(value,depth+1,num+[str(n)], ctype) + else: + tree_obj = None if type(tree_obj) is dict: if tree_obj['has_data'] and ctype != otype: contentlist.append([ctype,tree_obj]) @@ -626,7 +635,7 @@ def do_it(): prog = parse(data) sys.stderr.write(" formatting ...") sys.stderr.flush() - if prog: do_it() + #if prog: do_it() sys.stderr.write(" Done.\n") sys.stderr.flush() @@ -667,4 +676,4 @@ def do_it(): -if __name__ == "__main__": main() \ No newline at end of file +if __name__ == "__main__": main() From 3a3731d28c1228af2698f1b082a7321d90911ab8 Mon Sep 17 00:00:00 2001 From: yulieth Date: Thu, 16 Feb 2012 12:22:50 +0100 Subject: [PATCH 017/100] soporte de bloques vacios --- flscriptparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flscriptparse.py b/flscriptparse.py index 1e6e1dd..77ab4fa 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -250,6 +250,7 @@ def p_parse(token): statement_block : statement | LBRACE statement_list RBRACE + | LBRACE RBRACE optelse : ELSE statement_block | empty From b23d3db15cd07ea16dc84e0276d8a8cf0963065f Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 17 Feb 2012 13:03:35 +0100 Subject: [PATCH 018/100] Add script to analyze missing UI controls --- analyze_controls.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 analyze_controls.sh diff --git a/analyze_controls.sh b/analyze_controls.sh new file mode 100755 index 0000000..d9419d0 --- /dev/null +++ b/analyze_controls.sh @@ -0,0 +1,13 @@ + +for file_ui in $( git ls-files -- "*.ui" ); do + file_qs1="${file_ui%\.*}.qs" + file_qs="${file_qs1/forms/scripts}" + + + test -f "$file_qs" && { + for control in $(grep -Po "this\.child\(.(\w+).\)" "$file_qs" | sort -u | sed "s/\"/\t/g" | awk '{print $2}'); do + grep -Eq "$control" "$file_ui" || echo $file_qs $control; + done + } || echo "ERROR: $file_qs" + +done From 75920e52cfe5033f44b66652004afba2654a7adb Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 23 Feb 2012 11:29:16 +0100 Subject: [PATCH 019/100] Better parsing of QS --- flscriptparse.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 77ab4fa..43e4fa0 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -276,9 +276,12 @@ def p_parse(token): withstatement : WITH LPAREN variable RPAREN statement_block | error - forstatement : FOR LPAREN storeinstruction SEMI base_expression SEMI storeinstruction RPAREN statement_block - forstatement : FOR LPAREN VAR vardecl SEMI base_expression SEMI storeinstruction RPAREN statement_block - | FOR LPAREN SEMI base_expression SEMI storeinstruction RPAREN statement_block + storeormember : storeinstruction + | member_var + + forstatement : FOR LPAREN storeinstruction SEMI base_expression SEMI storeormember RPAREN statement_block + forstatement : FOR LPAREN VAR vardecl SEMI base_expression SEMI storeormember RPAREN statement_block + | FOR LPAREN SEMI base_expression SEMI storeormember RPAREN statement_block | error forinstatement : FOR LPAREN VAR vardecl IN base_expression RPAREN statement_block @@ -581,11 +584,14 @@ def parse(data): def main(): + global start parser = OptionParser() #parser.add_option("-f", "--file", dest="filename", # help="write report to FILE", metavar="FILE") parser.add_option("-O", "--output", dest="output", default = "xml", help="Set output TYPE: xml|hash", metavar="TYPE") + parser.add_option("--start", dest="start", default = None, + help="Set start block", metavar="STMT") parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True, help="don't print status messages to stdout") @@ -602,7 +608,10 @@ def main(): (options, args) = parser.parse_args() if options.optdebug: print options, args - + if options.start: + start = options.start + print "Start setted to:" , start + @@ -646,7 +655,18 @@ def do_it(): line = "" while 1: try: - line += raw_input("flscript> ") + line1 = raw_input("flscript> ") + if line1.startswith("#"): + comm = line1[1:].split(" ") + if comm[0] == "setstart": + start = comm[1] + print "Start setted to:" , start + if comm[0] == "parse": + print + prog = parse(line) + line = "" + else: + line += line1 except EOFError: break; line += "\n" From 214cd0363028b16a2b8bd75eefc0d98b75e4086a Mon Sep 17 00:00:00 2001 From: Omar El Mahi Date: Tue, 27 Mar 2012 11:22:40 +0200 Subject: [PATCH 020/100] Correccion Bug --- .gitignore | 10 ++++++++++ flscriptparse.py | 5 +++-- 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..900fe79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +actualiza +*~ +*.orig +*.directory +*.BASE.* +*.REMOTE.* +*.LOCAL.* +*.log +*.rej +*.pyc diff --git a/flscriptparse.py b/flscriptparse.py index 43e4fa0..848cfd2 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -588,7 +588,7 @@ def main(): parser = OptionParser() #parser.add_option("-f", "--file", dest="filename", # help="write report to FILE", metavar="FILE") - parser.add_option("-O", "--output", dest="output", default = "xml", + parser.add_option("-O", "--output", dest="output", default = "none", help="Set output TYPE: xml|hash", metavar="TYPE") parser.add_option("--start", dest="start", default = None, help="Set start block", metavar="STMT") @@ -616,6 +616,7 @@ def main(): def do_it(): + if options.output == "none": return tree_data = calctree(prog) if options.output == "hash": printtree(tree_data, mode = "hash") @@ -645,7 +646,7 @@ def do_it(): prog = parse(data) sys.stderr.write(" formatting ...") sys.stderr.flush() - #if prog: do_it() + if prog: do_it() sys.stderr.write(" Done.\n") sys.stderr.flush() From b0bdc12894563f1eecfd5df6796479c5b47ffc89 Mon Sep 17 00:00:00 2001 From: Gustavo Sanchis Date: Tue, 3 Apr 2012 12:57:24 +0200 Subject: [PATCH 021/100] Corregir error de install.sh al borrar enlaces antiguos --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 46786fe..0416d58 100644 --- a/install.sh +++ b/install.sh @@ -13,6 +13,6 @@ done < <(find /usr/local/ -type f -name designer) while read line; do bname=$(basename "$line") - test -e /usr/local/bin/$bname && unlink /usr/local/bin/$bname + test -h /usr/local/bin/$bname && unlink /usr/local/bin/$bname ln -s "$line" /usr/local/bin/$bname done < <(find $(pwd) -executable -type f \! -path "*/.*") \ No newline at end of file From 3a0028386b020ccd4bed0dbb0a3f2ed40f4bb571 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 11 Apr 2012 19:17:51 +0200 Subject: [PATCH 022/100] Pruebas de parseado alternativo --- flscriptparse.py | 58 +++++++----- postparse.py | 242 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 23 deletions(-) create mode 100644 postparse.py diff --git a/flscriptparse.py b/flscriptparse.py index 43e4fa0..5cf00ea 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -46,6 +46,8 @@ def p_parse(token): | variable | funccall | error + + identifier : ID array_value : LBRACKET RBRACKET @@ -160,8 +162,8 @@ def p_parse(token): | LPAREN member_call RPAREN | LPAREN funccall_1 RPAREN - funccall_1 : ID LPAREN callargs RPAREN - | ID LPAREN RPAREN + funccall_1 : identifier LPAREN callargs RPAREN + | identifier LPAREN RPAREN mathoperator : PLUS | MINUS @@ -179,15 +181,15 @@ def p_parse(token): | LPAREN variable_1 RPAREN | LPAREN member_var RPAREN - variable_1 : ID + variable_1 : identifier | inlinestoreinstruction | variable_1 LBRACKET base_expression RBRACKET | funccall_1 LBRACKET base_expression RBRACKET - inlinestoreinstruction : PLUSPLUS ID - | MINUSMINUS ID - | ID PLUSPLUS - | ID MINUSMINUS + inlinestoreinstruction : PLUSPLUS identifier + | MINUSMINUS identifier + | identifier PLUSPLUS + | identifier MINUSMINUS storeequalinstruction : variable EQUALS expression | variable EQUALS storeequalinstruction @@ -393,7 +395,7 @@ def print_tokentree(token, depth = 0): print_tokentree(tk.value, depth +1) -def calctree(obj, depth = 0, num = [], otype = "source"): +def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): #if depth > 5: return source_data = [ 'source', @@ -408,17 +410,24 @@ def calctree(obj, depth = 0, num = [], otype = "source"): has_data = 0 has_objects = 0 contentlist = [] - ctype_alias = { - "member_var" : "member", - "member_call" : "member", - "variable_1" : "variable", - "funccall_1" : "funccall", - "flowinstruction" : "instruction", - "storeequalinstruction" : "instruction", - "vardecl" : "vardeclaration", - #"vardecl_list" : "vardeclaration", + + if alias_mode == 0: + ctype_alias = {} + elif alias_mode == 1: + ctype_alias = { + "member_var" : "member", + "member_call" : "member", + "variable_1" : "variable", + "funccall_1" : "funccall", + "flowinstruction" : "instruction", + "storeequalinstruction" : "instruction", + "vardecl" : "vardeclaration", + #"vardecl_list" : "vardeclaration", + + } + else: + raise ValueError, "alias_mode unknown" - } if otype in ctype_alias: otype = ctype_alias[otype] #print " " * depth , obj['02-size'] @@ -435,12 +444,12 @@ def calctree(obj, depth = 0, num = [], otype = "source"): if type(value) is dict: #print "*" - if depth < 60: - tree_obj = calctree(value,depth+1,num+[str(n)], ctype) + if depth < 600: + tree_obj = calctree(value,depth+1,num+[str(n)], ctype, alias_mode=alias_mode) else: tree_obj = None if type(tree_obj) is dict: - if tree_obj['has_data'] and ctype != otype: + if (tree_obj['has_data'] or alias_mode == 0) and ctype != otype : contentlist.append([ctype,tree_obj]) has_objects += 1 else: @@ -588,7 +597,7 @@ def main(): parser = OptionParser() #parser.add_option("-f", "--file", dest="filename", # help="write report to FILE", metavar="FILE") - parser.add_option("-O", "--output", dest="output", default = "xml", + parser.add_option("-O", "--output", dest="output", default = "none", help="Set output TYPE: xml|hash", metavar="TYPE") parser.add_option("--start", dest="start", default = None, help="Set start block", metavar="STMT") @@ -629,6 +638,9 @@ def do_it(): f1_xml = open(filename+".xml","w") printtree(tree_data, mode = "xml", output = f1_xml) f1_xml.close() + elif options.output == "yaml": + import yaml + print yaml.dump(tree_data['content']) else: print "Unknown outputmode", options.output @@ -645,7 +657,7 @@ def do_it(): prog = parse(data) sys.stderr.write(" formatting ...") sys.stderr.flush() - #if prog: do_it() + if prog and options.output != "none": do_it() sys.stderr.write(" Done.\n") sys.stderr.flush() diff --git a/postparse.py b/postparse.py new file mode 100644 index 0000000..34109f1 --- /dev/null +++ b/postparse.py @@ -0,0 +1,242 @@ +from optparse import OptionParser +import flscriptparse +from lxml import etree +USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") + +KNOWN_PARSERS = {} +UNKNOWN_PARSERS = {} +def parse_for(*tagnames): + global KNOWN_PARSERS + def decorator(fn): + for n in tagnames: + KNOWN_PARSERS[n] = fn + return fn + return decorator + +def parse(tagname, treedata): + global KNOWN_PARSERS,UNKNOWN_PARSERS + if tagname not in KNOWN_PARSERS: + UNKNOWN_PARSERS[tagname] = 1 + fn = parse_unknown + else: + fn = KNOWN_PARSERS[tagname] + return fn(tagname,treedata) + + +def getxmltagname(tagname): + if tagname == "source": return "Source" + if tagname == "funcdeclaration": return "Function" + if tagname == "classdeclaration": return "Class" + if tagname == "vardeclaration": return "Variable" + return "Unknown.%s" % tagname + +xml_class_types = [] + +class TagObjectFactory(type): + def __init__(cls, name, bases, dct): + global xml_class_types + xml_class_types.append(cls) + super(TagObjectFactory, cls).__init__(name, bases, dct) + +class TagObject(object): + __metaclass__ = TagObjectFactory + tags = [] + set_child_argn = True + name_is_first_id = False + debug_other = True + adopt_childs_tags = [] + omit_tags = ['empty'] + callback_subelem = {} + + @classmethod + def tagname(self, tagname): return self.__name__ + + @classmethod + def can_process_tag(self, tagname): return tagname in self.tags + + def __init__(self, tagname): + self.astname = tagname + self.xml = etree.Element(self.tagname(tagname)) + self.xmlname = None + if self.name_is_first_id: + self.xml.set("name","") + + def adopt_children(self, argn, subelem): + for child in subelem.xml.iterchildren(): + self.xml.append(child) + + def omit_subelem(self, argn, subelem): + return + + def is_in(self, listobj): + return self.__class__ in listobj or self.astname in listobj + + def get(self, listobj, default = None): + if self.__class__ in listobj: return listobj[self.__class__] + if self.astname in listobj: return listobj[self.astname] + return default + + + def add_subelem(self, argn, subelem): + if subelem.is_in(self.omit_tags): return self.omit_subelem(argn, subelem) + if subelem.is_in(self.adopt_childs_tags): return self.adopt_children(argn, subelem) + callback = subelem.get(self.callback_subelem) + if callback: return getattr(self,callback)(argn,subelem) + + if self.set_child_argn: subelem.xml.set("argn",str(argn)) + self.xml.append(subelem.xml) + + def add_value(self, argn, vtype, value): + if vtype == "ID" and self.name_is_first_id and self.xmlname is None: + self.xmlname = value + self.xml.set("name",value) + return + + self.xml.set("arg%02d" % argn,vtype + ":" + repr(value)) + + def add_other(self, argn, vtype, data): + if self.debug_other: + self.xml.set("arg%02d" % argn,vtype) + + def polish(self): + return + + +class ListObject(TagObject): + set_child_argn = False + debug_other = False + +class NamedObject(TagObject): + name_is_first_id = True + debug_other = False + +class ListNamedObject(TagObject): + name_is_first_id = True + set_child_argn = False + debug_other = False + +class StatementList(ListObject): + tags = ["statement_list"] + adopt_childs_tags = ['statement'] + +class Instruction(ListObject): + tags = ["instruction"] + adopt_childs_tags = ['base_instruction'] + + +class Source(ListObject): + tags = ["source","basicsource"] + adopt_childs_tags = ['source_element','statement_list'] + +class Identifier(NamedObject): + tags = ["identifier"] + + +class Class(ListNamedObject): + tags = ["classdeclaration"] + +class Arguments(ListObject): + tags = ["arglist"] + adopt_childs_tags = ['vardecl_list'] + +class VariableType(NamedObject): + tags = ["optvartype"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + +class Function(ListNamedObject): + tags = ["funcdeclaration"] + callback_subelem = ListNamedObject.callback_subelem.copy() + callback_subelem[VariableType] = "add_vartype" + + def add_vartype(self, argn, subelem): + self.xml.set("returns", str(subelem.xmlname)) + +class Variable(NamedObject): + tags = ["vardecl"] + callback_subelem = NamedObject.callback_subelem.copy() + callback_subelem[VariableType] = "add_vartype" + + def add_vartype(self, argn, subelem): + self.xml.set("type", str(subelem.xmlname)) + +class DeclarationBlock(ListObject): + tags = ["vardeclaration"] + adopt_childs_tags = ['vardecl_list'] + def add_other(self, argn, vtype, value): + if argn == 0: + self.xml.set("mode", vtype) + + +class Unknown(TagObject): + set_child_argn = False + @classmethod + def tagname(self, tagname): return tagname + + @classmethod + def can_process_tag(self, tagname): return True + + +def create_xml(tagname): + classobj = None + for cls in xml_class_types: + if cls.can_process_tag(tagname): + classobj = cls + break + if classobj is None: return None + return classobj(tagname) + +def parse_unknown(tagname, treedata): + xmlelem = create_xml(tagname) + i = 0 + for k, v in treedata['content']: + if type(v) is dict: + instruction = parse(k,v) + xmlelem.add_subelem(i, instruction) + elif k in USEFUL_TOKENS: + xmlelem.add_value(i, k, v) + else: + xmlelem.add_other(i, k, v) + + i+=1 + xmlelem.polish() + return xmlelem + + + +def post_parse(treedata): + source = parse("source",treedata) + #print UNKNOWN_PARSERS.keys() + return source.xml + + +def main(): + parser = OptionParser() + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + + (options, args) = parser.parse_args() + if options.optdebug: + print options, args + + for filename in args: + prog = flscriptparse.parse(open(filename).read()) + tree_data = flscriptparse.calctree(prog, alias_mode = 0) + ast = post_parse(tree_data) + + print etree.tostring(ast, pretty_print = True) + + + +if __name__ == "__main__": main() From 972d34a91528be70018c583496d0afd57553f074 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 11 Apr 2012 23:36:55 +0200 Subject: [PATCH 023/100] Pruebas de parseado alternativo (II) --- flscriptparse.py | 52 +++++++------ postparse.py | 187 ++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 199 insertions(+), 40 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 5cf00ea..054c5f5 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -63,15 +63,24 @@ def p_parse(token): | base_expression mathoperator base_expression | base_expression cmp_symbol base_expression | base_expression boolcmp_symbol base_expression - | LPAREN base_expression RPAREN - | LNOT base_expression - | MINUS base_expression - | PLUS base_expression - | NEW funccall_1 - | NEW ID - | base_expression CONDITIONAL1 base_expression COLON base_expression + | inlinestoreinstruction + | parentheses + | unary_operator + | new_operator + | ternary_operator | array_value | dictobject_value + + parentheses : LPAREN base_expression RPAREN + + unary_operator : LNOT base_expression + | MINUS base_expression + | PLUS base_expression + + new_operator : NEW funccall_1 + | NEW identifier + + ternary_operator : base_expression CONDITIONAL1 base_expression COLON base_expression expression : base_expression | error @@ -79,7 +88,7 @@ def p_parse(token): case_cblock_list : case_block case_cblock_list : case_cblock_list case_block - case_block : CASE base_expression COLON statement_list + case_block : CASE expression COLON statement_list case_default : DEFAULT COLON statement_list @@ -162,8 +171,8 @@ def p_parse(token): | LPAREN member_call RPAREN | LPAREN funccall_1 RPAREN - funccall_1 : identifier LPAREN callargs RPAREN - | identifier LPAREN RPAREN + funccall_1 : ID LPAREN callargs RPAREN + | ID LPAREN RPAREN mathoperator : PLUS | MINUS @@ -182,14 +191,13 @@ def p_parse(token): | LPAREN member_var RPAREN variable_1 : identifier - | inlinestoreinstruction | variable_1 LBRACKET base_expression RBRACKET | funccall_1 LBRACKET base_expression RBRACKET - inlinestoreinstruction : PLUSPLUS identifier - | MINUSMINUS identifier - | identifier PLUSPLUS - | identifier MINUSMINUS + inlinestoreinstruction : PLUSPLUS variable + | MINUSMINUS variable + | variable PLUSPLUS + | variable MINUSMINUS storeequalinstruction : variable EQUALS expression | variable EQUALS storeequalinstruction @@ -216,12 +224,13 @@ def p_parse(token): instruction : base_instruction SEMI | SEMI | base_instruction + + callinstruction : funccall + | variable + base_instruction : storeinstruction - | funccall + | callinstruction | flowinstruction - | variable - | variable PLUSPLUS - | variable MINUSMINUS optextends : EXTENDS ID | empty @@ -311,8 +320,6 @@ def p_parse(token): def p_error(t): global error_count global last_error_token - - if repr(t) != repr(last_error_token): error_count += 1 if error_count>100 or t is None: @@ -337,6 +344,7 @@ def p_error(t): # yacc.errok() #else: if t is None: + print "ERROR: End of the file reached." yacc.errok() return t = yacc.token() @@ -349,7 +357,7 @@ def p_error(t): parser = yacc.yacc(method='LALR',debug=0, - optimize = 0, write_tables = 1, debugfile = '/tmp/yaccdebug.txt',outputdir='/tmp/') + optimize = 1, write_tables = 1, debugfile = '/tmp/yaccdebug.txt',outputdir='/tmp/') #profile.run("yacc.yacc(method='LALR')") diff --git a/postparse.py b/postparse.py index 34109f1..a4d25dd 100644 --- a/postparse.py +++ b/postparse.py @@ -41,12 +41,13 @@ def __init__(cls, name, bases, dct): class TagObject(object): __metaclass__ = TagObjectFactory tags = [] - set_child_argn = True + set_child_argn = False name_is_first_id = False debug_other = True adopt_childs_tags = [] omit_tags = ['empty'] callback_subelem = {} + promote_child_if_alone = False @classmethod def tagname(self, tagname): return self.__name__ @@ -58,11 +59,16 @@ def __init__(self, tagname): self.astname = tagname self.xml = etree.Element(self.tagname(tagname)) self.xmlname = None + self.subelems = [] + self.values = [] if self.name_is_first_id: self.xml.set("name","") def adopt_children(self, argn, subelem): for child in subelem.xml.iterchildren(): + if self.set_child_argn: child.set("argn",str(argn)) + else: + if 'argn' in child.attrib: del child.attrib['argn'] self.xml.append(child) def omit_subelem(self, argn, subelem): @@ -85,8 +91,10 @@ def add_subelem(self, argn, subelem): if self.set_child_argn: subelem.xml.set("argn",str(argn)) self.xml.append(subelem.xml) + self.subelems.append(subelem) def add_value(self, argn, vtype, value): + self.values.append( (vtype, value) ) if vtype == "ID" and self.name_is_first_id and self.xmlname is None: self.xmlname = value self.xml.set("name",value) @@ -99,7 +107,10 @@ def add_other(self, argn, vtype, data): self.xml.set("arg%02d" % argn,vtype) def polish(self): - return + if self.promote_child_if_alone: + if len(self.values) == 0 and len(self.subelems) == 1: + return self.subelems[0] + return self class ListObject(TagObject): @@ -115,25 +126,19 @@ class ListNamedObject(TagObject): set_child_argn = False debug_other = False -class StatementList(ListObject): - tags = ["statement_list"] - adopt_childs_tags = ['statement'] - -class Instruction(ListObject): - tags = ["instruction"] - adopt_childs_tags = ['base_instruction'] class Source(ListObject): - tags = ["source","basicsource"] - adopt_childs_tags = ['source_element','statement_list'] + tags = ["source","basicsource","classdeclarationsource","statement_list"] + adopt_childs_tags = ['source_element','statement_list','statement'] class Identifier(NamedObject): - tags = ["identifier"] - + tags = ["identifier","optid"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self -class Class(ListNamedObject): - tags = ["classdeclaration"] class Arguments(ListObject): tags = ["arglist"] @@ -144,6 +149,14 @@ class VariableType(NamedObject): def polish(self): if self.xmlname is None: self.astname = "empty" + return self + +class ExtendsType(NamedObject): + tags = ["optextends"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self class Function(ListNamedObject): tags = ["funcdeclaration"] @@ -167,9 +180,138 @@ class DeclarationBlock(ListObject): def add_other(self, argn, vtype, value): if argn == 0: self.xml.set("mode", vtype) + def polish(self): + if len(self.values) == 0 and len(self.subelems) == 1: + self.subelems[0].xml.set("mode",self.xml.get("mode")) + return self.subelems[0] + return self + + +class Class(ListNamedObject): + tags = ["classdeclaration"] + callback_subelem = ListNamedObject.callback_subelem.copy() + callback_subelem[ExtendsType] = "add_exttype" + def add_exttype(self, argn, subelem): + self.xml.set("extends", str(subelem.xmlname)) + +class Member(TagObject): + debug_other = False + set_child_argn = False + tags = ["member_var","member_call"] + adopt_childs_tags = ['varmemcall',"member_var","member_call"] + +class InstructionCall(TagObject): + debug_other = False + tags = ["callinstruction"] + +class InstructionStore(TagObject): + promote_child_if_alone = True + debug_other = False + tags = ["storeinstruction"] + +class InstructionFlow(TagObject): + debug_other = True + tags = ["flowinstruction"] + def add_other(self, argn, vtype, value): + if argn == 0: + self.xml.set("type", vtype) + +class Math(TagObject): + debug_other = True + tags = ["mathoperator"] + def add_other(self, argn, vtype, value): + if argn == 0: + self.xml.set("type", vtype) + +class Compare(TagObject): + debug_other = True + tags = ["cmp_symbol","boolcmp_symbol"] + def add_other(self, argn, vtype, value): + if argn == 0: + self.xml.set("type", vtype) + +class FunctionCall(NamedObject): + tags = ["funccall_1"] + +class CallArguments(ListObject): + tags = ["callargs"] + +class InstructionStoreEqual(ListObject): + tags = ["storeequalinstruction"] + +class Constant(ListObject): + tags = ["constant"] + def add_value(self, argn, vtype, value): + value = unicode(value,"ISO-8859-15","replace") + if vtype == "SCONST": + vtype = "String" + value = value[1:-1] + if vtype == "CCONST": vtype = "String" + if vtype == "RCONST": vtype = "Regex" + if vtype == "ICONST": vtype = "Number" + if vtype == "FCONST": vtype = "Number" + self.const_value = value + self.const_type = vtype + self.xml.set("type",vtype) + self.xml.set("value",value) + +class If(ListObject): + tags = ["ifstatement"] + +class Condition(ListObject): + tags = ["condition"] + +class Else(ListObject): + tags = ["optelse"] + adopt_childs_tags = ['statement_block'] + def polish(self): + if len(self.subelems) == 0: + self.astname = "empty" + return self + + +class ExpressionContainer(ListObject): + tags = ["expression"] + # adopt_childs_tags = ['base_expression'] + def polish(self): + if len(self.values) == 0 and len(self.subelems) == 1: + #if isinstance(self.subelems[0], Constant): + if self.subelems[0].xml.tag == "base_expression": + self.subelems[0].xml.tag = "Expression" + return self.subelems[0] + else: + self.xml.tag = "Value" + + return self + +class Switch(ListObject): + tags = ["switch"] + adopt_childs_tags = ['case_cblock_list'] + + +class Case(ListObject): + tags = ["case_block"] + +class New(ListObject): + tags = ["new_operator"] + +class Parentheses(ListObject): + tags = ["parentheses"] + adopt_childs_tags = ['base_expression'] + +class Unary(ListObject): + tags = ["unary_operator"] + def add_other(self, argn, vtype, value): + if argn == 0: + self.xml.set("type", vtype) + +class Ternary(ListObject): + tags = ["ternary_operator"] + class Unknown(TagObject): + promote_child_if_alone = True set_child_argn = False @classmethod def tagname(self, tagname): return tagname @@ -200,8 +342,7 @@ def parse_unknown(tagname, treedata): xmlelem.add_other(i, k, v) i+=1 - xmlelem.polish() - return xmlelem + return xmlelem.polish() @@ -231,10 +372,20 @@ def main(): print options, args for filename in args: + print filename prog = flscriptparse.parse(open(filename).read()) + if not prog: + print "No se pudo abrir el fichero." + continue tree_data = flscriptparse.calctree(prog, alias_mode = 0) - ast = post_parse(tree_data) + if not tree_data: + print "No se pudo parsear." + continue + ast = post_parse(tree_data) + if ast is None: + print "No se pudo analizar." + continue print etree.tostring(ast, pretty_print = True) From 2542f6a40815d0dca65ec853ccfd5c4b4849a42a Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 12 Apr 2012 09:30:25 +0200 Subject: [PATCH 024/100] Mejoras en parseo --- flscriptparse.py | 43 +++++++++++++++++++++---------------------- postparse.py | 17 +++++++++++++++-- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 9466587..1880eb9 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -312,37 +312,35 @@ def p_parse(token): lexspan = list(token.lexspan(0)) data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } + global ok_count + ok_count += 1 error_count = 0 last_error_token = None +last_error_line = -1 +ok_count = 0 def p_error(t): global error_count + global ok_count global last_error_token - if repr(t) != repr(last_error_token): - error_count += 1 - if error_count>100 or t is None: + global last_error_line + if t is not None: + if last_error_token is None or t.lexpos != last_error_token.lexpos: + error_count = 0 + if abs(last_error_line - t.lineno) > 3 and ok_count > 2: + try: print_context(t) + except: pass + print "ERROR" + last_error_line = t.lineno yacc.errok() + ok_count = 0 return - try: - print_context(t) - except: - pass - print "ERROR" - import sys - sys.exit() - #while 1: - # if t is None: - # print "*** Se encontro EOF intentando resolver el error *** " - # break - # if t.type == 'RBRACE': break - # if t.type == 'SEMI': break - # t = yacc.token() # Get the next token - - #if t is not None: - # yacc.errok() - #else: + else: + error_count += 1 + + ok_count = 0 if t is None: print "ERROR: End of the file reached." yacc.errok() @@ -365,7 +363,7 @@ def p_error(t): def print_context(token): global input_data - if not token: return + if token is None: return last_cr = input_data.rfind('\n',0,token.lexpos) next_cr = input_data.find('\n',token.lexpos) column = (token.lexpos - last_cr) @@ -595,6 +593,7 @@ def parse(data): global input_data parser.error = 0 input_data = data + flex.lexer.lineno = 0 p = parser.parse(data, debug = 0, tracking = 1, tokenfunc = my_tokenfunc) if parser.error: return None return p diff --git a/postparse.py b/postparse.py index a4d25dd..d68587e 100644 --- a/postparse.py +++ b/postparse.py @@ -1,4 +1,5 @@ from optparse import OptionParser +import os, os.path import flscriptparse from lxml import etree USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") @@ -366,17 +367,26 @@ def main(): action="store_true", dest="debug", default=False, help="prints lots of useless messages") + parser.add_option("--path", + dest="storepath", default=None, + help="store XML results in PATH") + (options, args) = parser.parse_args() if options.optdebug: print options, args for filename in args: - print filename + bname = os.path.basename(filename) + print "File:", bname prog = flscriptparse.parse(open(filename).read()) if not prog: print "No se pudo abrir el fichero." continue + if options.storepath is None: + # Si no se quiere guardar resultado, no hace falta calcular mas + continue + tree_data = flscriptparse.calctree(prog, alias_mode = 0) if not tree_data: print "No se pudo parsear." @@ -386,7 +396,10 @@ def main(): if ast is None: print "No se pudo analizar." continue - print etree.tostring(ast, pretty_print = True) + if options.storepath: + f1 = open(os.path.join(options.storepath,bname+".xml"),"w") + f1.write(etree.tostring(ast, pretty_print = True)) + f1.close() From f69b9b122ae7ab7ae3265dffb8e3c806bed26ad8 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 12 Apr 2012 09:34:34 +0200 Subject: [PATCH 025/100] Mejoras en parseo --- flscriptparse.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 1880eb9..9fca967 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -342,9 +342,10 @@ def p_error(t): ok_count = 0 if t is None: - print "ERROR: End of the file reached." - yacc.errok() - return + if last_error_token != "EOF": + print "ERROR: End of the file reached." + last_error_token = "EOF" + return t t = yacc.token() yacc.restart() last_error_token = t From 48acfae39d9fcd21a088b685920856c43949c160 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 12 Apr 2012 12:36:14 +0200 Subject: [PATCH 026/100] Mejoras parseo --- flscriptparse.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 9fca967..c745671 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -59,11 +59,11 @@ def p_parse(token): dictobject_value_elem : constant COLON exprval - base_expression : exprval + base_expression : inlinestoreinstruction + | exprval | base_expression mathoperator base_expression | base_expression cmp_symbol base_expression | base_expression boolcmp_symbol base_expression - | inlinestoreinstruction | parentheses | unary_operator | new_operator @@ -71,6 +71,7 @@ def p_parse(token): | array_value | dictobject_value + parentheses : LPAREN base_expression RPAREN unary_operator : LNOT base_expression @@ -158,7 +159,7 @@ def p_parse(token): | funccall_1 | member_call | member_var - | base_expression + | LPAREN base_expression RPAREN member_var : varmemcall PERIOD variable_1 member_call : LPAREN member_var RPAREN PERIOD funccall_1 @@ -272,6 +273,7 @@ def p_parse(token): | GE | EQ | NE + | IN boolcmp_symbol : LOR | LAND @@ -326,19 +328,16 @@ def p_error(t): global ok_count global last_error_token global last_error_line + error_count += 1 if t is not None: if last_error_token is None or t.lexpos != last_error_token.lexpos: - error_count = 0 if abs(last_error_line - t.lineno) > 3 and ok_count > 2: try: print_context(t) except: pass - print "ERROR" last_error_line = t.lineno yacc.errok() ok_count = 0 return - else: - error_count += 1 ok_count = 0 if t is None: @@ -375,7 +374,6 @@ def print_context(token): print input_data[last_cr:next_cr].replace("\t"," ") print (" " * (column-1)) + "^", column, "#ERROR#" , token - print def my_tokenfunc(*args, **kwargs): @@ -592,11 +590,16 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou def parse(data): global input_data + global error_count parser.error = 0 input_data = data - flex.lexer.lineno = 0 + flex.lexer.lineno = 1 + error_count = 0 p = parser.parse(data, debug = 0, tracking = 1, tokenfunc = my_tokenfunc) - if parser.error: return None + if error_count > 0: + print "ERRORS (%d)" % error_count + if parser.error: + return None return p From 5672fca71c7bc8c494dc475c4055851c94110229 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 12 Apr 2012 13:03:46 +0200 Subject: [PATCH 027/100] Agregar comando flscriptparser2 --- flex.py | 4 ++-- flscriptparse.py | 10 ++++++++-- flscriptparser2 | 4 ++++ postparse.py | 1 + 4 files changed, 15 insertions(+), 4 deletions(-) create mode 100755 flscriptparser2 mode change 100644 => 100755 postparse.py diff --git a/flex.py b/flex.py index 8b9bc6a..a86b4aa 100644 --- a/flex.py +++ b/flex.py @@ -17,7 +17,7 @@ #'STRUCT', 'SWITCH', 'WHILE', 'CLASS', 'VAR', 'FUNCTION', - 'EXTENDS', 'NEW','WITH','TRY','CATCH','THROW', 'DELETE' + 'EXTENDS', 'NEW','WITH','TRY','CATCH','THROW', 'DELETE', 'TYPEOF' ] token_literals = [ # Literals (identifier, integer constant, float constant, string constant, char const) @@ -157,7 +157,7 @@ def t_ID(t): t_SCONST = r'\"([^\\\n]|(\\.))*?\"' # Character constant 'c' or L'c' -t_CCONST = r'(L)?\'([^\\\n]|(\\.))*?\'' +t_CCONST = r'\'([^\\\n]|(\\.))*?\'' # REGEX constant t_RXCONST = r'/.+/g' diff --git a/flscriptparse.py b/flscriptparse.py index c745671..73687e7 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -57,7 +57,7 @@ def p_parse(token): dictobject_value_elemlist : dictobject_value_elem | dictobject_value_elemlist COMMA dictobject_value_elem - dictobject_value_elem : constant COLON exprval + dictobject_value_elem : exprval COLON expression base_expression : inlinestoreinstruction | exprval @@ -70,6 +70,7 @@ def p_parse(token): | ternary_operator | array_value | dictobject_value + | typeof_operator parentheses : LPAREN base_expression RPAREN @@ -80,10 +81,13 @@ def p_parse(token): new_operator : NEW funccall_1 | NEW identifier + + typeof_operator : TYPEOF variable ternary_operator : base_expression CONDITIONAL1 base_expression COLON base_expression expression : base_expression + | funcdeclaration_anon | error case_cblock_list : case_block @@ -148,6 +152,7 @@ def p_parse(token): | funcdeclaration : FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE + funcdeclaration_anon : FUNCTION LPAREN arglist RPAREN LBRACE basicsource RBRACE callarg : expression @@ -160,6 +165,7 @@ def p_parse(token): | member_call | member_var | LPAREN base_expression RPAREN + | LPAREN identifier RPAREN member_var : varmemcall PERIOD variable_1 member_call : LPAREN member_var RPAREN PERIOD funccall_1 @@ -330,7 +336,7 @@ def p_error(t): global last_error_line error_count += 1 if t is not None: - if last_error_token is None or t.lexpos != last_error_token.lexpos: + if last_error_token is None or t.lexpos != getattr(last_error_token,"lexpos",None): if abs(last_error_line - t.lineno) > 3 and ok_count > 2: try: print_context(t) except: pass diff --git a/flscriptparser2 b/flscriptparser2 new file mode 100755 index 0000000..b9a8956 --- /dev/null +++ b/flscriptparser2 @@ -0,0 +1,4 @@ +#!/usr/bin/python +import postparse +if __name__ == "__main__": + postparse.main() diff --git a/postparse.py b/postparse.py old mode 100644 new mode 100755 index d68587e..55f96ab --- a/postparse.py +++ b/postparse.py @@ -1,3 +1,4 @@ +#!/usr/bin/python from optparse import OptionParser import os, os.path import flscriptparse From ca0d984d3c005cc104c15dc150563f2fc94879ba Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 12 Apr 2012 23:24:53 +0200 Subject: [PATCH 028/100] Agregar boceto de pytnyzer.py --- flscriptparse.py | 47 ++++++---- postparse.py | 126 ++++++++++++++++---------- pytnyzer.py | 229 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 335 insertions(+), 67 deletions(-) create mode 100644 pytnyzer.py diff --git a/flscriptparse.py b/flscriptparse.py index 73687e7..92cd149 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -119,6 +119,7 @@ def p_parse(token): | vardeclaration | ifstatement | whilestatement + | dowhilestatement | withstatement | forstatement | forinstatement @@ -206,20 +207,21 @@ def p_parse(token): | variable PLUSPLUS | variable MINUSMINUS - storeequalinstruction : variable EQUALS expression - | variable EQUALS storeequalinstruction - + updateoperator : EQUALS + | PLUSEQUAL + | MINUSEQUAL + | MODEQUAL + | DIVEQUAL + | TIMESEQUAL + + updateinstruction : variable updateoperator expression + | variable updateoperator updateinstruction - + deleteinstruction : DELETE variable storeinstruction : inlinestoreinstruction - | storeequalinstruction - | variable PLUSEQUAL expression - | variable MINUSEQUAL expression - | variable MODEQUAL expression - | variable DIVEQUAL expression - | variable TIMESEQUAL expression - | DELETE variable + | updateinstruction + | deleteinstruction flowinstruction : RETURN expression @@ -289,22 +291,31 @@ def p_parse(token): ifstatement : IF LPAREN condition RPAREN statement_block optelse whilestatement : WHILE LPAREN condition RPAREN statement_block - whilestatement : DO statement_block WHILE LPAREN condition RPAREN SEMI - | error + dowhilestatement : DO statement_block WHILE LPAREN condition RPAREN SEMI withstatement : WITH LPAREN variable RPAREN statement_block | error storeormember : storeinstruction | member_var + + for_initialize : storeinstruction + | VAR vardecl + | for_initialize COMMA for_initialize + | empty - forstatement : FOR LPAREN storeinstruction SEMI base_expression SEMI storeormember RPAREN statement_block - forstatement : FOR LPAREN VAR vardecl SEMI base_expression SEMI storeormember RPAREN statement_block - | FOR LPAREN SEMI base_expression SEMI storeormember RPAREN statement_block + for_compare : expression + | empty + + for_increment : storeormember + | for_increment COMMA for_increment + | empty + + + forstatement : FOR LPAREN for_initialize SEMI for_compare SEMI for_increment RPAREN statement_block | error - forinstatement : FOR LPAREN VAR vardecl IN base_expression RPAREN statement_block - | FOR LPAREN variable IN base_expression RPAREN statement_block + forinstatement : FOR LPAREN for_initialize IN variable RPAREN statement_block | error switch : SWITCH LPAREN expression RPAREN LBRACE case_block_list RBRACE diff --git a/postparse.py b/postparse.py index 55f96ab..afdbb7a 100755 --- a/postparse.py +++ b/postparse.py @@ -128,6 +128,11 @@ class ListNamedObject(TagObject): set_child_argn = False debug_other = False +class TypedObject(ListObject): + type_arg = 0 + def add_other(self, argn, vtype, value): + if argn == self.type_arg: + self.xml.set("type", vtype) class Source(ListObject): @@ -183,12 +188,11 @@ def add_other(self, argn, vtype, value): if argn == 0: self.xml.set("mode", vtype) def polish(self): - if len(self.values) == 0 and len(self.subelems) == 1: - self.subelems[0].xml.set("mode",self.xml.get("mode")) - return self.subelems[0] + #if len(self.values) == 0 and len(self.subelems) == 1: + # self.subelems[0].xml.set("mode",self.xml.get("mode")) + # return self.subelems[0] return self - class Class(ListNamedObject): tags = ["classdeclaration"] callback_subelem = ListNamedObject.callback_subelem.copy() @@ -211,26 +215,17 @@ class InstructionStore(TagObject): debug_other = False tags = ["storeinstruction"] -class InstructionFlow(TagObject): +class InstructionFlow(TypedObject): debug_other = True tags = ["flowinstruction"] - def add_other(self, argn, vtype, value): - if argn == 0: - self.xml.set("type", vtype) -class Math(TagObject): +class OpMath(TypedObject): debug_other = True tags = ["mathoperator"] - def add_other(self, argn, vtype, value): - if argn == 0: - self.xml.set("type", vtype) -class Compare(TagObject): +class Compare(TypedObject): debug_other = True tags = ["cmp_symbol","boolcmp_symbol"] - def add_other(self, argn, vtype, value): - if argn == 0: - self.xml.set("type", vtype) class FunctionCall(NamedObject): tags = ["funccall_1"] @@ -238,9 +233,6 @@ class FunctionCall(NamedObject): class CallArguments(ListObject): tags = ["callargs"] -class InstructionStoreEqual(ListObject): - tags = ["storeequalinstruction"] - class Constant(ListObject): tags = ["constant"] def add_value(self, argn, vtype, value): @@ -287,14 +279,34 @@ def polish(self): return self +class InstructionUpdate(ListObject): + tags = ["updateinstruction"] + class Switch(ListObject): tags = ["switch"] adopt_childs_tags = ['case_cblock_list'] - class Case(ListObject): tags = ["case_block"] +class While(ListObject): + tags = ["whilestatement"] + +class For(ListObject): + tags = ["forstatement"] + +class DoWhile(ListObject): + tags = ["dowhilestatement"] + +class ForIn(ListObject): + tags = ["forinstatement"] + +class With(ListObject): + tags = ["withstatement"] + +class TryCatch(ListObject): + tags = ["trycatch"] + class New(ListObject): tags = ["new_operator"] @@ -302,16 +314,18 @@ class Parentheses(ListObject): tags = ["parentheses"] adopt_childs_tags = ['base_expression'] -class Unary(ListObject): +class OpUnary(TypedObject): tags = ["unary_operator"] - def add_other(self, argn, vtype, value): - if argn == 0: - self.xml.set("type", vtype) -class Ternary(ListObject): +class OpTernary(ListObject): tags = ["ternary_operator"] +class OpUpdate(TypedObject): + tags = ["updateoperator"] + + +# ----- keep this one at the end. class Unknown(TagObject): promote_child_if_alone = True set_child_argn = False @@ -320,7 +334,7 @@ def tagname(self, tagname): return tagname @classmethod def can_process_tag(self, tagname): return True - +# ----------------- def create_xml(tagname): classobj = None @@ -372,35 +386,49 @@ def main(): dest="storepath", default=None, help="store XML results in PATH") + parser.add_option("--topython", + action="store_true", dest="topython", default=False, + help="write python file from xml") + (options, args) = parser.parse_args() if options.optdebug: print options, args - - for filename in args: - bname = os.path.basename(filename) - print "File:", bname - prog = flscriptparse.parse(open(filename).read()) - if not prog: - print "No se pudo abrir el fichero." - continue - if options.storepath is None: - # Si no se quiere guardar resultado, no hace falta calcular mas - continue + if options.topython: + from pytnyzer import pythonize + for filename in args: + bname = os.path.basename(filename) + if options.storepath: + destname = os.path.join(options.storepath,bname+".py") + else: + destname = filename+".py" + pythonize(filename, destname) + + else: + for filename in args: + bname = os.path.basename(filename) + print "File:", bname + prog = flscriptparse.parse(open(filename).read()) + if not prog: + print "No se pudo abrir el fichero." + continue + if options.storepath is None: + # Si no se quiere guardar resultado, no hace falta calcular mas + continue - tree_data = flscriptparse.calctree(prog, alias_mode = 0) - if not tree_data: - print "No se pudo parsear." - continue + tree_data = flscriptparse.calctree(prog, alias_mode = 0) + if not tree_data: + print "No se pudo parsear." + continue - ast = post_parse(tree_data) - if ast is None: - print "No se pudo analizar." - continue - if options.storepath: - f1 = open(os.path.join(options.storepath,bname+".xml"),"w") - f1.write(etree.tostring(ast, pretty_print = True)) - f1.close() + ast = post_parse(tree_data) + if ast is None: + print "No se pudo analizar." + continue + if options.storepath: + f1 = open(os.path.join(options.storepath,bname+".xml"),"w") + f1.write(etree.tostring(ast, pretty_print = True)) + f1.close() diff --git a/pytnyzer.py b/pytnyzer.py new file mode 100644 index 0000000..9d709a1 --- /dev/null +++ b/pytnyzer.py @@ -0,0 +1,229 @@ +#!/usr/bin/python +# ------ Pythonyzer ... reads XML AST created by postparse.py and creates an equivalent Python file. +from optparse import OptionParser +import os, os.path +import flscriptparse +from lxml import etree + +ast_class_types = [] + +class ASTPythonFactory(type): + def __init__(cls, name, bases, dct): + global ast_class_types + ast_class_types.append(cls) + super(ASTPythonFactory, cls).__init__(name, bases, dct) + +class ASTPython(object): + __metaclass__ = ASTPythonFactory + tags = [] + + @classmethod + def can_process_tag(self, tagname): return self.__name__ == tagname or tagname in self.tags + + def __init__(self, elem): + self.elem = elem + + def polish(self): return self + + def generate(self): + yield "debug", etree.tostring(self.elem) + + +class Source(ASTPython): + def generate(self): + elems = 0 + for child in self.elem: + #yield "debug", "<%s %s>" % (child.tag, repr(child.attrib)) + for dtype, data in parse_ast(child).generate(): + if dtype == "line": + elems += 1 + yield dtype, data + if elems == 0: + yield "line", "pass" + +class Class(ASTPython): + def generate(self): + name = self.elem.get("name") + extends = self.elem.get("extends","object") + + yield "line", "class %s(%s):" % (name,extends) + yield "begin", "block-class-%s" % (name) + for source in self.elem.xpath("Source"): + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-class-%s" % (name) + +class Function(ASTPython): + def generate(self): + name = self.elem.get("name") + returns = self.elem.get("returns",None) + arguments = [] + for n,arg in enumerate(self.elem.xpath("Arguments/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + + + yield "line", "def %s(%s):" % (name,", ".join(arguments)) + yield "begin", "block-def-%s" % (name) + if returns: + yield "debug", "Returns: %s" % returns + for source in self.elem.xpath("Source"): + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-def-%s" % (name) + +class Variable(ASTPython): + def generate(self, force_value = False): + name = self.elem.get("name") + yield "expr", name + values = 0 + for value in self.elem.xpath("Value"): + values += 1 + yield "expr", "=" + expr = 0 + for dtype, data in parse_ast(value).generate(isolate = False): + if dtype == "expr": expr += 1 + yield dtype, data + if expr == 0: + yield "expr", "None" + + dtype = self.elem.get("type",None) + + if values == 0 and force_value == True: + yield "expr", "=" + if dtype is None: + yield "expr", "None" + elif dtype == "String": + yield "expr", "\"\"" + elif dtype == "Number": + yield "expr", "0" + else: + yield "expr", "qsatype.%s()" % dtype + + if dtype and force_value == False: yield "debug", "Variable %s:%s" % (name,dtype) + +class Value(ASTPython): + def generate(self, isolate = True): + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + if isolate: yield "expr", ")" + + +class Constant(ASTPython): + def generate(self): + ctype = self.elem.get("type") + value = self.elem.get("value") + if ctype == "String": yield "expr", "\"%s\"" % value + else: yield "expr", value + +class DeclarationBlock(ASTPython): + def generate(self): + mode = self.elem.get("mode") + if mode == "CONST": yield "debug", "Const Declaration:" + for var in self.elem: + expr = [] + for dtype, data in parse_ast(var).generate(force_value=True): + if dtype == "expr": expr.append(data) + else: yield dtype,data + yield "line", " ".join(expr) + + +# ----- keep this one at the end. +class Unknown(ASTPython): + @classmethod + def can_process_tag(self, tagname): return True +# ----------------- + +def astparser_for(elem): + classobj = None + for cls in ast_class_types: + if cls.can_process_tag(elem.tag): + classobj = cls + break + if classobj is None: return None + return classobj(elem) + +def parse_ast(elem): + elemparser = astparser_for(elem) + return elemparser.polish() + + +def write_python_file(fobj, ast): + indent = [] + indent_text = " " + for dtype, data in parse_ast(ast).generate(): + line = None + if dtype == "line": line = data + if dtype == "debug": line = "# DEBUG:: " + data + if dtype == "expr": line = "# EXPR??:: " + data + if dtype == "begin": + #line = "# BEGIN:: " + data + indent.append(data) + if dtype == "end": + line = "# END:: " + data + endblock = indent.pop() + if endblock != data: + line = "# END-ERROR!! was %s but %s found." % (endblock, data) + + if line is not None: + fobj.write((len(indent)*indent_text) + line + "\n") + + if dtype == "end": + fobj.write((len(indent)*indent_text) + "\n") + +def pythonize(filename, destfilename): + bname = os.path.basename(filename) + + parser = etree.XMLParser(remove_blank_text=True) + ast_tree = etree.parse(open(filename), parser) + ast = ast_tree.getroot() + + + f1 = open(destfilename,"w") + write_python_file(f1,ast) + f1.close() + +def main(): + parser = OptionParser() + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("--path", + dest="storepath", default=None, + help="store PY results in PATH") + + + (options, args) = parser.parse_args() + if options.optdebug: + print options, args + + for filename in args: + if options.storepath: + destname = os.path.join(options.storepath,bname+".py") + else: + destname = filename+".py" + pythonize(filename, destname) + + + +if __name__ == "__main__": main() From cea6469f2e667e0c4ab00ad7b5d11fd4161712cd Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 13 Apr 2012 00:39:21 +0200 Subject: [PATCH 029/100] Avances en pytnyzer --- postparse.py | 4 +- pytnyzer.py | 253 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 242 insertions(+), 15 deletions(-) diff --git a/postparse.py b/postparse.py index afdbb7a..e0b16f3 100755 --- a/postparse.py +++ b/postparse.py @@ -136,8 +136,8 @@ def add_other(self, argn, vtype, value): class Source(ListObject): - tags = ["source","basicsource","classdeclarationsource","statement_list"] - adopt_childs_tags = ['source_element','statement_list','statement'] + tags = ["source","basicsource","classdeclarationsource","statement_list","statement_block"] + adopt_childs_tags = ['source_element','statement_list','statement',"statement_block"] class Identifier(NamedObject): tags = ["identifier","optid"] diff --git a/pytnyzer.py b/pytnyzer.py index 9d709a1..7395f53 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -25,12 +25,12 @@ def __init__(self, elem): def polish(self): return self - def generate(self): + def generate(self, **kwargs): yield "debug", etree.tostring(self.elem) class Source(ASTPython): - def generate(self): + def generate(self, **kwargs): elems = 0 for child in self.elem: #yield "debug", "<%s %s>" % (child.tag, repr(child.attrib)) @@ -42,7 +42,7 @@ def generate(self): yield "line", "pass" class Class(ASTPython): - def generate(self): + def generate(self, **kwargs): name = self.elem.get("name") extends = self.elem.get("extends","object") @@ -53,10 +53,19 @@ def generate(self): yield "end", "block-class-%s" % (name) class Function(ASTPython): - def generate(self): + def generate(self, **kwargs): name = self.elem.get("name") returns = self.elem.get("returns",None) + parent = self.elem.getparent() + grandparent = None + if parent is not None: grandparent = parent.getparent() arguments = [] + + if grandparent is not None: + if grandparent.tag == "Class": + arguments.append("self") + else: + arguments.append("self") for n,arg in enumerate(self.elem.xpath("Arguments/*")): expr = [] for dtype, data in parse_ast(arg).generate(): @@ -75,14 +84,65 @@ def generate(self): yield "line", "def %s(%s):" % (name,", ".join(arguments)) yield "begin", "block-def-%s" % (name) - if returns: - yield "debug", "Returns: %s" % returns + # if returns: yield "debug", "Returns: %s" % returns for source in self.elem.xpath("Source"): for obj in parse_ast(source).generate(): yield obj yield "end", "block-def-%s" % (name) + +class FunctionCall(ASTPython): + def generate(self, **kwargs): + name = self.elem.get("name") + arguments = [] + for n,arg in enumerate(self.elem.xpath("CallArguments/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + + + yield "expr", "%s(%s)" % (name,", ".join(arguments)) + +class If(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + + yield "line", "if %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-if" + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-if" + + for source in self.elem.xpath("Else/Source"): + yield "line", "else:" + yield "begin", "block-else" + for obj in parse_ast(source).generate(): yield obj + yield "begin", "block-end" + class Variable(ASTPython): - def generate(self, force_value = False): + def generate(self, force_value = False, **kwargs): name = self.elem.get("name") yield "expr", name values = 0 @@ -109,26 +169,189 @@ def generate(self, force_value = False): else: yield "expr", "qsatype.%s()" % dtype - if dtype and force_value == False: yield "debug", "Variable %s:%s" % (name,dtype) + #if dtype and force_value == False: yield "debug", "Variable %s:%s" % (name,dtype) + +class InstructionUpdate(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "line", " ".join(arguments) + +class InstructionCall(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "line", " ".join(arguments) + +class InstructionFlow(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + ctype = self.elem.get("type") + kw = ctype + if ctype == "RETURN": kw = "return" + if ctype == "BREAK": kw = "break" + if ctype == "CONTINUE": kw = "continue" + + + yield "line", kw + " " + ", ".join(arguments) + + + +class Member(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", ".".join(arguments) + class Value(ASTPython): - def generate(self, isolate = True): + def generate(self, isolate = True, **kwargs): + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + if isolate: yield "expr", ")" + +class Expression(ASTPython): + def generate(self, isolate = True, **kwargs): if isolate: yield "expr", "(" for child in self.elem: for dtype, data in parse_ast(child).generate(): yield dtype, data if isolate: yield "expr", ")" + +class OpUnary(ASTPython): + def generate(self, isolate = False, **kwargs): + ctype = self.elem.get("type") + if ctype == "LNOT": yield "expr", "not" + else: yield "expr", ctype + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + if isolate: yield "expr", ")" + +class New(ASTPython): + def generate(self, **kwargs): + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + if dtype != "expr": + yield dtype, data + continue + if child.tag == "Identifier": data = data+"()" + if data[:data.find("(")].find(".") == -1: data = "qsatype." + data + yield dtype, data class Constant(ASTPython): - def generate(self): + def generate(self, **kwargs): ctype = self.elem.get("type") value = self.elem.get("value") if ctype == "String": yield "expr", "\"%s\"" % value else: yield "expr", value +class Identifier(ASTPython): + def generate(self, **kwargs): + name = self.elem.get("name") + if name == "false": name = "False" + if name == "true": name = "True" + if name == "null": name = "None" + if name == "unknown": name = "None" + if name == "this": name = "self" + yield "expr", name + +class OpUpdate(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "EQUALS": yield "expr", "=" + elif ctype == "PLUSEQUALS": yield "expr", "+=" + elif ctype == "MINUSEQUALS": yield "expr", "-=" + elif ctype == "TIMESEQUALS": yield "expr", "*=" + elif ctype == "DIVEQUALS": yield "expr", "/=" + else: yield "expr", ctype + +class Compare(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "GT": yield "expr", ">" + elif ctype == "LT": yield "expr", "<" + elif ctype == "LE": yield "expr", "<=" + elif ctype == "GE": yield "expr", ">=" + elif ctype == "EQ": yield "expr", "==" + elif ctype == "NE": yield "expr", "!=" + elif ctype == "IN": yield "expr", "in" + else: yield "expr", ctype + +class OpMath(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "PLUS": yield "expr", "+" + elif ctype == "MINUS": yield "expr", "-" + elif ctype == "TIMES": yield "expr", "*" + elif ctype == "DIVIDE": yield "expr", "/" + elif ctype == "MOD": yield "expr", "%" + elif ctype == "XOR": yield "expr", "^" + elif ctype == "LSHIFT": yield "expr", "<<" + elif ctype == "RSHIFT": yield "expr", ">>" + elif ctype == "AND": yield "expr", "&" + else: yield "expr", ctype + + class DeclarationBlock(ASTPython): - def generate(self): + def generate(self, **kwargs): mode = self.elem.get("mode") if mode == "CONST": yield "debug", "Const Declaration:" for var in self.elem: @@ -171,16 +394,20 @@ def write_python_file(fobj, ast): #line = "# BEGIN:: " + data indent.append(data) if dtype == "end": - line = "# END:: " + data + if data not in ["block-if"]: + #line = "# END:: " + data + pass endblock = indent.pop() if endblock != data: line = "# END-ERROR!! was %s but %s found." % (endblock, data) if line is not None: + if type(line) is unicode: line = line.encode("UTF-8","replace") fobj.write((len(indent)*indent_text) + line + "\n") if dtype == "end": - fobj.write((len(indent)*indent_text) + "\n") + if data not in ["block-if"]: + fobj.write((len(indent)*indent_text) + "\n") def pythonize(filename, destfilename): bname = os.path.basename(filename) From 8ae9abf798e6bd886b2197f64c4a90411372bc0c Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 13 Apr 2012 10:17:22 +0200 Subject: [PATCH 030/100] Mejoras pytnyzer --- flscriptparse.py | 2 +- postparse.py | 27 +++++- pytnyzer.py | 209 +++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 222 insertions(+), 16 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 92cd149..c12902b 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -318,7 +318,7 @@ def p_parse(token): forinstatement : FOR LPAREN for_initialize IN variable RPAREN statement_block | error - switch : SWITCH LPAREN expression RPAREN LBRACE case_block_list RBRACE + switch : SWITCH LPAREN condition RPAREN LBRACE case_block_list RBRACE optid : ID | empty diff --git a/postparse.py b/postparse.py index e0b16f3..98bd97e 100755 --- a/postparse.py +++ b/postparse.py @@ -248,6 +248,15 @@ def add_value(self, argn, vtype, value): self.const_type = vtype self.xml.set("type",vtype) self.xml.set("value",value) + +class InlineUpdate(ListObject): + tags = ["inlinestoreinstruction"] + def add_other(self, argn, vtype, value): + self.xml.set("type", vtype) + if argn == 0: + self.xml.set("mode", "update-read") + if argn == 1: + self.xml.set("mode", "read-update") class If(ListObject): tags = ["ifstatement"] @@ -284,17 +293,33 @@ class InstructionUpdate(ListObject): class Switch(ListObject): tags = ["switch"] - adopt_childs_tags = ['case_cblock_list'] + adopt_childs_tags = ['case_cblock_list','case_block_list'] + +class CaseList(ListObject): + tags = ["case_block_list"] + adopt_childs_tags = ['case_cblock_list','case_block_list'] class Case(ListObject): tags = ["case_block"] +class CaseDefault(ListObject): + tags = ["case_default"] + class While(ListObject): tags = ["whilestatement"] class For(ListObject): tags = ["forstatement"] +class ForInitialize(ListObject): + tags = ["for_initialize"] + +class ForCompare(ListObject): + tags = ["for_compare"] + +class ForIncrement(ListObject): + tags = ["for_increment"] + class DoWhile(ListObject): tags = ["dowhilestatement"] diff --git a/pytnyzer.py b/pytnyzer.py index 7395f53..40c47ec 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -1,7 +1,7 @@ #!/usr/bin/python # ------ Pythonyzer ... reads XML AST created by postparse.py and creates an equivalent Python file. from optparse import OptionParser -import os, os.path +import os, os.path, random import flscriptparse from lxml import etree @@ -30,15 +30,16 @@ def generate(self, **kwargs): class Source(ASTPython): - def generate(self, **kwargs): + def generate(self, break_mode = False, include_pass = True, **kwargs): elems = 0 for child in self.elem: #yield "debug", "<%s %s>" % (child.tag, repr(child.attrib)) - for dtype, data in parse_ast(child).generate(): + for dtype, data in parse_ast(child).generate(break_mode = break_mode): if dtype == "line": elems += 1 yield dtype, data - if elems == 0: + if dtype == "break": return + if elems == 0 and include_pass: yield "line", "pass" class Class(ASTPython): @@ -138,15 +139,152 @@ def generate(self, **kwargs): yield "line", "else:" yield "begin", "block-else" for obj in parse_ast(source).generate(): yield obj - yield "begin", "block-end" + yield "end", "block-else" +class While(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + + yield "line", "while %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-while" + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-while" + +class For(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("ForInitialize/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) > 0: + main_expr.append(" ".join(expr)) + if main_expr: + yield "line", " ".join(main_expr) + + incr_expr = [] + incr_lines = [] + for n,arg in enumerate(self.elem.xpath("ForIncrement/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + elif dtype == "line": + incr_lines.append(data) + else: + yield dtype, data + if len(expr) > 0: + incr_expr.append(" ".join(expr)) + + + main_expr = [] + for n,arg in enumerate(self.elem.xpath("ForCompare/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("True") + else: + main_expr.append(" ".join(expr)) + yield "debug", "FOR:" + yield "line", "while %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-for" + for obj in parse_ast(source).generate(include_pass=False): yield obj + if incr_lines: + for line in incr_lines: + yield "line", line + yield "end", "block-for" + +class Switch(ASTPython): + def generate(self, **kwargs): + key = "%02x" % random.randint(0,255) + name = "s%s_when" % key + name_pr = "s%s_do_work" % key + name_pr2 = "s%s_work_done" % key + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + yield "line", "%s = %s" % (name, " ".join(main_expr)) + yield "line", "%s,%s = %s,%s" % (name_pr,name_pr2, "False","False") + for scase in self.elem.xpath("Case"): + value_expr = [] + for n,arg in enumerate(scase.xpath("Value")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + value_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + value_expr.append(" ".join(expr)) + + yield "line", "if %s == %s: %s,%s = %s,%s" % (name," ".join(value_expr),name_pr,name_pr2, "True", "True") + yield "line", "if %s:" % (name_pr) + yield "begin", "block-if" + for source in scase.xpath("Source"): + for obj in parse_ast(source).generate(break_mode = True): + if obj[0] == "break": + yield "line", "%s = %s # BREAK" % (name_pr,"False") + else: + yield obj + yield "end", "block-if" + + for scasedefault in self.elem.xpath("CaseDefault"): + yield "line", "if not %s: %s,%s = %s,%s" % (name_pr2, name_pr,name_pr2, "True", "True") + yield "line", "if %s:" % (name_pr) + yield "begin", "block-if" + for source in scasedefault.xpath("Source"): + for obj in parse_ast(source).generate(break_mode = True): + if obj[0] == "break": + yield "line", "%s = %s # BREAK" % (name_pr,"False") + else: + yield obj + yield "end", "block-if" + yield "line", "assert( not %s )" % name_pr + # yield "line", "assert( %s )" % name_pr2 class Variable(ASTPython): def generate(self, force_value = False, **kwargs): name = self.elem.get("name") yield "expr", name values = 0 - for value in self.elem.xpath("Value"): + for value in self.elem.xpath("Value|Expression"): values += 1 yield "expr", "=" expr = 0 @@ -189,6 +327,32 @@ def generate(self, **kwargs): arguments.append(" ".join(expr)) yield "line", " ".join(arguments) + +class InlineUpdate(ASTPython): + def generate(self, plusplus_as_instruction = False, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + ctype = self.elem.get("type") + if not plusplus_as_instruction: + yield "expr", arguments[0] + if ctype == "PLUSPLUS": + yield "line", arguments[0] + "+= 1" + elif ctype == "MINUSMINUS": + yield "line", arguments[0] + "-= 1" + else: + yield "line", arguments[0] + "?= 1" class InstructionCall(ASTPython): def generate(self, **kwargs): @@ -210,7 +374,7 @@ def generate(self, **kwargs): yield "line", " ".join(arguments) class InstructionFlow(ASTPython): - def generate(self, **kwargs): + def generate(self, break_mode = False, **kwargs): arguments = [] for n,arg in enumerate(self.elem): expr = [] @@ -229,7 +393,9 @@ def generate(self, **kwargs): ctype = self.elem.get("type") kw = ctype if ctype == "RETURN": kw = "return" - if ctype == "BREAK": kw = "break" + if ctype == "BREAK": + kw = "break" + if break_mode: yield "break", kw + " " + ", ".join(arguments) if ctype == "CONTINUE": kw = "continue" @@ -273,6 +439,14 @@ def generate(self, isolate = True, **kwargs): yield dtype, data if isolate: yield "expr", ")" +class Parentheses(ASTPython): + def generate(self, **kwargs): + yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(isolate = False): + yield dtype, data + yield "expr", ")" + class OpUnary(ASTPython): def generate(self, isolate = False, **kwargs): ctype = self.elem.get("type") @@ -292,7 +466,10 @@ def generate(self, **kwargs): yield dtype, data continue if child.tag == "Identifier": data = data+"()" - if data[:data.find("(")].find(".") == -1: data = "qsatype." + data + ident = data[:data.find("(")] + if ident.find(".") == -1: + if len(self.elem.xpath("//Class[@name='%s']" % ident)) == 0: + data = "qsatype." + data yield dtype, data @@ -317,10 +494,10 @@ class OpUpdate(ASTPython): def generate(self, **kwargs): ctype = self.elem.get("type") if ctype == "EQUALS": yield "expr", "=" - elif ctype == "PLUSEQUALS": yield "expr", "+=" - elif ctype == "MINUSEQUALS": yield "expr", "-=" - elif ctype == "TIMESEQUALS": yield "expr", "*=" - elif ctype == "DIVEQUALS": yield "expr", "/=" + elif ctype == "PLUSEQUAL": yield "expr", "+=" + elif ctype == "MINUSEQUAL": yield "expr", "-=" + elif ctype == "TIMESEQUAL": yield "expr", "*=" + elif ctype == "DIVEQUAL": yield "expr", "/=" else: yield "expr", ctype class Compare(ASTPython): @@ -333,6 +510,8 @@ def generate(self, **kwargs): elif ctype == "EQ": yield "expr", "==" elif ctype == "NE": yield "expr", "!=" elif ctype == "IN": yield "expr", "in" + elif ctype == "LOR": yield "expr", "or" + elif ctype == "LAND": yield "expr", "and" else: yield "expr", ctype class OpMath(ASTPython): @@ -357,7 +536,9 @@ def generate(self, **kwargs): for var in self.elem: expr = [] for dtype, data in parse_ast(var).generate(force_value=True): - if dtype == "expr": expr.append(data) + if dtype == "expr": + if data is None: data = "[None]" + expr.append(data) else: yield dtype,data yield "line", " ".join(expr) From 9970bf1fd4e2ea9e664bc4ae12fc9e7214b9c282 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 13 Apr 2012 14:02:14 +0200 Subject: [PATCH 031/100] =?UTF-8?q?Agregar=20ejecuci=C3=B3n=20de=20python?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flscriptparse.py | 12 ++- postparse.py | 47 ++++++++++- pytnyzer.py | 203 +++++++++++++++++++++++++++++++++++++++++------ qsatype.py | 14 ++++ 4 files changed, 241 insertions(+), 35 deletions(-) create mode 100644 qsatype.py diff --git a/flscriptparse.py b/flscriptparse.py index c12902b..e1421f0 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -49,8 +49,6 @@ def p_parse(token): identifier : ID - array_value : LBRACKET RBRACKET - dictobject_value : LBRACE RBRACE | LBRACE dictobject_value_elemlist RBRACE @@ -68,7 +66,6 @@ def p_parse(token): | unary_operator | new_operator | ternary_operator - | array_value | dictobject_value | typeof_operator @@ -199,8 +196,10 @@ def p_parse(token): | LPAREN member_var RPAREN variable_1 : identifier - | variable_1 LBRACKET base_expression RBRACKET - | funccall_1 LBRACKET base_expression RBRACKET + | array_member + + array_member : variable_1 LBRACKET expression RBRACKET + | funccall_1 LBRACKET expression RBRACKET inlinestoreinstruction : PLUSPLUS variable | MINUSMINUS variable @@ -323,8 +322,7 @@ def p_parse(token): optid : ID | empty - trycatch : TRY LBRACE statement_list RBRACE CATCH LPAREN optid RPAREN LBRACE statement_list RBRACE - trycatch : TRY LBRACE statement_list RBRACE CATCH LPAREN optid RPAREN LBRACE RBRACE + trycatch : TRY statement_block CATCH LPAREN optid RPAREN statement_block empty : ''' diff --git a/postparse.py b/postparse.py index 98bd97e..a457940 100755 --- a/postparse.py +++ b/postparse.py @@ -2,6 +2,7 @@ from optparse import OptionParser import os, os.path import flscriptparse +import imp, traceback from lxml import etree USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") @@ -206,6 +207,12 @@ class Member(TagObject): tags = ["member_var","member_call"] adopt_childs_tags = ['varmemcall',"member_var","member_call"] +class ArrayMember(TagObject): + debug_other = False + set_child_argn = False + tags = ["array_member"] + adopt_childs_tags = ['variable_1',"func_call"] + class InstructionCall(TagObject): debug_other = False tags = ["callinstruction"] @@ -391,7 +398,27 @@ def post_parse(treedata): source = parse("source",treedata) #print UNKNOWN_PARSERS.keys() return source.xml - + +class Module: + def __init__(self, name, path): + self.name = name + self.path = path + def loadModule(self): + fp = None + try: + description = ('.py', 'U', 1) + pathname = os.path.join(self.path, self.name) + fp = open(pathname) + name = self.name[:self.name.find(".")] + # fp, pathname, description = imp.find_module(self.name,[self.path]) + self.module = imp.load_module(name, fp, pathname, description) + result = True + except Exception,e: + print traceback.format_exc() + result = False + if fp: + fp.close() + return result def main(): parser = OptionParser() @@ -415,11 +442,27 @@ def main(): action="store_true", dest="topython", default=False, help="write python file from xml") + parser.add_option("--py", + action="store_true", dest="exec_python", default=False, + help="try to execute python file") + (options, args) = parser.parse_args() if options.optdebug: print options, args - if options.topython: + if options.exec_python: + import qsatype + for filename in args: + realpath = os.path.realpath(filename) + path, name = os.path.split(realpath) + mod = Module(name, path) + if mod.loadModule(): + print mod.module + print mod.module.form + else: + print "Error cargando modulo %s" % name + + elif options.topython: from pytnyzer import pythonize for filename in args: bname = os.path.basename(filename) diff --git a/pytnyzer.py b/pytnyzer.py index 40c47ec..4f18f85 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -5,6 +5,16 @@ import flscriptparse from lxml import etree +def id_translate(name): + if name == "false": name = "False" + if name == "true": name = "True" + if name == "null": name = "None" + if name == "unknown": name = "None" + if name == "this": name = "self" + if name == "exec": name = "_exec" + if name == "id": name = "_id" + return name + ast_class_types = [] class ASTPythonFactory(type): @@ -32,13 +42,29 @@ def generate(self, **kwargs): class Source(ASTPython): def generate(self, break_mode = False, include_pass = True, **kwargs): elems = 0 + after_lines = [] for child in self.elem: #yield "debug", "<%s %s>" % (child.tag, repr(child.attrib)) - for dtype, data in parse_ast(child).generate(break_mode = break_mode): + for dtype, data in parse_ast(child).generate(break_mode = break_mode, plusplus_as_instruction = True): + if dtype == "line+1": + after_lines.append(data) + continue if dtype == "line": elems += 1 yield dtype, data - if dtype == "break": return + if dtype == "line" and after_lines: + for line in after_lines: + elems+=1 + yield dtype, line + after_lines = [] + if dtype == "break": + for line in after_lines: + elems+=1 + yield "line", line + + for line in after_lines: + elems+=1 + yield "line", line if elems == 0 and include_pass: yield "line", "pass" @@ -55,7 +81,7 @@ def generate(self, **kwargs): class Function(ASTPython): def generate(self, **kwargs): - name = self.elem.get("name") + name = id_translate(self.elem.get("name")) returns = self.elem.get("returns",None) parent = self.elem.getparent() grandparent = None @@ -65,6 +91,8 @@ def generate(self, **kwargs): if grandparent is not None: if grandparent.tag == "Class": arguments.append("self") + if name == grandparent.get("name"): + name = "__init__" else: arguments.append("self") for n,arg in enumerate(self.elem.xpath("Arguments/*")): @@ -92,7 +120,7 @@ def generate(self, **kwargs): class FunctionCall(ASTPython): def generate(self, **kwargs): - name = self.elem.get("name") + name = id_translate(self.elem.get("name")) arguments = [] for n,arg in enumerate(self.elem.xpath("CallArguments/*")): expr = [] @@ -107,13 +135,11 @@ def generate(self, **kwargs): yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) - - yield "expr", "%s(%s)" % (name,", ".join(arguments)) class If(ASTPython): - def generate(self, **kwargs): + def generate(self, break_mode = False, **kwargs): main_expr = [] for n,arg in enumerate(self.elem.xpath("Condition/*")): expr = [] @@ -132,15 +158,45 @@ def generate(self, **kwargs): yield "line", "if %s:" % (" ".join(main_expr)) for source in self.elem.xpath("Source"): yield "begin", "block-if" - for obj in parse_ast(source).generate(): yield obj + for obj in parse_ast(source).generate(break_mode = break_mode): yield obj yield "end", "block-if" for source in self.elem.xpath("Else/Source"): yield "line", "else:" yield "begin", "block-else" - for obj in parse_ast(source).generate(): yield obj + for obj in parse_ast(source).generate(break_mode = break_mode): yield obj yield "end", "block-else" +class TryCatch(ASTPython): + def generate(self, **kwargs): + + tryblock, catchblock = self.elem.xpath("Source") + + yield "line", "try:" + yield "begin", "block-try" + for obj in parse_ast(tryblock).generate(): yield obj + yield "end", "block-try" + + identifier = None + for ident in self.elem.xpath("Identifier"): + expr = [] + for dtype, data in parse_ast(ident).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + identifier = " ".join(expr) + if identifier: + yield "line", "except Exception, %s:" % (identifier) + else: + yield "line", "except Exception:" + yield "begin", "block-except" + if identifier: + yield "line", "%s = str(%s)" % (identifier, identifier) + for obj in parse_ast(tryblock).generate(include_pass = identifier is None): yield obj + yield "end", "block-except" + + class While(ASTPython): def generate(self, **kwargs): main_expr = [] @@ -186,7 +242,7 @@ def generate(self, **kwargs): for dtype, data in parse_ast(arg).generate(isolate = False): if dtype == "expr": expr.append(data) - elif dtype == "line": + elif dtype in ["line","line+1"]: incr_lines.append(data) else: yield dtype, data @@ -276,8 +332,31 @@ def generate(self, **kwargs): else: yield obj yield "end", "block-if" - yield "line", "assert( not %s )" % name_pr + # yield "line", "assert( not %s )" % name_pr # yield "line", "assert( %s )" % name_pr2 + +class With(ASTPython): + def generate(self, **kwargs): + key = "%02x" % random.randint(0,255) + name = "w%s_obj" % key + yield "debug", "WITH: %s" % key + variable, source = [ obj for obj in self.elem ] + var_expr = [] + for dtype, data in parse_ast(variable).generate(isolate = False): + if dtype == "expr": + var_expr.append(data) + else: + yield dtype, data + if len(var_expr) == 0: + var_expr.append("None") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + + yield "line", "%s = %s" % (name, " ".join(var_expr)) + + for obj in parse_ast(source).generate(break_mode = True): + yield obj + yield "line", "del %s" % name class Variable(ASTPython): def generate(self, force_value = False, **kwargs): @@ -316,6 +395,7 @@ def generate(self, **kwargs): expr = [] for dtype, data in parse_ast(arg).generate(isolate=False): if dtype == "expr": + if data is None: raise ValueError, etree.tostring(arg) expr.append(data) else: yield dtype, data @@ -345,14 +425,19 @@ def generate(self, plusplus_as_instruction = False, **kwargs): else: arguments.append(" ".join(expr)) ctype = self.elem.get("type") + mode = self.elem.get("mode") + linetype = "line" if not plusplus_as_instruction: + if mode == "read-update": + linetype = "line+1" + yield "expr", arguments[0] if ctype == "PLUSPLUS": - yield "line", arguments[0] + "+= 1" + yield linetype, arguments[0] + " += 1" elif ctype == "MINUSMINUS": - yield "line", arguments[0] + "-= 1" + yield linetype, arguments[0] + " -= 1" else: - yield "line", arguments[0] + "?= 1" + yield linetype, arguments[0] + " ?= 1" class InstructionCall(ASTPython): def generate(self, **kwargs): @@ -395,9 +480,14 @@ def generate(self, break_mode = False, **kwargs): if ctype == "RETURN": kw = "return" if ctype == "BREAK": kw = "break" - if break_mode: yield "break", kw + " " + ", ".join(arguments) + if break_mode: + yield "break", kw + " " + ", ".join(arguments) + return if ctype == "CONTINUE": kw = "continue" + if ctype == "THROW": + yield "line", "Exception(" + ", ".join(arguments) + ")" + return yield "line", kw + " " + ", ".join(arguments) @@ -421,6 +511,25 @@ def generate(self, **kwargs): arguments.append(" ".join(expr)) yield "expr", ".".join(arguments) + +class ArrayMember(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "%s[%s]" % (arguments[0],arguments[1]) class Value(ASTPython): @@ -428,6 +537,7 @@ def generate(self, isolate = True, **kwargs): if isolate: yield "expr", "(" for child in self.elem: for dtype, data in parse_ast(child).generate(): + if data is None: raise ValueError, etree.tostring(child) yield dtype, data if isolate: yield "expr", ")" @@ -451,6 +561,7 @@ class OpUnary(ASTPython): def generate(self, isolate = False, **kwargs): ctype = self.elem.get("type") if ctype == "LNOT": yield "expr", "not" + elif ctype == "MINUS": yield "expr", "-" else: yield "expr", ctype if isolate: yield "expr", "(" for child in self.elem: @@ -477,17 +588,32 @@ class Constant(ASTPython): def generate(self, **kwargs): ctype = self.elem.get("type") value = self.elem.get("value") + if ctype is None or value is None: + for child in self.elem: + if child.tag == "CallArguments": + arguments = [] + for n,arg in enumerate(child): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "qsatype.Array([%s])" % (", ".join(arguments)) + return if ctype == "String": yield "expr", "\"%s\"" % value else: yield "expr", value class Identifier(ASTPython): def generate(self, **kwargs): - name = self.elem.get("name") - if name == "false": name = "False" - if name == "true": name = "True" - if name == "null": name = "None" - if name == "unknown": name = "None" - if name == "this": name = "self" + name = id_translate(self.elem.get("name")) yield "expr", name class OpUpdate(ASTPython): @@ -537,7 +663,7 @@ def generate(self, **kwargs): expr = [] for dtype, data in parse_ast(var).generate(force_value=True): if dtype == "expr": - if data is None: data = "[None]" + if data is None: raise ValueError, etree.tostring(var) expr.append(data) else: yield dtype,data yield "line", " ".join(expr) @@ -563,32 +689,57 @@ def parse_ast(elem): return elemparser.polish() +def file_template(ast): + yield "line", "# encoding: UTF-8" + yield "line", "try: import qsatype" + yield "line", "except ImportError: qsatype = None" + yield "line", "" + yield "line", "class FormInternalObj(object):" + yield "begin", "block-class-FormInternalObj" + for dtype, data in parse_ast(ast).generate(): + yield dtype, data + yield "end", "block-class-FormInternalObj" + yield "line", "" + yield "line", "form = FormInternalObj()" + def write_python_file(fobj, ast): indent = [] indent_text = " " - for dtype, data in parse_ast(ast).generate(): + last_line_for_indent = {} + numline = 0 + for dtype, data in file_template(ast): line = None - if dtype == "line": line = data + if dtype == "line": + line = data + numline +=1 + try: lines_since_last_indent = numline - last_line_for_indent[len(indent)] + except KeyError: lines_since_last_indent = 0 + if lines_since_last_indent > 4: + fobj.write((len(indent)*indent_text) + "\n") + last_line_for_indent[len(indent)] = numline if dtype == "debug": line = "# DEBUG:: " + data if dtype == "expr": line = "# EXPR??:: " + data + if dtype == "line+1": line = "# LINE+1??:: " + data if dtype == "begin": #line = "# BEGIN:: " + data indent.append(data) + last_line_for_indent[len(indent)] = numline if dtype == "end": if data not in ["block-if"]: #line = "# END:: " + data pass endblock = indent.pop() if endblock != data: - line = "# END-ERROR!! was %s but %s found." % (endblock, data) + line = "# END-ERROR!! was %s but %s found. (%s)" % (endblock, data,repr(indent)) if line is not None: if type(line) is unicode: line = line.encode("UTF-8","replace") fobj.write((len(indent)*indent_text) + line + "\n") if dtype == "end": - if data not in ["block-if"]: + if data.split("-")[1] in ["class","def","else","except"]: fobj.write((len(indent)*indent_text) + "\n") + last_line_for_indent[len(indent)] = numline def pythonize(filename, destfilename): bname = os.path.basename(filename) diff --git a/qsatype.py b/qsatype.py new file mode 100644 index 0000000..b4e1e52 --- /dev/null +++ b/qsatype.py @@ -0,0 +1,14 @@ +# encoding: UTF-8 +import os + +class Object(object): + pass + +class Array(object): + pass + +class FLSqlCursor(object): + pass + + + From aa98853afdc9178973ff2733cdb4e77b0086f4ac Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 13 Apr 2012 15:09:54 +0200 Subject: [PATCH 032/100] Correccion de la plantilla python --- pytnyzer.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index 4f18f85..8c5c9ed 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -694,11 +694,25 @@ def file_template(ast): yield "line", "try: import qsatype" yield "line", "except ImportError: qsatype = None" yield "line", "" - yield "line", "class FormInternalObj(object):" - yield "begin", "block-class-FormInternalObj" - for dtype, data in parse_ast(ast).generate(): + sourceclasses = etree.Element("Source") + for cls in ast.xpath("Class"): + sourceclasses.append(cls) + + mainclass = etree.SubElement(sourceclasses,"Class",name="FormInternalObj") + mainsource = etree.SubElement(mainclass,"Source") + + + constructor = etree.SubElement(mainsource,"Function",name="__init__") + args = etree.SubElement(constructor,"Arguments") + csource = etree.SubElement(constructor,"Source") + for child in ast: + if child.tag != "Function": + csource.append(child) + else: + mainsource.append(child) + + for dtype, data in parse_ast(sourceclasses).generate(): yield dtype, data - yield "end", "block-class-FormInternalObj" yield "line", "" yield "line", "form = FormInternalObj()" From 0871e0895a08880a16df4a81fb58743f4c2717f3 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 13 Apr 2012 15:22:26 +0200 Subject: [PATCH 033/100] Add --full option to flscriptparser2 --- postparse.py | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/postparse.py b/postparse.py index a457940..3773fcc 100755 --- a/postparse.py +++ b/postparse.py @@ -442,15 +442,42 @@ def main(): action="store_true", dest="topython", default=False, help="write python file from xml") - parser.add_option("--py", + parser.add_option("--exec-py", action="store_true", dest="exec_python", default=False, help="try to execute python file") + parser.add_option("--toxml", + action="store_true", dest="toxml", default=False, + help="write xml file from qs") + + parser.add_option("--full", + action="store_true", dest="full", default=False, + help="write xml file from qs") (options, args) = parser.parse_args() + execute(options,args) + +def execute(options, args): if options.optdebug: print options, args - if options.exec_python: + if options.full: + options.full = False + options.toxml = True + print "Pass 1 - Parse and write XML file . . ." + execute(options,args) + + options.toxml = False + options.topython = True + print "Pass 2 - Pythonize and write PY file . . ." + execute(options,[ arg+".xml" for arg in args]) + + options.topython = False + options.exec_python = True + print "Pass 3 - Test PY file load . . ." + execute(options,[ arg.replace(".qs",".py") for arg in args]) + print "Done." + + elif options.exec_python: import qsatype for filename in args: realpath = os.path.realpath(filename) @@ -470,6 +497,7 @@ def main(): destname = os.path.join(options.storepath,bname+".py") else: destname = filename+".py" + destname = destname.replace(".qs.xml.py",".py") pythonize(filename, destname) else: @@ -480,7 +508,7 @@ def main(): if not prog: print "No se pudo abrir el fichero." continue - if options.storepath is None: + if options.toxml == False: # Si no se quiere guardar resultado, no hace falta calcular mas continue @@ -494,9 +522,12 @@ def main(): print "No se pudo analizar." continue if options.storepath: - f1 = open(os.path.join(options.storepath,bname+".xml"),"w") - f1.write(etree.tostring(ast, pretty_print = True)) - f1.close() + destname = os.path.join(options.storepath,bname+".xml") + else: + destname = filename+".xml" + f1 = open(destname,"w") + f1.write(etree.tostring(ast, pretty_print = True)) + f1.close() From 1ffb576aa7be2811580e317b5e0041373db35624 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sat, 21 Apr 2012 15:42:55 +0200 Subject: [PATCH 034/100] Fixes to Python code generation --- postparse.py | 7 ++++--- pytnyzer.py | 24 +++++++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/postparse.py b/postparse.py index 3773fcc..4017bf6 100755 --- a/postparse.py +++ b/postparse.py @@ -406,7 +406,8 @@ def __init__(self, name, path): def loadModule(self): fp = None try: - description = ('.py', 'U', 1) + description = ('.py', 'U', imp.PY_SOURCE) + #description = ('.pyc', 'U', PY_COMPILED) pathname = os.path.join(self.path, self.name) fp = open(pathname) name = self.name[:self.name.find(".")] @@ -473,8 +474,8 @@ def execute(options, args): options.topython = False options.exec_python = True - print "Pass 3 - Test PY file load . . ." - execute(options,[ arg.replace(".qs",".py") for arg in args]) + #print "Pass 3 - Test PY file load . . ." + #execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) print "Done." elif options.exec_python: diff --git a/pytnyzer.py b/pytnyzer.py index 8c5c9ed..6871fc3 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -121,6 +121,15 @@ def generate(self, **kwargs): class FunctionCall(ASTPython): def generate(self, **kwargs): name = id_translate(self.elem.get("name")) + parent = self.elem.getparent() + if parent.tag == "InstructionCall": + classes = parent.xpath("ancestor::Class") + if classes: + class_ = classes[-1] + extends = class_.get("extends") + if extends == name: + name = "super(%s, self).__init__" % class_.get("name") + arguments = [] for n,arg in enumerate(self.elem.xpath("CallArguments/*")): expr = [] @@ -486,7 +495,7 @@ def generate(self, break_mode = False, **kwargs): if ctype == "CONTINUE": kw = "continue" if ctype == "THROW": - yield "line", "Exception(" + ", ".join(arguments) + ")" + yield "line", "raise Exception(" + ", ".join(arguments) + ")" return yield "line", kw + " " + ", ".join(arguments) @@ -658,6 +667,7 @@ def generate(self, **kwargs): class DeclarationBlock(ASTPython): def generate(self, **kwargs): mode = self.elem.get("mode") + is_constructor = self.elem.get("constructor") if mode == "CONST": yield "debug", "Const Declaration:" for var in self.elem: expr = [] @@ -666,6 +676,8 @@ def generate(self, **kwargs): if data is None: raise ValueError, etree.tostring(var) expr.append(data) else: yield dtype,data + if is_constructor: + expr[0] = "self."+expr[0] yield "line", " ".join(expr) @@ -691,22 +703,24 @@ def parse_ast(elem): def file_template(ast): yield "line", "# encoding: UTF-8" - yield "line", "try: import qsatype" - yield "line", "except ImportError: qsatype = None" + yield "line", "import qsatype" + yield "line", "from qsaglobals import *" yield "line", "" sourceclasses = etree.Element("Source") for cls in ast.xpath("Class"): sourceclasses.append(cls) - mainclass = etree.SubElement(sourceclasses,"Class",name="FormInternalObj") + mainclass = etree.SubElement(sourceclasses,"Class",name="FormInternalObj",extends="qsatype.FormDBWidget") mainsource = etree.SubElement(mainclass,"Source") - constructor = etree.SubElement(mainsource,"Function",name="__init__") + constructor = etree.SubElement(mainsource,"Function",name="_class_init") args = etree.SubElement(constructor,"Arguments") csource = etree.SubElement(constructor,"Source") + for child in ast: if child.tag != "Function": + child.set("constructor","1") csource.append(child) else: mainsource.append(child) From 3b11d296462b6e78f1bf9d27ead757cf1866c08f Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 23 Apr 2012 16:28:45 +0200 Subject: [PATCH 035/100] Fixes on Python generation --- pytnyzer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index 6871fc3..36063ad 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -393,7 +393,13 @@ def generate(self, force_value = False, **kwargs): elif dtype == "Number": yield "expr", "0" else: - yield "expr", "qsatype.%s()" % dtype + parent1 = self.elem.getparent() + parent2 = parent1.getparent() + parent3 = parent2.getparent() + if parent2 == "Source" and parent3 == "Class": + yield "expr", "None" + else: + yield "expr", "qsatype.%s()" % dtype #if dtype and force_value == False: yield "debug", "Variable %s:%s" % (name,dtype) @@ -728,7 +734,7 @@ def file_template(ast): for dtype, data in parse_ast(sourceclasses).generate(): yield dtype, data yield "line", "" - yield "line", "form = FormInternalObj()" + yield "line", "form = None" def write_python_file(fobj, ast): indent = [] From 7017b344b24edc63853d18905b35cc786b530fc5 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 23 Apr 2012 23:07:37 +0200 Subject: [PATCH 036/100] Better error reporting on trycatch blocks --- pytnyzer.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index 36063ad..76cdcd9 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -201,8 +201,9 @@ def generate(self, **kwargs): yield "line", "except Exception:" yield "begin", "block-except" if identifier: - yield "line", "%s = str(%s)" % (identifier, identifier) - for obj in parse_ast(tryblock).generate(include_pass = identifier is None): yield obj + # yield "line", "%s = str(%s)" % (identifier, identifier) + yield "line", "%s = traceback.format_exc()" % (identifier) + for obj in parse_ast(catchblock).generate(include_pass = identifier is None): yield obj yield "end", "block-except" @@ -709,8 +710,9 @@ def parse_ast(elem): def file_template(ast): yield "line", "# encoding: UTF-8" - yield "line", "import qsatype" - yield "line", "from qsaglobals import *" + yield "line", "from pineboolib import qsatype" + yield "line", "from pineboolib.qsaglobals import *" + yield "line", "import traceback" yield "line", "" sourceclasses = etree.Element("Source") for cls in ast.xpath("Class"): From 12064375c07037564398c2743e94dd8231b3f11f Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 25 Apr 2012 09:43:26 +0200 Subject: [PATCH 037/100] Better Python conversion, unicode strings by default and super() added --- pytnyzer.py | 12 +++++++++++- qsatype.py | 3 +++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pytnyzer.py b/pytnyzer.py index 76cdcd9..0cb12a3 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -525,6 +525,16 @@ def generate(self, **kwargs): yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) + if arguments[0:2] == ["self","iface"] and arguments[2].startswith("__"): + # From: self.iface.__function() + # to: super(className, self.iface).function() + funs = self.elem.xpath("ancestor::Function") + if funs: + fun = funs[-1] + name_parts = fun.get("name").split("_") + classname = name_parts[0] + arguments[2] = arguments[2][2:] + arguments[0:2] = ["super(%s, %s)" % (classname,".".join(arguments[0:2]))] yield "expr", ".".join(arguments) @@ -624,7 +634,7 @@ def generate(self, **kwargs): yield "expr", "qsatype.Array([%s])" % (", ".join(arguments)) return - if ctype == "String": yield "expr", "\"%s\"" % value + if ctype == "String": yield "expr", "u\"%s\"" % value else: yield "expr", value class Identifier(ASTPython): diff --git a/qsatype.py b/qsatype.py index b4e1e52..951a975 100644 --- a/qsatype.py +++ b/qsatype.py @@ -9,6 +9,9 @@ class Array(object): class FLSqlCursor(object): pass + +class Boolean(object): + pass From 827f1b874c6eb668420efac22756f5fe9c2aeed0 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 25 Apr 2012 10:18:26 +0200 Subject: [PATCH 038/100] Add "startswith" ID permutation --- pytnyzer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytnyzer.py b/pytnyzer.py index 0cb12a3..5c7dadd 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -13,6 +13,8 @@ def id_translate(name): if name == "this": name = "self" if name == "exec": name = "_exec" if name == "id": name = "_id" + + if name == "startsWith": name = "startswith" return name ast_class_types = [] From df863e5ffd37738aad2bc2458bc66aeab29d20b3 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 25 Apr 2012 12:04:43 +0200 Subject: [PATCH 039/100] Correccion de bugs en parseo y conversion python --- flscriptparse.py | 2 +- pytnyzer.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/flscriptparse.py b/flscriptparse.py index e1421f0..16b77c9 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -32,8 +32,8 @@ def cnvrt(val): precedence = ( ('nonassoc', 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL'), ('left','LOR', 'LAND'), - ('right', 'LNOT'), ('left', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE'), + ('right', 'LNOT'), ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE', 'MOD'), ('left', 'OR', 'AND', 'XOR', 'LSHIFT', 'RSHIFT'), diff --git a/pytnyzer.py b/pytnyzer.py index 5c7dadd..70ddbf4 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -570,6 +570,7 @@ def generate(self, isolate = True, **kwargs): if isolate: yield "expr", ")" class Expression(ASTPython): + tags = ["base_expression"] def generate(self, isolate = True, **kwargs): if isolate: yield "expr", "(" for child in self.elem: From 514422f7f0c1666ddbfb26e75fd44e1d76634e62 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 25 Apr 2012 13:48:01 +0200 Subject: [PATCH 040/100] Added deleteinstruction to parser --- postparse.py | 3 +++ pytnyzer.py | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/postparse.py b/postparse.py index 4017bf6..1d27ad5 100755 --- a/postparse.py +++ b/postparse.py @@ -342,6 +342,9 @@ class TryCatch(ListObject): class New(ListObject): tags = ["new_operator"] +class Delete(ListObject): + tags = ["deleteinstruction"] + class Parentheses(ListObject): tags = ["parentheses"] adopt_childs_tags = ['base_expression'] diff --git a/pytnyzer.py b/pytnyzer.py index 70ddbf4..5ae8a2e 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -11,8 +11,9 @@ def id_translate(name): if name == "null": name = "None" if name == "unknown": name = "None" if name == "this": name = "self" - if name == "exec": name = "_exec" - if name == "id": name = "_id" + if name == "exec": name = "exec_" + if name == "id": name = "id_" + if name == "from": name = "from_" if name == "startsWith": name = "startswith" return name @@ -101,7 +102,7 @@ def generate(self, **kwargs): expr = [] for dtype, data in parse_ast(arg).generate(): if dtype == "expr": - expr.append(data) + expr.append(id_translate(data)) else: yield dtype, data if len(expr) == 0: @@ -573,9 +574,20 @@ class Expression(ASTPython): tags = ["base_expression"] def generate(self, isolate = True, **kwargs): if isolate: yield "expr", "(" + coerce_string_mode = False + if self.elem.xpath("OpMath[@type=\"PLUS\"]"): + if self.elem.xpath("Constant[@type=\"String\"]"): + coerce_string_mode = True for child in self.elem: + coerce_to_string = False + if coerce_string_mode == True: + if child.tag in ["Identifier"]: coerce_to_string = True + if coerce_to_string: + yield "expr", "ustr(" for dtype, data in parse_ast(child).generate(): yield dtype, data + if coerce_to_string: + yield "expr", ")" if isolate: yield "expr", ")" class Parentheses(ASTPython): From 8af610ef03b0d2c1d30a37ab775b574dc6965958 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 25 Apr 2012 20:00:12 +0200 Subject: [PATCH 041/100] Fixes to String conversion --- postparse.py | 6 +++++- pytnyzer.py | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/postparse.py b/postparse.py index 1d27ad5..602e640 100755 --- a/postparse.py +++ b/postparse.py @@ -247,7 +247,11 @@ def add_value(self, argn, vtype, value): if vtype == "SCONST": vtype = "String" value = value[1:-1] - if vtype == "CCONST": vtype = "String" + self.xml.set("delim",'"') + if vtype == "CCONST": + vtype = "String" + value = value[1:-1] + self.xml.set("delim","'") if vtype == "RCONST": vtype = "Regex" if vtype == "ICONST": vtype = "Number" if vtype == "FCONST": vtype = "Number" diff --git a/pytnyzer.py b/pytnyzer.py index 5ae8a2e..84157c3 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -6,14 +6,16 @@ from lxml import etree def id_translate(name): + python_keywords = ['and', 'del', 'for', 'is', 'raise', 'assert', 'elif', + 'from', 'lambda', 'return', 'break', 'else', 'global', 'not', 'try', + 'class', 'except', 'if', 'or', 'while', 'continue', + 'exec', 'import', 'pass', 'yield', 'def', 'finally', 'in', 'print'] + if name in python_keywords: return name + "_" if name == "false": name = "False" if name == "true": name = "True" if name == "null": name = "None" if name == "unknown": name = "None" if name == "this": name = "self" - if name == "exec": name = "exec_" - if name == "id": name = "id_" - if name == "from": name = "from_" if name == "startsWith": name = "startswith" return name @@ -110,6 +112,7 @@ def generate(self, **kwargs): yield "debug", "Argument %d not understood" % n yield "debug", etree.tostring(arg) else: + if len(expr) == 1: expr += ["=","None"] arguments.append(" ".join(expr)) @@ -581,7 +584,7 @@ def generate(self, isolate = True, **kwargs): for child in self.elem: coerce_to_string = False if coerce_string_mode == True: - if child.tag in ["Identifier"]: coerce_to_string = True + if child.tag in ["Identifier","Member"]: coerce_to_string = True if coerce_to_string: yield "expr", "ustr(" for dtype, data in parse_ast(child).generate(): @@ -649,7 +652,12 @@ def generate(self, **kwargs): yield "expr", "qsatype.Array([%s])" % (", ".join(arguments)) return - if ctype == "String": yield "expr", "u\"%s\"" % value + if ctype == "String": + delim = self.elem.get("delim") + if delim == "'": + yield "expr", "u'%s'" % value + else: + yield "expr", "u\"%s\"" % value else: yield "expr", value class Identifier(ASTPython): From 23e692db4f1680b2a122b5d08209dc5f4d7b48a1 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 26 Apr 2012 12:08:04 +0200 Subject: [PATCH 042/100] Corrected coercion to string --- pytnyzer.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index 84157c3..8b2594f 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -581,16 +581,19 @@ def generate(self, isolate = True, **kwargs): if self.elem.xpath("OpMath[@type=\"PLUS\"]"): if self.elem.xpath("Constant[@type=\"String\"]"): coerce_string_mode = True + if coerce_string_mode: + yield "expr", "ustr(" for child in self.elem: - coerce_to_string = False - if coerce_string_mode == True: - if child.tag in ["Identifier","Member"]: coerce_to_string = True - if coerce_to_string: - yield "expr", "ustr(" + if coerce_string_mode and child.tag == "OpMath": + if child.get("type") == "PLUS": + yield "expr","," + continue + for dtype, data in parse_ast(child).generate(): yield dtype, data - if coerce_to_string: - yield "expr", ")" + + if coerce_string_mode: + yield "expr", ")" if isolate: yield "expr", ")" class Parentheses(ASTPython): From 2974c3cbe951c27cf303a4f47cdefb1e18a07543 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 26 Apr 2012 14:02:52 +0200 Subject: [PATCH 043/100] Correccion de llamada a funcion sin this.iface. --- pytnyzer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pytnyzer.py b/pytnyzer.py index 8b2594f..4996e76 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -135,6 +135,11 @@ def generate(self, **kwargs): extends = class_.get("extends") if extends == name: name = "super(%s, self).__init__" % class_.get("name") + functions = parent.xpath("//Function[@name=\"%s\"]" % name) + for f in functions: + #yield "debug", "Function to:" + etree.tostring(f) + name = "self.%s" % name + break arguments = [] for n,arg in enumerate(self.elem.xpath("CallArguments/*")): @@ -477,7 +482,6 @@ def generate(self, **kwargs): yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) - yield "line", " ".join(arguments) class InstructionFlow(ASTPython): From f552b8ee83b3cd9bfae9db95d91f23f060a39225 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 31 Jul 2012 12:54:19 +0200 Subject: [PATCH 044/100] =?UTF-8?q?Salida=20de=20parseo=20m=C3=A1s=20inter?= =?UTF-8?q?activa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flscriptparse.py | 2 ++ postparse.py | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 16b77c9..c1dd3c4 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -343,6 +343,7 @@ def p_error(t): global ok_count global last_error_token global last_error_line + # if error_count == 0: print error_count += 1 if t is not None: if last_error_token is None or t.lexpos != getattr(last_error_token,"lexpos",None): @@ -613,6 +614,7 @@ def parse(data): p = parser.parse(data, debug = 0, tracking = 1, tokenfunc = my_tokenfunc) if error_count > 0: print "ERRORS (%d)" % error_count + p["error_count"] = error_count if parser.error: return None return p diff --git a/postparse.py b/postparse.py index 602e640..ccc6839 100755 --- a/postparse.py +++ b/postparse.py @@ -1,6 +1,6 @@ #!/usr/bin/python from optparse import OptionParser -import os, os.path +import os, os.path, sys import flscriptparse import imp, traceback from lxml import etree @@ -509,12 +509,18 @@ def execute(options, args): pythonize(filename, destname) else: - for filename in args: + nfs = len(args) + for nf, filename in enumerate(args): bname = os.path.basename(filename) - print "File:", bname + sys.stdout.write("Parsing File: %-40s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) + sys.stdout.flush(); prog = flscriptparse.parse(open(filename).read()) + sys.stdout.write("\r"); if not prog: - print "No se pudo abrir el fichero." + print "Error: No se pudo abrir %-40s \n" % (repr(filename)) + continue + if prog["error_count"] > 0: + print "Encontramos %d errores parseando: %-40s \n" % (prog["error_count"], repr(filename)) continue if options.toxml == False: # Si no se quiere guardar resultado, no hace falta calcular mas @@ -522,12 +528,11 @@ def execute(options, args): tree_data = flscriptparse.calctree(prog, alias_mode = 0) if not tree_data: - print "No se pudo parsear." + print "No se pudo parsear %-40s \n" % (repr(filename)) continue - ast = post_parse(tree_data) if ast is None: - print "No se pudo analizar." + print "No se pudo analizar %-40s \n" % (repr(filename)) continue if options.storepath: destname = os.path.join(options.storepath,bname+".xml") From 91af98d501074b7ce6f4fd222f0b6a4ae200c2bb Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 12 Apr 2013 09:46:48 +0200 Subject: [PATCH 045/100] Correcciones para evitar que deje de parsear por un error --- flscriptparse.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flscriptparse.py b/flscriptparse.py index c1dd3c4..377f95b 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -232,6 +232,7 @@ def p_parse(token): instruction : base_instruction SEMI | SEMI | base_instruction + | funcdeclaration callinstruction : funccall | variable @@ -315,6 +316,7 @@ def p_parse(token): | error forinstatement : FOR LPAREN for_initialize IN variable RPAREN statement_block + | FOR LPAREN variable IN variable RPAREN statement_block | error switch : SWITCH LPAREN condition RPAREN LBRACE case_block_list RBRACE @@ -614,7 +616,13 @@ def parse(data): p = parser.parse(data, debug = 0, tracking = 1, tokenfunc = my_tokenfunc) if error_count > 0: print "ERRORS (%d)" % error_count - p["error_count"] = error_count + if p is None: return p + try: + p["error_count"] = error_count + except Exception, e: + print e + return None + if parser.error: return None return p From 10c3b939307f410034ba2b3e5fbed601cb95c403 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 29 Apr 2013 11:58:09 +0200 Subject: [PATCH 046/100] Correcciones al evaluar expresiones --- flscriptparse.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 377f95b..7ef5684 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -70,7 +70,9 @@ def p_parse(token): | typeof_operator + parentheses : LPAREN base_expression RPAREN + | LPAREN variable_1 RPAREN unary_operator : LNOT base_expression | MINUS base_expression @@ -162,8 +164,7 @@ def p_parse(token): | funccall_1 | member_call | member_var - | LPAREN base_expression RPAREN - | LPAREN identifier RPAREN + | base_expression member_var : varmemcall PERIOD variable_1 member_call : LPAREN member_var RPAREN PERIOD funccall_1 From 60eeee255ba67b55dc14d803e8a05ce699532db6 Mon Sep 17 00:00:00 2001 From: Tarek Bouharrat Date: Thu, 4 Jul 2013 12:39:46 +0200 Subject: [PATCH 047/100] Agregar control de cierre de llave --- flscriptparse.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/flscriptparse.py b/flscriptparse.py index 7ef5684..57e460f 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -39,7 +39,8 @@ def cnvrt(val): ('left', 'OR', 'AND', 'XOR', 'LSHIFT', 'RSHIFT'), ) - +tokelines = {} +last_lexspan = None def p_parse(token): ''' exprval : constant @@ -334,6 +335,12 @@ def p_parse(token): token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } global ok_count ok_count += 1 + if lexspan[0] not in tokelines: + tokelines[lexspan[0]] = token.lexer.lineno + global last_lexspan + last_lexspan = lexspan + + error_count = 0 @@ -362,6 +369,12 @@ def p_error(t): if t is None: if last_error_token != "EOF": print "ERROR: End of the file reached." + if last_lexspan: + try: + print "HINT: Last lexspan:", last_lexspan + print "HINT: Last line:", tokelines[last_lexspan[0]] + except Exception, e: + print "ERROR:", e last_error_token = "EOF" return t t = yacc.token() From 8fc7223780f866d7738dab6a57327f0a47589dae Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 15 Jan 2014 12:05:47 +0100 Subject: [PATCH 048/100] Mejoras en salida --- postparse.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/postparse.py b/postparse.py index ccc6839..efc2846 100755 --- a/postparse.py +++ b/postparse.py @@ -512,15 +512,15 @@ def execute(options, args): nfs = len(args) for nf, filename in enumerate(args): bname = os.path.basename(filename) - sys.stdout.write("Parsing File: %-40s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) + sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) sys.stdout.flush(); prog = flscriptparse.parse(open(filename).read()) sys.stdout.write("\r"); if not prog: - print "Error: No se pudo abrir %-40s \n" % (repr(filename)) + print "Error: No se pudo abrir %-35s \n" % (repr(filename)) continue if prog["error_count"] > 0: - print "Encontramos %d errores parseando: %-40s \n" % (prog["error_count"], repr(filename)) + print "Encontramos %d errores parseando: %-35s \n" % (prog["error_count"], repr(filename)) continue if options.toxml == False: # Si no se quiere guardar resultado, no hace falta calcular mas @@ -528,11 +528,11 @@ def execute(options, args): tree_data = flscriptparse.calctree(prog, alias_mode = 0) if not tree_data: - print "No se pudo parsear %-40s \n" % (repr(filename)) + print "No se pudo parsear %-35s \n" % (repr(filename)) continue ast = post_parse(tree_data) if ast is None: - print "No se pudo analizar %-40s \n" % (repr(filename)) + print "No se pudo analizar %-35s \n" % (repr(filename)) continue if options.storepath: destname = os.path.join(options.storepath,bname+".xml") From 0c00b883d657230b49a0db35fe06ce206f707b8f Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 15 Jan 2014 12:05:58 +0100 Subject: [PATCH 049/100] Permitir TypeOf(var) --- flscriptparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flscriptparse.py b/flscriptparse.py index 7ef5684..f8437a7 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -82,6 +82,7 @@ def p_parse(token): | NEW identifier typeof_operator : TYPEOF variable + | TYPEOF parentheses ternary_operator : base_expression CONDITIONAL1 base_expression COLON base_expression From c81cf4f59c52a1acb71621deac0f578411995b0d Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 7 Mar 2014 08:22:13 +0100 Subject: [PATCH 050/100] Agregar plantilla para dar soporte al END of file reached --- flscriptparse.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flscriptparse.py b/flscriptparse.py index f8437a7..415de6b 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -332,6 +332,9 @@ def p_parse(token): ''' lexspan = list(token.lexspan(0)) data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) + if len(lexspan) == 2: + fromline = token.lineno(0) + #print fromline, lexspan, token.slice[0] token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } global ok_count ok_count += 1 From 5a96ce70d8208a5ad08c04d11c6170463c0bf69f Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 9 Sep 2014 13:49:14 +0200 Subject: [PATCH 051/100] =?UTF-8?q?Mejorar=20parseo=20de=20Regex=20y=20det?= =?UTF-8?q?ecci=C3=B3n=20de=20par=C3=A9ntesis=20sin=20cerrar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flex.py | 14 +++++++--- flscriptparse.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/flex.py b/flex.py index a86b4aa..4015b9b 100644 --- a/flex.py +++ b/flex.py @@ -21,7 +21,7 @@ ] token_literals = [ # Literals (identifier, integer constant, float constant, string constant, char const) - 'ID', 'ICONST', 'FCONST', 'SCONST', 'CCONST', 'RXCONST' + 'ID', 'ICONST', 'FCONST', 'SCONST', 'CCONST' #, 'RXCONST' ] tokens = reserved + token_literals + [ @@ -57,7 +57,11 @@ # 'ELLIPSIS', 'DOCSTRINGOPEN', # 'COMMENTOPEN', - 'COMMENTCLOSE', + 'COMMENTCLOSE', + 'DOLLAR', + 'SQOUTE', + 'DQOUTE', + 'BACKSLASH', ] # Completely ignored characters @@ -69,6 +73,10 @@ def t_NEWLINE(t): t.lexer.lineno += t.value.count("\n") # Operators +t_BACKSLASH = '\\\\' +t_DOLLAR = r'\$' +t_SQOUTE = '\'' +t_DQOUTE = '"' t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' @@ -160,7 +168,7 @@ def t_ID(t): t_CCONST = r'\'([^\\\n]|(\\.))*?\'' # REGEX constant -t_RXCONST = r'/.+/g' +#t_RXCONST = r'/[^/ ]+/g?' # Comments def t_comment(t): diff --git a/flscriptparse.py b/flscriptparse.py index 415de6b..92eb4d5 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -20,6 +20,8 @@ reserv=['nonassoc'] reserv+=list(flex.reserved) +endoffile = None + def cnvrt(val): val = str(val) val = val.replace('&','&') @@ -177,6 +179,7 @@ def p_parse(token): | member_call | LPAREN member_call RPAREN | LPAREN funccall_1 RPAREN + | LPAREN error RPAREN funccall_1 : ID LPAREN callargs RPAREN | ID LPAREN RPAREN @@ -235,6 +238,7 @@ def p_parse(token): | SEMI | base_instruction | funcdeclaration + | error SEMI callinstruction : funccall | variable @@ -265,9 +269,36 @@ def p_parse(token): | FCONST | CCONST | SCONST - | RXCONST + | regex | list_constant - | error + + regex : DIVIDE regexbody DIVIDE regexflags + | DIVIDE regexbody COMMENTCLOSE regexflags + + regexbody : regexchar + | regexbody regexchar + + regexchar : LPAREN + | RPAREN + | ID + | COMMA + | XOR + | LBRACKET + | RBRACKET + | ICONST + | PLUS + | MINUS + | LBRACE + | RBRACE + | DOLLAR + | SQOUTE + | DQOUTE + | BACKSLASH + | SCONST + | error + + regexflags : ID + | empty statement_block : statement @@ -330,12 +361,40 @@ def p_parse(token): empty : ''' + global input_data + lexspan = list(token.lexspan(0)) data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) if len(lexspan) == 2: fromline = token.lineno(0) + global endoffile + endoffile = fromline, lexspan, token.slice[0] #print fromline, lexspan, token.slice[0] token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } + numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) + + rspan = lexspan[0] + if str(token.slice[0]) == 'empty' or numelems == 0: token[0] = None + else: + rvalues = [] + for n,s in enumerate(token.slice[1:]): + if s.type != 'empty' and s.value is not None: + val = None + if isinstance(s.value,basestring): + val = token.lexspan(n+1)[0] + len(s.value) - 1 + else: + val = token.lexspan(n+1)[1] + rvalues.append(val) + rspan = max(rvalues) + lexspan[1] = rspan + + if str(token.slice[0]) == 'regexbody': + token[0] = { "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } + + #if str(token.slice[0]) == 'regex': + # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] + # print " " + "\n ".join([ "%s(%r): %r" % (s.type, token.lexspan(n+1), s.value) for n,s in enumerate(token.slice[1:]) ]) + global ok_count ok_count += 1 @@ -366,6 +425,9 @@ def p_error(t): if t is None: if last_error_token != "EOF": print "ERROR: End of the file reached." + global endoffile + print "Last data:", endoffile + last_error_token = "EOF" return t t = yacc.token() From 42351f6a9ea14ff6a32ad65b147ecb282e5f2488 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 6 Oct 2014 12:19:18 +0200 Subject: [PATCH 052/100] mejora al considerar operador typeof --- flscriptparse.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 92eb4d5..f3f6f30 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -41,6 +41,7 @@ def cnvrt(val): ('left', 'OR', 'AND', 'XOR', 'LSHIFT', 'RSHIFT'), ) +seen_tokens = [] def p_parse(token): ''' @@ -59,8 +60,8 @@ def p_parse(token): dictobject_value_elem : exprval COLON expression - base_expression : inlinestoreinstruction - | exprval + base_expression : exprval + | inlinestoreinstruction | base_expression mathoperator base_expression | base_expression cmp_symbol base_expression | base_expression boolcmp_symbol base_expression @@ -73,6 +74,7 @@ def p_parse(token): + parentheses : LPAREN base_expression RPAREN | LPAREN variable_1 RPAREN @@ -84,7 +86,7 @@ def p_parse(token): | NEW identifier typeof_operator : TYPEOF variable - | TYPEOF parentheses + | TYPEOF base_expression ternary_operator : base_expression CONDITIONAL1 base_expression COLON base_expression @@ -183,6 +185,7 @@ def p_parse(token): funccall_1 : ID LPAREN callargs RPAREN | ID LPAREN RPAREN + | TYPEOF LPAREN callargs RPAREN mathoperator : PLUS | MINUS @@ -247,6 +250,10 @@ def p_parse(token): | callinstruction | flowinstruction + varorcall : variable + | funccall + | base_expression + optextends : EXTENDS ID | empty @@ -319,7 +326,8 @@ def p_parse(token): boolcmp_symbol : LOR | LAND - condition : expression + condition : expression + | error ifstatement : IF LPAREN condition RPAREN statement_block optelse @@ -348,8 +356,8 @@ def p_parse(token): forstatement : FOR LPAREN for_initialize SEMI for_compare SEMI for_increment RPAREN statement_block | error - forinstatement : FOR LPAREN for_initialize IN variable RPAREN statement_block - | FOR LPAREN variable IN variable RPAREN statement_block + forinstatement : FOR LPAREN for_initialize IN varorcall RPAREN statement_block + | FOR LPAREN variable IN varorcall RPAREN statement_block | error switch : SWITCH LPAREN condition RPAREN LBRACE case_block_list RBRACE @@ -394,11 +402,13 @@ def p_parse(token): #if str(token.slice[0]) == 'regex': # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] # print " " + "\n ".join([ "%s(%r): %r" % (s.type, token.lexspan(n+1), s.value) for n,s in enumerate(token.slice[1:]) ]) - + global seen_tokens, last_ok_token + last_ok_token = token + seen_tokens.append((str(token.slice[0]), token.lineno(0),input_data[lexspan[0]:lexspan[1]+1] )) global ok_count ok_count += 1 - +last_ok_token = None error_count = 0 last_error_token = None last_error_line = -1 @@ -408,14 +418,22 @@ def p_error(t): global error_count global ok_count global last_error_token - global last_error_line + global last_error_line, seen_tokens , last_ok_token + debug = False # Poner a True para toneladas de debug. # if error_count == 0: print - error_count += 1 if t is not None: if last_error_token is None or t.lexpos != getattr(last_error_token,"lexpos",None): - if abs(last_error_line - t.lineno) > 3 and ok_count > 2: + if abs(last_error_line - t.lineno) > 4 and ok_count > 1 and error_count < 4: + error_count += 1 try: print_context(t) except: pass + if debug == True: + for tokname, tokln, tokdata in seen_tokens[-32:]: + if tokln == t.lineno: + print tokname, tokdata + print repr(last_ok_token[0]) + last_error_line = t.lineno + elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: last_error_line = t.lineno yacc.errok() ok_count = 0 @@ -676,6 +694,8 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou def parse(data): global input_data global error_count + global seen_tokens + seen_tokens[:] = [] parser.error = 0 input_data = data flex.lexer.lineno = 1 From e6485ca9349c336e9af194da94625aecac8c465f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mart=C3=ADnez=20Mart=C3=AD?= Date: Sat, 7 Feb 2015 03:23:05 +0100 Subject: [PATCH 053/100] agregar soporte parcial para palabra clave "static" --- flex.py | 2 +- flscriptparse.py | 2 ++ pytnyzer.py | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/flex.py b/flex.py index 4015b9b..171b5eb 100644 --- a/flex.py +++ b/flex.py @@ -11,7 +11,7 @@ # Reserved words reserved = [ - 'BREAK', 'CASE', 'CONST', 'CONTINUE', 'DEFAULT', 'DO', + 'BREAK', 'CASE', 'CONST', 'STATIC', 'CONTINUE', 'DEFAULT', 'DO', 'ELSE', 'FOR', 'IF', 'IN', 'RETURN', #'STRUCT', diff --git a/flscriptparse.py b/flscriptparse.py index f3f6f30..f35ddd2 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -146,6 +146,7 @@ def p_parse(token): | CONST vardecl_list SEMI vardeclaration : VAR vardecl_list | CONST vardecl_list + | STATIC VAR vardecl_list vardecl : ID optvartype EQUALS expression vardecl : ID optvartype @@ -157,6 +158,7 @@ def p_parse(token): | funcdeclaration : FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE + funcdeclaration : STATIC FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE funcdeclaration_anon : FUNCTION LPAREN arglist RPAREN LBRACE basicsource RBRACE callarg : expression diff --git a/pytnyzer.py b/pytnyzer.py index 4996e76..19c7b69 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -793,7 +793,9 @@ def write_python_file(fobj, ast): if lines_since_last_indent > 4: fobj.write((len(indent)*indent_text) + "\n") last_line_for_indent[len(indent)] = numline - if dtype == "debug": line = "# DEBUG:: " + data + if dtype == "debug": + line = "# DEBUG:: " + data + print numline, line if dtype == "expr": line = "# EXPR??:: " + data if dtype == "line+1": line = "# LINE+1??:: " + data if dtype == "begin": From 5e9f433f83d51286e883e498cf4c308300ab3427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mart=C3=ADnez=20Mart=C3=AD?= Date: Sun, 8 Feb 2015 13:09:33 +0100 Subject: [PATCH 054/100] add "pass" to empty if blocks for switches. --- pytnyzer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytnyzer.py b/pytnyzer.py index 19c7b69..60a907d 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -334,12 +334,16 @@ def generate(self, **kwargs): yield "line", "if %s == %s: %s,%s = %s,%s" % (name," ".join(value_expr),name_pr,name_pr2, "True", "True") yield "line", "if %s:" % (name_pr) yield "begin", "block-if" + count = 0 for source in scase.xpath("Source"): for obj in parse_ast(source).generate(break_mode = True): if obj[0] == "break": yield "line", "%s = %s # BREAK" % (name_pr,"False") + count += 1 else: yield obj + count += 1 + if count < 1: yield "line", "pass" yield "end", "block-if" for scasedefault in self.elem.xpath("CaseDefault"): From 05dfb3384bb12acdbc12b852501e9b2b15ebf0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Mart=C3=ADnez=20Mart=C3=AD?= Date: Sun, 8 Feb 2015 16:14:49 +0100 Subject: [PATCH 055/100] bugfix --- pytnyzer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index 60a907d..0ecc947 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -539,7 +539,7 @@ def generate(self, **kwargs): yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) - if arguments[0:2] == ["self","iface"] and arguments[2].startswith("__"): + if len(arguments) >= 3 and arguments[0:2] == ["self","iface"] and arguments[2].startswith("__"): # From: self.iface.__function() # to: super(className, self.iface).function() funs = self.elem.xpath("ancestor::Function") @@ -787,6 +787,7 @@ def write_python_file(fobj, ast): indent_text = " " last_line_for_indent = {} numline = 0 + last_dtype = None for dtype, data in file_template(ast): line = None if dtype == "line": @@ -806,7 +807,11 @@ def write_python_file(fobj, ast): #line = "# BEGIN:: " + data indent.append(data) last_line_for_indent[len(indent)] = numline - if dtype == "end": + if dtype == "end": + if last_dtype == "begin": + fobj.write((len(indent)*indent_text) + "pass\n") + last_line_for_indent[len(indent)] = numline + if data not in ["block-if"]: #line = "# END:: " + data pass @@ -822,6 +827,7 @@ def write_python_file(fobj, ast): if data.split("-")[1] in ["class","def","else","except"]: fobj.write((len(indent)*indent_text) + "\n") last_line_for_indent[len(indent)] = numline + last_dtype = dtype def pythonize(filename, destfilename): bname = os.path.basename(filename) From ad3d82a4e5ec0df4a4204932ee70da2b94ced479 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 30 Mar 2015 11:38:56 +0200 Subject: [PATCH 056/100] Add === and !== --- flex.py | 4 +++- flscriptparse.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/flex.py b/flex.py index 171b5eb..e9896bd 100644 --- a/flex.py +++ b/flex.py @@ -32,7 +32,7 @@ #'NOT', 'XOR', 'LSHIFT', 'RSHIFT', 'LOR', 'LAND', 'LNOT', - 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', + 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'EQQ', 'NEQ', # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', @@ -97,6 +97,8 @@ def t_NEWLINE(t): t_GE = r'>=' t_EQ = r'==' t_NE = r'!=' +t_EQQ = r'===' +t_NEQ = r'!==' t_CONDITIONAL1 = r'\?' # Assignment operators diff --git a/flscriptparse.py b/flscriptparse.py index f35ddd2..6408242 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -34,7 +34,7 @@ def cnvrt(val): precedence = ( ('nonassoc', 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL'), ('left','LOR', 'LAND'), - ('left', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE'), + ('left', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'EQQ', 'NEQ'), ('right', 'LNOT'), ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE', 'MOD'), @@ -323,6 +323,8 @@ def p_parse(token): | GE | EQ | NE + | EQQ + | NEQ | IN boolcmp_symbol : LOR From 9e00b5d027577784ca85ad17566a727d319aa543 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 19 Jul 2015 17:54:45 +0200 Subject: [PATCH 057/100] flparser: integrar flscriptparser dentro de pineboo --- __init__.py | 0 flalign.py | 489 ++++++++++++++++++++++++++ flclasses.py | 335 ++++++++++++++++++ flex.py | 208 +++++++++++ flpremerge.py | 662 +++++++++++++++++++++++++++++++++++ flscriptparse.py | 845 +++++++++++++++++++++++++++++++++++++++++++++ postparse.py | 564 ++++++++++++++++++++++++++++++ pytnyzer.py | 883 +++++++++++++++++++++++++++++++++++++++++++++++ qsatype.py | 18 + xml2json.py | 634 ++++++++++++++++++++++++++++++++++ xmlparse.py | 200 +++++++++++ 11 files changed, 4838 insertions(+) create mode 100644 __init__.py create mode 100644 flalign.py create mode 100644 flclasses.py create mode 100644 flex.py create mode 100644 flpremerge.py create mode 100644 flscriptparse.py create mode 100644 postparse.py create mode 100644 pytnyzer.py create mode 100644 qsatype.py create mode 100644 xml2json.py create mode 100644 xmlparse.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flalign.py b/flalign.py new file mode 100644 index 0000000..a5563d2 --- /dev/null +++ b/flalign.py @@ -0,0 +1,489 @@ +from __future__ import print_function +from builtins import next +from builtins import object +import os, os.path, sys +import math +from optparse import OptionParser +import difflib +import re + +class ProcessFile(object): + def __init__(self, filename): + self.filename = filename + self.process() + self.indexLines() + self.bnames = set(self.idxnames.keys()) + + def process(self): + self.blocks = [] + self.idxlines = {} + self.idxnames = {} + self.sortednames = [] + + fblocks = open(self.filename + ".blocks") + n = 0 + line = fblocks.readline() + while line: + start, end, name = line.strip().split("\t") + self.blocks.append( (start,end, name) ) + self.idxlines[start] = n + self.sortednames.append(name) + if name in self.idxnames: + self.idxnames[name] = None + else: + self.idxnames[name] = n + n+=1 + line = fblocks.readline() + + def indexLines(self): + fqs = open(self.filename) + self.lines = [] + for line in fqs: + self.lines.append(line) + + + + def diffTo(self,pfile2): + added = pfile2.bnames - self.bnames + deleted = self.bnames - pfile2.bnames + + return added, deleted + +class LineNumber(object): + def __init__(self, letter, lines): + self.nl=0 + if len(letter)>1: + self.diffFrom = letter[0] + else: + self.diffFrom = None + self.error = None + + self.letter = letter[-1] + self.lines = lines + + def line(self): + l = self.lines[self.nl] + if self.diffFrom: + return l[2:] + else: + return l[:] + + def symbol(self): + l = self.lines[self.nl] + if self.diffFrom: + return l[0] + else: + return " " + + def next(self,t=1): + self.nl+=int(t) + + def __iadd__(self,other): + self.nl+=int(other) + return self + + def __int__(self): + return self.nl + + def __index__(self): + return self.nl + + +def appliedDiff(C, A, B, prefer = "C", debug = False, quiet = False, swap = False): + diffAB = list(difflib.ndiff(A.sortednames, B.sortednames)) + diffAC = list(difflib.ndiff(A.sortednames, C.sortednames)) + + + nlA = LineNumber("A",A.sortednames) + nlB = LineNumber("B",B.sortednames) + nlC = LineNumber("C",C.sortednames) + nlAB = LineNumber("AB", diffAB) + nlAC = LineNumber("AC", diffAC) + + maxA = len(A.sortednames) + maxB = len(B.sortednames) + maxC = len(C.sortednames) + patchedResult = [] + + def AddPatchLine(mode): + modefrom = mode[0] + modetype = mode[1] + linefrom = None + linetext = "" + conflict = False + linenumbers = int(nlA),int(nlB),int(nlC) + if modetype == "-": + linetext = lineaA + linefrom = "A" + if modefrom == "B": + DeletedB.append(lineaA) + elif modefrom == "C": + DeletedC.append(lineaA) + elif modetype == "=": + modefrom = prefer + if modefrom == "A": + linetext = lineaA + elif modefrom == "B": + linetext = lineaB + elif modefrom == "C": + linetext = lineaC + linefrom = modefrom + elif modetype == "+": + if modefrom == "A": + linetext = lineaA + elif modefrom == "B": + linetext = lineaB + if lineaB in AddedB: conflict = True + if lineaB in AddedC: conflict = True + AddedB.append(lineaB) + elif modefrom == "C": + linetext = lineaC + if lineaC in AddedB: conflict = True + if lineaC in AddedC: conflict = True + AddedC.append(lineaC) + linefrom = modefrom + line = ( + modefrom, modetype, + linenumbers, + linefrom, + linetext + ) + #ConflictMode = False + if len(ConflictMode): + if linetext[0]!="#": ConflictMode.pop() + else: + #print ">>", len(ConflictMode),linetext + if len(ConflictMode)>1 and linetext.find("separator") == -1: + ConflictMode.pop() + elif linetext.find("separator") >= 0: + ConflictMode.pop() + return + if conflict and linetext[0]!="#": + if debug: + print("WARNING: Omitting previously added block <%s>" % linetext) + + while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") >= 0: + #print "sep//", + patchedResult.pop() + + while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") == -1: + #print "comm//", + patchedResult.pop() + #if patchedResult[-1][4][0]=="#": patchedResult.pop() + + ConflictMode.append(1) + ConflictMode.append(1) + else: + patchedResult.append(line) + + AddedB = [] + AddedC = [] + DeletedB = [] + DeletedC = [] + ConflictMode = [] + + def getVars(letter): + if letter=="B": + return nlA, nlB, nlC, nlAB, nlAC + if letter=="C": + return nlA, nlC, nlB, nlAC, nlAB + raise TypeError + + def Patch(code): + letter = code[0] + sign = code[1] + nBase, nLocal, nRemote, nDiffLocal, nDiffRemote = getVars(letter) + + def Minus(): + AddPatchLine(letter + "-") + if debug: + print(letter + "-%04d" % int(nBase), nBase.line()) + if nBase.line() != nDiffLocal.line() and not quiet: + print(letter + "! " , nDiffLocal.line()) + if nDiffRemote.symbol()!=" " and not quiet: + if nDiffRemote.symbol()=="-": + #Removing twice! + nBase.error = "removing-twice" + else: + print("??", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) + else: + next(nRemote) + + next(nDiffLocal) + next(nDiffRemote) + next(nBase) + + def Plus(): + AddPatchLine(letter + "+") + if debug: + print(letter + "+%04d" % int(nLocal), nLocal.line()) + if nLocal.line()!=nDiffLocal.line() and not quiet: + print(letter + "! " , nDiffLocal.line()) + if nBase.error == "removing-twice": + nBase.error = None + while nDiffRemote.symbol() == "?": + next(nDiffRemote) + if nDiffRemote.symbol() == "+": + if not quiet: + print("!~", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) + next(nDiffRemote) + next(nRemote) + + + next(nDiffLocal) + next(nLocal) + + def Info(): + #if debug: print letter + "?> " , nDiffLocal.line() + next(nDiffLocal) + + if sign == "-": Minus() + elif sign == "+": Plus() + elif sign == "?": Info() + + + + while True: + + if ( + int(nlA) >= maxA and + int(nlB) >= maxB and + int(nlC) >= maxC + ): break + lineaA = " " + sAB = cAB = lineaA = " " + sAC = cAC = lineaC = " " + if int(nlA) >= maxA : + if int(nlA) - maxA > 0 : print("A overflow!", int(nlA) - maxA) + lineaA = " " + else: lineaA = A.sortednames[nlA] + + if int(nlB) >= maxB : + sAB = cAB = lineaA = " " + if int(nlB) - maxB > 0 : print("B overflow!", int(nlB) - maxB) + else: + lineaB = B.sortednames[nlB] + + if int(nlAB) < len(diffAB): + sAB = diffAB[nlAB][0] + cAB = diffAB[nlAB][2:] + + if int(nlC) >= maxC : + sAC = cAC = lineaC = " " + if int(nlC) - maxC > 0 : print("C overflow!", int(nlC) - maxC) + else: + lineaC = C.sortednames[nlC] + + if int(nlAC) < len(diffAC): + sAC = diffAC[nlAC][0] + cAC = diffAC[nlAC][2:] + + #print nlA, nlB, nlC + if sAB == " " and sAC == " ": + AddPatchLine("A=") + if debug: + if prefer == "": + print("ABC=%04d%+d%+d" % (nlA,nlB-nlA,nlC-nlA),lineaA) + elif prefer == "A": + print("A=%04d" % nlA,lineaA) + elif prefer == "B": + print("B=%04d" % nlB,lineaB) + elif prefer == "C": + print("C=%04d" % nlC,lineaC) + else: + assert prefer in ["A","B","C"] + if not quiet: + if lineaA!=cAB: + print("A!=B " , cAB) + if lineaA!=cAC: + print("A!=C " , cAC) + if lineaA != lineaB or lineaC != lineaA: + print("wtf!?A:", lineaA) + print("wtf!?B:", lineaB) + print("wtf!?C:", lineaC) + + next(nlAB) + next(nlAC) + next(nlA) + next(nlB) + next(nlC) + elif swap and sAB=="+": Patch("B+") + elif sAC=="+": Patch("C+") + elif sAC=="?": Patch("C?") + elif sAB=="+": Patch("B+") + elif sAB=="?": Patch("B?") + elif swap and sAB=="-": Patch("B-") + elif sAC=="-": Patch("C-") + elif sAB=="-": Patch("B-") + else: + if not quiet: + print(sAB,"*", sAC) + break + + addedB = set([x for x in AddedB if x[0]!="#"]) + addedC = set([x for x in AddedC if x[0]!="#"]) + + deletedB = set([x for x in DeletedB if x[0]!="#"]) + deletedC = set([x for x in DeletedC if x[0]!="#"]) + + movedB = addedB & deletedB + movedC = addedC & deletedC + + conflictsAA = addedB & addedC + conflictsDD = deletedB & deletedC + conflictsAD = addedB & deletedC + conflictsDA = deletedB & addedC + + if movedB: + print("CONFLICTS BLOCK MOVED A(%s)->B(%s):" % (A.filename,B.filename)) + for name in movedB: print("-",name) + + if movedC: + print("CONFLICTS BLOCK MOVED A(%s)->C(%s):" % (A.filename,C.filename)) + for name in movedC: print("-",name) + + if conflictsAA: + print("CONFLICTS SAME BLOCK ADDED B(%s)-C(%s):" % (B.filename,C.filename)) + for name in conflictsAA: print("-",name) + + if conflictsDD: + print("CONFLICTS SAME BLOCK DELETED B(%s)-C(%s):" % (B.filename,C.filename)) + for name in conflictsDD: print("-",name) + + if conflictsAD: + print("CONFLICTS BLOCK ADDED BY %s , DELETED BY %s:" % (B.filename,C.filename)) + for name in conflictsAD: print("-",name) + + if conflictsDA: + print("CONFLICTS BLOCK DELETED BY %s , ADDED BY %s:" % (B.filename,C.filename)) + for name in conflictsDA: print("-",name) + + + + return patchedResult + """ + added, deleted = pfrom.diffTo(pto) + plist = [] + for start,end,name in ptarget.blocks: + if name in added: + n = pto.idxnames[name] + if n is not None: + start,end,name = pto.blocks[n] + bobject = pto.filename + else: + print "Conflict with element", name + elif name in deleted: + bobject = "deleted" + plist.append((bobject,start,end,name)) + else: + bobject = ptarget.filename + plist.append((bobject,start,end,name)) + + return plist +""" + +def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = False): + patchlist = appliedDiff(C, A, B, prefer , debug, quiet, swap) + F = {"A": A, "B": B, "C": C} + L = ["A", "B", "C"] + + fout = open(F[prefer].filename + ".aligned","w") + classlist = [] + for Fby,action, nlines, Fwhich, line in patchlist: + nlA, nlB, nlC = nlines + if action not in ("+","="): continue + nl = nlines[L.index(Fwhich)] + try: + linebegin, lineend, line = F[Fwhich].blocks[nl] + except IndexError: + print("!!!ERROR MERGING!!") + continue + + text = "".join( + F[Fwhich].lines[int(linebegin):int(lineend)] + ) + #text = text.replace("\t"," ") + sline = line.split(":") + if sline[0]=="classdeclaration": + if len(classlist): + lastclass = classlist[-1] + else: + lastclass = None + + thisclass = sline[1] + classlist.append(thisclass) + if lastclass: + rs1 = re.search("class (\w+) extends (\w+)",text) + if rs1: + if lastclass != rs1.group(2): + #print "INFO: Changing >> class", thisclass, "extends",rs1.group(2), "--> extends", lastclass + text = re.sub("class (\w+) extends (\w+)", "class %s extends %s" % (thisclass,lastclass),text) + rs2 = re.search("function .*%s\(.*context.*\) { (\w+)" % thisclass,text) + if rs2: + if lastclass != rs2.group(1): + badline = rs2.group(0) + goodline = badline.replace(rs2.group(1),lastclass) + #print "INFO: Changing >>", badline , "-->", goodline + text = text.replace(badline,goodline) + + else: + print(text[:64]) + + #if debug: + # fout.write("<<< %s || %s >>>\n" % (Fwhich, line)) + # fout.write("<<< (%d:%d) >>>\n" % (int(linebegin),int(lineend))) + fout.write(text) + + + fout.close() + +def main(): + parser = OptionParser() + #parser.add_option("-q", "--quiet", + # action="store_false", dest="verbose", default=True, + # help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("-q","--quiet", + action="store_true", dest="quiet", default=False, + help="don't print status messages to stdout") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + + filenames = [x for x in args if os.path.isfile(x)] + not_a_file = set(args) - set(filenames) + if len(not_a_file): + print("WARNING: Not a file:", ", ".join(not_a_file)) + return + + if len(filenames) != 3: + print("MUST have exactly 3 files to align.") + pfiles = [] + for file1 in filenames: + #print "Load File:", file1 + pf = ProcessFile(file1) + pfiles.append(pf) + + A = pfiles[0] + B = pfiles[1] + C = pfiles[2] + + #addedAB, deletedAB = A.diffTo(B) + #addedAC, deletedAC = A.diffTo(C) + is_debug = options.debug + writeAlignedFile(C, A, B, swap = True, debug= is_debug) + writeAlignedFile(B, A, C, debug= is_debug) + writeAlignedFile(B, A, C, prefer = "A", debug= is_debug) + #writeAlignedFile(A, A, C) + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/flclasses.py b/flclasses.py new file mode 100644 index 0000000..2010cd6 --- /dev/null +++ b/flclasses.py @@ -0,0 +1,335 @@ +from __future__ import print_function +from builtins import str +from builtins import object +debug = 0 + +class cBase(object): + def __init__(self): + self.type = ("Unknown","Unknown") + self.codedepth = 0 + + def setSubtype(self,newsubtype): + x, y = self.type + self.type = (x,newsubtype) + + def setType(self,newtype): + x, y = self.type + self.type = (newtype,y) + + def addCodeDepth(self): + self.codedepth += 1 + + def __len__(self): + return 1; + +class cBaseItem(cBase): + def __init__(self,value): + cBase.__init__(self) + self.type = ("Item","Unknown") + self.value = value + + def __str__(self): + if debug > 0 and debug < 1 and self.type == ("Item","Unknown"): return "*" + str(self.value) + "?" + return str(self.value) + +class cBaseItemList(cBase): + def __init__(self,itemList,prefix,suffix,subtype="Unknown"): + cBase.__init__(self) + self.type = ("ItemList",subtype) + if not isinstance(itemList,cBaseList): + iList = cBaseListInline() + iList.addAuto(itemList,subtype=subtype) + itemList = iList + t,st = itemList.slice[0].type + #itemList.setSubtype(st) + itemList.setSubtype("OneItem") + + + if not isinstance(itemList,cBaseList): + raise NameError("itemList no es un cBaseList: %s" % repr(itemList)) + self.itemList = itemList + self.prefix = prefix + self.suffix = suffix + if subtype=="Declaration": + for item in self.itemList.slice: + ctype, csubtype = item.type + ctype = subtype + item.type = (ctype, csubtype) + + def __str__(self): + global debug + txt = str(self.prefix) + str(self.itemList) + str(self.suffix) + txt = txt.strip() + if debug>=2: + txt = "{%s/%s:" % self.itemList.type + txt + "}" + return txt + + +class cBaseVarSpec(cBase): + def __init__(self,name,vartype=None,value=None): + cBase.__init__(self) + self.type = ("Item","Variable") + self.value = value + self.vartype = vartype + self.name = name + + def __str__(self): + txt=self.name + if self.vartype: txt+=":"+self.vartype + if self.value: txt+="="+str(self.value) + return txt + + + +class cBaseList(cBase): + def __init__(self): + cBase.__init__(self) + self.type = ("List","Unknown") + self.slice = [] + self.hidden = [] + self.byType = {} + self.bySubtype = {} + self.byDefName = {} + + def __len__(self): + return len(self.slice); + + def includeItem(self,child): + if not isinstance(child,cBase): + raise NameError("Child is not an instance of Base Class!") + + try: + ctype, csubtype = child.type + except: + raise NameError("Base Class doesn't have `type` atribute or is incorrect.") + if not hasattr(self.byType,ctype): + self.byType[ctype]=[] + + if not hasattr(self.bySubtype,ctype): + self.bySubtype["%s:%s" % (ctype,csubtype)]=[] + + self.byType[ctype].append(child) + self.bySubtype["%s:%s" % (ctype,csubtype)].append(child) + + if ctype == "Declaration": + try: + cname = child.name + except: + raise NameError("Declaration Class doesn't have `name` atribute.") + + if cname in self.byDefName: + print("#WARNING# Variable %s found, but previously defined in this block" % cname) + # self.byDefName[cname]=None + else: + self.byDefName[cname]=child + try: + child.addCodeDepth() + except: + print(repr(child)) + raise NameError("Base Class doesn't have `addCodeDepth` function.") + + if isinstance(child,cBaseItemList): + sslice = child.itemList.slice[:] + for e in child.itemList.hidden: + sslice.remove(e) + + + for item in sslice: + itype, isubtype = item.type + if not isinstance(item,cBaseItemList): + self.includeItem(item) + #self.hidden.append(item) + + def addCodeDepth(self): + cBase.addCodeDepth(self) + sslice = self.slice[:] + + for e in self.hidden: + sslice.remove(e) + + for child in sslice: + child.addCodeDepth() + + + + def addAuto(self,element,subtype=None): + if not isinstance(element,cBase): + element = cBaseItem(element) + if subtype: + element.setSubtype(subtype) + + self.addChild(element) + + def addChild(self,child,hidden=False): + self.includeItem(child) + self.slice.append(child) + if hidden: self.hidden.append(child) + + def __str__(self): + global debug + sslice = self.slice[:] + + for e in self.hidden: + sslice.remove(e) + + + if debug>=1: + txt = "\n" + # --------- DEBUG OUTPUT VARDECL --------- + if len(self.byDefName)>0: + txt += " " * self.codedepth + " /** Declared vars: " + for definition in self.byDefName: + txt += definition + "; " + txt += "**/\n" + n=0 + for c in sslice: + t1, t2 = c.type + n+=1 + txt += "%-17s:%d" % ("%-8s/%-8s" % (t1[:8],t2[:8]),n)+ ">" + ". " * c.codedepth + str(c) + txt += "\t<%d:\n" % n + return txt + + + + if len(sslice) == 1: + c = str(sslice[0]) + if "\n" not in c: + return " " + c + " " + txt = "\n" + + #for child in self.slice: + # if str(child.type[0]) != "ItemList": + # txt += str(child.type) + "\n" + # txt += str(child) + "\n" + # txt += "____"+ "\n" + + lastmargin = 0 + for c in sslice: + t1, t2 = c.type + line = " " + str(c).replace("\n","\n ") + linecount = line.count("\n") + margin = 0 + if linecount>2: margin += 1 + if linecount>25: margin += 1 + if linecount>50: margin += 1 + topmargin = margin - lastmargin + if topmargin < 0: topmargin = 0 + txt += "\n" * topmargin + line + "\n" * (margin+1) + lastmargin = margin + return txt + + + +class cBaseListInline(cBaseList): + def __init__(self,separator=", "): + cBaseList.__init__(self) + self.separator = separator + + def __str__(self): + global debug + txt = "" + for c in self.slice: + if debug>=3: + txt += "[%s/%s: " % c.type + + txt += str(c) + if debug>=3: + txt += "]" + + txt += self.separator + + if len(self.separator) == 0: + return txt + else: + return txt[:-len(self.separator)] + + +class cStatementList(cBaseList): + def __init__(self): + cBaseList.__init__(self) + self.type = ("List","Statement") + +class cBaseDecl(cBase): + def __init__(self,name): + cBase.__init__(self) + self.type = ("Declaration","Unknown") + self.name = name + + def __str__(self): + return "@unknown declaration %s" % self.name + + +class cFuncDecl(cBaseDecl): + def __init__(self,name,arglist,rettype,source): + cBaseDecl.__init__(self,name=name) + self.type = ("Declaration","Function") + self.arglist = arglist + self.rettype = rettype + if not isinstance(source,cBaseList): + iList = cBaseList() + iList.addAuto(source) + source = iList + + + if not isinstance(source,cBaseList): + raise NameError("source no es un cBaseList: %s" % repr(itemList)) + + self.source = source + + def addCodeDepth(self): + cBase.addCodeDepth(self) + try: + self.source.addCodeDepth() + except: + import traceback,sys + print("Exception in user code:") + print('-'*60) + traceback.print_exc(file=sys.stdout) + print('-'*60) + + + + + def __str__(self): + if self.rettype: + ret=" : " + self.rettype + else: + ret="" + return 'function %s(%s)%s {%s}' % (self.name,self.arglist,ret,self.source) + +class cClassDecl(cBaseDecl): + def __init__(self,name,extends,source): + cBaseDecl.__init__(self,name=name) + self.type = ("Declaration","Class") + self.extends = extends + self.childclasses = None + self.constructor = None + if not isinstance(source,cBaseList): + iList = cBaseList() + iList.addAuto(source) + source = iList + + + if not isinstance(source,cBaseList): + raise NameError("source no es un cBaseList: %s" % repr(itemList)) + self.source = source + + def addCodeDepth(self): + cBase.addCodeDepth(self) + try: + self.source.addCodeDepth() + except: + import traceback,sys + print("Exception in user code:") + print('-'*60) + traceback.print_exc(file=sys.stdout) + print('-'*60) + + def __str__(self): + if self.extends: + ext = " extends " + self.extends + else: + ext = "" + + return 'class %s%s {%s}' % (self.name,ext,self.source) + diff --git a/flex.py b/flex.py new file mode 100644 index 0000000..57b1539 --- /dev/null +++ b/flex.py @@ -0,0 +1,208 @@ +from __future__ import print_function +# ---------------------------------------------------------------------- +# clex.py +# +# A lexer for ANSI C. +# ---------------------------------------------------------------------- + +#import sys +#sys.path.insert(0,"../..") + +import ply.lex as lex + +# Reserved words +reserved = [ + 'BREAK', 'CASE', 'CONST', 'STATIC', 'CONTINUE', 'DEFAULT', 'DO', + 'ELSE', 'FOR', 'IF', 'IN', + 'RETURN', + #'STRUCT', + 'SWITCH', + 'WHILE', 'CLASS', 'VAR', 'FUNCTION', + 'EXTENDS', 'NEW','WITH','TRY','CATCH','THROW', 'DELETE', 'TYPEOF' + ] +token_literals = [ + # Literals (identifier, integer constant, float constant, string constant, char const) + 'ID', 'ICONST', 'FCONST', 'SCONST', 'CCONST' #, 'RXCONST' +] +tokens = reserved + token_literals + [ + + # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=) + 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', + 'OR', 'AND', + 'CONDITIONAL1','AT', + #'NOT', + 'XOR', 'LSHIFT', 'RSHIFT', + 'LOR', 'LAND', 'LNOT', + 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', + + # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) + 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', +# 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL', + + # Increment/decrement (++,--) + 'PLUSPLUS', 'MINUSMINUS', + + # Structure dereference (->) +# 'ARROW', + + # Conditional operator (?) +# 'CONDOP', + + # Delimeters ( ) [ ] { } , . ; : + 'LPAREN', 'RPAREN', + 'LBRACKET', 'RBRACKET', + 'LBRACE', 'RBRACE', + 'COMMA', 'PERIOD', 'SEMI', 'COLON', + + # Ellipsis (...) +# 'ELLIPSIS', + 'DOCSTRINGOPEN', + # 'COMMENTOPEN', + 'COMMENTCLOSE', + 'DOLLAR', + 'SQOUTE', + 'DQOUTE', + 'BACKSLASH', + ] + +# Completely ignored characters +t_ignore = ' \r\t\x0c' + +# Newlines +def t_NEWLINE(t): + r'\n+' + t.lexer.lineno += t.value.count("\n") + +# Operators +t_BACKSLASH = '\\\\' +t_DOLLAR = r'\$' +t_SQOUTE = '\'' +t_DQOUTE = '"' +t_PLUS = r'\+' +t_MINUS = r'-' +t_TIMES = r'\*' +t_DIVIDE = r'/' +t_MOD = r'%' +t_OR = r'\|' +t_AND = r'&' +#t_NOT = r'~' +t_XOR = r'\^' +t_LSHIFT = r'<<' +t_RSHIFT = r'>>' +t_LOR = r'\|\|' +t_LAND = r'&&' +t_LNOT = r'!' +t_LT = r'<' +t_GT = r'>' +t_LE = r'<=' +t_GE = r'>=' +t_EQ = r'==' +t_NE = r'!=' +t_CONDITIONAL1 = r'\?' + +# Assignment operators + +t_EQUALS = r'=' +t_TIMESEQUAL = r'\*=' +t_DIVEQUAL = r'/=' +t_MODEQUAL = r'%=' +t_PLUSEQUAL = r'\+=' +t_MINUSEQUAL = r'-=' +""" +t_LSHIFTEQUAL = r'<<=' +t_RSHIFTEQUAL = r'>>=' +t_ANDEQUAL = r'&=' +t_OREQUAL = r'\|=' +t_XOREQUAL = r'^=' +""" + +# Increment/decrement +t_PLUSPLUS = r'\+\+' +t_MINUSMINUS = r'--' + +# -> +#t_ARROW = r'->' + +# ? +#t_CONDOP = r'\?' + + +# Delimeters +t_LPAREN = r'\(' +t_RPAREN = r'\)' +t_LBRACKET = r'\[' +t_RBRACKET = r'\]' +t_LBRACE = r'\{' +t_RBRACE = r'\}' +t_COMMA = r',' +t_PERIOD = r'\.' +t_SEMI = r';' +t_COLON = r':' +#t_ELLIPSIS = r'\.\.\.' +t_AT = r'@' +# Identifiers and reserved words + +reserved_map = { } +for r in reserved: + reserved_map[r.lower()] = r + + + +def t_ID(t): +# r'[A-Za-z_]+([\.]{0,1}[\w_]*)+' + r'[A-Za-z_]+[\w_]*' + t.type = reserved_map.get(t.value,"ID") + return t + + + +# Integer literal +t_ICONST = r'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?' + +# Floating literal +t_FCONST = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' + +# String literal +t_SCONST = r'\"([^\\\n]|(\\.))*?\"' + +# Character constant 'c' or L'c' +t_CCONST = r'\'([^\\\n]|(\\.))*?\'' + +# REGEX constant +#t_RXCONST = r'/[^/ ]+/g?' + +# Comments +def t_comment(t): + r'(/\*( |\*\*)(.|\n)*?\*/)|(//.*)' + #r'/\*(.|\n)*?\*/' + t.lexer.lineno += t.value.count('\n') + + +def t_DOCSTRINGOPEN(t): + r'/\*\*[ ]+' + return t; + +#t_COMMENTOPEN = r'/\*' +t_COMMENTCLOSE = r'\*/' + + +# Preprocessor directive (ignored) +def t_preprocessor(t): + r'\#(.)*?\n' + t.lexer.lineno += 1 + + +def t_error(t): + print("Illegal character %s" % repr(t.value[0])) + t.lexer.skip(1) + + + +lexer = lex.lex(debug=False) +if __name__ == "__main__": + lex.runmain(lexer) + + + + + diff --git a/flpremerge.py b/flpremerge.py new file mode 100644 index 0000000..73c74b2 --- /dev/null +++ b/flpremerge.py @@ -0,0 +1,662 @@ +from __future__ import print_function +from __future__ import division +from builtins import zip +from builtins import range +from past.utils import old_div +from builtins import object +import os, os.path, sys +import math +from optparse import OptionParser +import difflib +import re + +class processedFile(object): + def __init__(self, filename, debug = False): + self.debug = debug + self.table = {} # Carga literal de la lista de hashes, pk: (startbyte, endbyte) = csvrow + self.idxdepth = {} + self.idxtree = {} # pk: objnum (1.5.5.1.1) = (startbyte, endbyte) + self.hashes = {} + self.list_hashes = [] + self.filename = filename + self.cacheFullQname = {} + self.processHashFile() + self.indexNames() + self.computeSortedBlocks() + + def processHashFile(self): + self.cacheFullQname = {} + self.table, self.idxdepth,self.idxtree, self.hashes, self.list_hashes = process(self.filename+".hash") + + + def indexNames(self): + self.names = {} + self.sortedNames = [] + for pk in self.idxtree: + if len(pk) > 1: continue + name = self.fullQName(pk) + if name not in self.names: self.names[name] = [] + self.names[name].append(pk) + self.sortedNames.append((self.idxtree[pk],name)) + + self.sNames = set(self.names.keys()) + + def Qname(self,p): + row = self.table[self.idxtree[p]] + return row["name"] + + + def fullQName(self,p): + dcache = self.cacheFullQname + if p not in dcache: + name=[] + + for n in range(len(p)): + ps1 = p[:n+1] + name.append(self.Qname(ps1)) + fullname = "/".join(name) + dcache[p] = fullname + else: + fullname = dcache[p] + + return fullname + + def computeSortedBlocks(self, fout=None): + self.computedBlocks = [] + if fout is None: + fout = open(self.filename + ".blocks","w") + + antdesde, anthasta = 0 , -1 + fB = open(self.filename) + pos = 0 + linePosChar = [pos] + fB.seek(0) + line = fB.readline() + while line: + pos += len(line) + linePosChar.append(pos) + line = fB.readline() + linenum = 0 + self.linePosChar = linePosChar + for pk, bl_name in list(sorted(self.sortedNames))+[((None,None),None)]: + desde, hasta = pk + if desde is None or desde >= anthasta +1: + + bdesde = anthasta + 1 + fB.seek(bdesde) + if desde: + bhasta = desde - 1 + sB = fB.read((bhasta-bdesde)+1) + else: + sB = fB.read() + bhasta = bdesde + len(sB) -1 + + + while linenum < len(linePosChar) and linePosChar[linenum+1]<=bdesde: linenum += 1 + startline = linenum + linesize=linePosChar[startline] - linePosChar[startline-1] + curpos = bdesde - linePosChar[startline-1] + #if curpos != linesize: print "****", linesize, curpos + + + while linenum < len(linePosChar) and linePosChar[linenum+1]<=bhasta: linenum += 1 + endline = linenum + #while startline > 1 and linePosChar[startline-1]-bdesde >=-2: startline-=1 + + #print linePosChar[startline-1]-bdesde, linePosChar[startline]-bdesde, linePosChar[startline+1]-bdesde + + #print (startline, endline), "BLOCK", (linePosChar[startline], linePosChar[endline]), (bdesde , bhasta), (bhasta-bdesde)+1 + + #print "<<<<" + mode = "" + nline = startline + beginline = startline + bblocks = [] + blockdesc = [] + if len(sB.splitlines(1)) != endline-startline+1: + print(startline, endline, repr(sB)) + print(linePosChar[endline-1]-bhasta, linePosChar[endline]-bhasta, linePosChar[endline+1]-bhasta) + + print("Block lines doesnt match:", len(sB.splitlines(1)), endline-startline, startline, endline) + print(linePosChar[startline-2:startline+3],bdesde) + print(linePosChar[endline-2:endline+3],bhasta) + print(repr(sB.splitlines(1))) + for line in sB.splitlines(1): + nline += 1 + if line[-1]!='\n': break + ltype = "junk" + isseparator = re.match(r'[ \t]*\n',line) + iscommentline1 = re.match(r'[ \t]*//.+\n',line) + iscommentline2 = re.match(r'[ \t]*/\*.+\*/\n',line) + iscommentbegin = re.match(r'[ \t]*/\*.+\n',line) + iscommentend = re.match(r'.+\*/[ \t]*\n',line) + + + + if isseparator: ltype = "separator" + elif iscommentline1: ltype = "comment_inline" + elif iscommentline2: ltype = "comment_block_inline" + elif iscommentbegin: ltype = "comment_block" + elif iscommentend: ltype = "comment_blockend" + #else: print "junk?", line, + + if mode == "comment_block" and ltype == "comment_blockend": + mode = "comment_blockend" + + if mode == "comment_block" and ltype != "comment_blockend": + ltype = "comment_block" + + + if mode != ltype: + if mode: + if mode == "comment_blockend": + mode = "comment_block" + thisblock = ( (beginline, nline-1), ""+mode , ".".join(blockdesc)[:32]) + blockdesc = [] + bblocks.append(thisblock) + mode = ltype + beginline = nline - 1 + + words = re.split(r'\W+',line) + if words: + text = " ".join(words) + text = text.strip() + text = text.replace(" ", "-") + if len(text)>1: + blockdesc.append(text) + + if mode: + if mode == "comment_blockend": + mode = "comment_block" + + thisblock = ( (beginline, nline),""+mode, ".".join(blockdesc)[:32] ) + bblocks.append(thisblock) + blockdesc = [] + #print ltype, line, + + for lines, bname, desc in bblocks: + #print lines, " ", "%s:%s" % (bname, desc) + startline, endline = lines + name = "#..%s:%s" % (bname, desc) + + self.computedBlocks.append((startline, endline,name)) + fout.write("%d\t%d\t%s\n" % (startline, endline,name)) + #print "#..%s:%s" % (bname, desc) + #print sB, + #print ">>>>" + if desde is None: break + initline = linenum + while linenum < len(linePosChar) and linePosChar[linenum+1] 2: + print(linePosChar[startline] - desde, linePosChar[endline-1] - hasta, linePosChar[endline] - hasta, bl_name) + name = bl_name + #print (startline, endline), name, (linePosChar[startline], linePosChar[endline]), pk, (hasta-desde)+1 + self.computedBlocks.append((startline, endline,name)) + fout.write("%d\t%d\t%s\n" % (startline, endline,name)) + #print "%s" % name + antdesde, anthasta = pk + if anthasta <= linePosChar[linenum-1] + 1: + anthasta = linePosChar[linenum-1] + 1 + elif anthasta <= linePosChar[linenum] + 1: + anthasta = linePosChar[linenum] + 1 + + fB.close() + fout.close() + + +def linejunk(line): + line = line.strip() + if len(line)<4: return True + if line[2] == "//": return True + return False + +def charjunk(char): + junk = [" ", "\t"] + if char in junk: return True + return False + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + + filenames = [x for x in args if os.path.isfile(x)] + not_a_file = set(args) - set(filenames) + if len(not_a_file): + print("WARNING: Not a file:", ", ".join(not_a_file)) + + if len(filenames): + pfiles = [] + for file1 in filenames: + #print "File:", file1 + pf = processedFile(file1, debug = options.debug) + pfiles.append(pf) + #pfA = pfiles[0] + #for pfB in pfiles[1:]: + # feq = FindEquivalences(pfA, pfB) + +def tree_parents(pk): + parents = [] + while (len(pk)>0): + parents.append(pk) + pk = pk[:-1] + return parents + +class FindEquivalences(object): + def __init__(self,pfA, pfB, autoCompute = True): + self.equivalences = {} + self.parent_equivalences = {} + self.max_known_eq = {} + self.pfA, self.pfB = pfA, pfB + if autoCompute: self.compute() + + def compute(self): + print("Finding equivalences between A (%s) -> B (%s):" % ( + self.pfA.filename, + self.pfB.filename + )) + print("Modified names:") + commonNames = sorted(list(self.pfA.sNames & self.pfB.sNames)) + for name in commonNames: + if len(self.pfA.names[name]) > 1 or len(self.pfB.names[name]) > 1: + print("-", name,"(%d,%d)" % (len(self.pfA.names[name]),len(self.pfB.names[name]))) + else: + keyA = self.pfA.idxtree[self.pfA.names[name][0]] + rowA = self.pfA.table[keyA] + keyB = self.pfB.idxtree[self.pfB.names[name][0]] + rowB = self.pfB.table[keyB] + if rowA['hash'] != rowB['hash']: + print("###", name,"###") + fileA = self.pfA.filename.replace(".hash","") + fileB = self.pfB.filename.replace(".hash","") + if os.path.isfile(fileA) and os.path.isfile(fileB): + fA = open(fileA) + fA.seek(keyA[0]) + sA = fA.read(keyA[1]-keyA[0]+1) + fA.close() + + fB = open(fileB) + fB.seek(keyB[0]) + sB = fB.read(keyB[1]-keyB[0]+1) + fB.close() + + sA = sA.replace("\t", " ") + sB = sB.replace("\t", " ") + lines = list(difflib.ndiff(sA.splitlines(1), sB.splitlines(1),linejunk,charjunk)) + modifiedlines = [] + for n,line in enumerate(lines): + if line[0] != " ": + modifiedlines.append(n) + ml = 0 + omit = [] + Context = 3 + n = 0 + for line in lines: + d = [] + for m in modifiedlines: + dt = abs(m-n) + d.append(dt) + if dt < Context: break + dmin = min(d) + if dmin < Context: + if len(omit) : + if len(omit)>= Context: + for line in omit: + if line[0] in (' ','+'): n+=1 + print(" ", "(... %d lines ommitted ...)" % len(omit)) + else: + for line in omit: + if line[0] in (' ','+'): n+=1 + print("%03d" % n , line, end=' ') + omit = [] + if line[0] in (' ','+'): + n+=1 + print("%03d" % n , line, end=' ') + else: + print("%03d" % (n+1) , line, end=' ') + else: + omit.append(line) + if len(omit) : + if len(omit)>= Context: + for line in omit: + if line[0] in (' ','+'): n+=1 + print(" ", "(... %d lines ommitted ...)" % len(omit)) + else: + for line in omit: + if line[0] in (' ','+'): n+=1 + print("%03d" % n , line, end=' ') + omit = [] + print() + else: + print("(diff ommitted because we couldn't find original files)") + + print() + print("Deleted names:") + deletedNames = sorted(list(self.pfA.sNames - self.pfB.sNames)) + for name in deletedNames: + print("-", name) + print() + print("Added names:") + addedNames = sorted(list(self.pfB.sNames - self.pfA.sNames)) + for name in addedNames: + print("-", name) + print() + + return + + + + + + + for key in self.pfA.list_hashes: + if key in self.pfB.hashes: + lpkA = self.pfA.hashes[key] + lpkB = self.pfB.hashes[key] + self.addEquivalences(lpkA,lpkB) + # print "Found:", self.pfA.hashes[key] , "==>", self.pfB.hashes[key] + #else: + # print "Lost:", self.pfA.hashes[key] , "==>", "???" + + for pkA in self.pfA.idxtree: + parentA = pkA[:-1] + if pkA not in self.equivalences: + self.equivalences[pkA] = {} + if parentA: + if parentA not in self.parent_equivalences: + self.parent_equivalences[parentA] = [] + + for pkA in sorted(self.pfA.idxtree): + parentsA = tree_parents(pkA) + for pkB, punt in self.equivalences[pkA].items(): + if len(pkA) != len(pkB): continue + parentsB = tree_parents(pkB) + parentsAB = list(zip(parentsA,parentsB)) + for pA, pB in parentsAB: + sz2a, sz2b = self.pfA.idxtree[pkA] + sz2 = sz2b - sz2a + 1 + sz1a, sz1b = self.pfA.idxtree[pA] + sz1 = sz1b - sz1a + 1 + lev2 = 1.0 + old_div(sz1, float(sz2)) + #lev2 = 2**(len(pkA) - len(pA)) + #lev2_list = [ pC for pC in self.pfA.idxtree if len(pC) >= len(pA) and pC[:len(pA)] == pA ] + #lev2_plist = set([]) + #for l in lev2_list: + # lev2_plist |= set(tree_parents(l)[1:]) + #lev2 = len(set(lev2_list) - set(lev2_plist)) + + pEq = (pB, old_div(float(punt),lev2)) + if pA not in self.parent_equivalences: + self.parent_equivalences[pA] = [] + self.parent_equivalences[pA].append(pEq) + + norepeat = (0,) + self.parent_equivalences2 = {} + for pA in sorted(self.parent_equivalences): + count = {} + if pA[:len(norepeat)] == norepeat: continue + for pB, punt in self.parent_equivalences[pA]: + if pB not in count: count[pB] = 0 + count[pB] += punt + rcount = [] + ppA = pA[:-1] + if ppA in self.parent_equivalences2: + ppB = self.parent_equivalences2[ppA] + else: + ppB = None + + rowA = self.pfA.table[self.pfA.idxtree[pA]] + for key, punt in count.copy().items(): + if ppB and key[:-1] != ppB: continue + rowB = self.pfB.table[self.pfB.idxtree[pB]] + nameA = self.pfA.fullQName(pA) #rowA['name'].split(":") + nameB = self.pfB.fullQName(pB) #rowB['name'].split(":") + s = difflib.SequenceMatcher() + s.set_seqs(nameA,nameB) + t = s.quick_ratio() + t -= 0.8 + if t < 0: t = 0 + t *= 10.0 + punt *= t + #if nameA[0] != nameB[0]: punt /=3.0 + #if nameA[1] != nameB[1]: punt /=1.0+len(nameA[1]) / 40.0+len(nameB[1]) / 40.0 + if punt >= 0.50: + rcount.append((round(punt*100),key)) + + if len(rcount): + punt, pB = max(rcount) + self.parent_equivalences2[pA] = pB + rowB = self.pfB.table[self.pfB.idxtree[pB]] + print("parent:", pA, self.pfA.fullQName(pA), "%d%%\t" % punt, pB, len(rcount) , self.pfB.fullQName(pB)) + if punt > 100: + norepeat = pA + else: + if len(pA) == 1: + print("parent:", pA, self.pfA.fullQName(pA), "0%\t ???") + + """ + norepeat = (0,) + prevprint = None + for pkA in sorted(self.pfA.idxtree): + if len(pkA) > 1: continue + if pkA[:len(norepeat)] == norepeat: continue + if len(self.equivalences[pkA])==0: + if len(pkA) > 1: + if prevprint is None: continue + else: + if len(pkA) > len(prevprint) and prevprint[:-1] == pkA[:len(prevprint)-1]: continue + elif pkA < len(prevprint): prevprint = None + elif prevprint[:-1] != pkA[:len(prevprint)-1]: prevprint = None + + print pkA,":", + + for pkB, punt in self.equivalences[pkA].iteritems(): + if punt > 0.96: + norepeat = pkA + if punt >= 0.1: + print pkB, punt, ";", + prevprint = pkA + print + """ + """ + for pkB, punt in self.equivalences[pkA].iteritems(): + if punt > 0.96: + norepeat = pkA + print ">>", ".".join(["%02d" % x for x in pkB]) + prevprint = pkA + """ + + + + def getMaxKnown(self,pkA): + pkA = tuple(pkA) + if len(pkA) == 0: return 1.0, None + if pkA not in self.max_known_eq: return 0.0, None + pkB = self.max_known_eq[pkA] + eq_prob = self.equivalences[pkA][pkB] + return eq_prob, pkB + + + def addEquivalences(self,lpkA,lpkB): + lstEquivalences = self.multiplyEquivalences(lpkA,lpkB) + base_probability = old_div(1.0, len(lstEquivalences)) + if base_probability < 0.01: return + + for pkA,pkB in lstEquivalences: + probability = base_probability + parentA = pkA[:-1] + parent_prob, parentB = self.getMaxKnown(parentA) + if parentB: + if parentB != pkB[:-1]: parent_prob = old_div((1-parent_prob), 2.0) + probability *= parent_prob + + parentB = pkB[:-1] + if probability < 0.01: continue + if pkA not in self.equivalences: + self.equivalences[pkA] = {} + if pkB in self.equivalences[pkA]: + print("DUPLICATE", pkA,pkB) + self.equivalences[pkA][pkB] = probability + previousMax, prevPkB = self.getMaxKnown(pkA) + if probability > previousMax: + self.max_known_eq[pkA] = pkB + + + + def multiplyEquivalences(self,lpkA,lpkB): + leq = set([]) + for pkA in lpkA: + for pkB in lpkB: + leq|=set([(pkA,pkB)]) + return list(leq) + + +def process(filename): + table, idxdepth,idxtree = load(filename) + #print table.items()[:10] + treebydepth = {} + + for k in sorted(idxtree.keys()): + td = len(k)-1 + if td not in treebydepth: treebydepth[td] = [] + treebydepth[td].append(k) + nitems = 0 + maxitems = 0 + maxd = 0 + hashes = {} + list_hashes = [] + + for d,idx in treebydepth.items(): + if d > 2: break + nitems = len(idx) + if maxitems < nitems: maxitems = nitems + elif nitems * 2 < maxitems: break + #print "Depth:", d, "(%d items)" % nitems + maxd = d + for k in idx: + #print ".".join([str(x) for x in k]) + pk = idxtree[k] + r = table[pk] + rhash = r["hash"] + if rhash not in hashes: + list_hashes.append(rhash) + hashes[rhash] = [] + hashes[rhash].append(k) + return table, idxdepth,idxtree, hashes, list_hashes + +def isinside(parent, child): + pfrom, pto = parent + cfrom, cto = child + pto += 1 + if cfrom >= pfrom and cto <= pto: return 0 + if cfrom < pfrom and cto < pfrom: return -1 + if cfrom > pto and cto > pto: return 1 + #print "ERROR:", child , " is superior to ", parent + return 0 + +def load(filename): + file = open(filename) + def getpk(row): + return (row["start"],row["end"]) + + fields = [ + "depth", + "hash", + "start", + "end", + "name", + "len" + ] + rows = {} + intfields = [ + "depth", + "start", + "end", + "len" + ] + bydepth = {} + for line in file: + row = dict(list(zip(fields,line[:-1].split("\t")))) + for f in intfields: row[f]=int(row[f]) + pk = getpk(row) + depth = row["depth"] + if depth not in bydepth: bydepth[depth] = [] + bydepth[depth].append(pk) + rows[pk]=row + + for dpth, items in bydepth.items(): + bydepth[dpth] = list(sorted(items)) + + idxtree = {} + for currdepth in bydepth: + n = 0 + #print "Depth:", currdepth + if currdepth == 0: + nparent = [] + else: + pdepth = currdepth - 1 + np = 0 + + for pk in bydepth[currdepth]: + n+=1 + if currdepth > 0: + offset = 99 + it = 0 + while offset != 0: + it += 1 + assert(np >=0) + assert(np = 0) + #if it > 100: + # print it,n,pk,np, ppk, offset + # if offset < 0: + # print list(enumerate(bydepth[pdepth])) + # assert(offset >= 0) + # assert(it < 250) + + prow = rows[ppk] + nparent = prow["tree_id"] + + + row = rows[pk] + tree_id = nparent + [n] + row["tree_id"] = tree_id + if tuple(tree_id) in idxtree: + print("ERROR:", tuple(tree_id), " is duplicated:") + print("previous:", idxtree[tuple(tree_id)]) + print("new:" , pk) + else: + idxtree[tuple(tree_id)] = pk + + + + + return rows, bydepth, idxtree + + +if __name__ == "__main__": main() \ No newline at end of file diff --git a/flscriptparse.py b/flscriptparse.py new file mode 100644 index 0000000..1c75733 --- /dev/null +++ b/flscriptparse.py @@ -0,0 +1,845 @@ +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from builtins import input +from builtins import str +from builtins import range +from past.utils import old_div +# ----------------------------------------------------------------------------- +# flscriptparse.py +# +# Simple parser for FacturaLUX SCripting Language (QSA). +# ----------------------------------------------------------------------------- +from optparse import OptionParser + +import sys, math +import hashlib +from pineboolib.flparser import flex +import ply.yacc as yacc +import ply.lex as lex + +from pineboolib.flparser.flclasses import * + +# Get the token map +tokens = flex.tokens +start = "source" + +reserv=['nonassoc'] +reserv+=list(flex.reserved) + +endoffile = None + +def cnvrt(val): + val = str(val) + val = val.replace('&','&') + val = val.replace('"','"') + val = val.replace("'",''') + val = val.replace("<",'<') + val = val.replace(">",'>') + return val + +precedence = ( + ('nonassoc', 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL'), + ('left','LOR', 'LAND'), + ('left', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE'), + ('right', 'LNOT'), + ('left', 'PLUS', 'MINUS'), + ('left', 'TIMES', 'DIVIDE', 'MOD'), + ('left', 'OR', 'AND', 'XOR', 'LSHIFT', 'RSHIFT'), + +) +seen_tokens = [] + +def p_parse(token): + ''' + exprval : constant + | variable + | funccall + | error + + identifier : ID + + dictobject_value : LBRACE RBRACE + | LBRACE dictobject_value_elemlist RBRACE + + dictobject_value_elemlist : dictobject_value_elem + | dictobject_value_elemlist COMMA dictobject_value_elem + + dictobject_value_elem : exprval COLON expression + + base_expression : exprval + | inlinestoreinstruction + | base_expression mathoperator base_expression + | base_expression cmp_symbol base_expression + | base_expression boolcmp_symbol base_expression + | parentheses + | unary_operator + | new_operator + | ternary_operator + | dictobject_value + | typeof_operator + + + + + parentheses : LPAREN base_expression RPAREN + | LPAREN variable_1 RPAREN + + unary_operator : LNOT base_expression + | MINUS base_expression + | PLUS base_expression + + new_operator : NEW funccall_1 + | NEW identifier + + typeof_operator : TYPEOF variable + | TYPEOF base_expression + + ternary_operator : base_expression CONDITIONAL1 base_expression COLON base_expression + + expression : base_expression + | funcdeclaration_anon + | error + + case_cblock_list : case_block + case_cblock_list : case_cblock_list case_block + + case_block : CASE expression COLON statement_list + + case_default : DEFAULT COLON statement_list + + case_block_list : empty + case_block_list : case_default + case_block_list : case_cblock_list + case_block_list : case_cblock_list case_default + + source_element : docstring + | vardeclaration + | classdeclaration + | funcdeclaration + + source : source_element + source : source source_element + | statement_list + + + basicsource : statement_list + | empty + + statement : instruction + | vardeclaration + | ifstatement + | whilestatement + | dowhilestatement + | withstatement + | forstatement + | forinstatement + | switch + | trycatch + + statement_list : statement_list statement + + statement_list : statement + + statement_list : LBRACE statement_list RBRACE + + statement_list : LBRACE RBRACE + statement_list : empty + + optvartype : COLON ID + | empty + + vardeclaration : VAR vardecl_list SEMI + | CONST vardecl_list SEMI + vardeclaration : VAR vardecl_list + | CONST vardecl_list + | STATIC VAR vardecl_list + + vardecl : ID optvartype EQUALS expression + vardecl : ID optvartype + + vardecl_list : vardecl + | vardecl_list COMMA vardecl + + arglist : vardecl_list + | + + funcdeclaration : FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE + funcdeclaration : STATIC FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE + funcdeclaration_anon : FUNCTION LPAREN arglist RPAREN LBRACE basicsource RBRACE + + callarg : expression + + callargs : callarg + | callargs COMMA callarg + | empty + + varmemcall : variable_1 + | funccall_1 + | member_call + | member_var + | base_expression + + member_var : varmemcall PERIOD variable_1 + member_call : LPAREN member_var RPAREN PERIOD funccall_1 + + member_call : varmemcall PERIOD funccall_1 + member_call : LPAREN member_call RPAREN PERIOD funccall_1 + + funccall : funccall_1 + | member_call + | LPAREN member_call RPAREN + | LPAREN funccall_1 RPAREN + | LPAREN error RPAREN + + funccall_1 : ID LPAREN callargs RPAREN + | ID LPAREN RPAREN + | TYPEOF LPAREN callargs RPAREN + + mathoperator : PLUS + | MINUS + | TIMES + | DIVIDE + | MOD + | XOR + | OR + | LSHIFT + | RSHIFT + | AND + + variable : variable_1 + | member_var + | LPAREN variable_1 RPAREN + | LPAREN member_var RPAREN + + variable_1 : identifier + | array_member + + array_member : variable_1 LBRACKET expression RBRACKET + | funccall_1 LBRACKET expression RBRACKET + + inlinestoreinstruction : PLUSPLUS variable + | MINUSMINUS variable + | variable PLUSPLUS + | variable MINUSMINUS + + updateoperator : EQUALS + | PLUSEQUAL + | MINUSEQUAL + | MODEQUAL + | DIVEQUAL + | TIMESEQUAL + + updateinstruction : variable updateoperator expression + | variable updateoperator updateinstruction + + deleteinstruction : DELETE variable + + storeinstruction : inlinestoreinstruction + | updateinstruction + | deleteinstruction + + + flowinstruction : RETURN expression + | THROW expression + | RETURN + | BREAK + | CONTINUE + + instruction : base_instruction SEMI + | SEMI + | base_instruction + | funcdeclaration + | error SEMI + + callinstruction : funccall + | variable + + base_instruction : storeinstruction + | callinstruction + | flowinstruction + + varorcall : variable + | funccall + | base_expression + + optextends : EXTENDS ID + | empty + + classdeclaration : CLASS ID optextends LBRACE classdeclarationsource RBRACE + + classdeclarationsource : vardeclaration + | funcdeclaration + | classdeclarationsource vardeclaration + | classdeclarationsource funcdeclaration + | SEMI + | classdeclarationsource SEMI + + docstring : DOCSTRINGOPEN AT ID COMMENTCLOSE + | DOCSTRINGOPEN AT ID ID COMMENTCLOSE + + list_constant : LBRACKET callargs RBRACKET + list_constant : LBRACKET callargs COMMA RBRACKET + + constant : ICONST + | FCONST + | CCONST + | SCONST + | regex + | list_constant + + regex : DIVIDE regexbody DIVIDE regexflags + | DIVIDE regexbody COMMENTCLOSE regexflags + + regexbody : regexchar + | regexbody regexchar + + regexchar : LPAREN + | RPAREN + | ID + | COMMA + | XOR + | LBRACKET + | RBRACKET + | ICONST + | PLUS + | MINUS + | LBRACE + | RBRACE + | DOLLAR + | SQOUTE + | DQOUTE + | BACKSLASH + | SCONST + | error + + regexflags : ID + | empty + + + statement_block : statement + | LBRACE statement_list RBRACE + | LBRACE RBRACE + + optelse : ELSE statement_block + | empty + + cmp_symbol : LT + | LE + | GT + | GE + | EQ + | NE + | IN + + boolcmp_symbol : LOR + | LAND + + condition : expression + | error + + ifstatement : IF LPAREN condition RPAREN statement_block optelse + + whilestatement : WHILE LPAREN condition RPAREN statement_block + dowhilestatement : DO statement_block WHILE LPAREN condition RPAREN SEMI + + withstatement : WITH LPAREN variable RPAREN statement_block + | error + + storeormember : storeinstruction + | member_var + + for_initialize : storeinstruction + | VAR vardecl + | for_initialize COMMA for_initialize + | empty + + for_compare : expression + | empty + + for_increment : storeormember + | for_increment COMMA for_increment + | empty + + + forstatement : FOR LPAREN for_initialize SEMI for_compare SEMI for_increment RPAREN statement_block + | error + + forinstatement : FOR LPAREN for_initialize IN varorcall RPAREN statement_block + | FOR LPAREN variable IN varorcall RPAREN statement_block + | error + + switch : SWITCH LPAREN condition RPAREN LBRACE case_block_list RBRACE + + optid : ID + | empty + + trycatch : TRY statement_block CATCH LPAREN optid RPAREN statement_block + + empty : + ''' + global input_data + + lexspan = list(token.lexspan(0)) + data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) + if len(lexspan) == 2: + fromline = token.lineno(0) + global endoffile + endoffile = fromline, lexspan, token.slice[0] + #print fromline, lexspan, token.slice[0] + token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } + numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) + + rspan = lexspan[0] + if str(token.slice[0]) == 'empty' or numelems == 0: token[0] = None + else: + rvalues = [] + for n,s in enumerate(token.slice[1:]): + if s.type != 'empty' and s.value is not None: + val = None + if isinstance(s.value,str): + val = token.lexspan(n+1)[0] + len(s.value) - 1 + else: + val = token.lexspan(n+1)[1] + rvalues.append(val) + rspan = max(rvalues) + lexspan[1] = rspan + + if str(token.slice[0]) == 'regexbody': + token[0] = { "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } + + #if str(token.slice[0]) == 'regex': + # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] + # print " " + "\n ".join([ "%s(%r): %r" % (s.type, token.lexspan(n+1), s.value) for n,s in enumerate(token.slice[1:]) ]) + global seen_tokens, last_ok_token + last_ok_token = token + seen_tokens.append((str(token.slice[0]), token.lineno(0),input_data[lexspan[0]:lexspan[1]+1] )) + global ok_count + ok_count += 1 + +last_ok_token = None +error_count = 0 +last_error_token = None +last_error_line = -1 +ok_count = 0 + +def p_error(t): + global error_count + global ok_count + global last_error_token + global last_error_line, seen_tokens , last_ok_token + debug = False # Poner a True para toneladas de debug. + # if error_count == 0: print + if t is not None: + if last_error_token is None or t.lexpos != getattr(last_error_token,"lexpos",None): + if abs(last_error_line - t.lineno) > 4 and ok_count > 1 and error_count < 4: + error_count += 1 + try: print_context(t) + except: pass + if debug == True: + for tokname, tokln, tokdata in seen_tokens[-32:]: + if tokln == t.lineno: + print(tokname, tokdata) + print(repr(last_ok_token[0])) + last_error_line = t.lineno + elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: + last_error_line = t.lineno + yacc.errok() + ok_count = 0 + return + + ok_count = 0 + if t is None: + if last_error_token != "EOF": + print("ERROR: End of the file reached.") + global endoffile + print("Last data:", endoffile) + + last_error_token = "EOF" + return t + t = yacc.token() + yacc.restart() + last_error_token = t + return t + + +# Build the grammar + + +parser = yacc.yacc(method='LALR',debug=0, + optimize = 1, write_tables = 1, debugfile = '/tmp/yaccdebug.txt',outputdir='/tmp/') + +#profile.run("yacc.yacc(method='LALR')") + +global input_data + +def print_context(token): + global input_data + if token is None: return + last_cr = input_data.rfind('\n',0,token.lexpos) + next_cr = input_data.find('\n',token.lexpos) + column = (token.lexpos - last_cr) + column1 = (token.lexpos - last_cr) + while column1 < 16: + column1 = (token.lexpos - last_cr) + last_cr = input_data.rfind('\n',0,last_cr-1) + + print(input_data[last_cr:next_cr].replace("\t"," ")) + print((" " * (column-1)) + "^", column, "#ERROR#" , token) + + +def my_tokenfunc(*args, **kwargs): + #print("Call token:" ,args, kwargs) + ret = lex.lexer.token(*args, **kwargs) + #print "Return (",args, kwargs,") = " , ret + return ret + + +def print_tokentree(token, depth = 0): + print(" " * depth, token.__class__ , "=" , token) + + if str(token.__class__) == "ply.yacc.YaccProduction": + print(token.lexer) + for tk in token.slice: + if tk.value == token: continue + print(" " * (depth+1), tk.type, end=' ') + try: + print(tk.lexpos, end=' ') + print(tk.endlexpos, end=' ') + except: + pass + print() + + print_tokentree(tk.value, depth +1) + +def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): + #if depth > 5: return + source_data = [ + 'source', + 'source_element', + 'statement_list', + 'statement', + 'classdeclarationsource', + 'vardecl_list', + ] + final_obj = {} + final_obj['range'] = obj['02-size'] + has_data = 0 + has_objects = 0 + contentlist = [] + + if alias_mode == 0: + ctype_alias = {} + elif alias_mode == 1: + ctype_alias = { + "member_var" : "member", + "member_call" : "member", + "variable_1" : "variable", + "funccall_1" : "funccall", + "flowinstruction" : "instruction", + "storeequalinstruction" : "instruction", + "vardecl" : "vardeclaration", + #"vardecl_list" : "vardeclaration", + + } + else: + raise ValueError("alias_mode unknown") + + if otype in ctype_alias: + otype = ctype_alias[otype] + #print " " * depth , obj['02-size'] + for n,content in enumerate(obj['50-contents']): + ctype = content['01-type'] + value = content['99-value'] + if ctype in ctype_alias: + ctype = ctype_alias[ctype] + #if ctype in source_data: + # if depth == 0: print "--" + # print_tree(value,depth,num) + # continue + #print " " * depth , "%s:" % ".".join(num+[str(n)]), ctype, + + if type(value) is dict: + #print "*" + if depth < 600: + tree_obj = calctree(value,depth+1,num+[str(n)], ctype, alias_mode=alias_mode) + else: + tree_obj = None + if type(tree_obj) is dict: + if (tree_obj['has_data'] or alias_mode == 0) and ctype != otype : + contentlist.append([ctype,tree_obj]) + has_objects += 1 + else: + contentlist+=tree_obj["content"] + has_data += tree_obj["has_data"] + has_objects += tree_obj["has_objects"] + else: + #print "=", repr(value) + contentlist.append([ctype,value]) + has_data += 1 + + final_obj['content'] = contentlist + final_obj['has_data'] = has_data + final_obj['has_objects'] = has_objects + + return final_obj + + +hashes = [] +ranges = [] +def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdout): + global hashes, ranges + if depth == 0: + hashes = [] + ranges = [] + + sep = " " + marginblocks = { + "classdeclaration" : 1, + "funcdeclaration" : 1, + "statement_block" : 1, + #"instruction" : 1, + } + closingtokens = [ + "RBRACE", + "RPAREN", + "RBRACKET", + "SEMI", + ] + nuevalinea = False + name = "" + lines = [] + + + + for ctype, value in tree['content']: + if nuevalinea and ctype in closingtokens: + nuevalinea = False + + if nuevalinea: + for i in range(int(math.ceil(old_div(l,2.0)))): lines.append(sep * depth) + nuevalinea = False + + if type(value) is dict and ctype == otype: + tname,tlines,trange = printtree(value, depth, ctype) + if name == "" and tname: + name = tname + + lines += tlines + elif type(value) is dict: + l = 0 + if ctype in marginblocks: l = marginblocks[ctype] + + for i in range(int(math.floor(old_div(l,2.0)))): lines.append(sep * depth) + tname,tlines,trange = printtree(value, depth+1, ctype) + # lines.append(sep * depth + "" % (len("".join(tlines)))) + + if value['has_data'] > 0 and value['has_objects'] == 0 and False: + # Do it inline! + if value['has_data']==1 and tname: + lines.append(sep * depth + "<%s id=\"%s\" />" % (ctype,tname)) + else: + txt = "".join([ x.strip() for x in tlines]) + lines.append(sep * depth + "<%s>%s" % (ctype,txt,ctype)) + else: + attrs = [] + if tname: + attrs.append(("id",tname)) + + txtinline = "".join([ line.strip() for line in tlines ]) + + #if len(tlines)>1: + txthash = hashlib.sha1(txtinline).hexdigest()[:16] + #hashes.append(("depth:",depth,"hash:",txthash,"element:",ctype+":"+tname)) + hashes.append((txthash,ctype+":"+tname+"(%d)"% len(txtinline))) + ranges.append([depth,txthash]+trange+[ctype+":"+tname,len(txtinline)]) + #,"start:",trange[0],"end:",trange[1])) + #attrs.append(("start",trange[0])) + #attrs.append(("end",trange[1])) + #attrs.append(("hash",txthash)) + + txtattrs="" + for name1, val1 in attrs: + txtattrs+=" %s=\"%s\"" % (name1,cnvrt(val1)) + + lines.append(sep * depth + "<%s%s>" % (ctype,txtattrs)) + if depth > 50: + lines.append(sep * (depth+1) + "...") + else: + if len(txtinline)<80: + lines.append(sep * (depth+1) + txtinline) + else: + lines+=tlines + if txtattrs: + txtattrs = "" % txtattrs + lines.append(sep * depth + "" % (ctype)) + + nuevalinea = True + else: + if ctype == "ID" and name == "": + name = value + if ctype in flex.token_literals: + lines.append(sep * depth + "<%s value=\"%s\" />" % (ctype,cnvrt(value))) + else: + lines.append(sep * depth + "<%s />" % (ctype)) + + + if mode == "hash": + #print "\n".join(lines) + for row in sorted(ranges): + output.write("\t".join([ str(x) for x in row])) + output.write("\n") + output.flush() + if mode == "xml": + for row in lines: + output.write(row) + output.write("\n") + output.flush() + + return name, lines, tree['range'] + + + +def parse(data): + global input_data + global error_count + global seen_tokens + seen_tokens[:] = [] + parser.error = 0 + input_data = data + flex.lexer.lineno = 1 + error_count = 0 + p = parser.parse(data, debug = 0, tracking = 1, tokenfunc = my_tokenfunc) + if error_count > 0: + print("ERRORS (%d)" % error_count) + if p is None: return p + try: + p["error_count"] = error_count + except Exception as e: + print(e) + return None + + if parser.error: + return None + return p + + +def main(): + global start + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-O", "--output", dest="output", default = "none", + help="Set output TYPE: xml|hash", metavar="TYPE") + parser.add_option("--start", dest="start", default = None, + help="Set start block", metavar="STMT") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + if options.start: + start = options.start + print("Start setted to:" , start) + + + + + def do_it(): + if options.output == "none": return + tree_data = calctree(prog) + if options.output == "hash": + printtree(tree_data, mode = "hash") + elif options.output == "xml": + printtree(tree_data, mode = "xml") + elif options.output == "file": + f1_hash = open(filename+".hash","w") + printtree(tree_data, mode = "hash", output = f1_hash) + f1_hash.close() + + f1_xml = open(filename+".xml","w") + printtree(tree_data, mode = "xml", output = f1_xml) + f1_xml.close() + elif options.output == "yaml": + import yaml + print(yaml.dump(tree_data['content'])) + + else: + print("Unknown outputmode", options.output) + + prog = "$$$" + if len(args) > 0 : + for filename in args: + fs = filename.split("/") + sys.stderr.write("Loading %s ..." % fs[-1]) + sys.stderr.flush() + data = open(filename).read() + sys.stderr.write(" parsing ...") + sys.stderr.flush() + prog = parse(data) + sys.stderr.write(" formatting ...") + sys.stderr.flush() + if prog: do_it() + sys.stderr.write(" Done.\n") + sys.stderr.flush() + + else: + + + line = "" + while 1: + try: + line1 = input("flscript> ") + if line1.startswith("#"): + comm = line1[1:].split(" ") + if comm[0] == "setstart": + start = comm[1] + print("Start setted to:" , start) + if comm[0] == "parse": + print() + prog = parse(line) + line = "" + else: + line += line1 + except EOFError: + break; + line += "\n" + print() + prog = parse(line) + do_it() + """ + import yaml + + print yaml.dump(tree_data) + """ + #print_tokentree(prog) + + + + + #for varName in prog.byDefName: + # var = prog.byDefName[varName] + # print "%-15s / %-15s > " % var.type , varName + + + #import tests.ifaceclass + #tests.ifaceclass.do_test(prog) + + + + + + + +if __name__ == "__main__": main() diff --git a/postparse.py b/postparse.py new file mode 100644 index 0000000..10afa18 --- /dev/null +++ b/postparse.py @@ -0,0 +1,564 @@ +#!/usr/bin/python +from __future__ import print_function +from __future__ import absolute_import +from builtins import str +from builtins import object +from optparse import OptionParser +import os, os.path, sys +from pineboolib.flparser import flscriptparse +import imp, traceback +from lxml import etree +from future.utils import with_metaclass +USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") + +KNOWN_PARSERS = {} +UNKNOWN_PARSERS = {} +def parse_for(*tagnames): + global KNOWN_PARSERS + def decorator(fn): + for n in tagnames: + KNOWN_PARSERS[n] = fn + return fn + return decorator + +def parse(tagname, treedata): + global KNOWN_PARSERS,UNKNOWN_PARSERS + if tagname not in KNOWN_PARSERS: + UNKNOWN_PARSERS[tagname] = 1 + fn = parse_unknown + else: + fn = KNOWN_PARSERS[tagname] + return fn(tagname,treedata) + + +def getxmltagname(tagname): + if tagname == "source": return "Source" + if tagname == "funcdeclaration": return "Function" + if tagname == "classdeclaration": return "Class" + if tagname == "vardeclaration": return "Variable" + return "Unknown.%s" % tagname + +xml_class_types = [] + +class TagObjectFactory(type): + def __init__(cls, name, bases, dct): + global xml_class_types + xml_class_types.append(cls) + super(TagObjectFactory, cls).__init__(name, bases, dct) + +class TagObject(with_metaclass(TagObjectFactory, object)): + tags = [] + set_child_argn = False + name_is_first_id = False + debug_other = True + adopt_childs_tags = [] + omit_tags = ['empty'] + callback_subelem = {} + promote_child_if_alone = False + + @classmethod + def tagname(self, tagname): return self.__name__ + + @classmethod + def can_process_tag(self, tagname): return tagname in self.tags + + def __init__(self, tagname): + self.astname = tagname + self.xml = etree.Element(self.tagname(tagname)) + self.xmlname = None + self.subelems = [] + self.values = [] + if self.name_is_first_id: + self.xml.set("name","") + + def adopt_children(self, argn, subelem): + for child in subelem.xml.iterchildren(): + if self.set_child_argn: child.set("argn",str(argn)) + else: + if 'argn' in child.attrib: del child.attrib['argn'] + self.xml.append(child) + + def omit_subelem(self, argn, subelem): + return + + def is_in(self, listobj): + return self.__class__ in listobj or self.astname in listobj + + def get(self, listobj, default = None): + if self.__class__ in listobj: return listobj[self.__class__] + if self.astname in listobj: return listobj[self.astname] + return default + + + def add_subelem(self, argn, subelem): + if subelem.is_in(self.omit_tags): return self.omit_subelem(argn, subelem) + if subelem.is_in(self.adopt_childs_tags): return self.adopt_children(argn, subelem) + callback = subelem.get(self.callback_subelem) + if callback: return getattr(self,callback)(argn,subelem) + + if self.set_child_argn: subelem.xml.set("argn",str(argn)) + self.xml.append(subelem.xml) + self.subelems.append(subelem) + + def add_value(self, argn, vtype, value): + self.values.append( (vtype, value) ) + if vtype == "ID" and self.name_is_first_id and self.xmlname is None: + self.xmlname = value + self.xml.set("name",value) + return + + self.xml.set("arg%02d" % argn,vtype + ":" + repr(value)) + + def add_other(self, argn, vtype, data): + if self.debug_other: + self.xml.set("arg%02d" % argn,vtype) + + def polish(self): + if self.promote_child_if_alone: + if len(self.values) == 0 and len(self.subelems) == 1: + return self.subelems[0] + return self + + +class ListObject(TagObject): + set_child_argn = False + debug_other = False + +class NamedObject(TagObject): + name_is_first_id = True + debug_other = False + +class ListNamedObject(TagObject): + name_is_first_id = True + set_child_argn = False + debug_other = False + +class TypedObject(ListObject): + type_arg = 0 + def add_other(self, argn, vtype, value): + if argn == self.type_arg: + self.xml.set("type", vtype) + + +class Source(ListObject): + tags = ["source","basicsource","classdeclarationsource","statement_list","statement_block"] + adopt_childs_tags = ['source_element','statement_list','statement',"statement_block"] + +class Identifier(NamedObject): + tags = ["identifier","optid"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + + +class Arguments(ListObject): + tags = ["arglist"] + adopt_childs_tags = ['vardecl_list'] + +class VariableType(NamedObject): + tags = ["optvartype"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + +class ExtendsType(NamedObject): + tags = ["optextends"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + +class Function(ListNamedObject): + tags = ["funcdeclaration"] + callback_subelem = ListNamedObject.callback_subelem.copy() + callback_subelem[VariableType] = "add_vartype" + + def add_vartype(self, argn, subelem): + self.xml.set("returns", str(subelem.xmlname)) + +class Variable(NamedObject): + tags = ["vardecl"] + callback_subelem = NamedObject.callback_subelem.copy() + callback_subelem[VariableType] = "add_vartype" + + def add_vartype(self, argn, subelem): + self.xml.set("type", str(subelem.xmlname)) + +class DeclarationBlock(ListObject): + tags = ["vardeclaration"] + adopt_childs_tags = ['vardecl_list'] + def add_other(self, argn, vtype, value): + if argn == 0: + self.xml.set("mode", vtype) + def polish(self): + #if len(self.values) == 0 and len(self.subelems) == 1: + # self.subelems[0].xml.set("mode",self.xml.get("mode")) + # return self.subelems[0] + return self + +class Class(ListNamedObject): + tags = ["classdeclaration"] + callback_subelem = ListNamedObject.callback_subelem.copy() + callback_subelem[ExtendsType] = "add_exttype" + def add_exttype(self, argn, subelem): + self.xml.set("extends", str(subelem.xmlname)) + +class Member(TagObject): + debug_other = False + set_child_argn = False + tags = ["member_var","member_call"] + adopt_childs_tags = ['varmemcall',"member_var","member_call"] + +class ArrayMember(TagObject): + debug_other = False + set_child_argn = False + tags = ["array_member"] + adopt_childs_tags = ['variable_1',"func_call"] + +class InstructionCall(TagObject): + debug_other = False + tags = ["callinstruction"] + +class InstructionStore(TagObject): + promote_child_if_alone = True + debug_other = False + tags = ["storeinstruction"] + +class InstructionFlow(TypedObject): + debug_other = True + tags = ["flowinstruction"] + +class OpMath(TypedObject): + debug_other = True + tags = ["mathoperator"] + +class Compare(TypedObject): + debug_other = True + tags = ["cmp_symbol","boolcmp_symbol"] + +class FunctionCall(NamedObject): + tags = ["funccall_1"] + +class CallArguments(ListObject): + tags = ["callargs"] + +class Constant(ListObject): + tags = ["constant"] + def add_value(self, argn, vtype, value): + value = str(value) #str(value,"ISO-8859-15","replace") + if vtype == "SCONST": + vtype = "String" + value = value[1:-1] + self.xml.set("delim",'"') + if vtype == "CCONST": + vtype = "String" + value = value[1:-1] + self.xml.set("delim","'") + if vtype == "RCONST": vtype = "Regex" + if vtype == "ICONST": vtype = "Number" + if vtype == "FCONST": vtype = "Number" + self.const_value = value + self.const_type = vtype + self.xml.set("type",vtype) + self.xml.set("value",value) + +class InlineUpdate(ListObject): + tags = ["inlinestoreinstruction"] + def add_other(self, argn, vtype, value): + self.xml.set("type", vtype) + if argn == 0: + self.xml.set("mode", "update-read") + if argn == 1: + self.xml.set("mode", "read-update") + +class If(ListObject): + tags = ["ifstatement"] + +class Condition(ListObject): + tags = ["condition"] + +class Else(ListObject): + tags = ["optelse"] + adopt_childs_tags = ['statement_block'] + def polish(self): + if len(self.subelems) == 0: + self.astname = "empty" + return self + + +class ExpressionContainer(ListObject): + tags = ["expression"] + # adopt_childs_tags = ['base_expression'] + + def polish(self): + if len(self.values) == 0 and len(self.subelems) == 1: + #if isinstance(self.subelems[0], Constant): + if self.subelems[0].xml.tag == "base_expression": + self.subelems[0].xml.tag = "Expression" + return self.subelems[0] + else: + self.xml.tag = "Value" + + return self + +class InstructionUpdate(ListObject): + tags = ["updateinstruction"] + +class Switch(ListObject): + tags = ["switch"] + adopt_childs_tags = ['case_cblock_list','case_block_list'] + +class CaseList(ListObject): + tags = ["case_block_list"] + adopt_childs_tags = ['case_cblock_list','case_block_list'] + +class Case(ListObject): + tags = ["case_block"] + +class CaseDefault(ListObject): + tags = ["case_default"] + +class While(ListObject): + tags = ["whilestatement"] + +class For(ListObject): + tags = ["forstatement"] + +class ForInitialize(ListObject): + tags = ["for_initialize"] + +class ForCompare(ListObject): + tags = ["for_compare"] + +class ForIncrement(ListObject): + tags = ["for_increment"] + +class DoWhile(ListObject): + tags = ["dowhilestatement"] + +class ForIn(ListObject): + tags = ["forinstatement"] + +class With(ListObject): + tags = ["withstatement"] + +class TryCatch(ListObject): + tags = ["trycatch"] + +class New(ListObject): + tags = ["new_operator"] + +class Delete(ListObject): + tags = ["deleteinstruction"] + +class Parentheses(ListObject): + tags = ["parentheses"] + adopt_childs_tags = ['base_expression'] + +class OpUnary(TypedObject): + tags = ["unary_operator"] + +class OpTernary(ListObject): + tags = ["ternary_operator"] + +class OpUpdate(TypedObject): + tags = ["updateoperator"] + + + +# ----- keep this one at the end. +class Unknown(TagObject): + promote_child_if_alone = True + set_child_argn = False + @classmethod + def tagname(self, tagname): return tagname + + @classmethod + def can_process_tag(self, tagname): return True +# ----------------- + +def create_xml(tagname): + classobj = None + for cls in xml_class_types: + if cls.can_process_tag(tagname): + classobj = cls + break + if classobj is None: return None + return classobj(tagname) + +def parse_unknown(tagname, treedata): + xmlelem = create_xml(tagname) + i = 0 + for k, v in treedata['content']: + if type(v) is dict: + instruction = parse(k,v) + xmlelem.add_subelem(i, instruction) + elif k in USEFUL_TOKENS: + xmlelem.add_value(i, k, v) + else: + xmlelem.add_other(i, k, v) + + i+=1 + return xmlelem.polish() + + + +def post_parse(treedata): + source = parse("source",treedata) + #print UNKNOWN_PARSERS.keys() + return source.xml + +class Module(object): + def __init__(self, name, path): + self.name = name + self.path = path + def loadModule(self): + fp = None + try: + description = ('.py', 'U', imp.PY_SOURCE) + #description = ('.pyc', 'U', PY_COMPILED) + pathname = os.path.join(self.path, self.name) + fp = open(pathname) + name = self.name[:self.name.find(".")] + # fp, pathname, description = imp.find_module(self.name,[self.path]) + self.module = imp.load_module(name, fp, pathname, description) + result = True + except Exception as e: + print(traceback.format_exc()) + result = False + if fp: + fp.close() + return result + +def parseArgs(argv): + parser = OptionParser() + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("--path", + dest="storepath", default=None, + help="store XML results in PATH") + + parser.add_option("--topython", + action="store_true", dest="topython", default=False, + help="write python file from xml") + + parser.add_option("--exec-py", + action="store_true", dest="exec_python", default=False, + help="try to execute python file") + + parser.add_option("--toxml", + action="store_true", dest="toxml", default=False, + help="write xml file from qs") + + parser.add_option("--full", + action="store_true", dest="full", default=False, + help="write xml file from qs") + + (options, args) = parser.parse_args(argv) + return (options, args) + +def main(): + options, args = parseArgs(sys.argv[1:]) + execute(options,args) + +def pythonify(filelist): + options, args = parseArgs([]) + options.full = True + if isinstance(filelist, str): filelist = [filelist] + execute(options,filelist) + + + + +def execute(options, args): + if options.optdebug: + print(options, args) + if options.full: + options.full = False + options.toxml = True + print("Pass 1 - Parse and write XML file . . .") + execute(options,args) + + options.toxml = False + options.topython = True + print("Pass 2 - Pythonize and write PY file . . .") + execute(options,[ arg+".xml" for arg in args]) + + options.topython = False + options.exec_python = True + #print "Pass 3 - Test PY file load . . ." + #execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) + print("Done.") + + elif options.exec_python: + from . import qsatype + for filename in args: + realpath = os.path.realpath(filename) + path, name = os.path.split(realpath) + mod = Module(name, path) + if mod.loadModule(): + print(mod.module) + print(mod.module.form) + else: + print("Error cargando modulo %s" % name) + + elif options.topython: + from .pytnyzer import pythonize + for filename in args: + bname = os.path.basename(filename) + if options.storepath: + destname = os.path.join(options.storepath,bname+".py") + else: + destname = filename+".py" + destname = destname.replace(".qs.xml.py",".py") + pythonize(filename, destname) + + else: + nfs = len(args) + for nf, filename in enumerate(args): + bname = os.path.basename(filename) + sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) + sys.stdout.flush(); + prog = flscriptparse.parse(open(filename,"r", encoding="latin-1").read()) + sys.stdout.write("\r"); + if not prog: + print("Error: No se pudo abrir %-35s \n" % (repr(filename))) + continue + if prog["error_count"] > 0: + print("Encontramos %d errores parseando: %-35s \n" % (prog["error_count"], repr(filename))) + continue + if options.toxml == False: + # Si no se quiere guardar resultado, no hace falta calcular mas + continue + + tree_data = flscriptparse.calctree(prog, alias_mode = 0) + if not tree_data: + print("No se pudo parsear %-35s \n" % (repr(filename))) + continue + ast = post_parse(tree_data) + if ast is None: + print("No se pudo analizar %-35s \n" % (repr(filename))) + continue + if options.storepath: + destname = os.path.join(options.storepath,bname+".xml") + else: + destname = filename+".xml" + f1 = open(destname,"wb") + f1.write(etree.tostring(ast, pretty_print = True)) + f1.close() + + + +if __name__ == "__main__": main() diff --git a/pytnyzer.py b/pytnyzer.py new file mode 100644 index 0000000..24afd98 --- /dev/null +++ b/pytnyzer.py @@ -0,0 +1,883 @@ +#!/usr/bin/python +# ------ Pythonyzer ... reads XML AST created by postparse.py and creates an equivalent Python file. +from __future__ import print_function +from __future__ import absolute_import +from builtins import object +from optparse import OptionParser +import os, os.path, random +from . import flscriptparse +from lxml import etree +from future.utils import with_metaclass + +def id_translate(name): + python_keywords = ['and', 'del', 'for', 'is', 'raise', 'assert', 'elif', + 'from', 'lambda', 'return', 'break', 'else', 'global', 'not', 'try', + 'class', 'except', 'if', 'or', 'while', 'continue', + 'exec', 'import', 'pass', 'yield', 'def', 'finally', 'in', 'print'] + if name in python_keywords: return name + "_" + if name == "false": name = "False" + if name == "true": name = "True" + if name == "null": name = "None" + if name == "unknown": name = "None" + if name == "this": name = "self" + + if name == "startsWith": name = "startswith" + return name + +ast_class_types = [] + +class ASTPythonFactory(type): + def __init__(cls, name, bases, dct): + global ast_class_types + ast_class_types.append(cls) + super(ASTPythonFactory, cls).__init__(name, bases, dct) + +class ASTPython(with_metaclass(ASTPythonFactory, object)): + tags = [] + + @classmethod + def can_process_tag(self, tagname): return self.__name__ == tagname or tagname in self.tags + + def __init__(self, elem): + self.elem = elem + + def polish(self): return self + + def generate(self, **kwargs): + yield "debug", etree.tostring(self.elem) + + +class Source(ASTPython): + def generate(self, break_mode = False, include_pass = True, **kwargs): + elems = 0 + after_lines = [] + for child in self.elem: + #yield "debug", "<%s %s>" % (child.tag, repr(child.attrib)) + for dtype, data in parse_ast(child).generate(break_mode = break_mode, plusplus_as_instruction = True): + if dtype == "line+1": + after_lines.append(data) + continue + if dtype == "line": + elems += 1 + yield dtype, data + if dtype == "line" and after_lines: + for line in after_lines: + elems+=1 + yield dtype, line + after_lines = [] + if dtype == "break": + for line in after_lines: + elems+=1 + yield "line", line + + for line in after_lines: + elems+=1 + yield "line", line + if elems == 0 and include_pass: + yield "line", "pass" + +class Class(ASTPython): + def generate(self, **kwargs): + name = self.elem.get("name") + extends = self.elem.get("extends","object") + + yield "line", "class %s(%s):" % (name,extends) + yield "begin", "block-class-%s" % (name) + for source in self.elem.xpath("Source"): + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-class-%s" % (name) + +class Function(ASTPython): + def generate(self, **kwargs): + name = id_translate(self.elem.get("name")) + returns = self.elem.get("returns",None) + parent = self.elem.getparent() + grandparent = None + if parent is not None: grandparent = parent.getparent() + arguments = [] + + if grandparent is not None: + if grandparent.tag == "Class": + arguments.append("self") + if name == grandparent.get("name"): + name = "__init__" + else: + arguments.append("self") + for n,arg in enumerate(self.elem.xpath("Arguments/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(id_translate(data)) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + if len(expr) == 1: expr += ["=","None"] + arguments.append(" ".join(expr)) + + + + yield "line", "def %s(%s):" % (name,", ".join(arguments)) + yield "begin", "block-def-%s" % (name) + # if returns: yield "debug", "Returns: %s" % returns + for source in self.elem.xpath("Source"): + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-def-%s" % (name) + +class FunctionCall(ASTPython): + def generate(self, **kwargs): + name = id_translate(self.elem.get("name")) + parent = self.elem.getparent() + if parent.tag == "InstructionCall": + classes = parent.xpath("ancestor::Class") + if classes: + class_ = classes[-1] + extends = class_.get("extends") + if extends == name: + name = "super(%s, self).__init__" % class_.get("name") + functions = parent.xpath("//Function[@name=\"%s\"]" % name) + for f in functions: + #yield "debug", "Function to:" + etree.tostring(f) + name = "self.%s" % name + break + + arguments = [] + for n,arg in enumerate(self.elem.xpath("CallArguments/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "%s(%s)" % (name,", ".join(arguments)) + +class If(ASTPython): + def generate(self, break_mode = False, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + + yield "line", "if %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-if" + for obj in parse_ast(source).generate(break_mode = break_mode): yield obj + yield "end", "block-if" + + for source in self.elem.xpath("Else/Source"): + yield "line", "else:" + yield "begin", "block-else" + for obj in parse_ast(source).generate(break_mode = break_mode): yield obj + yield "end", "block-else" + +class TryCatch(ASTPython): + def generate(self, **kwargs): + + tryblock, catchblock = self.elem.xpath("Source") + + yield "line", "try:" + yield "begin", "block-try" + for obj in parse_ast(tryblock).generate(): yield obj + yield "end", "block-try" + + identifier = None + for ident in self.elem.xpath("Identifier"): + expr = [] + for dtype, data in parse_ast(ident).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + identifier = " ".join(expr) + if identifier: + yield "line", "except Exception, %s:" % (identifier) + else: + yield "line", "except Exception:" + yield "begin", "block-except" + if identifier: + # yield "line", "%s = str(%s)" % (identifier, identifier) + yield "line", "%s = traceback.format_exc()" % (identifier) + for obj in parse_ast(catchblock).generate(include_pass = identifier is None): yield obj + yield "end", "block-except" + + +class While(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + + yield "line", "while %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-while" + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-while" + +class For(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("ForInitialize/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) > 0: + main_expr.append(" ".join(expr)) + if main_expr: + yield "line", " ".join(main_expr) + + incr_expr = [] + incr_lines = [] + for n,arg in enumerate(self.elem.xpath("ForIncrement/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + elif dtype in ["line","line+1"]: + incr_lines.append(data) + else: + yield dtype, data + if len(expr) > 0: + incr_expr.append(" ".join(expr)) + + + main_expr = [] + for n,arg in enumerate(self.elem.xpath("ForCompare/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("True") + else: + main_expr.append(" ".join(expr)) + yield "debug", "FOR:" + yield "line", "while %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-for" + for obj in parse_ast(source).generate(include_pass=False): yield obj + if incr_lines: + for line in incr_lines: + yield "line", line + yield "end", "block-for" + +class Switch(ASTPython): + def generate(self, **kwargs): + key = "%02x" % random.randint(0,255) + name = "s%s_when" % key + name_pr = "s%s_do_work" % key + name_pr2 = "s%s_work_done" % key + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + yield "line", "%s = %s" % (name, " ".join(main_expr)) + yield "line", "%s,%s = %s,%s" % (name_pr,name_pr2, "False","False") + for scase in self.elem.xpath("Case"): + value_expr = [] + for n,arg in enumerate(scase.xpath("Value")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + value_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + value_expr.append(" ".join(expr)) + + yield "line", "if %s == %s: %s,%s = %s,%s" % (name," ".join(value_expr),name_pr,name_pr2, "True", "True") + yield "line", "if %s:" % (name_pr) + yield "begin", "block-if" + count = 0 + for source in scase.xpath("Source"): + for obj in parse_ast(source).generate(break_mode = True): + if obj[0] == "break": + yield "line", "%s = %s # BREAK" % (name_pr,"False") + count += 1 + else: + yield obj + count += 1 + if count < 1: yield "line", "pass" + yield "end", "block-if" + + for scasedefault in self.elem.xpath("CaseDefault"): + yield "line", "if not %s: %s,%s = %s,%s" % (name_pr2, name_pr,name_pr2, "True", "True") + yield "line", "if %s:" % (name_pr) + yield "begin", "block-if" + for source in scasedefault.xpath("Source"): + for obj in parse_ast(source).generate(break_mode = True): + if obj[0] == "break": + yield "line", "%s = %s # BREAK" % (name_pr,"False") + else: + yield obj + yield "end", "block-if" + # yield "line", "assert( not %s )" % name_pr + # yield "line", "assert( %s )" % name_pr2 + +class With(ASTPython): + def generate(self, **kwargs): + key = "%02x" % random.randint(0,255) + name = "w%s_obj" % key + yield "debug", "WITH: %s" % key + variable, source = [ obj for obj in self.elem ] + var_expr = [] + for dtype, data in parse_ast(variable).generate(isolate = False): + if dtype == "expr": + var_expr.append(data) + else: + yield dtype, data + if len(var_expr) == 0: + var_expr.append("None") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + + yield "line", "%s = %s" % (name, " ".join(var_expr)) + + for obj in parse_ast(source).generate(break_mode = True): + yield obj + yield "line", "del %s" % name + +class Variable(ASTPython): + def generate(self, force_value = False, **kwargs): + name = self.elem.get("name") + yield "expr", name + values = 0 + for value in self.elem.xpath("Value|Expression"): + values += 1 + yield "expr", "=" + expr = 0 + for dtype, data in parse_ast(value).generate(isolate = False): + if dtype == "expr": expr += 1 + yield dtype, data + if expr == 0: + yield "expr", "None" + + dtype = self.elem.get("type",None) + + if values == 0 and force_value == True: + yield "expr", "=" + if dtype is None: + yield "expr", "None" + elif dtype == "String": + yield "expr", "\"\"" + elif dtype == "Number": + yield "expr", "0" + else: + parent1 = self.elem.getparent() + parent2 = parent1.getparent() + parent3 = parent2.getparent() + if parent2 == "Source" and parent3 == "Class": + yield "expr", "None" + else: + yield "expr", "qsatype.%s()" % dtype + + #if dtype and force_value == False: yield "debug", "Variable %s:%s" % (name,dtype) + +class InstructionUpdate(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + if data is None: raise ValueError(etree.tostring(arg)) + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "line", " ".join(arguments) + +class InlineUpdate(ASTPython): + def generate(self, plusplus_as_instruction = False, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + ctype = self.elem.get("type") + mode = self.elem.get("mode") + linetype = "line" + if not plusplus_as_instruction: + if mode == "read-update": + linetype = "line+1" + + yield "expr", arguments[0] + if ctype == "PLUSPLUS": + yield linetype, arguments[0] + " += 1" + elif ctype == "MINUSMINUS": + yield linetype, arguments[0] + " -= 1" + else: + yield linetype, arguments[0] + " ?= 1" + +class InstructionCall(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + yield "line", " ".join(arguments) + +class InstructionFlow(ASTPython): + def generate(self, break_mode = False, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + ctype = self.elem.get("type") + kw = ctype + if ctype == "RETURN": kw = "return" + if ctype == "BREAK": + kw = "break" + if break_mode: + yield "break", kw + " " + ", ".join(arguments) + return + if ctype == "CONTINUE": kw = "continue" + + if ctype == "THROW": + yield "line", "raise Exception(" + ", ".join(arguments) + ")" + return + + yield "line", kw + " " + ", ".join(arguments) + + + +class Member(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + if len(arguments) >= 3 and arguments[0:2] == ["self","iface"] and arguments[2].startswith("__"): + # From: self.iface.__function() + # to: super(className, self.iface).function() + funs = self.elem.xpath("ancestor::Function") + if funs: + fun = funs[-1] + name_parts = fun.get("name").split("_") + classname = name_parts[0] + arguments[2] = arguments[2][2:] + arguments[0:2] = ["super(%s, %s)" % (classname,".".join(arguments[0:2]))] + + yield "expr", ".".join(arguments) + +class ArrayMember(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "%s[%s]" % (arguments[0],arguments[1]) + + +class Value(ASTPython): + def generate(self, isolate = True, **kwargs): + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + if data is None: raise ValueError(etree.tostring(child)) + yield dtype, data + if isolate: yield "expr", ")" + +class Expression(ASTPython): + tags = ["base_expression"] + def generate(self, isolate = True, **kwargs): + if isolate: yield "expr", "(" + coerce_string_mode = False + if self.elem.xpath("OpMath[@type=\"PLUS\"]"): + if self.elem.xpath("Constant[@type=\"String\"]"): + coerce_string_mode = True + if coerce_string_mode: + yield "expr", "ustr(" + for child in self.elem: + if coerce_string_mode and child.tag == "OpMath": + if child.get("type") == "PLUS": + yield "expr","," + continue + + for dtype, data in parse_ast(child).generate(): + yield dtype, data + + if coerce_string_mode: + yield "expr", ")" + if isolate: yield "expr", ")" + +class Parentheses(ASTPython): + def generate(self, **kwargs): + yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(isolate = False): + yield dtype, data + yield "expr", ")" + +class OpUnary(ASTPython): + def generate(self, isolate = False, **kwargs): + ctype = self.elem.get("type") + if ctype == "LNOT": yield "expr", "not" + elif ctype == "MINUS": yield "expr", "-" + else: yield "expr", ctype + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + if isolate: yield "expr", ")" + +class New(ASTPython): + def generate(self, **kwargs): + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + if dtype != "expr": + yield dtype, data + continue + if child.tag == "Identifier": data = data+"()" + ident = data[:data.find("(")] + if ident.find(".") == -1: + if len(self.elem.xpath("//Class[@name='%s']" % ident)) == 0: + data = "qsatype." + data + yield dtype, data + + +class Constant(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + value = self.elem.get("value") + if ctype is None or value is None: + for child in self.elem: + if child.tag == "CallArguments": + arguments = [] + for n,arg in enumerate(child): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "qsatype.Array([%s])" % (", ".join(arguments)) + return + if ctype == "String": + delim = self.elem.get("delim") + if delim == "'": + yield "expr", "u'%s'" % value + else: + yield "expr", "u\"%s\"" % value + else: yield "expr", value + +class Identifier(ASTPython): + def generate(self, **kwargs): + name = id_translate(self.elem.get("name")) + yield "expr", name + +class OpUpdate(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "EQUALS": yield "expr", "=" + elif ctype == "PLUSEQUAL": yield "expr", "+=" + elif ctype == "MINUSEQUAL": yield "expr", "-=" + elif ctype == "TIMESEQUAL": yield "expr", "*=" + elif ctype == "DIVEQUAL": yield "expr", "/=" + else: yield "expr", ctype + +class Compare(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "GT": yield "expr", ">" + elif ctype == "LT": yield "expr", "<" + elif ctype == "LE": yield "expr", "<=" + elif ctype == "GE": yield "expr", ">=" + elif ctype == "EQ": yield "expr", "==" + elif ctype == "NE": yield "expr", "!=" + elif ctype == "IN": yield "expr", "in" + elif ctype == "LOR": yield "expr", "or" + elif ctype == "LAND": yield "expr", "and" + else: yield "expr", ctype + +class OpMath(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "PLUS": yield "expr", "+" + elif ctype == "MINUS": yield "expr", "-" + elif ctype == "TIMES": yield "expr", "*" + elif ctype == "DIVIDE": yield "expr", "/" + elif ctype == "MOD": yield "expr", "%" + elif ctype == "XOR": yield "expr", "^" + elif ctype == "LSHIFT": yield "expr", "<<" + elif ctype == "RSHIFT": yield "expr", ">>" + elif ctype == "AND": yield "expr", "&" + else: yield "expr", ctype + + +class DeclarationBlock(ASTPython): + def generate(self, **kwargs): + mode = self.elem.get("mode") + is_constructor = self.elem.get("constructor") + if mode == "CONST": yield "debug", "Const Declaration:" + for var in self.elem: + expr = [] + for dtype, data in parse_ast(var).generate(force_value=True): + if dtype == "expr": + if data is None: raise ValueError(etree.tostring(var)) + expr.append(data) + else: yield dtype,data + if is_constructor: + expr[0] = "self."+expr[0] + yield "line", " ".join(expr) + + +# ----- keep this one at the end. +class Unknown(ASTPython): + @classmethod + def can_process_tag(self, tagname): return True +# ----------------- + +def astparser_for(elem): + classobj = None + for cls in ast_class_types: + if cls.can_process_tag(elem.tag): + classobj = cls + break + if classobj is None: return None + return classobj(elem) + +def parse_ast(elem): + elemparser = astparser_for(elem) + return elemparser.polish() + + +def file_template(ast): + yield "line", "# encoding: UTF-8" + yield "line", "from pineboolib import qsatype" + yield "line", "from pineboolib.qsaglobals import *" + yield "line", "import traceback" + yield "line", "" + sourceclasses = etree.Element("Source") + for cls in ast.xpath("Class"): + sourceclasses.append(cls) + + mainclass = etree.SubElement(sourceclasses,"Class",name="FormInternalObj",extends="qsatype.FormDBWidget") + mainsource = etree.SubElement(mainclass,"Source") + + + constructor = etree.SubElement(mainsource,"Function",name="_class_init") + args = etree.SubElement(constructor,"Arguments") + csource = etree.SubElement(constructor,"Source") + + for child in ast: + if child.tag != "Function": + child.set("constructor","1") + csource.append(child) + else: + mainsource.append(child) + + for dtype, data in parse_ast(sourceclasses).generate(): + yield dtype, data + yield "line", "" + yield "line", "form = None" + +def write_python_file(fobj, ast): + indent = [] + indent_text = " " + last_line_for_indent = {} + numline = 0 + last_dtype = None + for dtype, data in file_template(ast): + if isinstance(data, bytes): data = data.decode("UTF-8","replace") + line = None + if dtype == "line": + line = data + numline +=1 + try: lines_since_last_indent = numline - last_line_for_indent[len(indent)] + except KeyError: lines_since_last_indent = 0 + if lines_since_last_indent > 4: + fobj.write((len(indent)*indent_text) + "\n") + last_line_for_indent[len(indent)] = numline + if dtype == "debug": + line = "# DEBUG:: " + data + print(numline, line) + if dtype == "expr": line = "# EXPR??:: " + data + if dtype == "line+1": line = "# LINE+1??:: " + data + if dtype == "begin": + #line = "# BEGIN:: " + data + indent.append(data) + last_line_for_indent[len(indent)] = numline + if dtype == "end": + if last_dtype == "begin": + fobj.write((len(indent)*indent_text) + "pass\n") + last_line_for_indent[len(indent)] = numline + + if data not in ["block-if"]: + #line = "# END:: " + data + pass + endblock = indent.pop() + if endblock != data: + line = "# END-ERROR!! was %s but %s found. (%s)" % (endblock, data,repr(indent)) + + if line is not None: + fobj.write((len(indent)*indent_text) + line + "\n") + + if dtype == "end": + if data.split("-")[1] in ["class","def","else","except"]: + fobj.write((len(indent)*indent_text) + "\n") + last_line_for_indent[len(indent)] = numline + last_dtype = dtype + +def pythonize(filename, destfilename): + bname = os.path.basename(filename) + + parser = etree.XMLParser(remove_blank_text=True) + try: + ast_tree = etree.parse(open(filename), parser) + except Exception: + print("filename:",filename) + raise + ast = ast_tree.getroot() + + + f1 = open(destfilename,"w") + write_python_file(f1,ast) + f1.close() + +def main(): + parser = OptionParser() + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("--path", + dest="storepath", default=None, + help="store PY results in PATH") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + + for filename in args: + if options.storepath: + destname = os.path.join(options.storepath,bname+".py") + else: + destname = filename+".py" + pythonize(filename, destname) + + + +if __name__ == "__main__": main() diff --git a/qsatype.py b/qsatype.py new file mode 100644 index 0000000..7f8b8a7 --- /dev/null +++ b/qsatype.py @@ -0,0 +1,18 @@ +from builtins import object +# encoding: UTF-8 +import os + +class Object(object): + pass + +class Array(object): + pass + +class FLSqlCursor(object): + pass + +class Boolean(object): + pass + + + diff --git a/xml2json.py b/xml2json.py new file mode 100644 index 0000000..fd267ee --- /dev/null +++ b/xml2json.py @@ -0,0 +1,634 @@ +#!/usr/bin/python + +from __future__ import print_function +from builtins import object +try: + from json import dumps as json_dumps + from json import loads as json_loads +except: + from json import write as json_dumps + from json import read as json_loads + + +import xml.parsers.expat +from optparse import OptionParser + +# json_string = json.dumps(python_variable) +# python_var = json.loads("string_encoded_jsonvar") + +def printr(*args): + return + print(args[0], end=' ') + for arg in args[1:]: + if type(arg) is str: + arg=arg.encode("utf-8") + print(repr(arg), end=' ') + print() + +def entity_rep(txt,entities=""): + entity_list = list("&'\"<>") + entity_dict = { + '"' : """, + "'" : "'", + '<' : "<", + '>' : ">", + "&" : "&", + } + if entities == "": entities = entity_list + entities = list(entities) + if "&" not in entities: entities.append("&") + + for entity in entity_list: + if entity not in entities: continue + #print entity,entity_dict[entity] + txt = txt.replace(entity,entity_dict[entity]) + return txt + +class xmlElement(object): + def __init__(self, parent, tagname, attrs = {}, ttype = "text", tdata = ""): + self.parent = parent + + self.tagname = tagname + self.attrs = attrs + self.ttype = ttype + self.tdata = tdata + + self.children = [] + + if self.parent: + self.depth = parent.depth + 1 + self.path = self.parent.path + [self.tagname] + self.parent.children.append(self) + else: + self.depth = 0 + self.path = [self.tagname] + + def append(self,text): + self.tdata += text + + def export(self,encoding): + depth = self.depth + tagname = self.tagname + attrs = [ [k,v] for k,v in self.attrs.items() ] + attrs.sort() + tdata = self.tdata.strip() + ttype = self.ttype + + if len(tdata) == 0: + tdata = "" + ttype = "" + + #v = [depth,tagname,attrs,ttype,tdata] + #vt1 = json_dumps(v) + vt2 = "%d)%s" % (depth,tagname) + if attrs: vt2 +="\tattrs:" + json_dumps(attrs) + if ttype: vt2 +="\t%s:%s" % ( ttype, json_dumps(tdata)) + + return vt2.encode(encoding) + + def exportXML(self): + if type(self.attrs) is dict: + attrs = [ [k,v] for k,v in self.attrs.items() ] + attrs.sort() + else: + attrs = self.attrs + + txtattrs = "" + + if attrs: + txtattrs=" " + for key,value in attrs: + txtattrs += '%s="%s" ' % (key,entity_rep(value,'&<"')) + + + depthpad = u" " * self.depth + output = u"" + if self.tagname == "#comment": + output += u"%s\n" % (depthpad, self.tdata) + elif self.tagname[0] == "!": + output += u"%s\n" % (depthpad, self.tagname[1:],txtattrs) + elif self.children: + output += u"%s<%s%s>\n" % (depthpad, self.tagname,txtattrs) + for child in self.children: + output += child.exportXML() + + output += u"%s\n" % (depthpad, self.tagname) + else: + if self.tdata == "": + if txtattrs=="": txtattrs = " " + output += u"%s<%s%s/>\n" % (depthpad, self.tagname,txtattrs) + else: + tdata = self.tdata + if tdata.find("\n")>-1: tdata = "\n%s\n" % tdata + if self.ttype=="cdata": tdata = "" % self.tdata + else: tdata = entity_rep(self.tdata) + + output += u"%s<%s%s>%s\n" % (depthpad, self.tagname,txtattrs, tdata, self.tagname) + + + return output + + + + + +class JSON_Base(object): + def __init__(self, finput, foutput, encoding): + self.finput = finput + self.foutput = foutput + self.encoding = encoding + + self.init_vars() + + def process(self): + print("Please define a process function.") + + def init_vars(self): + pass + + + +class JSON_Reverter(JSON_Base): + def init_vars(self): + self.cElement = None + self.rootXML = [] + + def processCmd(self,key,val): + if key == "encoding": + if self.encoding != "auto": + self.encoding = self.encoding.upper() + if val.upper() != self.encoding: + print(" ignoring %s=%s , using specified value '%s' instead" % (key,val,self.encoding)) + return + self.encoding = val.upper() + return + + print("ERROR: unknown key %s='%s'" % (key,val)) + + def newElement(self,depth,tagname,text,ttype,attrs): + parent = self.cElement + if parent: parentdepth = parent.depth + else: parentdepth = -1 + while parent and parentdepth > depth - 1: + parent = parent.parent + if parent: parentdepth = parent.depth + else: parentdepth = -1 + + self.cElement = xmlElement(parent, tagname, attrs, ttype , text) + if parent is None: self.rootXML.append(self.cElement) + + + def process(self): + for line in self.finput: + line = line.strip() + if len(line) == 0: continue + if line[0]=="!": + lstkeys = line[1:].split(":") + key, val = lstkeys + self.processCmd(key.strip(),val.strip()) + continue + + fields = line.split("\t") + + depth, tag = fields[0].split(")") + depth = int(depth) + text = "" + ttype = "text" + attrs = {} + + #self.foutput.write("%d\t%s\n" % (depth, tag)) + for field in fields[1:]: + tpos = field.find(":") + if tpos == -1: + print("unexpected character:", line) + return + ftype = field[:tpos] + try: + fvalue = json_loads(field[tpos+1:]) + except ValueError: + print("ValueError:", field[tpos+1:]) + + #if type(fvalue) is unicode: + # self.foutput.write("%s = %s\n" % (ftype, fvalue.encode(self.encoding))) + #else: + # self.foutput.write("%s = %s\n" % (ftype, repr(fvalue))) + if ftype == "text": + text = fvalue + ttype = "text" + if ftype == "cdata": + text = fvalue + ttype = "cdata" + if ftype == "attrs": attrs = fvalue + + self.newElement(depth,tag,text,ttype,attrs) + + for element in self.rootXML: + self.foutput.write(element.exportXML().encode(self.encoding)) + +""" + Possible Format: + + List-per-tag: + [ depth, tagname, attrs, ttype, tdata ] + + depth: 0,1,2,3,4...N + + tagname: \w+ -> ElementTag + tagname: !\w+ -> DoctypeTag + tagname: ?\w+ -> XmlDeclTag (always: xml) (attrs = version, encoding?, standalone?) + tagname: #\w+ -> CommentTag (always: comment) (attrs = []) (tdata = comment) + + attrs: dict { attr : val , attr2 : val2 } + + ttype: text|cdata|mixed -multiline? + + tdata: raw text + cdata combined. + + + Problems: + * Handling C-DATA + * Handling multiline texts + * Handling tabs and spaces at the start of each line of multiline text + * Handling comments + + Non-treated: + * NameSpaces + 32.18 + + * Entity Declarations + + + * Element Declarations + + + * Notation Declarations + + + * Attribute List Delcarations + + + + + + Example 1: AbanQ UI (UTF-8) + StartDoctypeDeclHandler: 'UI' None None 0 + EndDoctypeDeclHandler: + StartElementHandler: 'UI' {u'version': u'3.3', u'stdsetdef': u'1'} + CharacterDataHandler: '\n' + StartElementHandler: 'class' {} + CharacterDataHandler: 'formLineasAlbaranesCli' + EndElementHandler: 'class' + + Example 2: JasperReports JRXML (UTF-8) + XmlDeclHandler: '1.0' 'UTF-8' -1 + Unhandled data: '\n' + StartElementHandler: 'jasperReport' {u'xmlns': u'http://jasperreports.sourceforge.net/jasperreports', u'name': u'report1', u'language': u'groovy', u'pageWidth': u'842', u'columnWidth': u'802', u'topMargin': u'20', u'rightMargin': u'20', u'bottomMargin': u'20', u'xmlns:xsi': u'http://www.w3.org/2001/XMLSchema-instance', u'leftMargin': u'20', u'xsi:schemaLocation': u'http://jasperreports.sourceforge.net/jasperreports http://jasperreports.sourceforge.net/xsd/jasperreport.xsd', u'pageHeight': u'595', u'orientation': u'Landscape'} + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + StartElementHandler: 'style' {u'fontName': u'Times New Roman', u'name': u'Title', u'isBold': u'false', u'forecolor': u'#FFFFFF', u'fontSize': u'50', u'isDefault': u'false', u'pdfFontName': u'Times-Bold'} + EndElementHandler: 'style' + + Example 3: AbanQ Actions XML (ISO-8859-1) + StartElementHandler: 'ACTIONS' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'action' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'name' {} + CharacterDataHandler: 'albaranescli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'description' {} + CharacterDataHandler: 'QT_TRANSLATE_NOOP("MetaData","Son los documentos que justifican la entrega de una mercancia a un ciente")' + EndElementHandler: 'description' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + Example 4: AbanQ Tables MTD (ISO-8859-1) + StartDoctypeDeclHandler: 'TMD' None None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'TMD' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'name' {} + CharacterDataHandler: 'facturascli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CommentHandler: 'Facturas de cliente' + CharacterDataHandler: ' ' + StartElementHandler: 'alias' {} + CharacterDataHandler: 'QT_TRANSLATE_NOOP("MetaData","Facturas de Clientes")' + EndElementHandler: 'alias' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + Example 5: AbanQ Report QRY (ISO-8859-1) + StartDoctypeDeclHandler: 'QRY' None None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'QRY' {} + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + StartElementHandler: 'name' {} + CharacterDataHandler: 'presupuestoscli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + + Example 6: AbanQ Report KUT (ISO-8859-1) + XmlDeclHandler: '1.0' 'UTF-8' -1 + Unhandled data: '\n' + StartDoctypeDeclHandler: 'KugarTemplate' 'kugartemplate.dtd' None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'KugarTemplate' {u'TopMargin': u'50', u'PageSize': u'0', u'RightMargin': u'30', u'PageOrientation': u'0', u'BottomMargin': u'50', u'LeftMargin': u'30'} + CharacterDataHandler: '\n' + StartElementHandler: 'Detail' {u'Level': u'0', u'Height': u'0'} + EndElementHandler: 'Detail' + CharacterDataHandler: '\n' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + + + +""" + +class JSON_Converter(JSON_Base): + + def init_vars(self): + self.real_encoding = self.getRealEncoding() + self.xmltag = None + self.taglist = [] + self.p = xml.parsers.expat.ParserCreate(self.real_encoding) + + self.p.StartElementHandler = self.StartElementHandler + self.p.EndElementHandler = self.EndElementHandler + self.p.CharacterDataHandler = self.CharacterDataHandler + self.p.XmlDeclHandler = self.XmlDeclHandler + self.p.StartDoctypeDeclHandler = self.StartDoctypeDeclHandler + self.p.EndDoctypeDeclHandler = self.EndDoctypeDeclHandler + self.p.ElementDeclHandler = self.ElementDeclHandler + self.p.AttlistDeclHandler = self.AttlistDeclHandler + self.p.ProcessingInstructionHandler = self.ProcessingInstructionHandler + self.p.CharacterDataHandler = self.CharacterDataHandler + self.p.EntityDeclHandler = self.EntityDeclHandler + self.p.NotationDeclHandler = self.NotationDeclHandler + self.p.StartNamespaceDeclHandler = self.StartNamespaceDeclHandler + self.p.EndNamespaceDeclHandler = self.EndNamespaceDeclHandler + self.p.CommentHandler = self.CommentHandler + self.p.StartCdataSectionHandler = self.StartCdataSectionHandler + self.p.EndCdataSectionHandler = self.EndCdataSectionHandler + self.p.DefaultHandler = self.DefaultHandler + + def getRealEncoding(self): + validEncodings = ["UTF-8", "UTF-16", "ISO-8859-1"] + self.encoding = self.encoding.upper() + + if self.encoding in validEncodings: return self.encoding + if self.encoding.find("UTF")>=0: + if self.encoding.find("8"): + return "UTF-8" + if self.encoding.find("16"): + return "UTF-16" + return "UTF-8" + + if self.encoding.find("ISO")>=0: + return "ISO-8859-1" + + if self.encoding.find("1252")>=0: + return "ISO-8859-1" + + if self.encoding.find("CP")==0: + return "ISO-8859-1" + + if self.encoding.find("WIN")==0: + return "ISO-8859-1" + + return "UTF-8" + + + def process(self): + self.p.ParseFile(self.finput) + self.foutput.write("!encoding: "+ self.real_encoding+"\n") + for tag in self.taglist: + self.foutput.write(tag.export(self.real_encoding)+"\n") + + def startTag(self,*args): + newtag = xmlElement(self.xmltag, *args) + self.xmltag = newtag + self.taglist.append(newtag) + return newtag + + def endTag(self): + self.xmltag = self.xmltag.parent + return self.xmltag + + + + + def StartElementHandler(self, name, attributes): + printr( "StartElementHandler:", name, attributes) + # tagname: \w+ -> ElementTag + self.startTag(name, attributes) + + def CharacterDataHandler(self, data): + printr( "CharacterDataHandler:", data) + self.xmltag.append(data) + + def EndElementHandler(self, name): + printr( "EndElementHandler:", name) + self.endTag() + + def XmlDeclHandler(self, version, encoding, standalone): + printr( "XmlDeclHandler:", version, encoding, standalone) + # tagname: ?\w+ -> XmlDeclTag (always: xml) (attrs = version, encoding?, standalone?) + attrs = { 'version' : version } + if encoding: attrs['encoding'] = encoding + if standalone: attrs['standalone'] = standalone + + self.startTag("?xml", attrs) + + self.endTag() + + def CommentHandler(self, data): + printr( "CommentHandler:", data) + # tagname: #\w+ -> CommentTag (always: comment) (attrs = []) (tdata = comment) + self.startTag("#comment") + self.xmltag.append(data) + self.endTag() + + + def StartCdataSectionHandler(self): + printr( "StartCdataSectionHandler:") + self.xmltag.ttype = "cdata" + + def EndCdataSectionHandler(self): + printr( "EndCdataSectionHandler:") + # se descarta el cierre... + + def StartDoctypeDeclHandler(self, doctypeName, systemId, publicId, has_internal_subset): + printr( "StartDoctypeDeclHandler:", doctypeName, systemId, publicId, has_internal_subset) + # tagname: !\w+ -> DoctypeTag + attrs = {} + if systemId: attrs['systemId'] = systemId + if publicId: attrs['publicId'] = publicId + if has_internal_subset: attrs['has_internal_subset'] = has_internal_subset + + self.startTag("!"+doctypeName,attrs) + + + def EndDoctypeDeclHandler(self): + printr( "EndDoctypeDeclHandler:") + self.endTag() + + def ElementDeclHandler(self, name, model): + printr( "ElementDeclHandler:", name, model) + + def AttlistDeclHandler(self, elname, attname, type, default, required): + printr( "AttlistDeclHandler:", elname, attname, type, default, required) + + def ProcessingInstructionHandler(self, target, data): + printr( "ProcessingInstructionHandler:", target, data) + + def EntityDeclHandler(self, entityName, is_parameter_entity, value, base, systemId, publicId, notationName): + printr( "EntityDeclHandler:" , entityName, is_parameter_entity, value, base, systemId, publicId, notationName) + + def NotationDeclHandler(self, notationName, base, systemId, publicId): + printr( "NotationDeclHandler:", notationName, base, systemId, publicId) + + def StartNamespaceDeclHandler(self, prefix, uri): + printr( "StartNamespaceDeclHandler:", prefix, uri) + + def EndNamespaceDeclHandler(self, prefix): + printr( "EndNamespaceDeclHandler:", prefix) + + def DefaultHandler(self,data): + printr( "Unhandled data:", data) + + + + + + + +def autodetectXmlEncoding(rawtext): + lines = [ line.strip() for line in rawtext.split("\n") if line.strip() ] + if lines[0].find("")>=0: + # File is QtDesigner UI + return "UTF-8" + + if lines[0].find("UTF-8")>=0: + # Unkown, standard xml (like jrxml) + return "UTF-8" + + if lines[0].find("")>=0: + # AbanQ actions XML + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ table MTD + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ report Query + return "ISO-8859-15" + + if lines[0].find('')>=0: + # AbanQ report Kut + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ translations + return "ISO-8859-15" + + try: + import chardet + dictEncoding = chardet.detect(rawtext) + encoding = dictEncoding["encoding"] + return encoding + except ImportError: + print("python-chardet library is not installed. Assuming input file is UTF-8.") + #encoding= + #UTF-8, UTF-16, ISO-8859-1 + + + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("-E", "--encoding", dest="encoding", default = "auto", + help="Set encoding=ENC: auto,utf-8,iso-8859-15", metavar="ENC") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + if len(args) < 2: + print("xml2json needs at least an action and a file.") + print("xml2json (revert|convert) file1 [file2] [file3]") + return + action = args.pop(0) + + if action == "convert": + for fname in args: + + fhandler = open(fname) + fw = open(fname+".json","w") + rawtext = fhandler.read() + fhandler.seek(0) + if options.encoding == "auto": + encoding = autodetectXmlEncoding(rawtext) + else: + encoding = options.encoding + jconv = JSON_Converter(fhandler, fw, encoding) + jconv.process() + + fhandler.close() + fw.close() + elif action == "revert": + for fname in args: + lExt = fname.split(".") + ext = "xml" + if lExt[-1]=="json" and len(lExt[-2])>=1 and len(lExt[-2])<=6: + ext = lExt[-2] + fhandler = open(fname) + + fw = open(fname+"."+ext,"w") + jrev = JSON_Reverter(fhandler, fw, options.encoding) + jrev.process() + + fw.close() + fhandler.close() + + else: + print("Unkown action '%s'" % action) + + + + + + + + + +if __name__ == "__main__": main() diff --git a/xmlparse.py b/xmlparse.py new file mode 100644 index 0000000..df1dbd5 --- /dev/null +++ b/xmlparse.py @@ -0,0 +1,200 @@ +from __future__ import print_function +from builtins import str +import xml.parsers.expat +import sys +from optparse import OptionParser +import re + +elements = [] +show_end = True +lasttextdata = "" +lstelements = [] + +def reset(): + global elements, show_end, lstelements, lasttextdata + elements = [] + show_end = True + lasttextdata = "" + lstelements = [] + + +# 3 handler functions +def start_element(name, attrs): + global elements, show_end, lstelements, lasttextdata + lstattrs=list(sorted([ "%s=%s" % (k,v) for k,v in attrs.items() ])) + completename=name + if len(lstattrs): + completename+="&"+"&".join(lstattrs) + + show_end = False + elements.append(completename) + #print 'Start element:', name, attrs + lstelements.append("/".join(elements)) + #print "/".join(elements) + lasttextdata = "" + +def end_element(name): + global elements, show_end, lstelements, lasttextdata + lasttextdata = "" + if show_end: + #print "/".join(elements) + "/" + lstelements.append("/".join(elements) + ";") + + show_end = True + elements.pop() + #print 'End element:', name + +def char_data(data): + global elements, show_end, lstelements, lasttextdata + #data = data.strip() + lasttextdata+=data + if lasttextdata.strip(): + #show_end = True + lstelements.pop() + lstelements.append("/".join(elements)+"(%s)" % repr(lasttextdata.strip())) + + + #print "/".join(elements)+ "(%s)" % repr(data) + + + +def unmap(lines): + + runmap = re.compile(r"^(?P/*)(?P\w+)(?P&[^\(]+)*(?P\(.+\))?$") + # depthlevel + # tagname + elementpool = [] + text = [] + for line in lines: + line = line.strip() + if line[-1] == ";": continue + rg1 = runmap.match(line) + if not rg1: + print("error:") + print(line) + break + + depth = len(rg1.group('depth')) + tagname = str(rg1.group('tagname')) + t_attrs = rg1.group('attrs') + attrs = [] + if t_attrs: + lattrs = t_attrs[1:].split("&") + for attr in lattrs: + key, val = attr.split("=") + attrs.append( (key,val) ) + + t_txt = rg1.group('txt') + txt = "" + if t_txt: + txt = eval(t_txt[1:-1]) + + while depth < len(elementpool): + toclose = elementpool.pop() + text.append("" % toclose) + text.append("\n" + " " * len(elementpool)) + + if depth == len(elementpool): + #print depth, tagname, attrs, txt + txtattrs = "" + if attrs: + for k,v in attrs: + txtattrs+=" %s=\"%s\"" % (k,v) + + if txt: + txt = txt.encode("utf-8") + txt = txt.replace("&","&") + txt = txt.replace("<","<") + else: + txt = "" + text.append("<%s%s>%s" % (tagname, txtattrs,txt)) + elementpool.append(tagname) + + + else: + print("error:") + print(depth, len(elementpool)) + break + + while len(elementpool): + toclose = elementpool.pop() + text.append("" % toclose) + text.append("\n" + " " * len(elementpool)) + + return text + + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + if len(args) < 2: + print("Se necesita al menos una accion y un argumento extra.") + print("xmlparse (map|unmap) file1 [file2] [file3]") + return + action = args.pop(0) + + if action == "map": + global lstelements + separators = [ + "hbox", + "vbox", + "grid", + ] + r1 = re.compile("/widget") + for fname in args: + p = xml.parsers.expat.ParserCreate() + + p.StartElementHandler = start_element + p.EndElementHandler = end_element + p.CharacterDataHandler = char_data + fhandler = open(fname) + fw = open(fname+".map","w") + reset() + p.ParseFile(fhandler) + for t in lstelements: + elems = t.split("/") + lbox = [] + for n,e in enumerate(elems): + if e in separators: lbox.append(n) + if len(lbox)>1: + nlbox = lbox[-2] + while len(elems[nlbox:]) < 2: + nlbox -= 1 + else: + nlbox = 0 + fw.write("/"*(len(elems)-1) + "/".join(elems[-1:])+ "\n") + #print "/"*(len(elems)-1) + "/".join(elems[-1:]) + lstelements = [] + fhandler.close() + fw.close() + elif action == "unmap": + for fname in args: + fhandler = open(fname) + fw = open(fname+".ui","w") + for line in unmap(fhandler): + fw.write(line) + fw.close() + fhandler.close() + + else: + print("Unkown action '%s'" % action) + + +if __name__ == "__main__": main() \ No newline at end of file From 7f258c9da7ead29c168d422b95b4dc061b844c7b Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 20 Jul 2015 09:46:24 +0200 Subject: [PATCH 058/100] lint: correcciones en chequeo estatico --- flalign.py | 168 ++++++++++++++++++------------------ flclasses.py | 88 +++++++++---------- flex.py | 30 +++---- flpremerge.py | 220 +++++++++++++++++++++++------------------------ flscriptparse.py | 8 +- 5 files changed, 258 insertions(+), 256 deletions(-) diff --git a/flalign.py b/flalign.py index a5563d2..b16f80a 100644 --- a/flalign.py +++ b/flalign.py @@ -4,7 +4,7 @@ import os, os.path, sys import math from optparse import OptionParser -import difflib +import difflib import re class ProcessFile(object): @@ -19,7 +19,7 @@ def process(self): self.idxlines = {} self.idxnames = {} self.sortednames = [] - + fblocks = open(self.filename + ".blocks") n = 0 line = fblocks.readline() @@ -34,21 +34,21 @@ def process(self): self.idxnames[name] = n n+=1 line = fblocks.readline() - + def indexLines(self): fqs = open(self.filename) self.lines = [] for line in fqs: self.lines.append(line) - - - + + + def diffTo(self,pfile2): added = pfile2.bnames - self.bnames deleted = self.bnames - pfile2.bnames - + return added, deleted - + class LineNumber(object): def __init__(self, letter, lines): self.nl=0 @@ -57,49 +57,49 @@ def __init__(self, letter, lines): else: self.diffFrom = None self.error = None - + self.letter = letter[-1] self.lines = lines - + def line(self): l = self.lines[self.nl] if self.diffFrom: return l[2:] else: return l[:] - - def symbol(self): + + def symbol(self): l = self.lines[self.nl] if self.diffFrom: return l[0] else: return " " - + def next(self,t=1): self.nl+=int(t) - + def __iadd__(self,other): self.nl+=int(other) return self - + def __int__(self): return self.nl - + def __index__(self): return self.nl - - + + def appliedDiff(C, A, B, prefer = "C", debug = False, quiet = False, swap = False): diffAB = list(difflib.ndiff(A.sortednames, B.sortednames)) diffAC = list(difflib.ndiff(A.sortednames, C.sortednames)) - - + + nlA = LineNumber("A",A.sortednames) nlB = LineNumber("B",B.sortednames) nlC = LineNumber("C",C.sortednames) nlAB = LineNumber("AB", diffAB) nlAC = LineNumber("AC", diffAC) - + maxA = len(A.sortednames) maxB = len(B.sortednames) maxC = len(C.sortednames) @@ -111,7 +111,7 @@ def AddPatchLine(mode): linefrom = None linetext = "" conflict = False - linenumbers = int(nlA),int(nlB),int(nlC) + linenumbers = int(nlA),int(nlB),int(nlC) if modetype == "-": linetext = lineaA linefrom = "A" @@ -151,49 +151,49 @@ def AddPatchLine(mode): #ConflictMode = False if len(ConflictMode): if linetext[0]!="#": ConflictMode.pop() - else: + else: #print ">>", len(ConflictMode),linetext if len(ConflictMode)>1 and linetext.find("separator") == -1: ConflictMode.pop() - elif linetext.find("separator") >= 0: + elif linetext.find("separator") >= 0: ConflictMode.pop() return if conflict and linetext[0]!="#": if debug: print("WARNING: Omitting previously added block <%s>" % linetext) - - while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") >= 0: + + while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") >= 0: #print "sep//", patchedResult.pop() - - while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") == -1: + + while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") == -1: #print "comm//", patchedResult.pop() #if patchedResult[-1][4][0]=="#": patchedResult.pop() - + ConflictMode.append(1) ConflictMode.append(1) else: patchedResult.append(line) - + AddedB = [] AddedC = [] DeletedB = [] DeletedC = [] ConflictMode = [] - + def getVars(letter): if letter=="B": return nlA, nlB, nlC, nlAB, nlAC if letter=="C": return nlA, nlC, nlB, nlAC, nlAB raise TypeError - + def Patch(code): letter = code[0] sign = code[1] nBase, nLocal, nRemote, nDiffLocal, nDiffRemote = getVars(letter) - + def Minus(): AddPatchLine(letter + "-") if debug: @@ -208,11 +208,11 @@ def Minus(): print("??", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) else: next(nRemote) - + next(nDiffLocal) next(nDiffRemote) next(nBase) - + def Plus(): AddPatchLine(letter + "+") if debug: @@ -221,15 +221,15 @@ def Plus(): print(letter + "! " , nDiffLocal.line()) if nBase.error == "removing-twice": nBase.error = None - while nDiffRemote.symbol() == "?": + while nDiffRemote.symbol() == "?": next(nDiffRemote) if nDiffRemote.symbol() == "+": if not quiet: print("!~", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) next(nDiffRemote) next(nRemote) - - + + next(nDiffLocal) next(nLocal) @@ -240,35 +240,35 @@ def Info(): if sign == "-": Minus() elif sign == "+": Plus() elif sign == "?": Info() - - - + + + while True: - + if ( - int(nlA) >= maxA and - int(nlB) >= maxB and + int(nlA) >= maxA and + int(nlB) >= maxB and int(nlC) >= maxC ): break lineaA = " " sAB = cAB = lineaA = " " sAC = cAC = lineaC = " " - if int(nlA) >= maxA : + if int(nlA) >= maxA : if int(nlA) - maxA > 0 : print("A overflow!", int(nlA) - maxA) lineaA = " " else: lineaA = A.sortednames[nlA] - - if int(nlB) >= maxB : + + if int(nlB) >= maxB : sAB = cAB = lineaA = " " if int(nlB) - maxB > 0 : print("B overflow!", int(nlB) - maxB) else: lineaB = B.sortednames[nlB] - + if int(nlAB) < len(diffAB): sAB = diffAB[nlAB][0] cAB = diffAB[nlAB][2:] - - if int(nlC) >= maxC : + + if int(nlC) >= maxC : sAC = cAC = lineaC = " " if int(nlC) - maxC > 0 : print("C overflow!", int(nlC) - maxC) else: @@ -277,7 +277,7 @@ def Info(): if int(nlAC) < len(diffAC): sAC = diffAC[nlAC][0] cAC = diffAC[nlAC][2:] - + #print nlA, nlB, nlC if sAB == " " and sAC == " ": AddPatchLine("A=") @@ -301,7 +301,7 @@ def Info(): print("wtf!?A:", lineaA) print("wtf!?B:", lineaB) print("wtf!?C:", lineaC) - + next(nlAB) next(nlAC) next(nlA) @@ -319,47 +319,47 @@ def Info(): if not quiet: print(sAB,"*", sAC) break - + addedB = set([x for x in AddedB if x[0]!="#"]) addedC = set([x for x in AddedC if x[0]!="#"]) - + deletedB = set([x for x in DeletedB if x[0]!="#"]) deletedC = set([x for x in DeletedC if x[0]!="#"]) - + movedB = addedB & deletedB movedC = addedC & deletedC - + conflictsAA = addedB & addedC conflictsDD = deletedB & deletedC conflictsAD = addedB & deletedC conflictsDA = deletedB & addedC - + if movedB: print("CONFLICTS BLOCK MOVED A(%s)->B(%s):" % (A.filename,B.filename)) for name in movedB: print("-",name) - + if movedC: print("CONFLICTS BLOCK MOVED A(%s)->C(%s):" % (A.filename,C.filename)) for name in movedC: print("-",name) - + if conflictsAA: print("CONFLICTS SAME BLOCK ADDED B(%s)-C(%s):" % (B.filename,C.filename)) for name in conflictsAA: print("-",name) - + if conflictsDD: print("CONFLICTS SAME BLOCK DELETED B(%s)-C(%s):" % (B.filename,C.filename)) for name in conflictsDD: print("-",name) - + if conflictsAD: print("CONFLICTS BLOCK ADDED BY %s , DELETED BY %s:" % (B.filename,C.filename)) for name in conflictsAD: print("-",name) - + if conflictsDA: print("CONFLICTS BLOCK DELETED BY %s , ADDED BY %s:" % (B.filename,C.filename)) for name in conflictsDA: print("-",name) - - - + + + return patchedResult """ added, deleted = pfrom.diffTo(pto) @@ -378,15 +378,15 @@ def Info(): else: bobject = ptarget.filename plist.append((bobject,start,end,name)) - + return plist """ def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = False): patchlist = appliedDiff(C, A, B, prefer , debug, quiet, swap) - F = {"A": A, "B": B, "C": C} - L = ["A", "B", "C"] - + F = {"A": A, "B": B, "C": C} + L = ["A", "B", "C"] + fout = open(F[prefer].filename + ".aligned","w") classlist = [] for Fby,action, nlines, Fwhich, line in patchlist: @@ -398,7 +398,7 @@ def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = except IndexError: print("!!!ERROR MERGING!!") continue - + text = "".join( F[Fwhich].lines[int(linebegin):int(lineend)] ) @@ -409,7 +409,7 @@ def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = lastclass = classlist[-1] else: lastclass = None - + thisclass = sline[1] classlist.append(thisclass) if lastclass: @@ -417,26 +417,26 @@ def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = if rs1: if lastclass != rs1.group(2): #print "INFO: Changing >> class", thisclass, "extends",rs1.group(2), "--> extends", lastclass - text = re.sub("class (\w+) extends (\w+)", "class %s extends %s" % (thisclass,lastclass),text) - rs2 = re.search("function .*%s\(.*context.*\) { (\w+)" % thisclass,text) + text = re.sub(r"class (\w+) extends (\w+)", "class %s extends %s" % (thisclass,lastclass),text) + rs2 = re.search(r"function .*%s\(.*context.*\) { (\w+)" % thisclass,text) if rs2: if lastclass != rs2.group(1): badline = rs2.group(0) goodline = badline.replace(rs2.group(1),lastclass) #print "INFO: Changing >>", badline , "-->", goodline text = text.replace(badline,goodline) - + else: print(text[:64]) - + #if debug: # fout.write("<<< %s || %s >>>\n" % (Fwhich, line)) # fout.write("<<< (%d:%d) >>>\n" % (int(linebegin),int(lineend))) fout.write(text) - - + + fout.close() - + def main(): parser = OptionParser() #parser.add_option("-q", "--quiet", @@ -450,7 +450,7 @@ def main(): parser.add_option("-q","--quiet", action="store_true", dest="quiet", default=False, help="don't print status messages to stdout") - + parser.add_option("--debug", action="store_true", dest="debug", default=False, help="prints lots of useless messages") @@ -467,7 +467,7 @@ def main(): if len(filenames) != 3: print("MUST have exactly 3 files to align.") - pfiles = [] + pfiles = [] for file1 in filenames: #print "Load File:", file1 pf = ProcessFile(file1) @@ -476,7 +476,7 @@ def main(): A = pfiles[0] B = pfiles[1] C = pfiles[2] - + #addedAB, deletedAB = A.diffTo(B) #addedAC, deletedAC = A.diffTo(C) is_debug = options.debug @@ -484,6 +484,6 @@ def main(): writeAlignedFile(B, A, C, debug= is_debug) writeAlignedFile(B, A, C, prefer = "A", debug= is_debug) #writeAlignedFile(A, A, C) - - -if __name__ == "__main__": main() \ No newline at end of file + + +if __name__ == "__main__": main() diff --git a/flclasses.py b/flclasses.py index 2010cd6..2bb0f00 100644 --- a/flclasses.py +++ b/flclasses.py @@ -11,14 +11,14 @@ def __init__(self): def setSubtype(self,newsubtype): x, y = self.type self.type = (x,newsubtype) - + def setType(self,newtype): x, y = self.type self.type = (newtype,y) - + def addCodeDepth(self): self.codedepth += 1 - + def __len__(self): return 1; @@ -43,8 +43,8 @@ def __init__(self,itemList,prefix,suffix,subtype="Unknown"): t,st = itemList.slice[0].type #itemList.setSubtype(st) itemList.setSubtype("OneItem") - - + + if not isinstance(itemList,cBaseList): raise NameError("itemList no es un cBaseList: %s" % repr(itemList)) self.itemList = itemList @@ -55,15 +55,15 @@ def __init__(self,itemList,prefix,suffix,subtype="Unknown"): ctype, csubtype = item.type ctype = subtype item.type = (ctype, csubtype) - + def __str__(self): global debug txt = str(self.prefix) + str(self.itemList) + str(self.suffix) txt = txt.strip() if debug>=2: txt = "{%s/%s:" % self.itemList.type + txt + "}" - return txt - + return txt + class cBaseVarSpec(cBase): def __init__(self,name,vartype=None,value=None): @@ -79,8 +79,8 @@ def __str__(self): if self.value: txt+="="+str(self.value) return txt - - + + class cBaseList(cBase): def __init__(self): cBase.__init__(self) @@ -97,7 +97,7 @@ def __len__(self): def includeItem(self,child): if not isinstance(child,cBase): raise NameError("Child is not an instance of Base Class!") - + try: ctype, csubtype = child.type except: @@ -107,7 +107,7 @@ def includeItem(self,child): if not hasattr(self.bySubtype,ctype): self.bySubtype["%s:%s" % (ctype,csubtype)]=[] - + self.byType[ctype].append(child) self.bySubtype["%s:%s" % (ctype,csubtype)].append(child) @@ -116,7 +116,7 @@ def includeItem(self,child): cname = child.name except: raise NameError("Declaration Class doesn't have `name` atribute.") - + if cname in self.byDefName: print("#WARNING# Variable %s found, but previously defined in this block" % cname) # self.byDefName[cname]=None @@ -132,14 +132,14 @@ def includeItem(self,child): sslice = child.itemList.slice[:] for e in child.itemList.hidden: sslice.remove(e) - - + + for item in sslice: itype, isubtype = item.type if not isinstance(item,cBaseItemList): self.includeItem(item) #self.hidden.append(item) - + def addCodeDepth(self): cBase.addCodeDepth(self) sslice = self.slice[:] @@ -149,7 +149,7 @@ def addCodeDepth(self): for child in sslice: child.addCodeDepth() - + def addAuto(self,element,subtype=None): @@ -157,14 +157,14 @@ def addAuto(self,element,subtype=None): element = cBaseItem(element) if subtype: element.setSubtype(subtype) - + self.addChild(element) - + def addChild(self,child,hidden=False): self.includeItem(child) self.slice.append(child) if hidden: self.hidden.append(child) - + def __str__(self): global debug sslice = self.slice[:] @@ -175,7 +175,7 @@ def __str__(self): if debug>=1: txt = "\n" - # --------- DEBUG OUTPUT VARDECL --------- + # --------- DEBUG OUTPUT VARDECL --------- if len(self.byDefName)>0: txt += " " * self.codedepth + " /** Declared vars: " for definition in self.byDefName: @@ -190,7 +190,7 @@ def __str__(self): return txt - + if len(sslice) == 1: c = str(sslice[0]) if "\n" not in c: @@ -202,7 +202,7 @@ def __str__(self): # txt += str(child.type) + "\n" # txt += str(child) + "\n" # txt += "____"+ "\n" - + lastmargin = 0 for c in sslice: t1, t2 = c.type @@ -217,9 +217,9 @@ def __str__(self): txt += "\n" * topmargin + line + "\n" * (margin+1) lastmargin = margin return txt - - - + + + class cBaseListInline(cBaseList): def __init__(self,separator=", "): cBaseList.__init__(self) @@ -230,20 +230,20 @@ def __str__(self): txt = "" for c in self.slice: if debug>=3: - txt += "[%s/%s: " % c.type - + txt += "[%s/%s: " % c.type + txt += str(c) if debug>=3: txt += "]" - + txt += self.separator - + if len(self.separator) == 0: return txt else: return txt[:-len(self.separator)] - - + + class cStatementList(cBaseList): def __init__(self): cBaseList.__init__(self) @@ -254,7 +254,7 @@ def __init__(self,name): cBase.__init__(self) self.type = ("Declaration","Unknown") self.name = name - + def __str__(self): return "@unknown declaration %s" % self.name @@ -269,13 +269,13 @@ def __init__(self,name,arglist,rettype,source): iList = cBaseList() iList.addAuto(source) source = iList - - + + if not isinstance(source,cBaseList): - raise NameError("source no es un cBaseList: %s" % repr(itemList)) + raise NameError("source no es un cBaseList: %s" % repr(source)) self.source = source - + def addCodeDepth(self): cBase.addCodeDepth(self) try: @@ -287,8 +287,8 @@ def addCodeDepth(self): traceback.print_exc(file=sys.stdout) print('-'*60) - - + + def __str__(self): if self.rettype: @@ -308,12 +308,12 @@ def __init__(self,name,extends,source): iList = cBaseList() iList.addAuto(source) source = iList - - + + if not isinstance(source,cBaseList): - raise NameError("source no es un cBaseList: %s" % repr(itemList)) + raise NameError("source no es un cBaseList: %s" % repr(source)) self.source = source - + def addCodeDepth(self): cBase.addCodeDepth(self) try: @@ -330,6 +330,6 @@ def __str__(self): ext = " extends " + self.extends else: ext = "" - + return 'class %s%s {%s}' % (self.name,ext,self.source) diff --git a/flex.py b/flex.py index 57b1539..fa59c04 100644 --- a/flex.py +++ b/flex.py @@ -14,10 +14,10 @@ reserved = [ 'BREAK', 'CASE', 'CONST', 'STATIC', 'CONTINUE', 'DEFAULT', 'DO', 'ELSE', 'FOR', 'IF', 'IN', - 'RETURN', - #'STRUCT', - 'SWITCH', - 'WHILE', 'CLASS', 'VAR', 'FUNCTION', + 'RETURN', + #'STRUCT', + 'SWITCH', + 'WHILE', 'CLASS', 'VAR', 'FUNCTION', 'EXTENDS', 'NEW','WITH','TRY','CATCH','THROW', 'DELETE', 'TYPEOF' ] token_literals = [ @@ -28,13 +28,13 @@ # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=) 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', - 'OR', 'AND', + 'OR', 'AND', 'CONDITIONAL1','AT', - #'NOT', + #'NOT', 'XOR', 'LSHIFT', 'RSHIFT', 'LOR', 'LAND', 'LNOT', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', - + # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', # 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL', @@ -47,7 +47,7 @@ # Conditional operator (?) # 'CONDOP', - + # Delimeters ( ) [ ] { } , . ; : 'LPAREN', 'RPAREN', 'LBRACKET', 'RBRACKET', @@ -58,7 +58,7 @@ # 'ELLIPSIS', 'DOCSTRINGOPEN', # 'COMMENTOPEN', - 'COMMENTCLOSE', + 'COMMENTCLOSE', 'DOLLAR', 'SQOUTE', 'DQOUTE', @@ -72,7 +72,7 @@ def t_NEWLINE(t): r'\n+' t.lexer.lineno += t.value.count("\n") - + # Operators t_BACKSLASH = '\\\\' t_DOLLAR = r'\$' @@ -168,7 +168,7 @@ def t_ID(t): # Character constant 'c' or L'c' t_CCONST = r'\'([^\\\n]|(\\.))*?\'' -# REGEX constant +# REGEX constant #t_RXCONST = r'/[^/ ]+/g?' # Comments @@ -185,24 +185,24 @@ def t_DOCSTRINGOPEN(t): #t_COMMENTOPEN = r'/\*' t_COMMENTCLOSE = r'\*/' - + # Preprocessor directive (ignored) def t_preprocessor(t): r'\#(.)*?\n' t.lexer.lineno += 1 - + def t_error(t): print("Illegal character %s" % repr(t.value[0])) t.lexer.skip(1) - + lexer = lex.lex(debug=False) if __name__ == "__main__": lex.runmain(lexer) - + diff --git a/flpremerge.py b/flpremerge.py index 73c74b2..7339970 100644 --- a/flpremerge.py +++ b/flpremerge.py @@ -7,14 +7,14 @@ import os, os.path, sys import math from optparse import OptionParser -import difflib +import difflib import re class processedFile(object): def __init__(self, filename, debug = False): self.debug = debug - self.table = {} # Carga literal de la lista de hashes, pk: (startbyte, endbyte) = csvrow - self.idxdepth = {} + self.table = {} # Carga literal de la lista de hashes, pk: (startbyte, endbyte) = csvrow + self.idxdepth = {} self.idxtree = {} # pk: objnum (1.5.5.1.1) = (startbyte, endbyte) self.hashes = {} self.list_hashes = [] @@ -23,10 +23,10 @@ def __init__(self, filename, debug = False): self.processHashFile() self.indexNames() self.computeSortedBlocks() - + def processHashFile(self): self.cacheFullQname = {} - self.table, self.idxdepth,self.idxtree, self.hashes, self.list_hashes = process(self.filename+".hash") + self.table, self.idxdepth,self.idxtree, self.hashes, self.list_hashes = process(self.filename+".hash") def indexNames(self): @@ -34,23 +34,23 @@ def indexNames(self): self.sortedNames = [] for pk in self.idxtree: if len(pk) > 1: continue - name = self.fullQName(pk) + name = self.fullQName(pk) if name not in self.names: self.names[name] = [] self.names[name].append(pk) self.sortedNames.append((self.idxtree[pk],name)) - + self.sNames = set(self.names.keys()) - + def Qname(self,p): row = self.table[self.idxtree[p]] return row["name"] - - + + def fullQName(self,p): dcache = self.cacheFullQname if p not in dcache: name=[] - + for n in range(len(p)): ps1 = p[:n+1] name.append(self.Qname(ps1)) @@ -58,15 +58,15 @@ def fullQName(self,p): dcache[p] = fullname else: fullname = dcache[p] - + return fullname def computeSortedBlocks(self, fout=None): self.computedBlocks = [] if fout is None: fout = open(self.filename + ".blocks","w") - - antdesde, anthasta = 0 , -1 + + antdesde, anthasta = 0 , -1 fB = open(self.filename) pos = 0 linePosChar = [pos] @@ -81,7 +81,7 @@ def computeSortedBlocks(self, fout=None): for pk, bl_name in list(sorted(self.sortedNames))+[((None,None),None)]: desde, hasta = pk if desde is None or desde >= anthasta +1: - + bdesde = anthasta + 1 fB.seek(bdesde) if desde: @@ -90,23 +90,23 @@ def computeSortedBlocks(self, fout=None): else: sB = fB.read() bhasta = bdesde + len(sB) -1 - - + + while linenum < len(linePosChar) and linePosChar[linenum+1]<=bdesde: linenum += 1 startline = linenum linesize=linePosChar[startline] - linePosChar[startline-1] curpos = bdesde - linePosChar[startline-1] #if curpos != linesize: print "****", linesize, curpos - - + + while linenum < len(linePosChar) and linePosChar[linenum+1]<=bhasta: linenum += 1 - endline = linenum + endline = linenum #while startline > 1 and linePosChar[startline-1]-bdesde >=-2: startline-=1 - + #print linePosChar[startline-1]-bdesde, linePosChar[startline]-bdesde, linePosChar[startline+1]-bdesde - + #print (startline, endline), "BLOCK", (linePosChar[startline], linePosChar[endline]), (bdesde , bhasta), (bhasta-bdesde)+1 - + #print "<<<<" mode = "" nline = startline @@ -116,7 +116,7 @@ def computeSortedBlocks(self, fout=None): if len(sB.splitlines(1)) != endline-startline+1: print(startline, endline, repr(sB)) print(linePosChar[endline-1]-bhasta, linePosChar[endline]-bhasta, linePosChar[endline+1]-bhasta) - + print("Block lines doesnt match:", len(sB.splitlines(1)), endline-startline, startline, endline) print(linePosChar[startline-2:startline+3],bdesde) print(linePosChar[endline-2:endline+3],bhasta) @@ -130,24 +130,24 @@ def computeSortedBlocks(self, fout=None): iscommentline2 = re.match(r'[ \t]*/\*.+\*/\n',line) iscommentbegin = re.match(r'[ \t]*/\*.+\n',line) iscommentend = re.match(r'.+\*/[ \t]*\n',line) - - - + + + if isseparator: ltype = "separator" elif iscommentline1: ltype = "comment_inline" elif iscommentline2: ltype = "comment_block_inline" elif iscommentbegin: ltype = "comment_block" elif iscommentend: ltype = "comment_blockend" #else: print "junk?", line, - + if mode == "comment_block" and ltype == "comment_blockend": mode = "comment_blockend" - + if mode == "comment_block" and ltype != "comment_blockend": ltype = "comment_block" - - - if mode != ltype: + + + if mode != ltype: if mode: if mode == "comment_blockend": mode = "comment_block" @@ -156,7 +156,7 @@ def computeSortedBlocks(self, fout=None): bblocks.append(thisblock) mode = ltype beginline = nline - 1 - + words = re.split(r'\W+',line) if words: text = " ".join(words) @@ -164,7 +164,7 @@ def computeSortedBlocks(self, fout=None): text = text.replace(" ", "-") if len(text)>1: blockdesc.append(text) - + if mode: if mode == "comment_blockend": mode = "comment_block" @@ -178,7 +178,7 @@ def computeSortedBlocks(self, fout=None): #print lines, " ", "%s:%s" % (bname, desc) startline, endline = lines name = "#..%s:%s" % (bname, desc) - + self.computedBlocks.append((startline, endline,name)) fout.write("%d\t%d\t%s\n" % (startline, endline,name)) #print "#..%s:%s" % (bname, desc) @@ -187,10 +187,10 @@ def computeSortedBlocks(self, fout=None): if desde is None: break initline = linenum while linenum < len(linePosChar) and linePosChar[linenum+1] 2: print(linePosChar[startline] - desde, linePosChar[endline-1] - hasta, linePosChar[endline] - hasta, bl_name) name = bl_name @@ -203,17 +203,17 @@ def computeSortedBlocks(self, fout=None): anthasta = linePosChar[linenum-1] + 1 elif anthasta <= linePosChar[linenum] + 1: anthasta = linePosChar[linenum] + 1 - + fB.close() fout.close() - + def linejunk(line): line = line.strip() if len(line)<4: return True if line[2] == "//": return True return False - + def charjunk(char): junk = [" ", "\t"] if char in junk: return True @@ -234,7 +234,7 @@ def main(): parser.add_option("--debug", action="store_true", dest="debug", default=False, help="prints lots of useless messages") - + (options, args) = parser.parse_args() if options.optdebug: print(options, args) @@ -253,7 +253,7 @@ def main(): #pfA = pfiles[0] #for pfB in pfiles[1:]: # feq = FindEquivalences(pfA, pfB) - + def tree_parents(pk): parents = [] while (len(pk)>0): @@ -268,15 +268,15 @@ def __init__(self,pfA, pfB, autoCompute = True): self.max_known_eq = {} self.pfA, self.pfB = pfA, pfB if autoCompute: self.compute() - + def compute(self): print("Finding equivalences between A (%s) -> B (%s):" % ( - self.pfA.filename, + self.pfA.filename, self.pfB.filename )) print("Modified names:") commonNames = sorted(list(self.pfA.sNames & self.pfB.sNames)) - for name in commonNames: + for name in commonNames: if len(self.pfA.names[name]) > 1 or len(self.pfB.names[name]) > 1: print("-", name,"(%d,%d)" % (len(self.pfA.names[name]),len(self.pfB.names[name]))) else: @@ -293,12 +293,12 @@ def compute(self): fA.seek(keyA[0]) sA = fA.read(keyA[1]-keyA[0]+1) fA.close() - + fB = open(fileB) fB.seek(keyB[0]) sB = fB.read(keyB[1]-keyB[0]+1) fB.close() - + sA = sA.replace("\t", " ") sB = sB.replace("\t", " ") lines = list(difflib.ndiff(sA.splitlines(1), sB.splitlines(1),linejunk,charjunk)) @@ -326,13 +326,13 @@ def compute(self): else: for line in omit: if line[0] in (' ','+'): n+=1 - print("%03d" % n , line, end=' ') + print("%03d" % n , line, end=' ') omit = [] - if line[0] in (' ','+'): + if line[0] in (' ','+'): n+=1 - print("%03d" % n , line, end=' ') - else: - print("%03d" % (n+1) , line, end=' ') + print("%03d" % n , line, end=' ') + else: + print("%03d" % (n+1) , line, end=' ') else: omit.append(line) if len(omit) : @@ -343,31 +343,31 @@ def compute(self): else: for line in omit: if line[0] in (' ','+'): n+=1 - print("%03d" % n , line, end=' ') + print("%03d" % n , line, end=' ') omit = [] print() else: print("(diff ommitted because we couldn't find original files)") - + print() print("Deleted names:") deletedNames = sorted(list(self.pfA.sNames - self.pfB.sNames)) - for name in deletedNames: + for name in deletedNames: print("-", name) print() print("Added names:") addedNames = sorted(list(self.pfB.sNames - self.pfA.sNames)) - for name in addedNames: + for name in addedNames: print("-", name) print() return - - - - - + + + + + for key in self.pfA.list_hashes: if key in self.pfB.hashes: lpkA = self.pfA.hashes[key] @@ -379,14 +379,14 @@ def compute(self): for pkA in self.pfA.idxtree: parentA = pkA[:-1] - if pkA not in self.equivalences: + if pkA not in self.equivalences: self.equivalences[pkA] = {} if parentA: - if parentA not in self.parent_equivalences: + if parentA not in self.parent_equivalences: self.parent_equivalences[parentA] = [] - + for pkA in sorted(self.pfA.idxtree): - parentsA = tree_parents(pkA) + parentsA = tree_parents(pkA) for pkB, punt in self.equivalences[pkA].items(): if len(pkA) != len(pkB): continue parentsB = tree_parents(pkB) @@ -403,12 +403,12 @@ def compute(self): #for l in lev2_list: # lev2_plist |= set(tree_parents(l)[1:]) #lev2 = len(set(lev2_list) - set(lev2_plist)) - + pEq = (pB, old_div(float(punt),lev2)) - if pA not in self.parent_equivalences: + if pA not in self.parent_equivalences: self.parent_equivalences[pA] = [] self.parent_equivalences[pA].append(pEq) - + norepeat = (0,) self.parent_equivalences2 = {} for pA in sorted(self.parent_equivalences): @@ -417,16 +417,16 @@ def compute(self): for pB, punt in self.parent_equivalences[pA]: if pB not in count: count[pB] = 0 count[pB] += punt - rcount = [] + rcount = [] ppA = pA[:-1] if ppA in self.parent_equivalences2: ppB = self.parent_equivalences2[ppA] else: ppB = None - + rowA = self.pfA.table[self.pfA.idxtree[pA]] for key, punt in count.copy().items(): - if ppB and key[:-1] != ppB: continue + if ppB and key[:-1] != ppB: continue rowB = self.pfB.table[self.pfB.idxtree[pB]] nameA = self.pfA.fullQName(pA) #rowA['name'].split(":") nameB = self.pfB.fullQName(pB) #rowB['name'].split(":") @@ -441,7 +441,7 @@ def compute(self): #if nameA[1] != nameB[1]: punt /=1.0+len(nameA[1]) / 40.0+len(nameB[1]) / 40.0 if punt >= 0.50: rcount.append((round(punt*100),key)) - + if len(rcount): punt, pB = max(rcount) self.parent_equivalences2[pA] = pB @@ -449,10 +449,10 @@ def compute(self): print("parent:", pA, self.pfA.fullQName(pA), "%d%%\t" % punt, pB, len(rcount) , self.pfB.fullQName(pB)) if punt > 100: norepeat = pA - else: + else: if len(pA) == 1: print("parent:", pA, self.pfA.fullQName(pA), "0%\t ???") - + """ norepeat = (0,) prevprint = None @@ -466,36 +466,36 @@ def compute(self): if len(pkA) > len(prevprint) and prevprint[:-1] == pkA[:len(prevprint)-1]: continue elif pkA < len(prevprint): prevprint = None elif prevprint[:-1] != pkA[:len(prevprint)-1]: prevprint = None - + print pkA,":", - + for pkB, punt in self.equivalences[pkA].iteritems(): - if punt > 0.96: + if punt > 0.96: norepeat = pkA if punt >= 0.1: print pkB, punt, ";", prevprint = pkA - print - """ + print + """ """ for pkB, punt in self.equivalences[pkA].iteritems(): - if punt > 0.96: + if punt > 0.96: norepeat = pkA print ">>", ".".join(["%02d" % x for x in pkB]) prevprint = pkA - """ + """ + + - - def getMaxKnown(self,pkA): pkA = tuple(pkA) if len(pkA) == 0: return 1.0, None if pkA not in self.max_known_eq: return 0.0, None pkB = self.max_known_eq[pkA] - eq_prob = self.equivalences[pkA][pkB] + eq_prob = self.equivalences[pkA][pkB] return eq_prob, pkB - - + + def addEquivalences(self,lpkA,lpkB): lstEquivalences = self.multiplyEquivalences(lpkA,lpkB) base_probability = old_div(1.0, len(lstEquivalences)) @@ -507,8 +507,8 @@ def addEquivalences(self,lpkA,lpkB): parent_prob, parentB = self.getMaxKnown(parentA) if parentB: if parentB != pkB[:-1]: parent_prob = old_div((1-parent_prob), 2.0) - probability *= parent_prob - + probability *= parent_prob + parentB = pkB[:-1] if probability < 0.01: continue if pkA not in self.equivalences: @@ -519,22 +519,22 @@ def addEquivalences(self,lpkA,lpkB): previousMax, prevPkB = self.getMaxKnown(pkA) if probability > previousMax: self.max_known_eq[pkA] = pkB - - - + + + def multiplyEquivalences(self,lpkA,lpkB): leq = set([]) for pkA in lpkA: for pkB in lpkB: leq|=set([(pkA,pkB)]) return list(leq) - + def process(filename): table, idxdepth,idxtree = load(filename) #print table.items()[:10] treebydepth = {} - + for k in sorted(idxtree.keys()): td = len(k)-1 if td not in treebydepth: treebydepth[td] = [] @@ -544,7 +544,7 @@ def process(filename): maxd = 0 hashes = {} list_hashes = [] - + for d,idx in treebydepth.items(): if d > 2: break nitems = len(idx) @@ -557,7 +557,7 @@ def process(filename): pk = idxtree[k] r = table[pk] rhash = r["hash"] - if rhash not in hashes: + if rhash not in hashes: list_hashes.append(rhash) hashes[rhash] = [] hashes[rhash].append(k) @@ -572,12 +572,12 @@ def isinside(parent, child): if cfrom > pto and cto > pto: return 1 #print "ERROR:", child , " is superior to ", parent return 0 - + def load(filename): file = open(filename) def getpk(row): return (row["start"],row["end"]) - + fields = [ "depth", "hash", @@ -602,7 +602,7 @@ def getpk(row): if depth not in bydepth: bydepth[depth] = [] bydepth[depth].append(pk) rows[pk]=row - + for dpth, items in bydepth.items(): bydepth[dpth] = list(sorted(items)) @@ -615,7 +615,7 @@ def getpk(row): else: pdepth = currdepth - 1 np = 0 - + for pk in bydepth[currdepth]: n+=1 if currdepth > 0: @@ -638,25 +638,25 @@ def getpk(row): # print list(enumerate(bydepth[pdepth])) # assert(offset >= 0) # assert(it < 250) - + prow = rows[ppk] nparent = prow["tree_id"] - - + + row = rows[pk] tree_id = nparent + [n] row["tree_id"] = tree_id if tuple(tree_id) in idxtree: print("ERROR:", tuple(tree_id), " is duplicated:") - print("previous:", idxtree[tuple(tree_id)]) + print("previous:", idxtree[tuple(tree_id)]) print("new:" , pk) else: - idxtree[tuple(tree_id)] = pk - - - - + idxtree[tuple(tree_id)] = pk + + + + return rows, bydepth, idxtree - -if __name__ == "__main__": main() \ No newline at end of file + +if __name__ == "__main__": main() diff --git a/flscriptparse.py b/flscriptparse.py index 1c75733..df26e18 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -609,7 +609,7 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou nuevalinea = False name = "" lines = [] - + l = 0 for ctype, value in tree['content']: @@ -617,7 +617,8 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou nuevalinea = False if nuevalinea: - for i in range(int(math.ceil(old_div(l,2.0)))): lines.append(sep * depth) + for i in range(int(math.ceil(l/2.0))): + lines.append(sep * depth) nuevalinea = False if type(value) is dict and ctype == otype: @@ -630,7 +631,8 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou l = 0 if ctype in marginblocks: l = marginblocks[ctype] - for i in range(int(math.floor(old_div(l,2.0)))): lines.append(sep * depth) + for i in range(int(math.floor(l/2.0))): + lines.append(sep * depth) tname,tlines,trange = printtree(value, depth+1, ctype) # lines.append(sep * depth + "" % (len("".join(tlines)))) From ce488722905e7f65ca92469c8b07028886efb742 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 20 Jul 2015 10:26:59 +0200 Subject: [PATCH 059/100] =?UTF-8?q?Mejoras=20parseado=20e=20intepretaci?= =?UTF-8?q?=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pytnyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytnyzer.py b/pytnyzer.py index 24afd98..2d412f2 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -210,7 +210,7 @@ def generate(self, **kwargs): yield dtype, data identifier = " ".join(expr) if identifier: - yield "line", "except Exception, %s:" % (identifier) + yield "line", "except Exception as %s:" % (identifier) else: yield "line", "except Exception:" yield "begin", "block-except" From 1947ad049bf1f55c5709f7265c9fdc3891094d6f Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 20 Jul 2015 12:14:06 +0200 Subject: [PATCH 060/100] mas arreglos y mejoras al parseador --- pytnyzer.py | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index 2d412f2..a4a44c7 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -44,7 +44,7 @@ def __init__(self, elem): def polish(self): return self def generate(self, **kwargs): - yield "debug", etree.tostring(self.elem) + yield "debug", "* not-known-seq * " + etree.tounicode(self.elem) class Source(ASTPython): @@ -286,7 +286,7 @@ def generate(self, **kwargs): main_expr.append("True") else: main_expr.append(" ".join(expr)) - yield "debug", "FOR:" + yield "debug", "WHILE-FROM-QS-FOR: " + repr(main_expr) yield "line", "while %s:" % (" ".join(main_expr)) for source in self.elem.xpath("Source"): yield "begin", "block-for" @@ -296,6 +296,28 @@ def generate(self, **kwargs): yield "line", line yield "end", "block-for" +class ForIn(ASTPython): + def generate(self, **kwargs): + list_elem, main_list = "None", "None" + myelems = [] + for e in self.elem: + if e.tag == "Source": break + if e.tag == "ForInitialize": e = list(e)[0] + expr = [] + for dtype, data in parse_ast(e).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + myelems.append(" ".join(expr)) + list_elem, main_list = myelems + yield "debug", "FOR-IN: " + repr(myelems) + yield "line", "for %s in %s:" % (list_elem, main_list) + for source in self.elem.xpath("Source"): + yield "begin", "block-for-in" + for obj in parse_ast(source).generate(include_pass=False): yield obj + yield "end", "block-for-in" + class Switch(ASTPython): def generate(self, **kwargs): key = "%02x" % random.randint(0,255) @@ -542,6 +564,8 @@ def generate(self, **kwargs): yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) + + # Lectura del self.iface.__init if len(arguments) >= 3 and arguments[0:2] == ["self","iface"] and arguments[2].startswith("__"): # From: self.iface.__function() # to: super(className, self.iface).function() @@ -553,6 +577,18 @@ def generate(self, **kwargs): arguments[2] = arguments[2][2:] arguments[0:2] = ["super(%s, %s)" % (classname,".".join(arguments[0:2]))] + # Lectura del self.iface.__init() al nuevo estilo yeboyebo + if len(arguments) >= 2 and arguments[0:1] == ["_i"] and arguments[1].startswith("__"): + # From: self.iface.__function() + # to: super(className, self.iface).function() + funs = self.elem.xpath("ancestor::Function") + if funs: + fun = funs[-1] + name_parts = fun.get("name").split("_") + classname = name_parts[0] + arguments[1] = arguments[1][2:] + arguments[0:1] = ["super(%s, %s)" % (classname,".".join(arguments[0:1]))] + yield "expr", ".".join(arguments) class ArrayMember(ASTPython): From 29664b6f610369c6bc4ced9d03abe7d1e5674126 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 20 Jul 2015 16:18:14 +0200 Subject: [PATCH 061/100] mejoras --- postparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postparse.py b/postparse.py index 10afa18..327e691 100644 --- a/postparse.py +++ b/postparse.py @@ -478,7 +478,7 @@ def pythonify(filelist): options.full = True if isinstance(filelist, str): filelist = [filelist] execute(options,filelist) - + print(filelist) From 847e6f9d84f0a14ed33f5b4f5eb999c6480c502d Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 21 Jul 2015 17:17:14 +0200 Subject: [PATCH 062/100] mejora debug pythnyzer --- pytnyzer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index a4a44c7..580b64f 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -246,7 +246,7 @@ def generate(self, **kwargs): class For(ASTPython): def generate(self, **kwargs): - main_expr = [] + init_expr = [] for n,arg in enumerate(self.elem.xpath("ForInitialize/*")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): @@ -255,9 +255,9 @@ def generate(self, **kwargs): else: yield dtype, data if len(expr) > 0: - main_expr.append(" ".join(expr)) - if main_expr: - yield "line", " ".join(main_expr) + init_expr.append(" ".join(expr)) + if init_expr: + yield "line", " ".join(init_expr) incr_expr = [] incr_lines = [] @@ -286,7 +286,7 @@ def generate(self, **kwargs): main_expr.append("True") else: main_expr.append(" ".join(expr)) - yield "debug", "WHILE-FROM-QS-FOR: " + repr(main_expr) + yield "debug", "WHILE-FROM-QS-FOR: (%r;%r;%r)" % (init_expr,main_expr,incr_lines) yield "line", "while %s:" % (" ".join(main_expr)) for source in self.elem.xpath("Source"): yield "begin", "block-for" From 253985a85a0a74550b916f2c0bc1d22d5f3eb85a Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 21 Jul 2015 18:42:32 +0200 Subject: [PATCH 063/100] agregar forbidden fruit y otros cambios --- pytnyzer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pytnyzer.py b/pytnyzer.py index 580b64f..a6bf88a 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -564,7 +564,6 @@ def generate(self, **kwargs): yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) - # Lectura del self.iface.__init if len(arguments) >= 3 and arguments[0:2] == ["self","iface"] and arguments[2].startswith("__"): # From: self.iface.__function() @@ -589,6 +588,15 @@ def generate(self, **kwargs): arguments[1] = arguments[1][2:] arguments[0:1] = ["super(%s, %s)" % (classname,".".join(arguments[0:1]))] + if "length" in arguments: + idx = arguments.index("length") + part1 = arguments[:idx] + try: + part2 = arguments[idx+1] + except IndexError: + part2 = [] # Para aquellos que solo sea un len(x) sin más miembros a la derecha + arguments = ["len(%s)" % (".".join(part1))] + part2 + yield "expr", ".".join(arguments) class ArrayMember(ASTPython): From 819c3398787e59f96ca2e9f7ae0222168c8c9323 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 22 Jul 2015 11:01:33 +0200 Subject: [PATCH 064/100] mejoras varias --- pytnyzer.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index a6bf88a..c3eef98 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -588,15 +588,23 @@ def generate(self, **kwargs): arguments[1] = arguments[1][2:] arguments[0:1] = ["super(%s, %s)" % (classname,".".join(arguments[0:1]))] - if "length" in arguments: - idx = arguments.index("length") - part1 = arguments[:idx] - try: - part2 = arguments[idx+1] - except IndexError: - part2 = [] # Para aquellos que solo sea un len(x) sin más miembros a la derecha - arguments = ["len(%s)" % (".".join(part1))] + part2 - + replace_members = [ + "length", + "text", + "join", + "date", + ] + + for member in replace_members: + for idx,arg in enumerate(arguments): + if member == arg or arg.startswith(member+"("): + + part1 = arguments[:idx] + try: + part2 = arguments[idx+1] + except IndexError: + part2 = [] # Para los que son últimos y no tienen parte adicional + arguments = ["qsa(%s).%s" % (".".join(part1), arg)] + part2 yield "expr", ".".join(arguments) class ArrayMember(ASTPython): From 6d1de5af5806b633e58e0872fd8700be1b13fcd6 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 22 Jul 2015 19:10:04 +0200 Subject: [PATCH 065/100] pytnyzer: agregar soporte para diccionarios y operador ternario --- postparse.py | 12 ++++++++ pytnyzer.py | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/postparse.py b/postparse.py index 327e691..e3e2738 100644 --- a/postparse.py +++ b/postparse.py @@ -230,6 +230,11 @@ class InstructionFlow(TypedObject): debug_other = True tags = ["flowinstruction"] +class Instruction(TagObject): + promote_child_if_alone = True + debug_other = False + tags = ["instruction"] + class OpMath(TypedObject): debug_other = True tags = ["mathoperator"] @@ -287,6 +292,13 @@ def polish(self): self.astname = "empty" return self +class DictObject(ListObject): + tags = ["dictobject_value_elemlist","dictobject_value"] + adopt_childs_tags = ['dictobject_value_elemlist',"dictobject_value"] + +class DictElem(ListObject): + tags = ["dictobject_value_elem"] + class ExpressionContainer(ListObject): tags = ["expression"] diff --git a/pytnyzer.py b/pytnyzer.py index c3eef98..f5b39c4 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -286,7 +286,7 @@ def generate(self, **kwargs): main_expr.append("True") else: main_expr.append(" ".join(expr)) - yield "debug", "WHILE-FROM-QS-FOR: (%r;%r;%r)" % (init_expr,main_expr,incr_lines) + #yield "debug", "WHILE-FROM-QS-FOR: (%r;%r;%r)" % (init_expr,main_expr,incr_lines) yield "line", "while %s:" % (" ".join(main_expr)) for source in self.elem.xpath("Source"): yield "begin", "block-for" @@ -389,7 +389,7 @@ class With(ASTPython): def generate(self, **kwargs): key = "%02x" % random.randint(0,255) name = "w%s_obj" % key - yield "debug", "WITH: %s" % key + #yield "debug", "WITH: %s" % key variable, source = [ obj for obj in self.elem ] var_expr = [] for dtype, data in parse_ast(variable).generate(isolate = False): @@ -513,6 +513,26 @@ def generate(self, **kwargs): arguments.append(" ".join(expr)) yield "line", " ".join(arguments) +class Instruction(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + if arguments: + yield "debug", "Instruction: Maybe parse-error. This class is only for non-understood instructions or empty ones" + yield "line", " ".join(arguments) + class InstructionFlow(ASTPython): def generate(self, break_mode = False, **kwargs): arguments = [] @@ -667,6 +687,60 @@ def generate(self, **kwargs): yield dtype, data yield "expr", ")" +class Delete(ASTPython): + def generate(self, **kwargs): + yield "expr", "del" + for child in self.elem: + for dtype, data in parse_ast(child).generate(isolate = False): + yield dtype, data + + +class OpTernary(ASTPython): + def generate(self, isolate = False, **kwargs): + """ + Ejemplo op. ternario + + + + + + + + + + """ + if_cond = self.elem[0] + then_val = self.elem[1] + else_val = self.elem[2] + yield "expr", "(" # Por seguridad, unos paréntesis + for dtype, data in parse_ast(then_val).generate(): yield dtype, data + yield "expr", "if" + for dtype, data in parse_ast(if_cond).generate(): yield dtype, data + yield "expr", "else" + for dtype, data in parse_ast(else_val).generate(): yield dtype, data + yield "expr", ")" # Por seguridad, unos paréntesis + + +class DictObject(ASTPython): + def generate(self, isolate = False, **kwargs): + yield "expr", "{" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + yield "expr", "," # Como en Python la coma final la ignora, pues la ponemos. + yield "expr", "}" + +class DictElem(ASTPython): + def generate(self, isolate = False, **kwargs): + # Clave: + for dtype, data in parse_ast(self.elem[0]).generate(): + yield dtype, data + yield "expr", ":" + # Valor: + for dtype, data in parse_ast(self.elem[1]).generate(): + yield dtype, data + + class OpUnary(ASTPython): def generate(self, isolate = False, **kwargs): ctype = self.elem.get("type") From 8ef866de5907bbaeb80965bccb414dad854acdd2 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 18 Sep 2015 09:02:07 +0200 Subject: [PATCH 066/100] Permitir trabajar sin "future" --- flscriptparse.py | 1 - postparse.py | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index df26e18..09856bd 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -4,7 +4,6 @@ from builtins import input from builtins import str from builtins import range -from past.utils import old_div # ----------------------------------------------------------------------------- # flscriptparse.py # diff --git a/postparse.py b/postparse.py index e3e2738..d4c4ac6 100644 --- a/postparse.py +++ b/postparse.py @@ -8,7 +8,11 @@ from pineboolib.flparser import flscriptparse import imp, traceback from lxml import etree -from future.utils import with_metaclass +try: + from future.utils import with_metaclass +except ImportError: + pass + USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") KNOWN_PARSERS = {} From 3af1870fca9afa67d88b5c09eac12b616688bbfb Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 25 Nov 2015 12:45:43 +0100 Subject: [PATCH 067/100] bugfix --- flscriptparse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 6408242..b73270e 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -439,7 +439,7 @@ def p_error(t): last_error_line = t.lineno elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: last_error_line = t.lineno - yacc.errok() + parser.errok() ok_count = 0 return @@ -452,8 +452,8 @@ def p_error(t): last_error_token = "EOF" return t - t = yacc.token() - yacc.restart() + t = parser.token() + parser.restart() last_error_token = t return t From 7e42cc427c4f710a2ef7e939ce269894d6d108ca Mon Sep 17 00:00:00 2001 From: Juan Jose Pablos Date: Wed, 2 Mar 2016 01:25:04 +0100 Subject: [PATCH 068/100] Cambio puramente decorativo. Primera y segunda linea define UTF-8 La sintaxis # -*- coding: -*- es la aceptada por editores https://www.python.org/dev/peps/pep-0263/ --- pytnyzer.py | 3 ++- qsatype.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pytnyzer.py b/pytnyzer.py index f5b39c4..0e12c78 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -1,4 +1,5 @@ #!/usr/bin/python +# -*- coding: utf-8 -*- # ------ Pythonyzer ... reads XML AST created by postparse.py and creates an equivalent Python file. from __future__ import print_function from __future__ import absolute_import @@ -882,7 +883,7 @@ def parse_ast(elem): def file_template(ast): - yield "line", "# encoding: UTF-8" + yield "line", "# -*- coding: utf-8 -*-" yield "line", "from pineboolib import qsatype" yield "line", "from pineboolib.qsaglobals import *" yield "line", "import traceback" diff --git a/qsatype.py b/qsatype.py index 7f8b8a7..3d5cc6a 100644 --- a/qsatype.py +++ b/qsatype.py @@ -1,5 +1,6 @@ +# -*- coding: utf-8 -*- + from builtins import object -# encoding: UTF-8 import os class Object(object): From d5ce88e3cb2fcda15ee75692c91f1b87606a3b50 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 18 Mar 2016 12:57:48 +0100 Subject: [PATCH 069/100] mejoras flscriptparser --- flscriptparse.py | 18 ++++++++++++------ postparse.py | 5 +++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index b73270e..3a83eb9 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -4,7 +4,7 @@ # Simple parser for FacturaLUX SCripting Language (QSA). # ----------------------------------------------------------------------------- from optparse import OptionParser - +import pprint import sys, math import hashlib import flex @@ -92,6 +92,8 @@ def p_parse(token): expression : base_expression | funcdeclaration_anon + | funcdeclaration_anon_exec + | LPAREN expression RPAREN | error case_cblock_list : case_block @@ -160,6 +162,9 @@ def p_parse(token): funcdeclaration : FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE funcdeclaration : STATIC FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE funcdeclaration_anon : FUNCTION LPAREN arglist RPAREN LBRACE basicsource RBRACE + | FUNCTION LPAREN RPAREN LBRACE basicsource RBRACE + funcdeclaration_anon_exec : funcdeclaration_anon LPAREN RPAREN + | funcdeclaration_anon LPAREN arglist RPAREN callarg : expression @@ -384,7 +389,6 @@ def p_parse(token): #print fromline, lexspan, token.slice[0] token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) - rspan = lexspan[0] if str(token.slice[0]) == 'empty' or numelems == 0: token[0] = None else: @@ -432,10 +436,12 @@ def p_error(t): try: print_context(t) except: pass if debug == True: - for tokname, tokln, tokdata in seen_tokens[-32:]: - if tokln == t.lineno: - print tokname, tokdata - print repr(last_ok_token[0]) + error_count += 20 # no imprimir mas de un error en debug. + print + for s in last_ok_token.slice[:]: + print ">>>" , s.lineno, repr(s), pprint.pformat(s.value,depth=3) + #print "LAST TOKEN:" + #print pprint.pformat(last_ok_token[0],depth=10) last_error_line = t.lineno elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: last_error_line = t.lineno diff --git a/postparse.py b/postparse.py index efc2846..7143a48 100755 --- a/postparse.py +++ b/postparse.py @@ -514,6 +514,11 @@ def execute(options, args): bname = os.path.basename(filename) sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) sys.stdout.flush(); + try: + filecontent = open(filename).read() + except Exception: + print "Error: No se pudo abrir fichero %-35s \n" % (repr(filename)) + continue prog = flscriptparse.parse(open(filename).read()) sys.stdout.write("\r"); if not prog: From 093aa065861fca9e2d0f00fcdd8a447d836abec1 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 28 Apr 2016 12:53:57 +0200 Subject: [PATCH 070/100] notas --- flscriptparser_automatic.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 flscriptparser_automatic.txt diff --git a/flscriptparser_automatic.txt b/flscriptparser_automatic.txt new file mode 100644 index 0000000..99a4d14 --- /dev/null +++ b/flscriptparser_automatic.txt @@ -0,0 +1,16 @@ +analisis completo: +git ls-files -- *.qs | xargs -t -P8 -IARG -n1 bash -c "eneboo-mergetool file-check qs-classes ARG >ARG.err 2>&1"; find * -iname "*err" -empty -delete + +analisis de los antiguos errores: +git ls-files -- *.qs | xargs -t -P8 -IARG -n1 bash -c "test -f ARG.err && eneboo-mergetool file-check qs-classes ARG >ARG.err 2>&1"; find * -iname "*err" -empty -delete + +lectura del analisis resumido: +grep -vE "(no corresponde|>>function |clase ifaceCtx|no heredó iface|para diferentes clases|más abajo en el código)" $(find * -iname "*err" ) + + +limpiar: +git clean -fx -- "*.qs.err" + + +auto-corregir "asumiendo class_definition": (PELIGROSO!!) +grep -h "asum" $(find * -iname "*err" ) | awk 'BEGIN { FS = " " } ; { print $16 "\t" $7 " " $8 "\t" $20 " " $21 }' | sed 's/[@\)]/ /g' | sed 's/:/\t/g' | awk '{ print "sed -i '"'"'" $2-3 "," $2+3 "s/" $3 " " $4 "/" $5 " " $6 "/g'"'"' " $1 }' From e70f6d6208cebf074e4fbe8247297e6f519c28e7 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 8 Sep 2016 09:27:14 +0200 Subject: [PATCH 071/100] pytnyzr: bugfix. --- pytnyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytnyzer.py b/pytnyzer.py index 0e12c78..9b9caa2 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -622,7 +622,7 @@ def generate(self, **kwargs): part1 = arguments[:idx] try: - part2 = arguments[idx+1] + part2 = arguments[idx+1:] except IndexError: part2 = [] # Para los que son últimos y no tienen parte adicional arguments = ["qsa(%s).%s" % (".".join(part1), arg)] + part2 From f609c528b0a4ebb5bd7f952e428e55115e185c47 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 20 Sep 2016 18:12:26 +0200 Subject: [PATCH 072/100] =?UTF-8?q?flparser:=20integraci=C3=B3n=20inicial?= =?UTF-8?q?=20en=20pineboo=20y=20conversion=20a=20Py3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- __init__.py | 0 flalign.py | 121 +++++++++--------- flclasses.py | 35 ++--- flex.py | 3 +- flpremerge.py | 104 ++++++++------- flscriptparse.py | 325 ++++++++++++++++++++++++----------------------- postparse.py | 175 +++++++++++++------------ pytnyzer.py | 273 ++++++++++++++++++++------------------- qsatype.py | 1 + xml2json.py | 38 +++--- xmlparse.py | 20 +-- 11 files changed, 572 insertions(+), 523 deletions(-) create mode 100644 __init__.py mode change 100755 => 100644 postparse.py mode change 100755 => 100644 xml2json.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/flalign.py b/flalign.py index 1f8a939..a5563d2 100644 --- a/flalign.py +++ b/flalign.py @@ -1,10 +1,13 @@ +from __future__ import print_function +from builtins import next +from builtins import object import os, os.path, sys import math from optparse import OptionParser import difflib import re -class ProcessFile: +class ProcessFile(object): def __init__(self, filename): self.filename = filename self.process() @@ -46,7 +49,7 @@ def diffTo(self,pfile2): return added, deleted -class LineNumber: +class LineNumber(object): def __init__(self, letter, lines): self.nl=0 if len(letter)>1: @@ -157,7 +160,7 @@ def AddPatchLine(mode): return if conflict and linetext[0]!="#": if debug: - print "WARNING: Omitting previously added block <%s>" % linetext + print("WARNING: Omitting previously added block <%s>" % linetext) while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") >= 0: #print "sep//", @@ -194,45 +197,45 @@ def Patch(code): def Minus(): AddPatchLine(letter + "-") if debug: - print letter + "-%04d" % int(nBase), nBase.line() + print(letter + "-%04d" % int(nBase), nBase.line()) if nBase.line() != nDiffLocal.line() and not quiet: - print letter + "! " , nDiffLocal.line() + print(letter + "! " , nDiffLocal.line()) if nDiffRemote.symbol()!=" " and not quiet: if nDiffRemote.symbol()=="-": #Removing twice! nBase.error = "removing-twice" else: - print "??", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line() + print("??", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) else: - nRemote.next() + next(nRemote) - nDiffLocal.next() - nDiffRemote.next() - nBase.next() + next(nDiffLocal) + next(nDiffRemote) + next(nBase) def Plus(): AddPatchLine(letter + "+") if debug: - print letter + "+%04d" % int(nLocal), nLocal.line() + print(letter + "+%04d" % int(nLocal), nLocal.line()) if nLocal.line()!=nDiffLocal.line() and not quiet: - print letter + "! " , nDiffLocal.line() + print(letter + "! " , nDiffLocal.line()) if nBase.error == "removing-twice": nBase.error = None while nDiffRemote.symbol() == "?": - nDiffRemote.next() + next(nDiffRemote) if nDiffRemote.symbol() == "+": if not quiet: - print "!~", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line() - nDiffRemote.next() - nRemote.next() + print("!~", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) + next(nDiffRemote) + next(nRemote) - nDiffLocal.next() - nLocal.next() + next(nDiffLocal) + next(nLocal) def Info(): #if debug: print letter + "?> " , nDiffLocal.line() - nDiffLocal.next() + next(nDiffLocal) if sign == "-": Minus() elif sign == "+": Plus() @@ -251,13 +254,13 @@ def Info(): sAB = cAB = lineaA = " " sAC = cAC = lineaC = " " if int(nlA) >= maxA : - if int(nlA) - maxA > 0 : print "A overflow!", int(nlA) - maxA + if int(nlA) - maxA > 0 : print("A overflow!", int(nlA) - maxA) lineaA = " " else: lineaA = A.sortednames[nlA] if int(nlB) >= maxB : sAB = cAB = lineaA = " " - if int(nlB) - maxB > 0 : print "B overflow!", int(nlB) - maxB + if int(nlB) - maxB > 0 : print("B overflow!", int(nlB) - maxB) else: lineaB = B.sortednames[nlB] @@ -267,7 +270,7 @@ def Info(): if int(nlC) >= maxC : sAC = cAC = lineaC = " " - if int(nlC) - maxC > 0 : print "C overflow!", int(nlC) - maxC + if int(nlC) - maxC > 0 : print("C overflow!", int(nlC) - maxC) else: lineaC = C.sortednames[nlC] @@ -280,30 +283,30 @@ def Info(): AddPatchLine("A=") if debug: if prefer == "": - print "ABC=%04d%+d%+d" % (nlA,nlB-nlA,nlC-nlA),lineaA + print("ABC=%04d%+d%+d" % (nlA,nlB-nlA,nlC-nlA),lineaA) elif prefer == "A": - print "A=%04d" % nlA,lineaA + print("A=%04d" % nlA,lineaA) elif prefer == "B": - print "B=%04d" % nlB,lineaB + print("B=%04d" % nlB,lineaB) elif prefer == "C": - print "C=%04d" % nlC,lineaC + print("C=%04d" % nlC,lineaC) else: assert prefer in ["A","B","C"] if not quiet: if lineaA!=cAB: - print "A!=B " , cAB + print("A!=B " , cAB) if lineaA!=cAC: - print "A!=C " , cAC + print("A!=C " , cAC) if lineaA != lineaB or lineaC != lineaA: - print "wtf!?A:", lineaA - print "wtf!?B:", lineaB - print "wtf!?C:", lineaC + print("wtf!?A:", lineaA) + print("wtf!?B:", lineaB) + print("wtf!?C:", lineaC) - nlAB.next() - nlAC.next() - nlA.next() - nlB.next() - nlC.next() + next(nlAB) + next(nlAC) + next(nlA) + next(nlB) + next(nlC) elif swap and sAB=="+": Patch("B+") elif sAC=="+": Patch("C+") elif sAC=="?": Patch("C?") @@ -314,14 +317,14 @@ def Info(): elif sAB=="-": Patch("B-") else: if not quiet: - print sAB,"*", sAC + print(sAB,"*", sAC) break - addedB = set(filter(lambda x: x[0]!="#",AddedB)) - addedC = set(filter(lambda x: x[0]!="#",AddedC)) + addedB = set([x for x in AddedB if x[0]!="#"]) + addedC = set([x for x in AddedC if x[0]!="#"]) - deletedB = set(filter(lambda x: x[0]!="#",DeletedB)) - deletedC = set(filter(lambda x: x[0]!="#",DeletedC)) + deletedB = set([x for x in DeletedB if x[0]!="#"]) + deletedC = set([x for x in DeletedC if x[0]!="#"]) movedB = addedB & deletedB movedC = addedC & deletedC @@ -332,28 +335,28 @@ def Info(): conflictsDA = deletedB & addedC if movedB: - print "CONFLICTS BLOCK MOVED A(%s)->B(%s):" % (A.filename,B.filename) - for name in movedB: print "-",name + print("CONFLICTS BLOCK MOVED A(%s)->B(%s):" % (A.filename,B.filename)) + for name in movedB: print("-",name) if movedC: - print "CONFLICTS BLOCK MOVED A(%s)->C(%s):" % (A.filename,C.filename) - for name in movedC: print "-",name + print("CONFLICTS BLOCK MOVED A(%s)->C(%s):" % (A.filename,C.filename)) + for name in movedC: print("-",name) if conflictsAA: - print "CONFLICTS SAME BLOCK ADDED B(%s)-C(%s):" % (B.filename,C.filename) - for name in conflictsAA: print "-",name + print("CONFLICTS SAME BLOCK ADDED B(%s)-C(%s):" % (B.filename,C.filename)) + for name in conflictsAA: print("-",name) if conflictsDD: - print "CONFLICTS SAME BLOCK DELETED B(%s)-C(%s):" % (B.filename,C.filename) - for name in conflictsDD: print "-",name + print("CONFLICTS SAME BLOCK DELETED B(%s)-C(%s):" % (B.filename,C.filename)) + for name in conflictsDD: print("-",name) if conflictsAD: - print "CONFLICTS BLOCK ADDED BY %s , DELETED BY %s:" % (B.filename,C.filename) - for name in conflictsAD: print "-",name + print("CONFLICTS BLOCK ADDED BY %s , DELETED BY %s:" % (B.filename,C.filename)) + for name in conflictsAD: print("-",name) if conflictsDA: - print "CONFLICTS BLOCK DELETED BY %s , ADDED BY %s:" % (B.filename,C.filename) - for name in conflictsDA: print "-",name + print("CONFLICTS BLOCK DELETED BY %s , ADDED BY %s:" % (B.filename,C.filename)) + for name in conflictsDA: print("-",name) @@ -393,7 +396,7 @@ def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = try: linebegin, lineend, line = F[Fwhich].blocks[nl] except IndexError: - print "!!!ERROR MERGING!!" + print("!!!ERROR MERGING!!") continue text = "".join( @@ -424,7 +427,7 @@ def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = text = text.replace(badline,goodline) else: - print text[:64] + print(text[:64]) #if debug: # fout.write("<<< %s || %s >>>\n" % (Fwhich, line)) @@ -454,16 +457,16 @@ def main(): (options, args) = parser.parse_args() if options.optdebug: - print options, args + print(options, args) - filenames = filter(lambda x: os.path.isfile(x) , args) + filenames = [x for x in args if os.path.isfile(x)] not_a_file = set(args) - set(filenames) if len(not_a_file): - print "WARNING: Not a file:", ", ".join(not_a_file) + print("WARNING: Not a file:", ", ".join(not_a_file)) return if len(filenames) != 3: - print "MUST have exactly 3 files to align." + print("MUST have exactly 3 files to align.") pfiles = [] for file1 in filenames: #print "Load File:", file1 diff --git a/flclasses.py b/flclasses.py index 5f3a0ae..2010cd6 100644 --- a/flclasses.py +++ b/flclasses.py @@ -1,6 +1,9 @@ +from __future__ import print_function +from builtins import str +from builtins import object debug = 0 -class cBase: +class cBase(object): def __init__(self): self.type = ("Unknown","Unknown") self.codedepth = 0 @@ -43,7 +46,7 @@ def __init__(self,itemList,prefix,suffix,subtype="Unknown"): if not isinstance(itemList,cBaseList): - raise NameError, "itemList no es un cBaseList: %s" % repr(itemList) + raise NameError("itemList no es un cBaseList: %s" % repr(itemList)) self.itemList = itemList self.prefix = prefix self.suffix = suffix @@ -93,12 +96,12 @@ def __len__(self): def includeItem(self,child): if not isinstance(child,cBase): - raise NameError, "Child is not an instance of Base Class!" + raise NameError("Child is not an instance of Base Class!") try: ctype, csubtype = child.type except: - raise NameError, "Base Class doesn't have `type` atribute or is incorrect." + raise NameError("Base Class doesn't have `type` atribute or is incorrect.") if not hasattr(self.byType,ctype): self.byType[ctype]=[] @@ -112,18 +115,18 @@ def includeItem(self,child): try: cname = child.name except: - raise NameError, "Declaration Class doesn't have `name` atribute." + raise NameError("Declaration Class doesn't have `name` atribute.") if cname in self.byDefName: - print "#WARNING# Variable %s found, but previously defined in this block" % cname + print("#WARNING# Variable %s found, but previously defined in this block" % cname) # self.byDefName[cname]=None else: self.byDefName[cname]=child try: child.addCodeDepth() except: - print repr(child) - raise NameError, "Base Class doesn't have `addCodeDepth` function." + print(repr(child)) + raise NameError("Base Class doesn't have `addCodeDepth` function.") if isinstance(child,cBaseItemList): sslice = child.itemList.slice[:] @@ -269,7 +272,7 @@ def __init__(self,name,arglist,rettype,source): if not isinstance(source,cBaseList): - raise NameError, "source no es un cBaseList: %s" % repr(itemList) + raise NameError("source no es un cBaseList: %s" % repr(itemList)) self.source = source @@ -279,10 +282,10 @@ def addCodeDepth(self): self.source.addCodeDepth() except: import traceback,sys - print "Exception in user code:" - print '-'*60 + print("Exception in user code:") + print('-'*60) traceback.print_exc(file=sys.stdout) - print '-'*60 + print('-'*60) @@ -308,7 +311,7 @@ def __init__(self,name,extends,source): if not isinstance(source,cBaseList): - raise NameError, "source no es un cBaseList: %s" % repr(itemList) + raise NameError("source no es un cBaseList: %s" % repr(itemList)) self.source = source def addCodeDepth(self): @@ -317,10 +320,10 @@ def addCodeDepth(self): self.source.addCodeDepth() except: import traceback,sys - print "Exception in user code:" - print '-'*60 + print("Exception in user code:") + print('-'*60) traceback.print_exc(file=sys.stdout) - print '-'*60 + print('-'*60) def __str__(self): if self.extends: diff --git a/flex.py b/flex.py index 171b5eb..57b1539 100644 --- a/flex.py +++ b/flex.py @@ -1,3 +1,4 @@ +from __future__ import print_function # ---------------------------------------------------------------------- # clex.py # @@ -192,7 +193,7 @@ def t_preprocessor(t): def t_error(t): - print "Illegal character %s" % repr(t.value[0]) + print("Illegal character %s" % repr(t.value[0])) t.lexer.skip(1) diff --git a/flpremerge.py b/flpremerge.py index 67dfeb9..73c74b2 100644 --- a/flpremerge.py +++ b/flpremerge.py @@ -1,10 +1,16 @@ +from __future__ import print_function +from __future__ import division +from builtins import zip +from builtins import range +from past.utils import old_div +from builtins import object import os, os.path, sys import math from optparse import OptionParser import difflib import re -class processedFile: +class processedFile(object): def __init__(self, filename, debug = False): self.debug = debug self.table = {} # Carga literal de la lista de hashes, pk: (startbyte, endbyte) = csvrow @@ -108,13 +114,13 @@ def computeSortedBlocks(self, fout=None): bblocks = [] blockdesc = [] if len(sB.splitlines(1)) != endline-startline+1: - print startline, endline, repr(sB) - print linePosChar[endline-1]-bhasta, linePosChar[endline]-bhasta, linePosChar[endline+1]-bhasta + print(startline, endline, repr(sB)) + print(linePosChar[endline-1]-bhasta, linePosChar[endline]-bhasta, linePosChar[endline+1]-bhasta) - print "Block lines doesnt match:", len(sB.splitlines(1)), endline-startline, startline, endline - print linePosChar[startline-2:startline+3],bdesde - print linePosChar[endline-2:endline+3],bhasta - print repr(sB.splitlines(1)) + print("Block lines doesnt match:", len(sB.splitlines(1)), endline-startline, startline, endline) + print(linePosChar[startline-2:startline+3],bdesde) + print(linePosChar[endline-2:endline+3],bhasta) + print(repr(sB.splitlines(1))) for line in sB.splitlines(1): nline += 1 if line[-1]!='\n': break @@ -186,7 +192,7 @@ def computeSortedBlocks(self, fout=None): #linenum += 1 endline = linenum if linePosChar[startline] - desde != 0 or linePosChar[endline] - hasta < 1 or linePosChar[endline] - hasta > 2: - print linePosChar[startline] - desde, linePosChar[endline-1] - hasta, linePosChar[endline] - hasta, bl_name + print(linePosChar[startline] - desde, linePosChar[endline-1] - hasta, linePosChar[endline] - hasta, bl_name) name = bl_name #print (startline, endline), name, (linePosChar[startline], linePosChar[endline]), pk, (hasta-desde)+1 self.computedBlocks.append((startline, endline,name)) @@ -231,12 +237,12 @@ def main(): (options, args) = parser.parse_args() if options.optdebug: - print options, args + print(options, args) - filenames = filter(lambda x: os.path.isfile(x) , args) + filenames = [x for x in args if os.path.isfile(x)] not_a_file = set(args) - set(filenames) if len(not_a_file): - print "WARNING: Not a file:", ", ".join(not_a_file) + print("WARNING: Not a file:", ", ".join(not_a_file)) if len(filenames): pfiles = [] @@ -255,7 +261,7 @@ def tree_parents(pk): pk = pk[:-1] return parents -class FindEquivalences: +class FindEquivalences(object): def __init__(self,pfA, pfB, autoCompute = True): self.equivalences = {} self.parent_equivalences = {} @@ -264,22 +270,22 @@ def __init__(self,pfA, pfB, autoCompute = True): if autoCompute: self.compute() def compute(self): - print "Finding equivalences between A (%s) -> B (%s):" % ( + print("Finding equivalences between A (%s) -> B (%s):" % ( self.pfA.filename, self.pfB.filename - ) - print "Modified names:" + )) + print("Modified names:") commonNames = sorted(list(self.pfA.sNames & self.pfB.sNames)) for name in commonNames: if len(self.pfA.names[name]) > 1 or len(self.pfB.names[name]) > 1: - print "-", name,"(%d,%d)" % (len(self.pfA.names[name]),len(self.pfB.names[name])) + print("-", name,"(%d,%d)" % (len(self.pfA.names[name]),len(self.pfB.names[name]))) else: keyA = self.pfA.idxtree[self.pfA.names[name][0]] rowA = self.pfA.table[keyA] keyB = self.pfB.idxtree[self.pfB.names[name][0]] rowB = self.pfB.table[keyB] if rowA['hash'] != rowB['hash']: - print "###", name,"###" + print("###", name,"###") fileA = self.pfA.filename.replace(".hash","") fileB = self.pfB.filename.replace(".hash","") if os.path.isfile(fileA) and os.path.isfile(fileB): @@ -316,44 +322,44 @@ def compute(self): if len(omit)>= Context: for line in omit: if line[0] in (' ','+'): n+=1 - print " ", "(... %d lines ommitted ...)" % len(omit) + print(" ", "(... %d lines ommitted ...)" % len(omit)) else: for line in omit: if line[0] in (' ','+'): n+=1 - print "%03d" % n , line , + print("%03d" % n , line, end=' ') omit = [] if line[0] in (' ','+'): n+=1 - print "%03d" % n , line , + print("%03d" % n , line, end=' ') else: - print "%03d" % (n+1) , line , + print("%03d" % (n+1) , line, end=' ') else: omit.append(line) if len(omit) : if len(omit)>= Context: for line in omit: if line[0] in (' ','+'): n+=1 - print " ", "(... %d lines ommitted ...)" % len(omit) + print(" ", "(... %d lines ommitted ...)" % len(omit)) else: for line in omit: if line[0] in (' ','+'): n+=1 - print "%03d" % n , line , + print("%03d" % n , line, end=' ') omit = [] - print + print() else: - print "(diff ommitted because we couldn't find original files)" + print("(diff ommitted because we couldn't find original files)") - print - print "Deleted names:" + print() + print("Deleted names:") deletedNames = sorted(list(self.pfA.sNames - self.pfB.sNames)) for name in deletedNames: - print "-", name - print - print "Added names:" + print("-", name) + print() + print("Added names:") addedNames = sorted(list(self.pfB.sNames - self.pfA.sNames)) for name in addedNames: - print "-", name - print + print("-", name) + print() return @@ -381,16 +387,16 @@ def compute(self): for pkA in sorted(self.pfA.idxtree): parentsA = tree_parents(pkA) - for pkB, punt in self.equivalences[pkA].iteritems(): + for pkB, punt in self.equivalences[pkA].items(): if len(pkA) != len(pkB): continue parentsB = tree_parents(pkB) - parentsAB = zip(parentsA,parentsB) + parentsAB = list(zip(parentsA,parentsB)) for pA, pB in parentsAB: sz2a, sz2b = self.pfA.idxtree[pkA] sz2 = sz2b - sz2a + 1 sz1a, sz1b = self.pfA.idxtree[pA] sz1 = sz1b - sz1a + 1 - lev2 = 1.0 + sz1 / float(sz2) + lev2 = 1.0 + old_div(sz1, float(sz2)) #lev2 = 2**(len(pkA) - len(pA)) #lev2_list = [ pC for pC in self.pfA.idxtree if len(pC) >= len(pA) and pC[:len(pA)] == pA ] #lev2_plist = set([]) @@ -398,7 +404,7 @@ def compute(self): # lev2_plist |= set(tree_parents(l)[1:]) #lev2 = len(set(lev2_list) - set(lev2_plist)) - pEq = (pB, float(punt)/lev2) + pEq = (pB, old_div(float(punt),lev2)) if pA not in self.parent_equivalences: self.parent_equivalences[pA] = [] self.parent_equivalences[pA].append(pEq) @@ -419,7 +425,7 @@ def compute(self): ppB = None rowA = self.pfA.table[self.pfA.idxtree[pA]] - for key, punt in count.copy().iteritems(): + for key, punt in count.copy().items(): if ppB and key[:-1] != ppB: continue rowB = self.pfB.table[self.pfB.idxtree[pB]] nameA = self.pfA.fullQName(pA) #rowA['name'].split(":") @@ -440,12 +446,12 @@ def compute(self): punt, pB = max(rcount) self.parent_equivalences2[pA] = pB rowB = self.pfB.table[self.pfB.idxtree[pB]] - print "parent:", pA, self.pfA.fullQName(pA), "%d%%\t" % punt, pB, len(rcount) , self.pfB.fullQName(pB) + print("parent:", pA, self.pfA.fullQName(pA), "%d%%\t" % punt, pB, len(rcount) , self.pfB.fullQName(pB)) if punt > 100: norepeat = pA else: if len(pA) == 1: - print "parent:", pA, self.pfA.fullQName(pA), "0%\t ???" + print("parent:", pA, self.pfA.fullQName(pA), "0%\t ???") """ norepeat = (0,) @@ -492,7 +498,7 @@ def getMaxKnown(self,pkA): def addEquivalences(self,lpkA,lpkB): lstEquivalences = self.multiplyEquivalences(lpkA,lpkB) - base_probability = 1.0 / len(lstEquivalences) + base_probability = old_div(1.0, len(lstEquivalences)) if base_probability < 0.01: return for pkA,pkB in lstEquivalences: @@ -500,7 +506,7 @@ def addEquivalences(self,lpkA,lpkB): parentA = pkA[:-1] parent_prob, parentB = self.getMaxKnown(parentA) if parentB: - if parentB != pkB[:-1]: parent_prob = (1-parent_prob) / 2.0 + if parentB != pkB[:-1]: parent_prob = old_div((1-parent_prob), 2.0) probability *= parent_prob parentB = pkB[:-1] @@ -508,7 +514,7 @@ def addEquivalences(self,lpkA,lpkB): if pkA not in self.equivalences: self.equivalences[pkA] = {} if pkB in self.equivalences[pkA]: - print "DUPLICATE", pkA,pkB + print("DUPLICATE", pkA,pkB) self.equivalences[pkA][pkB] = probability previousMax, prevPkB = self.getMaxKnown(pkA) if probability > previousMax: @@ -539,7 +545,7 @@ def process(filename): hashes = {} list_hashes = [] - for d,idx in treebydepth.iteritems(): + for d,idx in treebydepth.items(): if d > 2: break nitems = len(idx) if maxitems < nitems: maxitems = nitems @@ -589,7 +595,7 @@ def getpk(row): ] bydepth = {} for line in file: - row = dict(zip(fields,line[:-1].split("\t"))) + row = dict(list(zip(fields,line[:-1].split("\t")))) for f in intfields: row[f]=int(row[f]) pk = getpk(row) depth = row["depth"] @@ -597,7 +603,7 @@ def getpk(row): bydepth[depth].append(pk) rows[pk]=row - for dpth, items in bydepth.iteritems(): + for dpth, items in bydepth.items(): bydepth[dpth] = list(sorted(items)) idxtree = {} @@ -624,7 +630,7 @@ def getpk(row): np += offset if offset: n = 1 if offset < 0: - print list(enumerate(bydepth[pdepth])) + print(list(enumerate(bydepth[pdepth]))) assert(offset >= 0) #if it > 100: # print it,n,pk,np, ppk, offset @@ -641,9 +647,9 @@ def getpk(row): tree_id = nparent + [n] row["tree_id"] = tree_id if tuple(tree_id) in idxtree: - print "ERROR:", tuple(tree_id), " is duplicated:" - print "previous:", idxtree[tuple(tree_id)] - print "new:" , pk + print("ERROR:", tuple(tree_id), " is duplicated:") + print("previous:", idxtree[tuple(tree_id)]) + print("new:" , pk) else: idxtree[tuple(tree_id)] = pk diff --git a/flscriptparse.py b/flscriptparse.py index f35ddd2..1c75733 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -1,17 +1,24 @@ +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from builtins import input +from builtins import str +from builtins import range +from past.utils import old_div # ----------------------------------------------------------------------------- # flscriptparse.py # -# Simple parser for FacturaLUX SCripting Language (QSA). +# Simple parser for FacturaLUX SCripting Language (QSA). # ----------------------------------------------------------------------------- from optparse import OptionParser import sys, math import hashlib -import flex +from pineboolib.flparser import flex import ply.yacc as yacc import ply.lex as lex - -from flclasses import * + +from pineboolib.flparser.flclasses import * # Get the token map tokens = flex.tokens @@ -49,7 +56,7 @@ def p_parse(token): | variable | funccall | error - + identifier : ID dictobject_value : LBRACE RBRACE @@ -57,9 +64,9 @@ def p_parse(token): dictobject_value_elemlist : dictobject_value_elem | dictobject_value_elemlist COMMA dictobject_value_elem - + dictobject_value_elem : exprval COLON expression - + base_expression : exprval | inlinestoreinstruction | base_expression mathoperator base_expression @@ -71,39 +78,39 @@ def p_parse(token): | ternary_operator | dictobject_value | typeof_operator - - - - + + + + parentheses : LPAREN base_expression RPAREN | LPAREN variable_1 RPAREN - + unary_operator : LNOT base_expression | MINUS base_expression | PLUS base_expression - + new_operator : NEW funccall_1 | NEW identifier - + typeof_operator : TYPEOF variable | TYPEOF base_expression - + ternary_operator : base_expression CONDITIONAL1 base_expression COLON base_expression expression : base_expression | funcdeclaration_anon | error - case_cblock_list : case_block - case_cblock_list : case_cblock_list case_block + case_cblock_list : case_block + case_cblock_list : case_cblock_list case_block - case_block : CASE expression COLON statement_list + case_block : CASE expression COLON statement_list case_default : DEFAULT COLON statement_list case_block_list : empty case_block_list : case_default - case_block_list : case_cblock_list + case_block_list : case_cblock_list case_block_list : case_cblock_list case_default source_element : docstring @@ -114,7 +121,7 @@ def p_parse(token): source : source_element source : source source_element | statement_list - + basicsource : statement_list | empty @@ -132,7 +139,7 @@ def p_parse(token): statement_list : statement_list statement - statement_list : statement + statement_list : statement statement_list : LBRACE statement_list RBRACE @@ -144,11 +151,11 @@ def p_parse(token): vardeclaration : VAR vardecl_list SEMI | CONST vardecl_list SEMI - vardeclaration : VAR vardecl_list - | CONST vardecl_list - | STATIC VAR vardecl_list + vardeclaration : VAR vardecl_list + | CONST vardecl_list + | STATIC VAR vardecl_list - vardecl : ID optvartype EQUALS expression + vardecl : ID optvartype EQUALS expression vardecl : ID optvartype vardecl_list : vardecl @@ -202,19 +209,19 @@ def p_parse(token): variable : variable_1 | member_var - | LPAREN variable_1 RPAREN + | LPAREN variable_1 RPAREN | LPAREN member_var RPAREN - variable_1 : identifier + variable_1 : identifier | array_member - + array_member : variable_1 LBRACKET expression RBRACKET | funccall_1 LBRACKET expression RBRACKET - inlinestoreinstruction : PLUSPLUS variable - | MINUSMINUS variable - | variable PLUSPLUS - | variable MINUSMINUS + inlinestoreinstruction : PLUSPLUS variable + | MINUSMINUS variable + | variable PLUSPLUS + | variable MINUSMINUS updateoperator : EQUALS | PLUSEQUAL @@ -222,39 +229,39 @@ def p_parse(token): | MODEQUAL | DIVEQUAL | TIMESEQUAL - + updateinstruction : variable updateoperator expression | variable updateoperator updateinstruction - deleteinstruction : DELETE variable + deleteinstruction : DELETE variable storeinstruction : inlinestoreinstruction | updateinstruction | deleteinstruction - - flowinstruction : RETURN expression - | THROW expression - | RETURN - | BREAK - | CONTINUE + + flowinstruction : RETURN expression + | THROW expression + | RETURN + | BREAK + | CONTINUE instruction : base_instruction SEMI | SEMI - | base_instruction + | base_instruction | funcdeclaration | error SEMI - + callinstruction : funccall | variable - + base_instruction : storeinstruction | callinstruction - | flowinstruction + | flowinstruction varorcall : variable | funccall - | base_expression + | base_expression optextends : EXTENDS ID | empty @@ -280,13 +287,13 @@ def p_parse(token): | SCONST | regex | list_constant - + regex : DIVIDE regexbody DIVIDE regexflags | DIVIDE regexbody COMMENTCLOSE regexflags - + regexbody : regexchar | regexbody regexchar - + regexchar : LPAREN | RPAREN | ID @@ -305,10 +312,10 @@ def p_parse(token): | BACKSLASH | SCONST | error - + regexflags : ID | empty - + statement_block : statement | LBRACE statement_list RBRACE @@ -329,14 +336,14 @@ def p_parse(token): | LAND condition : expression - | error + | error ifstatement : IF LPAREN condition RPAREN statement_block optelse - whilestatement : WHILE LPAREN condition RPAREN statement_block + whilestatement : WHILE LPAREN condition RPAREN statement_block dowhilestatement : DO statement_block WHILE LPAREN condition RPAREN SEMI - withstatement : WITH LPAREN variable RPAREN statement_block + withstatement : WITH LPAREN variable RPAREN statement_block | error storeormember : storeinstruction @@ -346,20 +353,20 @@ def p_parse(token): | VAR vardecl | for_initialize COMMA for_initialize | empty - + for_compare : expression | empty - + for_increment : storeormember | for_increment COMMA for_increment | empty - - - forstatement : FOR LPAREN for_initialize SEMI for_compare SEMI for_increment RPAREN statement_block + + + forstatement : FOR LPAREN for_initialize SEMI for_compare SEMI for_increment RPAREN statement_block | error - forinstatement : FOR LPAREN for_initialize IN varorcall RPAREN statement_block - | FOR LPAREN variable IN varorcall RPAREN statement_block + forinstatement : FOR LPAREN for_initialize IN varorcall RPAREN statement_block + | FOR LPAREN variable IN varorcall RPAREN statement_block | error switch : SWITCH LPAREN condition RPAREN LBRACE case_block_list RBRACE @@ -369,40 +376,40 @@ def p_parse(token): trycatch : TRY statement_block CATCH LPAREN optid RPAREN statement_block - empty : + empty : ''' global input_data - + lexspan = list(token.lexspan(0)) data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) if len(lexspan) == 2: fromline = token.lineno(0) - global endoffile + global endoffile endoffile = fromline, lexspan, token.slice[0] #print fromline, lexspan, token.slice[0] - token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } + token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) - + rspan = lexspan[0] if str(token.slice[0]) == 'empty' or numelems == 0: token[0] = None else: rvalues = [] - for n,s in enumerate(token.slice[1:]): + for n,s in enumerate(token.slice[1:]): if s.type != 'empty' and s.value is not None: val = None - if isinstance(s.value,basestring): + if isinstance(s.value,str): val = token.lexspan(n+1)[0] + len(s.value) - 1 else: val = token.lexspan(n+1)[1] rvalues.append(val) rspan = max(rvalues) lexspan[1] = rspan - + if str(token.slice[0]) == 'regexbody': - token[0] = { "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } - - #if str(token.slice[0]) == 'regex': - # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] + token[0] = { "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } + + #if str(token.slice[0]) == 'regex': + # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] # print " " + "\n ".join([ "%s(%r): %r" % (s.type, token.lexspan(n+1), s.value) for n,s in enumerate(token.slice[1:]) ]) global seen_tokens, last_ok_token last_ok_token = token @@ -432,8 +439,8 @@ def p_error(t): if debug == True: for tokname, tokln, tokdata in seen_tokens[-32:]: if tokln == t.lineno: - print tokname, tokdata - print repr(last_ok_token[0]) + print(tokname, tokdata) + print(repr(last_ok_token[0])) last_error_line = t.lineno elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: last_error_line = t.lineno @@ -443,23 +450,23 @@ def p_error(t): ok_count = 0 if t is None: - if last_error_token != "EOF": - print "ERROR: End of the file reached." - global endoffile - print "Last data:", endoffile - + if last_error_token != "EOF": + print("ERROR: End of the file reached.") + global endoffile + print("Last data:", endoffile) + last_error_token = "EOF" return t - t = yacc.token() + t = yacc.token() yacc.restart() last_error_token = t return t - - + + # Build the grammar -parser = yacc.yacc(method='LALR',debug=0, +parser = yacc.yacc(method='LALR',debug=0, optimize = 1, write_tables = 1, debugfile = '/tmp/yaccdebug.txt',outputdir='/tmp/') #profile.run("yacc.yacc(method='LALR')") @@ -477,32 +484,32 @@ def print_context(token): column1 = (token.lexpos - last_cr) last_cr = input_data.rfind('\n',0,last_cr-1) - print input_data[last_cr:next_cr].replace("\t"," ") - print (" " * (column-1)) + "^", column, "#ERROR#" , token - - + print(input_data[last_cr:next_cr].replace("\t"," ")) + print((" " * (column-1)) + "^", column, "#ERROR#" , token) + + def my_tokenfunc(*args, **kwargs): - #print "Call token:" ,args, kwargs - ret = lex.lexer.token(*args, **kwargs) + #print("Call token:" ,args, kwargs) + ret = lex.lexer.token(*args, **kwargs) #print "Return (",args, kwargs,") = " , ret return ret def print_tokentree(token, depth = 0): - print " " * depth, token.__class__ , "=" , token - + print(" " * depth, token.__class__ , "=" , token) + if str(token.__class__) == "ply.yacc.YaccProduction": - print token.lexer - for tk in token.slice: + print(token.lexer) + for tk in token.slice: if tk.value == token: continue - print " " * (depth+1), tk.type, + print(" " * (depth+1), tk.type, end=' ') try: - print tk.lexpos, - print tk.endlexpos, + print(tk.lexpos, end=' ') + print(tk.endlexpos, end=' ') except: pass - print - + print() + print_tokentree(tk.value, depth +1) def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): @@ -516,11 +523,11 @@ def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): 'vardecl_list', ] final_obj = {} - final_obj['range'] = obj['02-size'] + final_obj['range'] = obj['02-size'] has_data = 0 has_objects = 0 contentlist = [] - + if alias_mode == 0: ctype_alias = {} elif alias_mode == 1: @@ -533,11 +540,11 @@ def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): "storeequalinstruction" : "instruction", "vardecl" : "vardeclaration", #"vardecl_list" : "vardeclaration", - + } else: - raise ValueError, "alias_mode unknown" - + raise ValueError("alias_mode unknown") + if otype in ctype_alias: otype = ctype_alias[otype] #print " " * depth , obj['02-size'] @@ -551,7 +558,7 @@ def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): # print_tree(value,depth,num) # continue #print " " * depth , "%s:" % ".".join(num+[str(n)]), ctype, - + if type(value) is dict: #print "*" if depth < 600: @@ -574,9 +581,9 @@ def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): final_obj['content'] = contentlist final_obj['has_data'] = has_data final_obj['has_objects'] = has_objects - + return final_obj - + hashes = [] ranges = [] @@ -585,7 +592,7 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou if depth == 0: hashes = [] ranges = [] - + sep = " " marginblocks = { "classdeclaration" : 1, @@ -602,31 +609,31 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou nuevalinea = False name = "" lines = [] - - - + + + for ctype, value in tree['content']: if nuevalinea and ctype in closingtokens: nuevalinea = False - - if nuevalinea: - for i in range(int(math.ceil(l/2.0))): lines.append(sep * depth) + + if nuevalinea: + for i in range(int(math.ceil(old_div(l,2.0)))): lines.append(sep * depth) nuevalinea = False - + if type(value) is dict and ctype == otype: tname,tlines,trange = printtree(value, depth, ctype) if name == "" and tname: name = tname - + lines += tlines elif type(value) is dict: l = 0 if ctype in marginblocks: l = marginblocks[ctype] - - for i in range(int(math.floor(l/2.0))): lines.append(sep * depth) + + for i in range(int(math.floor(old_div(l,2.0)))): lines.append(sep * depth) tname,tlines,trange = printtree(value, depth+1, ctype) # lines.append(sep * depth + "" % (len("".join(tlines)))) - + if value['has_data'] > 0 and value['has_objects'] == 0 and False: # Do it inline! if value['has_data']==1 and tname: @@ -638,23 +645,23 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou attrs = [] if tname: attrs.append(("id",tname)) - + txtinline = "".join([ line.strip() for line in tlines ]) - + #if len(tlines)>1: txthash = hashlib.sha1(txtinline).hexdigest()[:16] - #hashes.append(("depth:",depth,"hash:",txthash,"element:",ctype+":"+tname)) - hashes.append((txthash,ctype+":"+tname+"(%d)"% len(txtinline))) + #hashes.append(("depth:",depth,"hash:",txthash,"element:",ctype+":"+tname)) + hashes.append((txthash,ctype+":"+tname+"(%d)"% len(txtinline))) ranges.append([depth,txthash]+trange+[ctype+":"+tname,len(txtinline)]) #,"start:",trange[0],"end:",trange[1])) #attrs.append(("start",trange[0])) #attrs.append(("end",trange[1])) #attrs.append(("hash",txthash)) - + txtattrs="" for name1, val1 in attrs: txtattrs+=" %s=\"%s\"" % (name1,cnvrt(val1)) - + lines.append(sep * depth + "<%s%s>" % (ctype,txtattrs)) if depth > 50: lines.append(sep * (depth+1) + "...") @@ -666,7 +673,7 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou if txtattrs: txtattrs = "" % txtattrs lines.append(sep * depth + "" % (ctype)) - + nuevalinea = True else: if ctype == "ID" and name == "": @@ -675,8 +682,8 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou lines.append(sep * depth + "<%s value=\"%s\" />" % (ctype,cnvrt(value))) else: lines.append(sep * depth + "<%s />" % (ctype)) - - + + if mode == "hash": #print "\n".join(lines) for row in sorted(ranges): @@ -688,9 +695,9 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou output.write(row) output.write("\n") output.flush() - + return name, lines, tree['range'] - + def parse(data): @@ -704,15 +711,15 @@ def parse(data): error_count = 0 p = parser.parse(data, debug = 0, tracking = 1, tokenfunc = my_tokenfunc) if error_count > 0: - print "ERRORS (%d)" % error_count + print("ERRORS (%d)" % error_count) if p is None: return p try: p["error_count"] = error_count - except Exception, e: - print e + except Exception as e: + print(e) return None - - if parser.error: + + if parser.error: return None return p @@ -733,26 +740,26 @@ def main(): parser.add_option("--optdebug", action="store_true", dest="optdebug", default=False, help="debug optparse module") - + parser.add_option("--debug", action="store_true", dest="debug", default=False, help="prints lots of useless messages") - + (options, args) = parser.parse_args() if options.optdebug: - print options, args + print(options, args) if options.start: start = options.start - print "Start setted to:" , start - + print("Start setted to:" , start) + def do_it(): - if options.output == "none": return + if options.output == "none": return tree_data = calctree(prog) - if options.output == "hash": + if options.output == "hash": printtree(tree_data, mode = "hash") elif options.output == "xml": printtree(tree_data, mode = "xml") @@ -760,56 +767,56 @@ def do_it(): f1_hash = open(filename+".hash","w") printtree(tree_data, mode = "hash", output = f1_hash) f1_hash.close() - + f1_xml = open(filename+".xml","w") printtree(tree_data, mode = "xml", output = f1_xml) f1_xml.close() elif options.output == "yaml": import yaml - print yaml.dump(tree_data['content']) - + print(yaml.dump(tree_data['content'])) + else: - print "Unknown outputmode", options.output + print("Unknown outputmode", options.output) prog = "$$$" - if len(args) > 0 : + if len(args) > 0 : for filename in args: fs = filename.split("/") sys.stderr.write("Loading %s ..." % fs[-1]) sys.stderr.flush() - data = open(filename).read() + data = open(filename).read() sys.stderr.write(" parsing ...") sys.stderr.flush() - prog = parse(data) + prog = parse(data) sys.stderr.write(" formatting ...") sys.stderr.flush() if prog: do_it() sys.stderr.write(" Done.\n") sys.stderr.flush() - + else: line = "" while 1: try: - line1 = raw_input("flscript> ") + line1 = input("flscript> ") if line1.startswith("#"): comm = line1[1:].split(" ") if comm[0] == "setstart": start = comm[1] - print "Start setted to:" , start + print("Start setted to:" , start) if comm[0] == "parse": - print - prog = parse(line) + print() + prog = parse(line) line = "" else: line += line1 - except EOFError: + except EOFError: break; - line += "\n" - print - prog = parse(line) + line += "\n" + print() + prog = parse(line) do_it() """ import yaml @@ -824,9 +831,9 @@ def do_it(): #for varName in prog.byDefName: # var = prog.byDefName[varName] # print "%-15s / %-15s > " % var.type , varName - - #import tests.ifaceclass + + #import tests.ifaceclass #tests.ifaceclass.do_test(prog) diff --git a/postparse.py b/postparse.py old mode 100755 new mode 100644 index efc2846..10afa18 --- a/postparse.py +++ b/postparse.py @@ -1,9 +1,14 @@ #!/usr/bin/python +from __future__ import print_function +from __future__ import absolute_import +from builtins import str +from builtins import object from optparse import OptionParser import os, os.path, sys -import flscriptparse +from pineboolib.flparser import flscriptparse import imp, traceback from lxml import etree +from future.utils import with_metaclass USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") KNOWN_PARSERS = {} @@ -18,13 +23,13 @@ def decorator(fn): def parse(tagname, treedata): global KNOWN_PARSERS,UNKNOWN_PARSERS - if tagname not in KNOWN_PARSERS: + if tagname not in KNOWN_PARSERS: UNKNOWN_PARSERS[tagname] = 1 fn = parse_unknown else: fn = KNOWN_PARSERS[tagname] return fn(tagname,treedata) - + def getxmltagname(tagname): if tagname == "source": return "Source" @@ -34,15 +39,14 @@ def getxmltagname(tagname): return "Unknown.%s" % tagname xml_class_types = [] - + class TagObjectFactory(type): def __init__(cls, name, bases, dct): global xml_class_types xml_class_types.append(cls) super(TagObjectFactory, cls).__init__(name, bases, dct) - -class TagObject(object): - __metaclass__ = TagObjectFactory + +class TagObject(with_metaclass(TagObjectFactory, object)): tags = [] set_child_argn = False name_is_first_id = False @@ -51,13 +55,13 @@ class TagObject(object): omit_tags = ['empty'] callback_subelem = {} promote_child_if_alone = False - + @classmethod def tagname(self, tagname): return self.__name__ - + @classmethod def can_process_tag(self, tagname): return tagname in self.tags - + def __init__(self, tagname): self.astname = tagname self.xml = etree.Element(self.tagname(tagname)) @@ -66,17 +70,17 @@ def __init__(self, tagname): self.values = [] if self.name_is_first_id: self.xml.set("name","") - + def adopt_children(self, argn, subelem): for child in subelem.xml.iterchildren(): if self.set_child_argn: child.set("argn",str(argn)) - else: + else: if 'argn' in child.attrib: del child.attrib['argn'] self.xml.append(child) def omit_subelem(self, argn, subelem): return - + def is_in(self, listobj): return self.__class__ in listobj or self.astname in listobj @@ -84,31 +88,31 @@ def get(self, listobj, default = None): if self.__class__ in listobj: return listobj[self.__class__] if self.astname in listobj: return listobj[self.astname] return default - - + + def add_subelem(self, argn, subelem): if subelem.is_in(self.omit_tags): return self.omit_subelem(argn, subelem) if subelem.is_in(self.adopt_childs_tags): return self.adopt_children(argn, subelem) callback = subelem.get(self.callback_subelem) if callback: return getattr(self,callback)(argn,subelem) - + if self.set_child_argn: subelem.xml.set("argn",str(argn)) self.xml.append(subelem.xml) self.subelems.append(subelem) - + def add_value(self, argn, vtype, value): self.values.append( (vtype, value) ) if vtype == "ID" and self.name_is_first_id and self.xmlname is None: self.xmlname = value self.xml.set("name",value) return - + self.xml.set("arg%02d" % argn,vtype + ":" + repr(value)) - + def add_other(self, argn, vtype, data): if self.debug_other: self.xml.set("arg%02d" % argn,vtype) - + def polish(self): if self.promote_child_if_alone: if len(self.values) == 0 and len(self.subelems) == 1: @@ -119,7 +123,7 @@ def polish(self): class ListObject(TagObject): set_child_argn = False debug_other = False - + class NamedObject(TagObject): name_is_first_id = True debug_other = False @@ -170,7 +174,7 @@ class Function(ListNamedObject): tags = ["funcdeclaration"] callback_subelem = ListNamedObject.callback_subelem.copy() callback_subelem[VariableType] = "add_vartype" - + def add_vartype(self, argn, subelem): self.xml.set("returns", str(subelem.xmlname)) @@ -178,10 +182,10 @@ class Variable(NamedObject): tags = ["vardecl"] callback_subelem = NamedObject.callback_subelem.copy() callback_subelem[VariableType] = "add_vartype" - + def add_vartype(self, argn, subelem): self.xml.set("type", str(subelem.xmlname)) - + class DeclarationBlock(ListObject): tags = ["vardeclaration"] adopt_childs_tags = ['vardecl_list'] @@ -234,7 +238,7 @@ class Compare(TypedObject): debug_other = True tags = ["cmp_symbol","boolcmp_symbol"] -class FunctionCall(NamedObject): +class FunctionCall(NamedObject): tags = ["funccall_1"] class CallArguments(ListObject): @@ -243,12 +247,12 @@ class CallArguments(ListObject): class Constant(ListObject): tags = ["constant"] def add_value(self, argn, vtype, value): - value = unicode(value,"ISO-8859-15","replace") - if vtype == "SCONST": + value = str(value) #str(value,"ISO-8859-15","replace") + if vtype == "SCONST": vtype = "String" value = value[1:-1] self.xml.set("delim",'"') - if vtype == "CCONST": + if vtype == "CCONST": vtype = "String" value = value[1:-1] self.xml.set("delim","'") @@ -259,7 +263,7 @@ def add_value(self, argn, vtype, value): self.const_type = vtype self.xml.set("type",vtype) self.xml.set("value",value) - + class InlineUpdate(ListObject): tags = ["inlinestoreinstruction"] def add_other(self, argn, vtype, value): @@ -271,7 +275,7 @@ def add_other(self, argn, vtype, value): class If(ListObject): tags = ["ifstatement"] - + class Condition(ListObject): tags = ["condition"] @@ -287,16 +291,16 @@ def polish(self): class ExpressionContainer(ListObject): tags = ["expression"] # adopt_childs_tags = ['base_expression'] - + def polish(self): if len(self.values) == 0 and len(self.subelems) == 1: - #if isinstance(self.subelems[0], Constant): + #if isinstance(self.subelems[0], Constant): if self.subelems[0].xml.tag == "base_expression": self.subelems[0].xml.tag = "Expression" return self.subelems[0] - else: + else: self.xml.tag = "Value" - + return self class InstructionUpdate(ListObject): @@ -329,8 +333,8 @@ class ForCompare(ListObject): tags = ["for_compare"] class ForIncrement(ListObject): - tags = ["for_increment"] - + tags = ["for_increment"] + class DoWhile(ListObject): tags = ["dowhilestatement"] @@ -345,14 +349,14 @@ class TryCatch(ListObject): class New(ListObject): tags = ["new_operator"] - + class Delete(ListObject): tags = ["deleteinstruction"] - + class Parentheses(ListObject): tags = ["parentheses"] adopt_childs_tags = ['base_expression'] - + class OpUnary(TypedObject): tags = ["unary_operator"] @@ -361,7 +365,7 @@ class OpTernary(ListObject): class OpUpdate(TypedObject): tags = ["updateoperator"] - + # ----- keep this one at the end. @@ -370,7 +374,7 @@ class Unknown(TagObject): set_child_argn = False @classmethod def tagname(self, tagname): return tagname - + @classmethod def can_process_tag(self, tagname): return True # ----------------- @@ -395,7 +399,7 @@ def parse_unknown(tagname, treedata): xmlelem.add_value(i, k, v) else: xmlelem.add_other(i, k, v) - + i+=1 return xmlelem.polish() @@ -405,8 +409,8 @@ def post_parse(treedata): source = parse("source",treedata) #print UNKNOWN_PARSERS.keys() return source.xml - -class Module: + +class Module(object): def __init__(self, name, path): self.name = name self.path = path @@ -421,14 +425,14 @@ def loadModule(self): # fp, pathname, description = imp.find_module(self.name,[self.path]) self.module = imp.load_module(name, fp, pathname, description) result = True - except Exception,e: - print traceback.format_exc() + except Exception as e: + print(traceback.format_exc()) result = False if fp: - fp.close() + fp.close() return result -def main(): +def parseArgs(argv): parser = OptionParser() parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True, @@ -437,23 +441,23 @@ def main(): parser.add_option("--optdebug", action="store_true", dest="optdebug", default=False, help="debug optparse module") - + parser.add_option("--debug", action="store_true", dest="debug", default=False, help="prints lots of useless messages") - + parser.add_option("--path", dest="storepath", default=None, help="store XML results in PATH") - + parser.add_option("--topython", action="store_true", dest="topython", default=False, help="write python file from xml") - + parser.add_option("--exec-py", action="store_true", dest="exec_python", default=False, help="try to execute python file") - + parser.add_option("--toxml", action="store_true", dest="toxml", default=False, help="write xml file from qs") @@ -462,83 +466,96 @@ def main(): action="store_true", dest="full", default=False, help="write xml file from qs") - (options, args) = parser.parse_args() + (options, args) = parser.parse_args(argv) + return (options, args) + +def main(): + options, args = parseArgs(sys.argv[1:]) execute(options,args) - + +def pythonify(filelist): + options, args = parseArgs([]) + options.full = True + if isinstance(filelist, str): filelist = [filelist] + execute(options,filelist) + + + + def execute(options, args): if options.optdebug: - print options, args + print(options, args) if options.full: options.full = False options.toxml = True - print "Pass 1 - Parse and write XML file . . ." + print("Pass 1 - Parse and write XML file . . .") execute(options,args) - + options.toxml = False options.topython = True - print "Pass 2 - Pythonize and write PY file . . ." + print("Pass 2 - Pythonize and write PY file . . .") execute(options,[ arg+".xml" for arg in args]) options.topython = False options.exec_python = True #print "Pass 3 - Test PY file load . . ." #execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) - print "Done." - + print("Done.") + elif options.exec_python: - import qsatype + from . import qsatype for filename in args: realpath = os.path.realpath(filename) path, name = os.path.split(realpath) mod = Module(name, path) if mod.loadModule(): - print mod.module - print mod.module.form + print(mod.module) + print(mod.module.form) else: - print "Error cargando modulo %s" % name - + print("Error cargando modulo %s" % name) + elif options.topython: - from pytnyzer import pythonize + from .pytnyzer import pythonize for filename in args: bname = os.path.basename(filename) if options.storepath: - destname = os.path.join(options.storepath,bname+".py") + destname = os.path.join(options.storepath,bname+".py") else: destname = filename+".py" destname = destname.replace(".qs.xml.py",".py") pythonize(filename, destname) - - else: + + else: nfs = len(args) for nf, filename in enumerate(args): bname = os.path.basename(filename) sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) sys.stdout.flush(); - prog = flscriptparse.parse(open(filename).read()) + prog = flscriptparse.parse(open(filename,"r", encoding="latin-1").read()) sys.stdout.write("\r"); if not prog: - print "Error: No se pudo abrir %-35s \n" % (repr(filename)) + print("Error: No se pudo abrir %-35s \n" % (repr(filename))) continue if prog["error_count"] > 0: - print "Encontramos %d errores parseando: %-35s \n" % (prog["error_count"], repr(filename)) + print("Encontramos %d errores parseando: %-35s \n" % (prog["error_count"], repr(filename))) continue - if options.toxml == False: + if options.toxml == False: # Si no se quiere guardar resultado, no hace falta calcular mas continue - + tree_data = flscriptparse.calctree(prog, alias_mode = 0) if not tree_data: - print "No se pudo parsear %-35s \n" % (repr(filename)) + print("No se pudo parsear %-35s \n" % (repr(filename))) continue ast = post_parse(tree_data) if ast is None: - print "No se pudo analizar %-35s \n" % (repr(filename)) + print("No se pudo analizar %-35s \n" % (repr(filename))) continue if options.storepath: - destname = os.path.join(options.storepath,bname+".xml") + destname = os.path.join(options.storepath,bname+".xml") else: destname = filename+".xml" - f1 = open(destname,"w") + f1 = open(destname,"wb") f1.write(etree.tostring(ast, pretty_print = True)) f1.close() diff --git a/pytnyzer.py b/pytnyzer.py index 0ecc947..24afd98 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -1,14 +1,18 @@ #!/usr/bin/python # ------ Pythonyzer ... reads XML AST created by postparse.py and creates an equivalent Python file. +from __future__ import print_function +from __future__ import absolute_import +from builtins import object from optparse import OptionParser import os, os.path, random -import flscriptparse +from . import flscriptparse from lxml import etree +from future.utils import with_metaclass def id_translate(name): - python_keywords = ['and', 'del', 'for', 'is', 'raise', 'assert', 'elif', - 'from', 'lambda', 'return', 'break', 'else', 'global', 'not', 'try', - 'class', 'except', 'if', 'or', 'while', 'continue', + python_keywords = ['and', 'del', 'for', 'is', 'raise', 'assert', 'elif', + 'from', 'lambda', 'return', 'break', 'else', 'global', 'not', 'try', + 'class', 'except', 'if', 'or', 'while', 'continue', 'exec', 'import', 'pass', 'yield', 'def', 'finally', 'in', 'print'] if name in python_keywords: return name + "_" if name == "false": name = "False" @@ -27,22 +31,21 @@ def __init__(cls, name, bases, dct): global ast_class_types ast_class_types.append(cls) super(ASTPythonFactory, cls).__init__(name, bases, dct) - -class ASTPython(object): - __metaclass__ = ASTPythonFactory + +class ASTPython(with_metaclass(ASTPythonFactory, object)): tags = [] - + @classmethod def can_process_tag(self, tagname): return self.__name__ == tagname or tagname in self.tags - + def __init__(self, elem): self.elem = elem def polish(self): return self - + def generate(self, **kwargs): yield "debug", etree.tostring(self.elem) - + class Source(ASTPython): def generate(self, break_mode = False, include_pass = True, **kwargs): @@ -62,11 +65,11 @@ def generate(self, break_mode = False, include_pass = True, **kwargs): elems+=1 yield dtype, line after_lines = [] - if dtype == "break": + if dtype == "break": for line in after_lines: elems+=1 yield "line", line - + for line in after_lines: elems+=1 yield "line", line @@ -77,7 +80,7 @@ class Class(ASTPython): def generate(self, **kwargs): name = self.elem.get("name") extends = self.elem.get("extends","object") - + yield "line", "class %s(%s):" % (name,extends) yield "begin", "block-class-%s" % (name) for source in self.elem.xpath("Source"): @@ -92,21 +95,21 @@ def generate(self, **kwargs): grandparent = None if parent is not None: grandparent = parent.getparent() arguments = [] - + if grandparent is not None: if grandparent.tag == "Class": arguments.append("self") if name == grandparent.get("name"): name = "__init__" - else: + else: arguments.append("self") for n,arg in enumerate(self.elem.xpath("Arguments/*")): expr = [] for dtype, data in parse_ast(arg).generate(): - if dtype == "expr": + if dtype == "expr": expr.append(id_translate(data)) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n @@ -114,10 +117,10 @@ def generate(self, **kwargs): else: if len(expr) == 1: expr += ["=","None"] arguments.append(" ".join(expr)) - - - - yield "line", "def %s(%s):" % (name,", ".join(arguments)) + + + + yield "line", "def %s(%s):" % (name,", ".join(arguments)) yield "begin", "block-def-%s" % (name) # if returns: yield "debug", "Returns: %s" % returns for source in self.elem.xpath("Source"): @@ -140,23 +143,23 @@ def generate(self, **kwargs): #yield "debug", "Function to:" + etree.tostring(f) name = "self.%s" % name break - + arguments = [] for n,arg in enumerate(self.elem.xpath("CallArguments/*")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) - - yield "expr", "%s(%s)" % (name,", ".join(arguments)) + + yield "expr", "%s(%s)" % (name,", ".join(arguments)) class If(ASTPython): def generate(self, break_mode = False, **kwargs): @@ -164,23 +167,23 @@ def generate(self, break_mode = False, **kwargs): for n,arg in enumerate(self.elem.xpath("Condition/*")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: main_expr.append("False") yield "debug", "Expression %d not understood" % n yield "debug", etree.tostring(arg) else: main_expr.append(" ".join(expr)) - + yield "line", "if %s:" % (" ".join(main_expr)) for source in self.elem.xpath("Source"): yield "begin", "block-if" for obj in parse_ast(source).generate(break_mode = break_mode): yield obj yield "end", "block-if" - + for source in self.elem.xpath("Else/Source"): yield "line", "else:" yield "begin", "block-else" @@ -191,32 +194,32 @@ class TryCatch(ASTPython): def generate(self, **kwargs): tryblock, catchblock = self.elem.xpath("Source") - - yield "line", "try:" + + yield "line", "try:" yield "begin", "block-try" for obj in parse_ast(tryblock).generate(): yield obj yield "end", "block-try" - + identifier = None for ident in self.elem.xpath("Identifier"): expr = [] for dtype, data in parse_ast(ident).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data identifier = " ".join(expr) if identifier: yield "line", "except Exception, %s:" % (identifier) else: - yield "line", "except Exception:" + yield "line", "except Exception:" yield "begin", "block-except" if identifier: # yield "line", "%s = str(%s)" % (identifier, identifier) yield "line", "%s = traceback.format_exc()" % (identifier) for obj in parse_ast(catchblock).generate(include_pass = identifier is None): yield obj yield "end", "block-except" - + class While(ASTPython): def generate(self, **kwargs): @@ -224,17 +227,17 @@ def generate(self, **kwargs): for n,arg in enumerate(self.elem.xpath("Condition/*")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: main_expr.append("False") yield "debug", "Expression %d not understood" % n yield "debug", etree.tostring(arg) else: main_expr.append(" ".join(expr)) - + yield "line", "while %s:" % (" ".join(main_expr)) for source in self.elem.xpath("Source"): yield "begin", "block-while" @@ -247,10 +250,10 @@ def generate(self, **kwargs): for n,arg in enumerate(self.elem.xpath("ForInitialize/*")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) > 0: main_expr.append(" ".join(expr)) if main_expr: @@ -261,24 +264,24 @@ def generate(self, **kwargs): for n,arg in enumerate(self.elem.xpath("ForIncrement/*")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) - elif dtype in ["line","line+1"]: + elif dtype in ["line","line+1"]: incr_lines.append(data) else: - yield dtype, data + yield dtype, data if len(expr) > 0: incr_expr.append(" ".join(expr)) - + main_expr = [] for n,arg in enumerate(self.elem.xpath("ForCompare/*")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: main_expr.append("True") else: @@ -288,11 +291,11 @@ def generate(self, **kwargs): for source in self.elem.xpath("Source"): yield "begin", "block-for" for obj in parse_ast(source).generate(include_pass=False): yield obj - if incr_lines: + if incr_lines: for line in incr_lines: yield "line", line yield "end", "block-for" - + class Switch(ASTPython): def generate(self, **kwargs): key = "%02x" % random.randint(0,255) @@ -303,10 +306,10 @@ def generate(self, **kwargs): for n,arg in enumerate(self.elem.xpath("Condition/*")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: main_expr.append("False") yield "debug", "Expression %d not understood" % n @@ -320,10 +323,10 @@ def generate(self, **kwargs): for n,arg in enumerate(scase.xpath("Value")): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: value_expr.append("False") yield "debug", "Expression %d not understood" % n @@ -336,7 +339,7 @@ def generate(self, **kwargs): yield "begin", "block-if" count = 0 for source in scase.xpath("Source"): - for obj in parse_ast(source).generate(break_mode = True): + for obj in parse_ast(source).generate(break_mode = True): if obj[0] == "break": yield "line", "%s = %s # BREAK" % (name_pr,"False") count += 1 @@ -345,13 +348,13 @@ def generate(self, **kwargs): count += 1 if count < 1: yield "line", "pass" yield "end", "block-if" - + for scasedefault in self.elem.xpath("CaseDefault"): yield "line", "if not %s: %s,%s = %s,%s" % (name_pr2, name_pr,name_pr2, "True", "True") yield "line", "if %s:" % (name_pr) yield "begin", "block-if" for source in scasedefault.xpath("Source"): - for obj in parse_ast(source).generate(break_mode = True): + for obj in parse_ast(source).generate(break_mode = True): if obj[0] == "break": yield "line", "%s = %s # BREAK" % (name_pr,"False") else: @@ -368,21 +371,21 @@ def generate(self, **kwargs): variable, source = [ obj for obj in self.elem ] var_expr = [] for dtype, data in parse_ast(variable).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": var_expr.append(data) else: - yield dtype, data + yield dtype, data if len(var_expr) == 0: var_expr.append("None") yield "debug", "Expression %d not understood" % n yield "debug", etree.tostring(arg) - + yield "line", "%s = %s" % (name, " ".join(var_expr)) - - for obj in parse_ast(source).generate(break_mode = True): + + for obj in parse_ast(source).generate(break_mode = True): yield obj yield "line", "del %s" % name - + class Variable(ASTPython): def generate(self, force_value = False, **kwargs): name = self.elem.get("name") @@ -397,7 +400,7 @@ def generate(self, force_value = False, **kwargs): yield dtype, data if expr == 0: yield "expr", "None" - + dtype = self.elem.get("type",None) if values == 0 and force_value == True: @@ -416,7 +419,7 @@ def generate(self, force_value = False, **kwargs): yield "expr", "None" else: yield "expr", "qsatype.%s()" % dtype - + #if dtype and force_value == False: yield "debug", "Variable %s:%s" % (name,dtype) class InstructionUpdate(ASTPython): @@ -425,18 +428,18 @@ def generate(self, **kwargs): for n,arg in enumerate(self.elem): expr = [] for dtype, data in parse_ast(arg).generate(isolate=False): - if dtype == "expr": - if data is None: raise ValueError, etree.tostring(arg) + if dtype == "expr": + if data is None: raise ValueError(etree.tostring(arg)) expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) - + yield "line", " ".join(arguments) class InlineUpdate(ASTPython): @@ -445,10 +448,10 @@ def generate(self, plusplus_as_instruction = False, **kwargs): for n,arg in enumerate(self.elem): expr = [] for dtype, data in parse_ast(arg).generate(isolate=False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n @@ -458,10 +461,10 @@ def generate(self, plusplus_as_instruction = False, **kwargs): ctype = self.elem.get("type") mode = self.elem.get("mode") linetype = "line" - if not plusplus_as_instruction: + if not plusplus_as_instruction: if mode == "read-update": linetype = "line+1" - + yield "expr", arguments[0] if ctype == "PLUSPLUS": yield linetype, arguments[0] + " += 1" @@ -469,17 +472,17 @@ def generate(self, plusplus_as_instruction = False, **kwargs): yield linetype, arguments[0] + " -= 1" else: yield linetype, arguments[0] + " ?= 1" - + class InstructionCall(ASTPython): def generate(self, **kwargs): arguments = [] for n,arg in enumerate(self.elem): expr = [] for dtype, data in parse_ast(arg).generate(): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n @@ -494,10 +497,10 @@ def generate(self, break_mode = False, **kwargs): for n,arg in enumerate(self.elem): expr = [] for dtype, data in parse_ast(arg).generate(isolate=False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n @@ -508,20 +511,20 @@ def generate(self, break_mode = False, **kwargs): ctype = self.elem.get("type") kw = ctype if ctype == "RETURN": kw = "return" - if ctype == "BREAK": + if ctype == "BREAK": kw = "break" - if break_mode: + if break_mode: yield "break", kw + " " + ", ".join(arguments) return if ctype == "CONTINUE": kw = "continue" - + if ctype == "THROW": yield "line", "raise Exception(" + ", ".join(arguments) + ")" return - + yield "line", kw + " " + ", ".join(arguments) - - + + class Member(ASTPython): def generate(self, **kwargs): @@ -529,10 +532,10 @@ def generate(self, **kwargs): for n,arg in enumerate(self.elem): expr = [] for dtype, data in parse_ast(arg).generate(): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n @@ -549,7 +552,7 @@ def generate(self, **kwargs): classname = name_parts[0] arguments[2] = arguments[2][2:] arguments[0:2] = ["super(%s, %s)" % (classname,".".join(arguments[0:2]))] - + yield "expr", ".".join(arguments) class ArrayMember(ASTPython): @@ -558,26 +561,26 @@ def generate(self, **kwargs): for n,arg in enumerate(self.elem): expr = [] for dtype, data in parse_ast(arg).generate(isolate=False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) - + yield "expr", "%s[%s]" % (arguments[0],arguments[1]) - + class Value(ASTPython): def generate(self, isolate = True, **kwargs): if isolate: yield "expr", "(" for child in self.elem: for dtype, data in parse_ast(child).generate(): - if data is None: raise ValueError, etree.tostring(child) + if data is None: raise ValueError(etree.tostring(child)) yield dtype, data if isolate: yield "expr", ")" @@ -596,7 +599,7 @@ def generate(self, isolate = True, **kwargs): if child.get("type") == "PLUS": yield "expr","," continue - + for dtype, data in parse_ast(child).generate(): yield dtype, data @@ -633,11 +636,11 @@ def generate(self, **kwargs): continue if child.tag == "Identifier": data = data+"()" ident = data[:data.find("(")] - if ident.find(".") == -1: + if ident.find(".") == -1: if len(self.elem.xpath("//Class[@name='%s']" % ident)) == 0: data = "qsatype." + data yield dtype, data - + class Constant(ASTPython): def generate(self, **kwargs): @@ -650,20 +653,20 @@ def generate(self, **kwargs): for n,arg in enumerate(child): expr = [] for dtype, data in parse_ast(arg).generate(isolate = False): - if dtype == "expr": + if dtype == "expr": expr.append(data) else: - yield dtype, data + yield dtype, data if len(expr) == 0: arguments.append("unknownarg") yield "debug", "Argument %d not understood" % n yield "debug", etree.tostring(arg) else: arguments.append(" ".join(expr)) - - yield "expr", "qsatype.Array([%s])" % (", ".join(arguments)) + + yield "expr", "qsatype.Array([%s])" % (", ".join(arguments)) return - if ctype == "String": + if ctype == "String": delim = self.elem.get("delim") if delim == "'": yield "expr", "u'%s'" % value @@ -699,7 +702,7 @@ def generate(self, **kwargs): elif ctype == "LOR": yield "expr", "or" elif ctype == "LAND": yield "expr", "and" else: yield "expr", ctype - + class OpMath(ASTPython): def generate(self, **kwargs): ctype = self.elem.get("type") @@ -707,14 +710,14 @@ def generate(self, **kwargs): elif ctype == "MINUS": yield "expr", "-" elif ctype == "TIMES": yield "expr", "*" elif ctype == "DIVIDE": yield "expr", "/" - elif ctype == "MOD": yield "expr", "%" + elif ctype == "MOD": yield "expr", "%" elif ctype == "XOR": yield "expr", "^" elif ctype == "LSHIFT": yield "expr", "<<" elif ctype == "RSHIFT": yield "expr", ">>" elif ctype == "AND": yield "expr", "&" else: yield "expr", ctype - + class DeclarationBlock(ASTPython): def generate(self, **kwargs): mode = self.elem.get("mode") @@ -723,14 +726,14 @@ def generate(self, **kwargs): for var in self.elem: expr = [] for dtype, data in parse_ast(var).generate(force_value=True): - if dtype == "expr": - if data is None: raise ValueError, etree.tostring(var) + if dtype == "expr": + if data is None: raise ValueError(etree.tostring(var)) expr.append(data) else: yield dtype,data if is_constructor: expr[0] = "self."+expr[0] yield "line", " ".join(expr) - + # ----- keep this one at the end. class Unknown(ASTPython): @@ -769,14 +772,14 @@ def file_template(ast): constructor = etree.SubElement(mainsource,"Function",name="_class_init") args = etree.SubElement(constructor,"Arguments") csource = etree.SubElement(constructor,"Source") - + for child in ast: if child.tag != "Function": child.set("constructor","1") csource.append(child) else: mainsource.append(child) - + for dtype, data in parse_ast(sourceclasses).generate(): yield dtype, data yield "line", "" @@ -789,54 +792,58 @@ def write_python_file(fobj, ast): numline = 0 last_dtype = None for dtype, data in file_template(ast): + if isinstance(data, bytes): data = data.decode("UTF-8","replace") line = None - if dtype == "line": + if dtype == "line": line = data numline +=1 - try: lines_since_last_indent = numline - last_line_for_indent[len(indent)] + try: lines_since_last_indent = numline - last_line_for_indent[len(indent)] except KeyError: lines_since_last_indent = 0 if lines_since_last_indent > 4: - fobj.write((len(indent)*indent_text) + "\n") + fobj.write((len(indent)*indent_text) + "\n") last_line_for_indent[len(indent)] = numline - if dtype == "debug": - line = "# DEBUG:: " + data - print numline, line + if dtype == "debug": + line = "# DEBUG:: " + data + print(numline, line) if dtype == "expr": line = "# EXPR??:: " + data if dtype == "line+1": line = "# LINE+1??:: " + data - if dtype == "begin": + if dtype == "begin": #line = "# BEGIN:: " + data indent.append(data) last_line_for_indent[len(indent)] = numline if dtype == "end": if last_dtype == "begin": - fobj.write((len(indent)*indent_text) + "pass\n") + fobj.write((len(indent)*indent_text) + "pass\n") last_line_for_indent[len(indent)] = numline - + if data not in ["block-if"]: #line = "# END:: " + data pass endblock = indent.pop() if endblock != data: line = "# END-ERROR!! was %s but %s found. (%s)" % (endblock, data,repr(indent)) - + if line is not None: - if type(line) is unicode: line = line.encode("UTF-8","replace") - fobj.write((len(indent)*indent_text) + line + "\n") + fobj.write((len(indent)*indent_text) + line + "\n") - if dtype == "end": + if dtype == "end": if data.split("-")[1] in ["class","def","else","except"]: - fobj.write((len(indent)*indent_text) + "\n") + fobj.write((len(indent)*indent_text) + "\n") last_line_for_indent[len(indent)] = numline last_dtype = dtype def pythonize(filename, destfilename): bname = os.path.basename(filename) - + parser = etree.XMLParser(remove_blank_text=True) - ast_tree = etree.parse(open(filename), parser) + try: + ast_tree = etree.parse(open(filename), parser) + except Exception: + print("filename:",filename) + raise ast = ast_tree.getroot() - - + + f1 = open(destfilename,"w") write_python_file(f1,ast) f1.close() @@ -850,23 +857,23 @@ def main(): parser.add_option("--optdebug", action="store_true", dest="optdebug", default=False, help="debug optparse module") - + parser.add_option("--debug", action="store_true", dest="debug", default=False, help="prints lots of useless messages") - + parser.add_option("--path", dest="storepath", default=None, help="store PY results in PATH") - + (options, args) = parser.parse_args() if options.optdebug: - print options, args - + print(options, args) + for filename in args: if options.storepath: - destname = os.path.join(options.storepath,bname+".py") + destname = os.path.join(options.storepath,bname+".py") else: destname = filename+".py" pythonize(filename, destname) diff --git a/qsatype.py b/qsatype.py index 951a975..7f8b8a7 100644 --- a/qsatype.py +++ b/qsatype.py @@ -1,3 +1,4 @@ +from builtins import object # encoding: UTF-8 import os diff --git a/xml2json.py b/xml2json.py old mode 100755 new mode 100644 index e6b4e24..fd267ee --- a/xml2json.py +++ b/xml2json.py @@ -1,5 +1,7 @@ #!/usr/bin/python +from __future__ import print_function +from builtins import object try: from json import dumps as json_dumps from json import loads as json_loads @@ -16,12 +18,12 @@ def printr(*args): return - print args[0], + print(args[0], end=' ') for arg in args[1:]: - if type(arg) is unicode: + if type(arg) is str: arg=arg.encode("utf-8") - print repr(arg), - print + print(repr(arg), end=' ') + print() def entity_rep(txt,entities=""): entity_list = list("&'\"<>") @@ -42,7 +44,7 @@ def entity_rep(txt,entities=""): txt = txt.replace(entity,entity_dict[entity]) return txt -class xmlElement: +class xmlElement(object): def __init__(self, parent, tagname, attrs = {}, ttype = "text", tdata = ""): self.parent = parent @@ -67,7 +69,7 @@ def append(self,text): def export(self,encoding): depth = self.depth tagname = self.tagname - attrs = [ [k,v] for k,v in self.attrs.iteritems() ] + attrs = [ [k,v] for k,v in self.attrs.items() ] attrs.sort() tdata = self.tdata.strip() ttype = self.ttype @@ -86,7 +88,7 @@ def export(self,encoding): def exportXML(self): if type(self.attrs) is dict: - attrs = [ [k,v] for k,v in self.attrs.iteritems() ] + attrs = [ [k,v] for k,v in self.attrs.items() ] attrs.sort() else: attrs = self.attrs @@ -130,7 +132,7 @@ def exportXML(self): -class JSON_Base: +class JSON_Base(object): def __init__(self, finput, foutput, encoding): self.finput = finput self.foutput = foutput @@ -139,7 +141,7 @@ def __init__(self, finput, foutput, encoding): self.init_vars() def process(self): - print "Please define a process function." + print("Please define a process function.") def init_vars(self): pass @@ -156,12 +158,12 @@ def processCmd(self,key,val): if self.encoding != "auto": self.encoding = self.encoding.upper() if val.upper() != self.encoding: - print " ignoring %s=%s , using specified value '%s' instead" % (key,val,self.encoding) + print(" ignoring %s=%s , using specified value '%s' instead" % (key,val,self.encoding)) return self.encoding = val.upper() return - print "ERROR: unknown key %s='%s'" % (key,val) + print("ERROR: unknown key %s='%s'" % (key,val)) def newElement(self,depth,tagname,text,ttype,attrs): parent = self.cElement @@ -198,13 +200,13 @@ def process(self): for field in fields[1:]: tpos = field.find(":") if tpos == -1: - print "unexpected character:", line + print("unexpected character:", line) return ftype = field[:tpos] try: fvalue = json_loads(field[tpos+1:]) except ValueError: - print "ValueError:", field[tpos+1:] + print("ValueError:", field[tpos+1:]) #if type(fvalue) is unicode: # self.foutput.write("%s = %s\n" % (ftype, fvalue.encode(self.encoding))) @@ -552,7 +554,7 @@ def autodetectXmlEncoding(rawtext): encoding = dictEncoding["encoding"] return encoding except ImportError: - print "python-chardet library is not installed. Assuming input file is UTF-8." + print("python-chardet library is not installed. Assuming input file is UTF-8.") #encoding= #UTF-8, UTF-16, ISO-8859-1 @@ -580,10 +582,10 @@ def main(): (options, args) = parser.parse_args() if options.optdebug: - print options, args + print(options, args) if len(args) < 2: - print "xml2json needs at least an action and a file." - print "xml2json (revert|convert) file1 [file2] [file3]" + print("xml2json needs at least an action and a file.") + print("xml2json (revert|convert) file1 [file2] [file3]") return action = args.pop(0) @@ -619,7 +621,7 @@ def main(): fhandler.close() else: - print "Unkown action '%s'" % action + print("Unkown action '%s'" % action) diff --git a/xmlparse.py b/xmlparse.py index 706176b..df1dbd5 100644 --- a/xmlparse.py +++ b/xmlparse.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from builtins import str import xml.parsers.expat import sys from optparse import OptionParser @@ -19,7 +21,7 @@ def reset(): # 3 handler functions def start_element(name, attrs): global elements, show_end, lstelements, lasttextdata - lstattrs=list(sorted([ "%s=%s" % (k,v) for k,v in attrs.iteritems() ])) + lstattrs=list(sorted([ "%s=%s" % (k,v) for k,v in attrs.items() ])) completename=name if len(lstattrs): completename+="&"+"&".join(lstattrs) @@ -68,8 +70,8 @@ def unmap(lines): if line[-1] == ";": continue rg1 = runmap.match(line) if not rg1: - print "error:" - print line + print("error:") + print(line) break depth = len(rg1.group('depth')) @@ -110,8 +112,8 @@ def unmap(lines): else: - print "error:" - print depth, len(elementpool) + print("error:") + print(depth, len(elementpool)) break while len(elementpool): @@ -141,10 +143,10 @@ def main(): (options, args) = parser.parse_args() if options.optdebug: - print options, args + print(options, args) if len(args) < 2: - print "Se necesita al menos una accion y un argumento extra." - print "xmlparse (map|unmap) file1 [file2] [file3]" + print("Se necesita al menos una accion y un argumento extra.") + print("xmlparse (map|unmap) file1 [file2] [file3]") return action = args.pop(0) @@ -192,7 +194,7 @@ def main(): fhandler.close() else: - print "Unkown action '%s'" % action + print("Unkown action '%s'" % action) if __name__ == "__main__": main() \ No newline at end of file From 4f2903445af8bb2d7519cbd06aafc5fa303c8e82 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 20 Sep 2016 18:29:12 +0200 Subject: [PATCH 073/100] flparser: adaptaciones para que flscriptparser siga funcionando como standalone --- flscriptparse.py | 8 ++++++-- flscriptparser2 | 6 +++--- postparse.py | 7 ++++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 1c75733..67d3eab 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -14,11 +14,15 @@ import sys, math import hashlib -from pineboolib.flparser import flex import ply.yacc as yacc import ply.lex as lex -from pineboolib.flparser.flclasses import * +try: + from pineboolib.flparser import flex + from pineboolib.flparser.flclasses import * +except ImportError: + import flex + from flclasses import * # Get the token map tokens = flex.tokens diff --git a/flscriptparser2 b/flscriptparser2 index b9a8956..99cb558 100755 --- a/flscriptparser2 +++ b/flscriptparser2 @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import postparse -if __name__ == "__main__": - postparse.main() +if __name__ == "__main__": + postparse.main() diff --git a/postparse.py b/postparse.py index 10afa18..ef8b014 100644 --- a/postparse.py +++ b/postparse.py @@ -5,10 +5,15 @@ from builtins import object from optparse import OptionParser import os, os.path, sys -from pineboolib.flparser import flscriptparse import imp, traceback from lxml import etree from future.utils import with_metaclass + +try: + from pineboolib.flparser import flscriptparse +except ImportError: + import flscriptparse + USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") KNOWN_PARSERS = {} From fc989894b9db5cc15697926060862f7fc911092e Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 20 Sep 2016 18:34:13 +0200 Subject: [PATCH 074/100] bugfix: correcciones de mezcla --- flscriptparse.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 6625eb7..2c135e2 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -422,12 +422,12 @@ def p_parse(token): seen_tokens.append((str(token.slice[0]), token.lineno(0),input_data[lexspan[0]:lexspan[1]+1] )) global ok_count ok_count += 1 - if lexspan[0] not in tokelines: + if lexspan[0] not in tokelines: tokelines[lexspan[0]] = token.lexer.lineno global last_lexspan last_lexspan = lexspan - - + + last_ok_token = None error_count = 0 @@ -447,7 +447,7 @@ def p_error(t): if abs(last_error_line - t.lineno) > 4 and ok_count > 1 and error_count < 4: error_count += 1 try: print_context(t) - except: pass + except Exception: pass if debug == True: for tokname, tokln, tokdata in seen_tokens[-32:]: if tokln == t.lineno: @@ -469,13 +469,13 @@ def p_error(t): if last_lexspan: try: - print "HINT: Last lexspan:", last_lexspan - print "HINT: Last line:", tokelines[last_lexspan[0]] - except Exception, e: - print "ERROR:", e + print("HINT: Last lexspan:", last_lexspan) + print("HINT: Last line:", tokelines[last_lexspan[0]]) + except Exception as e: + print("ERROR:", e) last_error_token = "EOF" return t - t = parser.token() + t = parser.token() parser.restart() last_error_token = t return t From 1d7bb5fdd7425db8f96233419d5b6fb428424e5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20A=2E=20Fern=C3=A1ndez?= Date: Tue, 20 Sep 2016 19:46:55 +0200 Subject: [PATCH 075/100] =?UTF-8?q?ocultar=20bot=C3=B3n=20en=20FLFieldDB?= =?UTF-8?q?=20tipo=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- COPYRIGHT | 21 + INSTALL | 43 ++ INSTALL-ES | 44 ++ LICENSE | 505 ++++++++++++++++++++++ Makefile | 7 + README | 10 + __init__.py | 0 analyze_controls.sh | 13 + dependencies.debian | 3 + flalign.py | 489 +++++++++++++++++++++ flclasses.py | 335 ++++++++++++++ flex.py | 210 +++++++++ flmergetool | 199 +++++++++ flpatcher | 356 +++++++++++++++ flpremerge.py | 662 ++++++++++++++++++++++++++++ flscriptparse.py | 865 ++++++++++++++++++++++++++++++++++++ flscriptparser2 | 4 + install.sh | 18 + postparse.py | 585 +++++++++++++++++++++++++ pytnyzer.py | 1010 +++++++++++++++++++++++++++++++++++++++++++ qsatype.py | 19 + test/README | 5 + test/flscriptparser | 82 ++++ tests/__init__.py | 1 + tests/ifaceclass.py | 102 +++++ xml2json.py | 634 +++++++++++++++++++++++++++ xmlparse.py | 200 +++++++++ 27 files changed, 6422 insertions(+) create mode 100644 COPYRIGHT create mode 100644 INSTALL create mode 100644 INSTALL-ES create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README create mode 100644 __init__.py create mode 100755 analyze_controls.sh create mode 100644 dependencies.debian create mode 100644 flalign.py create mode 100644 flclasses.py create mode 100644 flex.py create mode 100755 flmergetool create mode 100755 flpatcher create mode 100644 flpremerge.py create mode 100644 flscriptparse.py create mode 100755 flscriptparser2 create mode 100644 install.sh create mode 100644 postparse.py create mode 100644 pytnyzer.py create mode 100644 qsatype.py create mode 100644 test/README create mode 100755 test/flscriptparser create mode 100644 tests/__init__.py create mode 100644 tests/ifaceclass.py create mode 100644 xml2json.py create mode 100644 xmlparse.py diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..1e0340e --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,21 @@ +FLScriptParser - A program that parses QtScript Files from AbanQ projects. +- Based on the work of the "AnsiC" example from python-ply documentation - + +Original work Copyright: 2001-2006 David M. Beazley +Copyright: 2009 - Gestiweb, Integración de Soluciones Web S.L. +Copyright: 2009 - David Martínez Martí + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..8ebfe02 --- /dev/null +++ b/INSTALL @@ -0,0 +1,43 @@ +parser for javascript-like sources for AbanQ QtScript files. +------------------------------------------------------- + +DEPENDS: + +- Requires python PLY to be installed. + +With Debian, you can install it typing: +$ sudo apt-get install python-ply + +RUNNING LOCALLY: + +Start up the script "flscriptparser" from the ./test/ folder + +INSTALL: + +You should create a symbolic link of ./test/flscriptparser to a folder inside +something in your PATH. For example: + +ln -s /path/to/project/flscriptparser/test/flscriptparser \ + /usr/local/bin/flscriptparser + + +USE: + +flscriptparser [file1.qs] [file2.qs] [file3.qs] [..] + +Run flscriptparser without arguments to parse all files named *.qs in the +current folder. If there are arguments, these are treated as files to parse. + +It generates, for each input file, one output named .parse-filename.qs.log +where filename.qs is the filename for the file that you are trying to parse. + + +KNOWN BUGS: + +It doesn't handle (yet) several structures, like array definitions [a,b,c] or +extensive use of dots and functions. + +* Fixed * It hangs up unexpectedly when trying to parse some files with errors. +(You can try to hit Ctrl-C) +* Fixed * There are several problems if you try to pass a file with path as an argument. + diff --git a/INSTALL-ES b/INSTALL-ES new file mode 100644 index 0000000..88e6e43 --- /dev/null +++ b/INSTALL-ES @@ -0,0 +1,44 @@ +Parseador de Código fuente tipo Javascript para AbanQ +------------------------------------------------------- + +DEPENDENCIAS: +- python-ply +- git-core +- realpath + +flscriptparser depende únicamente del proyecto PLY de Python. + +En debian, puede instalarla con: +$ sudo apt-get install python-ply + +EJECUCION LOCAL: + +Lance el programa "flscriptparser" de la carpeta ./test/ + +INSTALACION: + +Debe crear un enlace simbólico de ./test/flscriptparser a una carpeta dentro de +su PATH. Por ejemplo: + +ln -s /ruta/al/proyecto/flscriptparser/test/flscriptparser \ + /usr/local/bin/flscriptparser + + +USO: + +flscriptparser [file1.qs] [file2.qs] [file3.qs] [..] + +Ejecute flscriptparser sin argumentos para parsear todos los .qs de la carpeta +en que se encuentra. Si le pasa argumentos, éste los interpretará como ficheros +a parsear. + +Genera para cada fichero, una salida en .parse-filename.qs.log donde +filename.qs es el nombre del fichero que intentaba parsear. + +BUGS CONOCIDOS: + +Tiene problemas si se le pasan ficheros con ruta como argumento. +Se cuelga sin explicación al parsear algunos ficheros con errores. +(Pulse Control+C para seguir con el siguiente fichero en tal caso) + + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..02a8b59 --- /dev/null +++ b/LICENSE @@ -0,0 +1,505 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c9cf0d1 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ + +all: + @echo "> run sudo make install to install." + +install: + bash install.sh + diff --git a/README b/README new file mode 100644 index 0000000..754f015 --- /dev/null +++ b/README @@ -0,0 +1,10 @@ +Example of a Javascript-like parser. +This parser is intended to enhnace good practices when coding; so it's only +cares about structure. It's not intended to execute any data. + +It was developed to aid the developers when coding something for AbanQ/FacturaLUX. + +NOTE: Don't try to execute the parser from this folder, it will get filled + with trash files. Instead, enter to the "test" folder and run it from there. + + diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/analyze_controls.sh b/analyze_controls.sh new file mode 100755 index 0000000..d9419d0 --- /dev/null +++ b/analyze_controls.sh @@ -0,0 +1,13 @@ + +for file_ui in $( git ls-files -- "*.ui" ); do + file_qs1="${file_ui%\.*}.qs" + file_qs="${file_qs1/forms/scripts}" + + + test -f "$file_qs" && { + for control in $(grep -Po "this\.child\(.(\w+).\)" "$file_qs" | sort -u | sed "s/\"/\t/g" | awk '{print $2}'); do + grep -Eq "$control" "$file_ui" || echo $file_qs $control; + done + } || echo "ERROR: $file_qs" + +done diff --git a/dependencies.debian b/dependencies.debian new file mode 100644 index 0000000..8fb8480 --- /dev/null +++ b/dependencies.debian @@ -0,0 +1,3 @@ +python-ply +git-core +realpath \ No newline at end of file diff --git a/flalign.py b/flalign.py new file mode 100644 index 0000000..b16f80a --- /dev/null +++ b/flalign.py @@ -0,0 +1,489 @@ +from __future__ import print_function +from builtins import next +from builtins import object +import os, os.path, sys +import math +from optparse import OptionParser +import difflib +import re + +class ProcessFile(object): + def __init__(self, filename): + self.filename = filename + self.process() + self.indexLines() + self.bnames = set(self.idxnames.keys()) + + def process(self): + self.blocks = [] + self.idxlines = {} + self.idxnames = {} + self.sortednames = [] + + fblocks = open(self.filename + ".blocks") + n = 0 + line = fblocks.readline() + while line: + start, end, name = line.strip().split("\t") + self.blocks.append( (start,end, name) ) + self.idxlines[start] = n + self.sortednames.append(name) + if name in self.idxnames: + self.idxnames[name] = None + else: + self.idxnames[name] = n + n+=1 + line = fblocks.readline() + + def indexLines(self): + fqs = open(self.filename) + self.lines = [] + for line in fqs: + self.lines.append(line) + + + + def diffTo(self,pfile2): + added = pfile2.bnames - self.bnames + deleted = self.bnames - pfile2.bnames + + return added, deleted + +class LineNumber(object): + def __init__(self, letter, lines): + self.nl=0 + if len(letter)>1: + self.diffFrom = letter[0] + else: + self.diffFrom = None + self.error = None + + self.letter = letter[-1] + self.lines = lines + + def line(self): + l = self.lines[self.nl] + if self.diffFrom: + return l[2:] + else: + return l[:] + + def symbol(self): + l = self.lines[self.nl] + if self.diffFrom: + return l[0] + else: + return " " + + def next(self,t=1): + self.nl+=int(t) + + def __iadd__(self,other): + self.nl+=int(other) + return self + + def __int__(self): + return self.nl + + def __index__(self): + return self.nl + + +def appliedDiff(C, A, B, prefer = "C", debug = False, quiet = False, swap = False): + diffAB = list(difflib.ndiff(A.sortednames, B.sortednames)) + diffAC = list(difflib.ndiff(A.sortednames, C.sortednames)) + + + nlA = LineNumber("A",A.sortednames) + nlB = LineNumber("B",B.sortednames) + nlC = LineNumber("C",C.sortednames) + nlAB = LineNumber("AB", diffAB) + nlAC = LineNumber("AC", diffAC) + + maxA = len(A.sortednames) + maxB = len(B.sortednames) + maxC = len(C.sortednames) + patchedResult = [] + + def AddPatchLine(mode): + modefrom = mode[0] + modetype = mode[1] + linefrom = None + linetext = "" + conflict = False + linenumbers = int(nlA),int(nlB),int(nlC) + if modetype == "-": + linetext = lineaA + linefrom = "A" + if modefrom == "B": + DeletedB.append(lineaA) + elif modefrom == "C": + DeletedC.append(lineaA) + elif modetype == "=": + modefrom = prefer + if modefrom == "A": + linetext = lineaA + elif modefrom == "B": + linetext = lineaB + elif modefrom == "C": + linetext = lineaC + linefrom = modefrom + elif modetype == "+": + if modefrom == "A": + linetext = lineaA + elif modefrom == "B": + linetext = lineaB + if lineaB in AddedB: conflict = True + if lineaB in AddedC: conflict = True + AddedB.append(lineaB) + elif modefrom == "C": + linetext = lineaC + if lineaC in AddedB: conflict = True + if lineaC in AddedC: conflict = True + AddedC.append(lineaC) + linefrom = modefrom + line = ( + modefrom, modetype, + linenumbers, + linefrom, + linetext + ) + #ConflictMode = False + if len(ConflictMode): + if linetext[0]!="#": ConflictMode.pop() + else: + #print ">>", len(ConflictMode),linetext + if len(ConflictMode)>1 and linetext.find("separator") == -1: + ConflictMode.pop() + elif linetext.find("separator") >= 0: + ConflictMode.pop() + return + if conflict and linetext[0]!="#": + if debug: + print("WARNING: Omitting previously added block <%s>" % linetext) + + while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") >= 0: + #print "sep//", + patchedResult.pop() + + while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") == -1: + #print "comm//", + patchedResult.pop() + #if patchedResult[-1][4][0]=="#": patchedResult.pop() + + ConflictMode.append(1) + ConflictMode.append(1) + else: + patchedResult.append(line) + + AddedB = [] + AddedC = [] + DeletedB = [] + DeletedC = [] + ConflictMode = [] + + def getVars(letter): + if letter=="B": + return nlA, nlB, nlC, nlAB, nlAC + if letter=="C": + return nlA, nlC, nlB, nlAC, nlAB + raise TypeError + + def Patch(code): + letter = code[0] + sign = code[1] + nBase, nLocal, nRemote, nDiffLocal, nDiffRemote = getVars(letter) + + def Minus(): + AddPatchLine(letter + "-") + if debug: + print(letter + "-%04d" % int(nBase), nBase.line()) + if nBase.line() != nDiffLocal.line() and not quiet: + print(letter + "! " , nDiffLocal.line()) + if nDiffRemote.symbol()!=" " and not quiet: + if nDiffRemote.symbol()=="-": + #Removing twice! + nBase.error = "removing-twice" + else: + print("??", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) + else: + next(nRemote) + + next(nDiffLocal) + next(nDiffRemote) + next(nBase) + + def Plus(): + AddPatchLine(letter + "+") + if debug: + print(letter + "+%04d" % int(nLocal), nLocal.line()) + if nLocal.line()!=nDiffLocal.line() and not quiet: + print(letter + "! " , nDiffLocal.line()) + if nBase.error == "removing-twice": + nBase.error = None + while nDiffRemote.symbol() == "?": + next(nDiffRemote) + if nDiffRemote.symbol() == "+": + if not quiet: + print("!~", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) + next(nDiffRemote) + next(nRemote) + + + next(nDiffLocal) + next(nLocal) + + def Info(): + #if debug: print letter + "?> " , nDiffLocal.line() + next(nDiffLocal) + + if sign == "-": Minus() + elif sign == "+": Plus() + elif sign == "?": Info() + + + + while True: + + if ( + int(nlA) >= maxA and + int(nlB) >= maxB and + int(nlC) >= maxC + ): break + lineaA = " " + sAB = cAB = lineaA = " " + sAC = cAC = lineaC = " " + if int(nlA) >= maxA : + if int(nlA) - maxA > 0 : print("A overflow!", int(nlA) - maxA) + lineaA = " " + else: lineaA = A.sortednames[nlA] + + if int(nlB) >= maxB : + sAB = cAB = lineaA = " " + if int(nlB) - maxB > 0 : print("B overflow!", int(nlB) - maxB) + else: + lineaB = B.sortednames[nlB] + + if int(nlAB) < len(diffAB): + sAB = diffAB[nlAB][0] + cAB = diffAB[nlAB][2:] + + if int(nlC) >= maxC : + sAC = cAC = lineaC = " " + if int(nlC) - maxC > 0 : print("C overflow!", int(nlC) - maxC) + else: + lineaC = C.sortednames[nlC] + + if int(nlAC) < len(diffAC): + sAC = diffAC[nlAC][0] + cAC = diffAC[nlAC][2:] + + #print nlA, nlB, nlC + if sAB == " " and sAC == " ": + AddPatchLine("A=") + if debug: + if prefer == "": + print("ABC=%04d%+d%+d" % (nlA,nlB-nlA,nlC-nlA),lineaA) + elif prefer == "A": + print("A=%04d" % nlA,lineaA) + elif prefer == "B": + print("B=%04d" % nlB,lineaB) + elif prefer == "C": + print("C=%04d" % nlC,lineaC) + else: + assert prefer in ["A","B","C"] + if not quiet: + if lineaA!=cAB: + print("A!=B " , cAB) + if lineaA!=cAC: + print("A!=C " , cAC) + if lineaA != lineaB or lineaC != lineaA: + print("wtf!?A:", lineaA) + print("wtf!?B:", lineaB) + print("wtf!?C:", lineaC) + + next(nlAB) + next(nlAC) + next(nlA) + next(nlB) + next(nlC) + elif swap and sAB=="+": Patch("B+") + elif sAC=="+": Patch("C+") + elif sAC=="?": Patch("C?") + elif sAB=="+": Patch("B+") + elif sAB=="?": Patch("B?") + elif swap and sAB=="-": Patch("B-") + elif sAC=="-": Patch("C-") + elif sAB=="-": Patch("B-") + else: + if not quiet: + print(sAB,"*", sAC) + break + + addedB = set([x for x in AddedB if x[0]!="#"]) + addedC = set([x for x in AddedC if x[0]!="#"]) + + deletedB = set([x for x in DeletedB if x[0]!="#"]) + deletedC = set([x for x in DeletedC if x[0]!="#"]) + + movedB = addedB & deletedB + movedC = addedC & deletedC + + conflictsAA = addedB & addedC + conflictsDD = deletedB & deletedC + conflictsAD = addedB & deletedC + conflictsDA = deletedB & addedC + + if movedB: + print("CONFLICTS BLOCK MOVED A(%s)->B(%s):" % (A.filename,B.filename)) + for name in movedB: print("-",name) + + if movedC: + print("CONFLICTS BLOCK MOVED A(%s)->C(%s):" % (A.filename,C.filename)) + for name in movedC: print("-",name) + + if conflictsAA: + print("CONFLICTS SAME BLOCK ADDED B(%s)-C(%s):" % (B.filename,C.filename)) + for name in conflictsAA: print("-",name) + + if conflictsDD: + print("CONFLICTS SAME BLOCK DELETED B(%s)-C(%s):" % (B.filename,C.filename)) + for name in conflictsDD: print("-",name) + + if conflictsAD: + print("CONFLICTS BLOCK ADDED BY %s , DELETED BY %s:" % (B.filename,C.filename)) + for name in conflictsAD: print("-",name) + + if conflictsDA: + print("CONFLICTS BLOCK DELETED BY %s , ADDED BY %s:" % (B.filename,C.filename)) + for name in conflictsDA: print("-",name) + + + + return patchedResult + """ + added, deleted = pfrom.diffTo(pto) + plist = [] + for start,end,name in ptarget.blocks: + if name in added: + n = pto.idxnames[name] + if n is not None: + start,end,name = pto.blocks[n] + bobject = pto.filename + else: + print "Conflict with element", name + elif name in deleted: + bobject = "deleted" + plist.append((bobject,start,end,name)) + else: + bobject = ptarget.filename + plist.append((bobject,start,end,name)) + + return plist +""" + +def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = False): + patchlist = appliedDiff(C, A, B, prefer , debug, quiet, swap) + F = {"A": A, "B": B, "C": C} + L = ["A", "B", "C"] + + fout = open(F[prefer].filename + ".aligned","w") + classlist = [] + for Fby,action, nlines, Fwhich, line in patchlist: + nlA, nlB, nlC = nlines + if action not in ("+","="): continue + nl = nlines[L.index(Fwhich)] + try: + linebegin, lineend, line = F[Fwhich].blocks[nl] + except IndexError: + print("!!!ERROR MERGING!!") + continue + + text = "".join( + F[Fwhich].lines[int(linebegin):int(lineend)] + ) + #text = text.replace("\t"," ") + sline = line.split(":") + if sline[0]=="classdeclaration": + if len(classlist): + lastclass = classlist[-1] + else: + lastclass = None + + thisclass = sline[1] + classlist.append(thisclass) + if lastclass: + rs1 = re.search("class (\w+) extends (\w+)",text) + if rs1: + if lastclass != rs1.group(2): + #print "INFO: Changing >> class", thisclass, "extends",rs1.group(2), "--> extends", lastclass + text = re.sub(r"class (\w+) extends (\w+)", "class %s extends %s" % (thisclass,lastclass),text) + rs2 = re.search(r"function .*%s\(.*context.*\) { (\w+)" % thisclass,text) + if rs2: + if lastclass != rs2.group(1): + badline = rs2.group(0) + goodline = badline.replace(rs2.group(1),lastclass) + #print "INFO: Changing >>", badline , "-->", goodline + text = text.replace(badline,goodline) + + else: + print(text[:64]) + + #if debug: + # fout.write("<<< %s || %s >>>\n" % (Fwhich, line)) + # fout.write("<<< (%d:%d) >>>\n" % (int(linebegin),int(lineend))) + fout.write(text) + + + fout.close() + +def main(): + parser = OptionParser() + #parser.add_option("-q", "--quiet", + # action="store_false", dest="verbose", default=True, + # help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("-q","--quiet", + action="store_true", dest="quiet", default=False, + help="don't print status messages to stdout") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + + filenames = [x for x in args if os.path.isfile(x)] + not_a_file = set(args) - set(filenames) + if len(not_a_file): + print("WARNING: Not a file:", ", ".join(not_a_file)) + return + + if len(filenames) != 3: + print("MUST have exactly 3 files to align.") + pfiles = [] + for file1 in filenames: + #print "Load File:", file1 + pf = ProcessFile(file1) + pfiles.append(pf) + + A = pfiles[0] + B = pfiles[1] + C = pfiles[2] + + #addedAB, deletedAB = A.diffTo(B) + #addedAC, deletedAC = A.diffTo(C) + is_debug = options.debug + writeAlignedFile(C, A, B, swap = True, debug= is_debug) + writeAlignedFile(B, A, C, debug= is_debug) + writeAlignedFile(B, A, C, prefer = "A", debug= is_debug) + #writeAlignedFile(A, A, C) + + +if __name__ == "__main__": main() diff --git a/flclasses.py b/flclasses.py new file mode 100644 index 0000000..2bb0f00 --- /dev/null +++ b/flclasses.py @@ -0,0 +1,335 @@ +from __future__ import print_function +from builtins import str +from builtins import object +debug = 0 + +class cBase(object): + def __init__(self): + self.type = ("Unknown","Unknown") + self.codedepth = 0 + + def setSubtype(self,newsubtype): + x, y = self.type + self.type = (x,newsubtype) + + def setType(self,newtype): + x, y = self.type + self.type = (newtype,y) + + def addCodeDepth(self): + self.codedepth += 1 + + def __len__(self): + return 1; + +class cBaseItem(cBase): + def __init__(self,value): + cBase.__init__(self) + self.type = ("Item","Unknown") + self.value = value + + def __str__(self): + if debug > 0 and debug < 1 and self.type == ("Item","Unknown"): return "*" + str(self.value) + "?" + return str(self.value) + +class cBaseItemList(cBase): + def __init__(self,itemList,prefix,suffix,subtype="Unknown"): + cBase.__init__(self) + self.type = ("ItemList",subtype) + if not isinstance(itemList,cBaseList): + iList = cBaseListInline() + iList.addAuto(itemList,subtype=subtype) + itemList = iList + t,st = itemList.slice[0].type + #itemList.setSubtype(st) + itemList.setSubtype("OneItem") + + + if not isinstance(itemList,cBaseList): + raise NameError("itemList no es un cBaseList: %s" % repr(itemList)) + self.itemList = itemList + self.prefix = prefix + self.suffix = suffix + if subtype=="Declaration": + for item in self.itemList.slice: + ctype, csubtype = item.type + ctype = subtype + item.type = (ctype, csubtype) + + def __str__(self): + global debug + txt = str(self.prefix) + str(self.itemList) + str(self.suffix) + txt = txt.strip() + if debug>=2: + txt = "{%s/%s:" % self.itemList.type + txt + "}" + return txt + + +class cBaseVarSpec(cBase): + def __init__(self,name,vartype=None,value=None): + cBase.__init__(self) + self.type = ("Item","Variable") + self.value = value + self.vartype = vartype + self.name = name + + def __str__(self): + txt=self.name + if self.vartype: txt+=":"+self.vartype + if self.value: txt+="="+str(self.value) + return txt + + + +class cBaseList(cBase): + def __init__(self): + cBase.__init__(self) + self.type = ("List","Unknown") + self.slice = [] + self.hidden = [] + self.byType = {} + self.bySubtype = {} + self.byDefName = {} + + def __len__(self): + return len(self.slice); + + def includeItem(self,child): + if not isinstance(child,cBase): + raise NameError("Child is not an instance of Base Class!") + + try: + ctype, csubtype = child.type + except: + raise NameError("Base Class doesn't have `type` atribute or is incorrect.") + if not hasattr(self.byType,ctype): + self.byType[ctype]=[] + + if not hasattr(self.bySubtype,ctype): + self.bySubtype["%s:%s" % (ctype,csubtype)]=[] + + self.byType[ctype].append(child) + self.bySubtype["%s:%s" % (ctype,csubtype)].append(child) + + if ctype == "Declaration": + try: + cname = child.name + except: + raise NameError("Declaration Class doesn't have `name` atribute.") + + if cname in self.byDefName: + print("#WARNING# Variable %s found, but previously defined in this block" % cname) + # self.byDefName[cname]=None + else: + self.byDefName[cname]=child + try: + child.addCodeDepth() + except: + print(repr(child)) + raise NameError("Base Class doesn't have `addCodeDepth` function.") + + if isinstance(child,cBaseItemList): + sslice = child.itemList.slice[:] + for e in child.itemList.hidden: + sslice.remove(e) + + + for item in sslice: + itype, isubtype = item.type + if not isinstance(item,cBaseItemList): + self.includeItem(item) + #self.hidden.append(item) + + def addCodeDepth(self): + cBase.addCodeDepth(self) + sslice = self.slice[:] + + for e in self.hidden: + sslice.remove(e) + + for child in sslice: + child.addCodeDepth() + + + + def addAuto(self,element,subtype=None): + if not isinstance(element,cBase): + element = cBaseItem(element) + if subtype: + element.setSubtype(subtype) + + self.addChild(element) + + def addChild(self,child,hidden=False): + self.includeItem(child) + self.slice.append(child) + if hidden: self.hidden.append(child) + + def __str__(self): + global debug + sslice = self.slice[:] + + for e in self.hidden: + sslice.remove(e) + + + if debug>=1: + txt = "\n" + # --------- DEBUG OUTPUT VARDECL --------- + if len(self.byDefName)>0: + txt += " " * self.codedepth + " /** Declared vars: " + for definition in self.byDefName: + txt += definition + "; " + txt += "**/\n" + n=0 + for c in sslice: + t1, t2 = c.type + n+=1 + txt += "%-17s:%d" % ("%-8s/%-8s" % (t1[:8],t2[:8]),n)+ ">" + ". " * c.codedepth + str(c) + txt += "\t<%d:\n" % n + return txt + + + + if len(sslice) == 1: + c = str(sslice[0]) + if "\n" not in c: + return " " + c + " " + txt = "\n" + + #for child in self.slice: + # if str(child.type[0]) != "ItemList": + # txt += str(child.type) + "\n" + # txt += str(child) + "\n" + # txt += "____"+ "\n" + + lastmargin = 0 + for c in sslice: + t1, t2 = c.type + line = " " + str(c).replace("\n","\n ") + linecount = line.count("\n") + margin = 0 + if linecount>2: margin += 1 + if linecount>25: margin += 1 + if linecount>50: margin += 1 + topmargin = margin - lastmargin + if topmargin < 0: topmargin = 0 + txt += "\n" * topmargin + line + "\n" * (margin+1) + lastmargin = margin + return txt + + + +class cBaseListInline(cBaseList): + def __init__(self,separator=", "): + cBaseList.__init__(self) + self.separator = separator + + def __str__(self): + global debug + txt = "" + for c in self.slice: + if debug>=3: + txt += "[%s/%s: " % c.type + + txt += str(c) + if debug>=3: + txt += "]" + + txt += self.separator + + if len(self.separator) == 0: + return txt + else: + return txt[:-len(self.separator)] + + +class cStatementList(cBaseList): + def __init__(self): + cBaseList.__init__(self) + self.type = ("List","Statement") + +class cBaseDecl(cBase): + def __init__(self,name): + cBase.__init__(self) + self.type = ("Declaration","Unknown") + self.name = name + + def __str__(self): + return "@unknown declaration %s" % self.name + + +class cFuncDecl(cBaseDecl): + def __init__(self,name,arglist,rettype,source): + cBaseDecl.__init__(self,name=name) + self.type = ("Declaration","Function") + self.arglist = arglist + self.rettype = rettype + if not isinstance(source,cBaseList): + iList = cBaseList() + iList.addAuto(source) + source = iList + + + if not isinstance(source,cBaseList): + raise NameError("source no es un cBaseList: %s" % repr(source)) + + self.source = source + + def addCodeDepth(self): + cBase.addCodeDepth(self) + try: + self.source.addCodeDepth() + except: + import traceback,sys + print("Exception in user code:") + print('-'*60) + traceback.print_exc(file=sys.stdout) + print('-'*60) + + + + + def __str__(self): + if self.rettype: + ret=" : " + self.rettype + else: + ret="" + return 'function %s(%s)%s {%s}' % (self.name,self.arglist,ret,self.source) + +class cClassDecl(cBaseDecl): + def __init__(self,name,extends,source): + cBaseDecl.__init__(self,name=name) + self.type = ("Declaration","Class") + self.extends = extends + self.childclasses = None + self.constructor = None + if not isinstance(source,cBaseList): + iList = cBaseList() + iList.addAuto(source) + source = iList + + + if not isinstance(source,cBaseList): + raise NameError("source no es un cBaseList: %s" % repr(source)) + self.source = source + + def addCodeDepth(self): + cBase.addCodeDepth(self) + try: + self.source.addCodeDepth() + except: + import traceback,sys + print("Exception in user code:") + print('-'*60) + traceback.print_exc(file=sys.stdout) + print('-'*60) + + def __str__(self): + if self.extends: + ext = " extends " + self.extends + else: + ext = "" + + return 'class %s%s {%s}' % (self.name,ext,self.source) + diff --git a/flex.py b/flex.py new file mode 100644 index 0000000..dc6f1f7 --- /dev/null +++ b/flex.py @@ -0,0 +1,210 @@ +from __future__ import print_function +# ---------------------------------------------------------------------- +# clex.py +# +# A lexer for ANSI C. +# ---------------------------------------------------------------------- + +#import sys +#sys.path.insert(0,"../..") + +import ply.lex as lex + +# Reserved words +reserved = [ + 'BREAK', 'CASE', 'CONST', 'STATIC', 'CONTINUE', 'DEFAULT', 'DO', + 'ELSE', 'FOR', 'IF', 'IN', + 'RETURN', + #'STRUCT', + 'SWITCH', + 'WHILE', 'CLASS', 'VAR', 'FUNCTION', + 'EXTENDS', 'NEW','WITH','TRY','CATCH','THROW', 'DELETE', 'TYPEOF' + ] +token_literals = [ + # Literals (identifier, integer constant, float constant, string constant, char const) + 'ID', 'ICONST', 'FCONST', 'SCONST', 'CCONST' #, 'RXCONST' +] +tokens = reserved + token_literals + [ + + # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=) + 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', + 'OR', 'AND', + 'CONDITIONAL1','AT', + #'NOT', + 'XOR', 'LSHIFT', 'RSHIFT', + 'LOR', 'LAND', 'LNOT', + 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'EQQ', 'NEQ', + + # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) + 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', +# 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL', + + # Increment/decrement (++,--) + 'PLUSPLUS', 'MINUSMINUS', + + # Structure dereference (->) +# 'ARROW', + + # Conditional operator (?) +# 'CONDOP', + + # Delimeters ( ) [ ] { } , . ; : + 'LPAREN', 'RPAREN', + 'LBRACKET', 'RBRACKET', + 'LBRACE', 'RBRACE', + 'COMMA', 'PERIOD', 'SEMI', 'COLON', + + # Ellipsis (...) +# 'ELLIPSIS', + 'DOCSTRINGOPEN', + # 'COMMENTOPEN', + 'COMMENTCLOSE', + 'DOLLAR', + 'SQOUTE', + 'DQOUTE', + 'BACKSLASH', + ] + +# Completely ignored characters +t_ignore = ' \r\t\x0c' + +# Newlines +def t_NEWLINE(t): + r'\n+' + t.lexer.lineno += t.value.count("\n") + +# Operators +t_BACKSLASH = '\\\\' +t_DOLLAR = r'\$' +t_SQOUTE = '\'' +t_DQOUTE = '"' +t_PLUS = r'\+' +t_MINUS = r'-' +t_TIMES = r'\*' +t_DIVIDE = r'/' +t_MOD = r'%' +t_OR = r'\|' +t_AND = r'&' +#t_NOT = r'~' +t_XOR = r'\^' +t_LSHIFT = r'<<' +t_RSHIFT = r'>>' +t_LOR = r'\|\|' +t_LAND = r'&&' +t_LNOT = r'!' +t_LT = r'<' +t_GT = r'>' +t_LE = r'<=' +t_GE = r'>=' +t_EQ = r'==' +t_NE = r'!=' +t_EQQ = r'===' +t_NEQ = r'!==' +t_CONDITIONAL1 = r'\?' + +# Assignment operators + +t_EQUALS = r'=' +t_TIMESEQUAL = r'\*=' +t_DIVEQUAL = r'/=' +t_MODEQUAL = r'%=' +t_PLUSEQUAL = r'\+=' +t_MINUSEQUAL = r'-=' +""" +t_LSHIFTEQUAL = r'<<=' +t_RSHIFTEQUAL = r'>>=' +t_ANDEQUAL = r'&=' +t_OREQUAL = r'\|=' +t_XOREQUAL = r'^=' +""" + +# Increment/decrement +t_PLUSPLUS = r'\+\+' +t_MINUSMINUS = r'--' + +# -> +#t_ARROW = r'->' + +# ? +#t_CONDOP = r'\?' + + +# Delimeters +t_LPAREN = r'\(' +t_RPAREN = r'\)' +t_LBRACKET = r'\[' +t_RBRACKET = r'\]' +t_LBRACE = r'\{' +t_RBRACE = r'\}' +t_COMMA = r',' +t_PERIOD = r'\.' +t_SEMI = r';' +t_COLON = r':' +#t_ELLIPSIS = r'\.\.\.' +t_AT = r'@' +# Identifiers and reserved words + +reserved_map = { } +for r in reserved: + reserved_map[r.lower()] = r + + + +def t_ID(t): +# r'[A-Za-z_]+([\.]{0,1}[\w_]*)+' + r'[A-Za-z_]+[\w_]*' + t.type = reserved_map.get(t.value,"ID") + return t + + + +# Integer literal +t_ICONST = r'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?' + +# Floating literal +t_FCONST = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' + +# String literal +t_SCONST = r'\"([^\\\n]|(\\.))*?\"' + +# Character constant 'c' or L'c' +t_CCONST = r'\'([^\\\n]|(\\.))*?\'' + +# REGEX constant +#t_RXCONST = r'/[^/ ]+/g?' + +# Comments +def t_comment(t): + r'(/\*( |\*\*)(.|\n)*?\*/)|(//.*)' + #r'/\*(.|\n)*?\*/' + t.lexer.lineno += t.value.count('\n') + + +def t_DOCSTRINGOPEN(t): + r'/\*\*[ ]+' + return t; + +#t_COMMENTOPEN = r'/\*' +t_COMMENTCLOSE = r'\*/' + + +# Preprocessor directive (ignored) +def t_preprocessor(t): + r'\#(.)*?\n' + t.lexer.lineno += 1 + + +def t_error(t): + print("Illegal character %s" % repr(t.value[0])) + t.lexer.skip(1) + + + +lexer = lex.lex(debug=False) +if __name__ == "__main__": + lex.runmain(lexer) + + + + + diff --git a/flmergetool b/flmergetool new file mode 100755 index 0000000..4d2845a --- /dev/null +++ b/flmergetool @@ -0,0 +1,199 @@ +#!/bin/bash + +# FLMERGETOOL. +# To add this tool to git, do the following: +# flmergetool INSTALL +editors=(medit kate gedit kwrite nano vim vi) +EDITOR_FILEPATH= +for editor in "${editors[@]}"; do + EDITOR_FILEPATH=$(command -v $editor) + [[ $EDITOR_FILEPATH ]] && break +done +echo $EDITOR_FILEPATH + +thisprg=`realpath $0` +thisdir=`dirname $thisprg` +if test "$1" = "INSTALL" +then + sudo ln -s $thisprg /usr/local/bin/flmergetool + git config --global mergetool.flmergetool.cmd \ + "flmergetool \$MERGED \$BASE \$LOCAL \$REMOTE" + git config --global mergetool.flmergetool.trustExitCode true + exit 0 +fi +if [[ $1 == "--debug" ]]; then + DEBUG="--debug" + shift +else + DEBUG="" +fi +MERGE=$1 +BASE=$2 +LOCAL=$4 +REMOTE=$3 +if test -f $LOCAL +then + OK=1 +else + echo "File $LOCAL does not exist." + exit 1 +fi + +if test -f $REMOTE +then + OK=1 +else + echo "File $REMOTE does not exist." + exit 1 +fi + +if test -f $BASE +then + OK=1 +else + if kdiff3 $LOCAL $REMOTE -o $MERGE >/dev/null 2>&1 + then + #echo "KDIFF3 OK" + OK=1 + exit 0 + else + echo "KDIFF3 FAILED" + exit 1 + fi +fi + + +merge_other() { + kdiff3 "$BASE" "$LOCAL" "$REMOTE" -o "$MERGE" --auto >/dev/null 2>&1 + return $? +} + +merge_xml() { + python $thisdir/xml2json.py convert "$BASE" "$LOCAL" "$REMOTE" && \ + kdiff3 "$BASE.json" "$LOCAL.json" "$REMOTE.json" -o "$MERGE.json" --auto >/dev/null 2>&1 && \ + python $thisdir/xml2json.py revert "$MERGE.json" && \ + cp $MERGE.json.* $MERGE + for f in "$BASE" "$LOCAL" "$REMOTE" "$MERGE"; do + rm "$f".json "$f".json.* + done + return $? +} + +merge_qs() { + if echo $MERGE | grep -E '\.qs' + then + OK=1 + else + echo "File $MERGE does not have .qs" + exit 1 + fi + + ( + for file1 in "$BASE" "$LOCAL" "$REMOTE"; do + is_done=1 + until [[ $is_done == 0 ]] + do + if python $thisdir/flscriptparse.py -O file "$file1" && test -f "$file1.xml" + then + #echo "FLScriptParse OK" + is_done=0 + else + patched=0 + for kpatch in /tmp/knownpatch.*; do + [[ $kpatch == "/tmp/knownpatch.*" ]] && break + patch "$file1" "$kpatch" -f -F4 -l && { + patched=1 + } + done + + read -p "Fallo en el parseo del fichero, desea abrir un editor para corregirlo manualmente? [Y/n]:" editor + [[ $editor == "n" ]] && { + echo "FLScriptParse FAILED" + exit 1 + } + bname=$(basename "$file1") + cp $file1 /tmp/$bname.orig + "$EDITOR_FILEPATH" "$file1" + read -p "Pulse Intro si ha terminado." + diff -U6 -p --minimal -d /tmp/$bname.orig $file1 >/tmp/knownpatch.$bname.patch.tmp + if [[ $? == 1 ]] && [[ $(wc -l /tmp/knownpatch.$bname.patch.tmp) > 0 ]] ; then + mv /tmp/knownpatch.$bname.patch.tmp /tmp/knownpatch.$bname.patch + else + unlink /tmp/knownpatch.$bname.patch.tmp + fi + + fi + done + done + + + if python $thisdir/flpremerge.py $DEBUG $BASE $LOCAL $REMOTE + then + #echo "FLPreMerge OK" + OK=1 + else + echo "FLPreMerge FAILED" + exit 1 + fi + + if python $thisdir/flalign.py $DEBUG $BASE $LOCAL $REMOTE + then + #echo "FLAlign OK" + OK=1 + else + echo "FLAlign FAILED" + exit 1 + fi + + if test -f $BASE.aligned && test -f $LOCAL.aligned && test -f $REMOTE.aligned + then + OK=1 + else + echo "FLAlign FAILED" + exit 1 + fi + unlink $MERGE + #echo "kdiff3 $BASE.aligned $LOCAL.aligned $REMOTE.aligned -o $MERGE --auto" + if kdiff3 --auto $BASE.aligned $LOCAL.aligned $REMOTE.aligned -o $MERGE >/dev/null 2>&1 + then + #echo "KDIFF3 OK" + OK=1 + else + echo "KDIFF3 FAILED" + exit 1 + fi + ) + + for file1 in "$BASE" "$LOCAL" "$REMOTE"; do + unlink "$file1.xml" + unlink "$file1.blocks" + unlink "$file1.aligned" + unlink "$file1.hash" + done + + + return $?; +} + + + + + + +if echo $MERGE | grep -E '\.qs' ; then + merge_qs + exit $? +fi +if echo $MERGE | grep -E '\.(xml|mtd|ui)' ; then + merge_xml + exit $? +fi +if echo $MERGE | grep -E '\.(kut|qry)' ; then + merge_other + exit $? +fi + +print "Unknown extension." +exit 3 + + diff --git a/flpatcher b/flpatcher new file mode 100755 index 0000000..290df58 --- /dev/null +++ b/flpatcher @@ -0,0 +1,356 @@ +#!/bin/bash +APP_DIR=$(dirname $(realpath $0)) + + +SRC_PROJECT="" +SRC_START_COMMIT="" +SRC_END_COMMIT="HEAD" +DST_PROJECT=$(pwd) +DST_APPLY_COMMIT="HEAD" +TYPE_CS="" +TYPE_BL="" + +show_help() { + echo " $(basename $0) [options]" + echo + echo " ## options ##" + echo + echo " --src=FOLDER -- sets FOLDER as the source project" + echo " --dst=FOLDER -- sets FOLDER as the dest project" + echo " --start=COMMIT -- sets COMMIT as the source start commit" + echo " --end=COMMIT -- sets COMMIT as the source end commit" + echo " --apply=COMMIT -- sets COMMIT as the dest apply commit" + echo " --help -- shows this help message" + echo " --cs=(m|l|r) -- sets the default action for Both Created" + echo " m: 2-way merge l: use local r: use remote" + echo " --bl=(b|i) -- sets the default action for Local Deleted" + echo " b: delete i: include remote" + echo + echo " ## examples ## " + echo + echo " Default Invocation:" + echo " $(basename $0) --src=../otherprj --start=origin/base-code --end=dev-code " + echo + echo + + +} + + +process_argument() { + local argname="$1" + local value="$2" + + case $argname in + src) SRC_PROJECT="$value" ;; + dst) DST_PROJECT="$value" ;; + start) SRC_START_COMMIT="$value" ;; + end) SRC_END_COMMIT="$value" ;; + apply) DST_APPLY_COMMIT="$value" ;; + cs) TYPE_CS="$value" ;; + bl) TYPE_BL="$value" ;; + help) show_help; exit 0;; + *) echo "Unexpected argument '$argname'"; return 1 ;; + esac + + #echo "* $argname -> '$value'" + return 0 +} + +let n=0 + +ARGS=() + +for arg in "$@"; do + let n+=1 + if [[ $arg == --* ]] ; then + argname="${arg#--}" + value="" + + if [[ $argname == *"="* ]] ; then + value="${argname#*=}" + argname="${argname%%=*}" + fi + + process_argument "$argname" "$value" || exit 1 + + else + ARGS+=( "$arg" ) + fi +done + +let n=0 +for arg in "${ARGS[@]}"; do + let n+=1 + echo "Unexpected positional argument '$arg'" + exit 1 +done + +is_valid_git_folder() { + local gitfolder="$1" + test -d "$gitfolder" || return 1 + ( + cd "$gitfolder" || exit 1 + test -d ".git" || exit 1 + local line="" + + while read line; do + local status="${line%% *}" + local filename="${line#* }" + case $status in + ??) continue ;; + *) echo "Modified file in working directory '$filename' ($status)"; exit 1;; + esac + # echo "*$status* '$filename'" + done < <(git status --porcelain 2>/dev/null) + + ) || return 1 + +} +LAST_VALID_COMMIT_ID="" +LAST_VALID_COMMIT_MSG="" + +is_valid_git_commit() { + local gitfolder="$1" + local commit="$2" + + local COMMIT_LINE="" + pushd "$gitfolder" || exit 1 + [[ "$commit" ]] || exit 2 + COMMIT_LINE=$(git log "$commit" --pretty=oneline --abbrev=8 --abbrev-commit -1 ) + [[ $? != 0 ]] && { echo "$(pwd) git log salio con estado $? " >&2; exit 3; } + LAST_VALID_COMMIT_ID=${COMMIT_LINE:0:8} + LAST_VALID_COMMIT_MSG=${COMMIT_LINE:9} + popd + + return $? +} + +# VALIDAR -------------- +is_valid_git_folder "$SRC_PROJECT" || { + echo "El proyecto de origen '$SRC_PROJECT' no es valido." >&2; + exit 1; +} + +is_valid_git_folder "$DST_PROJECT" || { + echo "El proyecto de destino '$DST_PROJECT' no es valido." >&2; + exit 1; +} + +is_valid_git_commit "$SRC_PROJECT" "$SRC_START_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; + echo "El commit inicial <$SRC_START_COMMIT> no es valido." >&2; + exit 1; +} +echo "<$LAST_VALID_COMMIT_ID>" + +SRC_START_COMMIT=$LAST_VALID_COMMIT_ID +SRC_START_COMMIT_MSG=$LAST_VALID_COMMIT_MSG + +is_valid_git_commit "$SRC_PROJECT" "$SRC_END_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; + echo "El commit final <$SRC_END_COMMIT> no es valido." >&2; + exit 1; +} +SRC_END_COMMIT=$LAST_VALID_COMMIT_ID +SRC_END_COMMIT_MSG=$LAST_VALID_COMMIT_MSG + +is_valid_git_commit "$DST_PROJECT" "$DST_APPLY_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; + echo "El commit a aplicar <$DST_APPLY_COMMIT> no es valido." >&2; + exit 1; +} +DST_APPLY_COMMIT=$LAST_VALID_COMMIT_ID +DST_APPLY_COMMIT_MSG=$LAST_VALID_COMMIT_MSG + + +echo " [$SRC_START_COMMIT] - [$SRC_END_COMMIT] -> [$DST_APPLY_COMMIT]" + +# 1.- Calcular lista de ficheros que componen la diferencia en SRC + +FILE_LIST=() +pushd "$SRC_PROJECT" >/dev/null +IFS=$'\t' +while read -a line; do + FILE_LIST+=("${line[2]}") + #echo "${line[2]}" +done < <(git diff --numstat "$SRC_START_COMMIT" "$SRC_END_COMMIT") +unset IFS + +TMP_FILES=() + +git_show() { + local commit="$1" + local src="$2" + local dst="$3" + if git show "$commit":"$src" >"$dst" 2>/dev/null + then + TMP_FILES+=("$dst") + else + unlink "$dst" + fi + +} +# 2.- Para cada fichero en la diferencia, extraer las versiones BASE y REMOTE +for filename in "${FILE_LIST[@]}"; do + basename=$(basename "$filename") + if [[ $basename =~ .+\..+ ]] ; then + fileext=".${basename##*.}" + else + fileext="" + fi + # Ojo! si el fichero ha sido creado, aparece en REMOTE pero NO aparece en BASE + # Ojo^2! si el fichero ha sido borrado, aparece en BASE pero NO aparece en REMOTE + git_show $SRC_START_COMMIT "$filename" "$DST_PROJECT/$filename.BASE$fileext" + git_show $SRC_END_COMMIT "$filename" "$DST_PROJECT/$filename.REMOTE$fileext" +done + + +popd >/dev/null +# 3.- Para cada fichero en la diferencia, extraer la versión LOCAL +pushd "$DST_PROJECT" >/dev/null +for filename in "${FILE_LIST[@]}"; do + basename=$(basename "$filename") + if [[ $basename =~ .+\..+ ]] ; then + fileext=".${basename##*.}" + else + fileext="" + fi + + git_show $DST_APPLY_COMMIT "$filename" "$DST_PROJECT/$filename.LOCAL$fileext" +done + + +# 4.- Para cada fichero en la diferencia, ejecutar el +# sistema de mezcla adecuado ** segun EXTENSION** +for filename in "${FILE_LIST[@]}"; do + basename=$(basename "$filename") + if [[ $basename =~ .+\..+ ]] ; then + fileext=".${basename##*.}" + else + fileext="" + fi + + MERGE="$filename" + BASE="$filename.BASE$fileext" + LOCAL="$filename.LOCAL$fileext" + REMOTE="$filename.REMOTE$fileext" + + if test \( -f "$BASE" \) -a \( -f "$LOCAL" \) -a \( -f "$REMOTE" \) ; then + + # MEZCLA A 3 + #echo "EXT: $fileext;" + case $fileext in + .qs) flmergetool "$MERGE" "$BASE" "$LOCAL" "$REMOTE" ;; + .ui) cp "$LOCAL" "$MERGE"; fldesigner "$BASE" "$REMOTE" "$MERGE" ;; + # mtd) + # xml) + # kut) + # qry) + *) kdiff3 --auto "$BASE" "$LOCAL" "$REMOTE" -o "$MERGE" >/dev/null 2>&1 ;; + esac + if [[ $? == 0 ]] ; then + git add "$MERGE" + else + echo "Fallo en la mezcla" + # aqui podemos preguntar al usuario + fi + elif test \( \! -f "$BASE" \) -a \( \! -f "$LOCAL" \) -a \( -f "$REMOTE" \) ; then + # FICHERO CREADO + cp "$REMOTE" "$MERGE" + git add "$MERGE" + elif test \( -f "$BASE" \) -a \( -f "$LOCAL" \) -a \( \! -f "$REMOTE" \) ; then + # FICHERO BORRADO + git rm "$MERGE" + elif test \( -f "$BASE" \) -a \( \! -f "$LOCAL" \) -a \( \! -f "$REMOTE" \) ; then + # FICHERO BORRADO EN LOS DOS EXTREMOS + rm "$BASE" "$REMOTE" # igoramos realmente el fichero. + elif test \( \! -f "$BASE" \) -a \( -f "$LOCAL" \) -a \( -f "$REMOTE" \) ; then + # CREACION SIMULTANEA + if [[ $TYPE_CS ]] ; then + type=$TYPE_CS + else + echo "El fichero $MERGE ha sido creado simultaneamente en ambos proyectos" + echo "(para recordar esta respuesta las proximas veces escriba: m* l* r*)" + question="Elija una opcion [(m)erge|(l)ocal|(r)emoto]: " + read -p "$question" answer + type="${answer:0:1}" + [[ ${answer:1:1} == "*" ]] && TYPE_CS=$type + fi + case "$type" in + m) kdiff3 "$LOCAL" "$REMOTE" -o "$MERGE" >/dev/null 2>&1 ;; + l) cp "$LOCAL" "$MERGE" ;; + r) cp "$REMOTE" "$MERGE" ;; + *) echo "Unexpected type"; [[ "" ]] ;; + esac + if [[ $? == 0 ]] ; then + git add "$MERGE" + else + echo "Fallo en la mezcla" + # aqui podemos preguntar al usuario + fi + + elif test \( -f "$BASE" \) -a \( \! -f "$LOCAL" \) -a \( -f "$REMOTE" \) ; then + # BORRADO LOCAL + if [[ $TYPE_BL ]] ; then + type=$TYPE_BL + else + echo "El fichero $MERGE no existe en este proyecto pero si fue modificado en el parche" + echo "(para recordar esta respuesta las proximas veces escriba: b* i*)" + question="Elija una opcion [(b)orrar|(i)ncluir]: " + read -p "$question" answer + type="${answer:0:1}" + [[ ${answer:1:1} == "*" ]] && TYPE_BL=$type + fi + + case "${type}" in + b) rm "$BASE" "$REMOTE" && git rm "$MERGE" ;; + i) cp "$REMOTE" "$MERGE" && git add "$MERGE";; + *) echo "Unexpected type" ;; + esac + + else + echo "Ha ocurrido algun error anteriormente" + fi +done + +for f in ${TMP_FILES[@]}; do + unlink "$f" +done + +# 5.- Para cada mezcla finalizada, añadirla al proyecto DST (git add) + + + +# 6.- Si todos los ficheros se mezclaron sin error: +# hacer un commit, proponer un mensaje de commit. +GIT_MSGFILE="$DST_PROJECT/.git/GITGUI_MSG" +echo "patch: $SRC_START_COMMIT..$SRC_END_COMMIT > $DST_APPLY_COMMIT -- $SRC_PROJECT" > $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +echo "project: $SRC_PROJECT" >> $GIT_MSGFILE +echo "start: $SRC_START_COMMIT $SRC_START_COMMIT_MSG" >> $GIT_MSGFILE +echo "end: $SRC_END_COMMIT $SRC_END_COMMIT_MSG" >> $GIT_MSGFILE +echo "applied: $DST_APPLY_COMMIT $DST_APPLY_COMMIT_MSG" >> $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +echo " -- files patched: " >> $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +for filename in "${FILE_LIST[@]}"; do + echo " $filename" >> $GIT_MSGFILE +done +echo "" >> $GIT_MSGFILE + + +popd >/dev/null + + + + + + + + + + + + + diff --git a/flpremerge.py b/flpremerge.py new file mode 100644 index 0000000..7339970 --- /dev/null +++ b/flpremerge.py @@ -0,0 +1,662 @@ +from __future__ import print_function +from __future__ import division +from builtins import zip +from builtins import range +from past.utils import old_div +from builtins import object +import os, os.path, sys +import math +from optparse import OptionParser +import difflib +import re + +class processedFile(object): + def __init__(self, filename, debug = False): + self.debug = debug + self.table = {} # Carga literal de la lista de hashes, pk: (startbyte, endbyte) = csvrow + self.idxdepth = {} + self.idxtree = {} # pk: objnum (1.5.5.1.1) = (startbyte, endbyte) + self.hashes = {} + self.list_hashes = [] + self.filename = filename + self.cacheFullQname = {} + self.processHashFile() + self.indexNames() + self.computeSortedBlocks() + + def processHashFile(self): + self.cacheFullQname = {} + self.table, self.idxdepth,self.idxtree, self.hashes, self.list_hashes = process(self.filename+".hash") + + + def indexNames(self): + self.names = {} + self.sortedNames = [] + for pk in self.idxtree: + if len(pk) > 1: continue + name = self.fullQName(pk) + if name not in self.names: self.names[name] = [] + self.names[name].append(pk) + self.sortedNames.append((self.idxtree[pk],name)) + + self.sNames = set(self.names.keys()) + + def Qname(self,p): + row = self.table[self.idxtree[p]] + return row["name"] + + + def fullQName(self,p): + dcache = self.cacheFullQname + if p not in dcache: + name=[] + + for n in range(len(p)): + ps1 = p[:n+1] + name.append(self.Qname(ps1)) + fullname = "/".join(name) + dcache[p] = fullname + else: + fullname = dcache[p] + + return fullname + + def computeSortedBlocks(self, fout=None): + self.computedBlocks = [] + if fout is None: + fout = open(self.filename + ".blocks","w") + + antdesde, anthasta = 0 , -1 + fB = open(self.filename) + pos = 0 + linePosChar = [pos] + fB.seek(0) + line = fB.readline() + while line: + pos += len(line) + linePosChar.append(pos) + line = fB.readline() + linenum = 0 + self.linePosChar = linePosChar + for pk, bl_name in list(sorted(self.sortedNames))+[((None,None),None)]: + desde, hasta = pk + if desde is None or desde >= anthasta +1: + + bdesde = anthasta + 1 + fB.seek(bdesde) + if desde: + bhasta = desde - 1 + sB = fB.read((bhasta-bdesde)+1) + else: + sB = fB.read() + bhasta = bdesde + len(sB) -1 + + + while linenum < len(linePosChar) and linePosChar[linenum+1]<=bdesde: linenum += 1 + startline = linenum + linesize=linePosChar[startline] - linePosChar[startline-1] + curpos = bdesde - linePosChar[startline-1] + #if curpos != linesize: print "****", linesize, curpos + + + while linenum < len(linePosChar) and linePosChar[linenum+1]<=bhasta: linenum += 1 + endline = linenum + #while startline > 1 and linePosChar[startline-1]-bdesde >=-2: startline-=1 + + #print linePosChar[startline-1]-bdesde, linePosChar[startline]-bdesde, linePosChar[startline+1]-bdesde + + #print (startline, endline), "BLOCK", (linePosChar[startline], linePosChar[endline]), (bdesde , bhasta), (bhasta-bdesde)+1 + + #print "<<<<" + mode = "" + nline = startline + beginline = startline + bblocks = [] + blockdesc = [] + if len(sB.splitlines(1)) != endline-startline+1: + print(startline, endline, repr(sB)) + print(linePosChar[endline-1]-bhasta, linePosChar[endline]-bhasta, linePosChar[endline+1]-bhasta) + + print("Block lines doesnt match:", len(sB.splitlines(1)), endline-startline, startline, endline) + print(linePosChar[startline-2:startline+3],bdesde) + print(linePosChar[endline-2:endline+3],bhasta) + print(repr(sB.splitlines(1))) + for line in sB.splitlines(1): + nline += 1 + if line[-1]!='\n': break + ltype = "junk" + isseparator = re.match(r'[ \t]*\n',line) + iscommentline1 = re.match(r'[ \t]*//.+\n',line) + iscommentline2 = re.match(r'[ \t]*/\*.+\*/\n',line) + iscommentbegin = re.match(r'[ \t]*/\*.+\n',line) + iscommentend = re.match(r'.+\*/[ \t]*\n',line) + + + + if isseparator: ltype = "separator" + elif iscommentline1: ltype = "comment_inline" + elif iscommentline2: ltype = "comment_block_inline" + elif iscommentbegin: ltype = "comment_block" + elif iscommentend: ltype = "comment_blockend" + #else: print "junk?", line, + + if mode == "comment_block" and ltype == "comment_blockend": + mode = "comment_blockend" + + if mode == "comment_block" and ltype != "comment_blockend": + ltype = "comment_block" + + + if mode != ltype: + if mode: + if mode == "comment_blockend": + mode = "comment_block" + thisblock = ( (beginline, nline-1), ""+mode , ".".join(blockdesc)[:32]) + blockdesc = [] + bblocks.append(thisblock) + mode = ltype + beginline = nline - 1 + + words = re.split(r'\W+',line) + if words: + text = " ".join(words) + text = text.strip() + text = text.replace(" ", "-") + if len(text)>1: + blockdesc.append(text) + + if mode: + if mode == "comment_blockend": + mode = "comment_block" + + thisblock = ( (beginline, nline),""+mode, ".".join(blockdesc)[:32] ) + bblocks.append(thisblock) + blockdesc = [] + #print ltype, line, + + for lines, bname, desc in bblocks: + #print lines, " ", "%s:%s" % (bname, desc) + startline, endline = lines + name = "#..%s:%s" % (bname, desc) + + self.computedBlocks.append((startline, endline,name)) + fout.write("%d\t%d\t%s\n" % (startline, endline,name)) + #print "#..%s:%s" % (bname, desc) + #print sB, + #print ">>>>" + if desde is None: break + initline = linenum + while linenum < len(linePosChar) and linePosChar[linenum+1] 2: + print(linePosChar[startline] - desde, linePosChar[endline-1] - hasta, linePosChar[endline] - hasta, bl_name) + name = bl_name + #print (startline, endline), name, (linePosChar[startline], linePosChar[endline]), pk, (hasta-desde)+1 + self.computedBlocks.append((startline, endline,name)) + fout.write("%d\t%d\t%s\n" % (startline, endline,name)) + #print "%s" % name + antdesde, anthasta = pk + if anthasta <= linePosChar[linenum-1] + 1: + anthasta = linePosChar[linenum-1] + 1 + elif anthasta <= linePosChar[linenum] + 1: + anthasta = linePosChar[linenum] + 1 + + fB.close() + fout.close() + + +def linejunk(line): + line = line.strip() + if len(line)<4: return True + if line[2] == "//": return True + return False + +def charjunk(char): + junk = [" ", "\t"] + if char in junk: return True + return False + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + + filenames = [x for x in args if os.path.isfile(x)] + not_a_file = set(args) - set(filenames) + if len(not_a_file): + print("WARNING: Not a file:", ", ".join(not_a_file)) + + if len(filenames): + pfiles = [] + for file1 in filenames: + #print "File:", file1 + pf = processedFile(file1, debug = options.debug) + pfiles.append(pf) + #pfA = pfiles[0] + #for pfB in pfiles[1:]: + # feq = FindEquivalences(pfA, pfB) + +def tree_parents(pk): + parents = [] + while (len(pk)>0): + parents.append(pk) + pk = pk[:-1] + return parents + +class FindEquivalences(object): + def __init__(self,pfA, pfB, autoCompute = True): + self.equivalences = {} + self.parent_equivalences = {} + self.max_known_eq = {} + self.pfA, self.pfB = pfA, pfB + if autoCompute: self.compute() + + def compute(self): + print("Finding equivalences between A (%s) -> B (%s):" % ( + self.pfA.filename, + self.pfB.filename + )) + print("Modified names:") + commonNames = sorted(list(self.pfA.sNames & self.pfB.sNames)) + for name in commonNames: + if len(self.pfA.names[name]) > 1 or len(self.pfB.names[name]) > 1: + print("-", name,"(%d,%d)" % (len(self.pfA.names[name]),len(self.pfB.names[name]))) + else: + keyA = self.pfA.idxtree[self.pfA.names[name][0]] + rowA = self.pfA.table[keyA] + keyB = self.pfB.idxtree[self.pfB.names[name][0]] + rowB = self.pfB.table[keyB] + if rowA['hash'] != rowB['hash']: + print("###", name,"###") + fileA = self.pfA.filename.replace(".hash","") + fileB = self.pfB.filename.replace(".hash","") + if os.path.isfile(fileA) and os.path.isfile(fileB): + fA = open(fileA) + fA.seek(keyA[0]) + sA = fA.read(keyA[1]-keyA[0]+1) + fA.close() + + fB = open(fileB) + fB.seek(keyB[0]) + sB = fB.read(keyB[1]-keyB[0]+1) + fB.close() + + sA = sA.replace("\t", " ") + sB = sB.replace("\t", " ") + lines = list(difflib.ndiff(sA.splitlines(1), sB.splitlines(1),linejunk,charjunk)) + modifiedlines = [] + for n,line in enumerate(lines): + if line[0] != " ": + modifiedlines.append(n) + ml = 0 + omit = [] + Context = 3 + n = 0 + for line in lines: + d = [] + for m in modifiedlines: + dt = abs(m-n) + d.append(dt) + if dt < Context: break + dmin = min(d) + if dmin < Context: + if len(omit) : + if len(omit)>= Context: + for line in omit: + if line[0] in (' ','+'): n+=1 + print(" ", "(... %d lines ommitted ...)" % len(omit)) + else: + for line in omit: + if line[0] in (' ','+'): n+=1 + print("%03d" % n , line, end=' ') + omit = [] + if line[0] in (' ','+'): + n+=1 + print("%03d" % n , line, end=' ') + else: + print("%03d" % (n+1) , line, end=' ') + else: + omit.append(line) + if len(omit) : + if len(omit)>= Context: + for line in omit: + if line[0] in (' ','+'): n+=1 + print(" ", "(... %d lines ommitted ...)" % len(omit)) + else: + for line in omit: + if line[0] in (' ','+'): n+=1 + print("%03d" % n , line, end=' ') + omit = [] + print() + else: + print("(diff ommitted because we couldn't find original files)") + + print() + print("Deleted names:") + deletedNames = sorted(list(self.pfA.sNames - self.pfB.sNames)) + for name in deletedNames: + print("-", name) + print() + print("Added names:") + addedNames = sorted(list(self.pfB.sNames - self.pfA.sNames)) + for name in addedNames: + print("-", name) + print() + + return + + + + + + + for key in self.pfA.list_hashes: + if key in self.pfB.hashes: + lpkA = self.pfA.hashes[key] + lpkB = self.pfB.hashes[key] + self.addEquivalences(lpkA,lpkB) + # print "Found:", self.pfA.hashes[key] , "==>", self.pfB.hashes[key] + #else: + # print "Lost:", self.pfA.hashes[key] , "==>", "???" + + for pkA in self.pfA.idxtree: + parentA = pkA[:-1] + if pkA not in self.equivalences: + self.equivalences[pkA] = {} + if parentA: + if parentA not in self.parent_equivalences: + self.parent_equivalences[parentA] = [] + + for pkA in sorted(self.pfA.idxtree): + parentsA = tree_parents(pkA) + for pkB, punt in self.equivalences[pkA].items(): + if len(pkA) != len(pkB): continue + parentsB = tree_parents(pkB) + parentsAB = list(zip(parentsA,parentsB)) + for pA, pB in parentsAB: + sz2a, sz2b = self.pfA.idxtree[pkA] + sz2 = sz2b - sz2a + 1 + sz1a, sz1b = self.pfA.idxtree[pA] + sz1 = sz1b - sz1a + 1 + lev2 = 1.0 + old_div(sz1, float(sz2)) + #lev2 = 2**(len(pkA) - len(pA)) + #lev2_list = [ pC for pC in self.pfA.idxtree if len(pC) >= len(pA) and pC[:len(pA)] == pA ] + #lev2_plist = set([]) + #for l in lev2_list: + # lev2_plist |= set(tree_parents(l)[1:]) + #lev2 = len(set(lev2_list) - set(lev2_plist)) + + pEq = (pB, old_div(float(punt),lev2)) + if pA not in self.parent_equivalences: + self.parent_equivalences[pA] = [] + self.parent_equivalences[pA].append(pEq) + + norepeat = (0,) + self.parent_equivalences2 = {} + for pA in sorted(self.parent_equivalences): + count = {} + if pA[:len(norepeat)] == norepeat: continue + for pB, punt in self.parent_equivalences[pA]: + if pB not in count: count[pB] = 0 + count[pB] += punt + rcount = [] + ppA = pA[:-1] + if ppA in self.parent_equivalences2: + ppB = self.parent_equivalences2[ppA] + else: + ppB = None + + rowA = self.pfA.table[self.pfA.idxtree[pA]] + for key, punt in count.copy().items(): + if ppB and key[:-1] != ppB: continue + rowB = self.pfB.table[self.pfB.idxtree[pB]] + nameA = self.pfA.fullQName(pA) #rowA['name'].split(":") + nameB = self.pfB.fullQName(pB) #rowB['name'].split(":") + s = difflib.SequenceMatcher() + s.set_seqs(nameA,nameB) + t = s.quick_ratio() + t -= 0.8 + if t < 0: t = 0 + t *= 10.0 + punt *= t + #if nameA[0] != nameB[0]: punt /=3.0 + #if nameA[1] != nameB[1]: punt /=1.0+len(nameA[1]) / 40.0+len(nameB[1]) / 40.0 + if punt >= 0.50: + rcount.append((round(punt*100),key)) + + if len(rcount): + punt, pB = max(rcount) + self.parent_equivalences2[pA] = pB + rowB = self.pfB.table[self.pfB.idxtree[pB]] + print("parent:", pA, self.pfA.fullQName(pA), "%d%%\t" % punt, pB, len(rcount) , self.pfB.fullQName(pB)) + if punt > 100: + norepeat = pA + else: + if len(pA) == 1: + print("parent:", pA, self.pfA.fullQName(pA), "0%\t ???") + + """ + norepeat = (0,) + prevprint = None + for pkA in sorted(self.pfA.idxtree): + if len(pkA) > 1: continue + if pkA[:len(norepeat)] == norepeat: continue + if len(self.equivalences[pkA])==0: + if len(pkA) > 1: + if prevprint is None: continue + else: + if len(pkA) > len(prevprint) and prevprint[:-1] == pkA[:len(prevprint)-1]: continue + elif pkA < len(prevprint): prevprint = None + elif prevprint[:-1] != pkA[:len(prevprint)-1]: prevprint = None + + print pkA,":", + + for pkB, punt in self.equivalences[pkA].iteritems(): + if punt > 0.96: + norepeat = pkA + if punt >= 0.1: + print pkB, punt, ";", + prevprint = pkA + print + """ + """ + for pkB, punt in self.equivalences[pkA].iteritems(): + if punt > 0.96: + norepeat = pkA + print ">>", ".".join(["%02d" % x for x in pkB]) + prevprint = pkA + """ + + + + def getMaxKnown(self,pkA): + pkA = tuple(pkA) + if len(pkA) == 0: return 1.0, None + if pkA not in self.max_known_eq: return 0.0, None + pkB = self.max_known_eq[pkA] + eq_prob = self.equivalences[pkA][pkB] + return eq_prob, pkB + + + def addEquivalences(self,lpkA,lpkB): + lstEquivalences = self.multiplyEquivalences(lpkA,lpkB) + base_probability = old_div(1.0, len(lstEquivalences)) + if base_probability < 0.01: return + + for pkA,pkB in lstEquivalences: + probability = base_probability + parentA = pkA[:-1] + parent_prob, parentB = self.getMaxKnown(parentA) + if parentB: + if parentB != pkB[:-1]: parent_prob = old_div((1-parent_prob), 2.0) + probability *= parent_prob + + parentB = pkB[:-1] + if probability < 0.01: continue + if pkA not in self.equivalences: + self.equivalences[pkA] = {} + if pkB in self.equivalences[pkA]: + print("DUPLICATE", pkA,pkB) + self.equivalences[pkA][pkB] = probability + previousMax, prevPkB = self.getMaxKnown(pkA) + if probability > previousMax: + self.max_known_eq[pkA] = pkB + + + + def multiplyEquivalences(self,lpkA,lpkB): + leq = set([]) + for pkA in lpkA: + for pkB in lpkB: + leq|=set([(pkA,pkB)]) + return list(leq) + + +def process(filename): + table, idxdepth,idxtree = load(filename) + #print table.items()[:10] + treebydepth = {} + + for k in sorted(idxtree.keys()): + td = len(k)-1 + if td not in treebydepth: treebydepth[td] = [] + treebydepth[td].append(k) + nitems = 0 + maxitems = 0 + maxd = 0 + hashes = {} + list_hashes = [] + + for d,idx in treebydepth.items(): + if d > 2: break + nitems = len(idx) + if maxitems < nitems: maxitems = nitems + elif nitems * 2 < maxitems: break + #print "Depth:", d, "(%d items)" % nitems + maxd = d + for k in idx: + #print ".".join([str(x) for x in k]) + pk = idxtree[k] + r = table[pk] + rhash = r["hash"] + if rhash not in hashes: + list_hashes.append(rhash) + hashes[rhash] = [] + hashes[rhash].append(k) + return table, idxdepth,idxtree, hashes, list_hashes + +def isinside(parent, child): + pfrom, pto = parent + cfrom, cto = child + pto += 1 + if cfrom >= pfrom and cto <= pto: return 0 + if cfrom < pfrom and cto < pfrom: return -1 + if cfrom > pto and cto > pto: return 1 + #print "ERROR:", child , " is superior to ", parent + return 0 + +def load(filename): + file = open(filename) + def getpk(row): + return (row["start"],row["end"]) + + fields = [ + "depth", + "hash", + "start", + "end", + "name", + "len" + ] + rows = {} + intfields = [ + "depth", + "start", + "end", + "len" + ] + bydepth = {} + for line in file: + row = dict(list(zip(fields,line[:-1].split("\t")))) + for f in intfields: row[f]=int(row[f]) + pk = getpk(row) + depth = row["depth"] + if depth not in bydepth: bydepth[depth] = [] + bydepth[depth].append(pk) + rows[pk]=row + + for dpth, items in bydepth.items(): + bydepth[dpth] = list(sorted(items)) + + idxtree = {} + for currdepth in bydepth: + n = 0 + #print "Depth:", currdepth + if currdepth == 0: + nparent = [] + else: + pdepth = currdepth - 1 + np = 0 + + for pk in bydepth[currdepth]: + n+=1 + if currdepth > 0: + offset = 99 + it = 0 + while offset != 0: + it += 1 + assert(np >=0) + assert(np = 0) + #if it > 100: + # print it,n,pk,np, ppk, offset + # if offset < 0: + # print list(enumerate(bydepth[pdepth])) + # assert(offset >= 0) + # assert(it < 250) + + prow = rows[ppk] + nparent = prow["tree_id"] + + + row = rows[pk] + tree_id = nparent + [n] + row["tree_id"] = tree_id + if tuple(tree_id) in idxtree: + print("ERROR:", tuple(tree_id), " is duplicated:") + print("previous:", idxtree[tuple(tree_id)]) + print("new:" , pk) + else: + idxtree[tuple(tree_id)] = pk + + + + + return rows, bydepth, idxtree + + +if __name__ == "__main__": main() diff --git a/flscriptparse.py b/flscriptparse.py new file mode 100644 index 0000000..2c135e2 --- /dev/null +++ b/flscriptparse.py @@ -0,0 +1,865 @@ +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from builtins import input +from builtins import str +from builtins import range +# ----------------------------------------------------------------------------- +# flscriptparse.py +# +# Simple parser for FacturaLUX SCripting Language (QSA). +# ----------------------------------------------------------------------------- +from optparse import OptionParser + +import sys, math +import hashlib +import ply.yacc as yacc +import ply.lex as lex + +try: + from pineboolib.flparser import flex + from pineboolib.flparser.flclasses import * +except ImportError: + import flex + from flclasses import * + +# Get the token map +tokens = flex.tokens +start = "source" + +reserv=['nonassoc'] +reserv+=list(flex.reserved) + +endoffile = None + +def cnvrt(val): + val = str(val) + val = val.replace('&','&') + val = val.replace('"','"') + val = val.replace("'",''') + val = val.replace("<",'<') + val = val.replace(">",'>') + return val + +precedence = ( + ('nonassoc', 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL'), + ('left','LOR', 'LAND'), + ('left', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'EQQ', 'NEQ'), + ('right', 'LNOT'), + ('left', 'PLUS', 'MINUS'), + ('left', 'TIMES', 'DIVIDE', 'MOD'), + ('left', 'OR', 'AND', 'XOR', 'LSHIFT', 'RSHIFT'), + +) +seen_tokens = [] +tokelines = {} +last_lexspan = None +def p_parse(token): + ''' + exprval : constant + | variable + | funccall + | error + + identifier : ID + + dictobject_value : LBRACE RBRACE + | LBRACE dictobject_value_elemlist RBRACE + + dictobject_value_elemlist : dictobject_value_elem + | dictobject_value_elemlist COMMA dictobject_value_elem + + dictobject_value_elem : exprval COLON expression + + base_expression : exprval + | inlinestoreinstruction + | base_expression mathoperator base_expression + | base_expression cmp_symbol base_expression + | base_expression boolcmp_symbol base_expression + | parentheses + | unary_operator + | new_operator + | ternary_operator + | dictobject_value + | typeof_operator + + + + + parentheses : LPAREN base_expression RPAREN + | LPAREN variable_1 RPAREN + + unary_operator : LNOT base_expression + | MINUS base_expression + | PLUS base_expression + + new_operator : NEW funccall_1 + | NEW identifier + + typeof_operator : TYPEOF variable + | TYPEOF base_expression + + ternary_operator : base_expression CONDITIONAL1 base_expression COLON base_expression + + expression : base_expression + | funcdeclaration_anon + | error + + case_cblock_list : case_block + case_cblock_list : case_cblock_list case_block + + case_block : CASE expression COLON statement_list + + case_default : DEFAULT COLON statement_list + + case_block_list : empty + case_block_list : case_default + case_block_list : case_cblock_list + case_block_list : case_cblock_list case_default + + source_element : docstring + | vardeclaration + | classdeclaration + | funcdeclaration + + source : source_element + source : source source_element + | statement_list + + + basicsource : statement_list + | empty + + statement : instruction + | vardeclaration + | ifstatement + | whilestatement + | dowhilestatement + | withstatement + | forstatement + | forinstatement + | switch + | trycatch + + statement_list : statement_list statement + + statement_list : statement + + statement_list : LBRACE statement_list RBRACE + + statement_list : LBRACE RBRACE + statement_list : empty + + optvartype : COLON ID + | empty + + vardeclaration : VAR vardecl_list SEMI + | CONST vardecl_list SEMI + vardeclaration : VAR vardecl_list + | CONST vardecl_list + | STATIC VAR vardecl_list + + vardecl : ID optvartype EQUALS expression + vardecl : ID optvartype + + vardecl_list : vardecl + | vardecl_list COMMA vardecl + + arglist : vardecl_list + | + + funcdeclaration : FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE + funcdeclaration : STATIC FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE + funcdeclaration_anon : FUNCTION LPAREN arglist RPAREN LBRACE basicsource RBRACE + + callarg : expression + + callargs : callarg + | callargs COMMA callarg + | empty + + varmemcall : variable_1 + | funccall_1 + | member_call + | member_var + | base_expression + + member_var : varmemcall PERIOD variable_1 + member_call : LPAREN member_var RPAREN PERIOD funccall_1 + + member_call : varmemcall PERIOD funccall_1 + member_call : LPAREN member_call RPAREN PERIOD funccall_1 + + funccall : funccall_1 + | member_call + | LPAREN member_call RPAREN + | LPAREN funccall_1 RPAREN + | LPAREN error RPAREN + + funccall_1 : ID LPAREN callargs RPAREN + | ID LPAREN RPAREN + | TYPEOF LPAREN callargs RPAREN + + mathoperator : PLUS + | MINUS + | TIMES + | DIVIDE + | MOD + | XOR + | OR + | LSHIFT + | RSHIFT + | AND + + variable : variable_1 + | member_var + | LPAREN variable_1 RPAREN + | LPAREN member_var RPAREN + + variable_1 : identifier + | array_member + + array_member : variable_1 LBRACKET expression RBRACKET + | funccall_1 LBRACKET expression RBRACKET + + inlinestoreinstruction : PLUSPLUS variable + | MINUSMINUS variable + | variable PLUSPLUS + | variable MINUSMINUS + + updateoperator : EQUALS + | PLUSEQUAL + | MINUSEQUAL + | MODEQUAL + | DIVEQUAL + | TIMESEQUAL + + updateinstruction : variable updateoperator expression + | variable updateoperator updateinstruction + + deleteinstruction : DELETE variable + + storeinstruction : inlinestoreinstruction + | updateinstruction + | deleteinstruction + + + flowinstruction : RETURN expression + | THROW expression + | RETURN + | BREAK + | CONTINUE + + instruction : base_instruction SEMI + | SEMI + | base_instruction + | funcdeclaration + | error SEMI + + callinstruction : funccall + | variable + + base_instruction : storeinstruction + | callinstruction + | flowinstruction + + varorcall : variable + | funccall + | base_expression + + optextends : EXTENDS ID + | empty + + classdeclaration : CLASS ID optextends LBRACE classdeclarationsource RBRACE + + classdeclarationsource : vardeclaration + | funcdeclaration + | classdeclarationsource vardeclaration + | classdeclarationsource funcdeclaration + | SEMI + | classdeclarationsource SEMI + + docstring : DOCSTRINGOPEN AT ID COMMENTCLOSE + | DOCSTRINGOPEN AT ID ID COMMENTCLOSE + + list_constant : LBRACKET callargs RBRACKET + list_constant : LBRACKET callargs COMMA RBRACKET + + constant : ICONST + | FCONST + | CCONST + | SCONST + | regex + | list_constant + + regex : DIVIDE regexbody DIVIDE regexflags + | DIVIDE regexbody COMMENTCLOSE regexflags + + regexbody : regexchar + | regexbody regexchar + + regexchar : LPAREN + | RPAREN + | ID + | COMMA + | XOR + | LBRACKET + | RBRACKET + | ICONST + | PLUS + | MINUS + | LBRACE + | RBRACE + | DOLLAR + | SQOUTE + | DQOUTE + | BACKSLASH + | SCONST + | error + + regexflags : ID + | empty + + + statement_block : statement + | LBRACE statement_list RBRACE + | LBRACE RBRACE + + optelse : ELSE statement_block + | empty + + cmp_symbol : LT + | LE + | GT + | GE + | EQ + | NE + | EQQ + | NEQ + | IN + + boolcmp_symbol : LOR + | LAND + + condition : expression + | error + + ifstatement : IF LPAREN condition RPAREN statement_block optelse + + whilestatement : WHILE LPAREN condition RPAREN statement_block + dowhilestatement : DO statement_block WHILE LPAREN condition RPAREN SEMI + + withstatement : WITH LPAREN variable RPAREN statement_block + | error + + storeormember : storeinstruction + | member_var + + for_initialize : storeinstruction + | VAR vardecl + | for_initialize COMMA for_initialize + | empty + + for_compare : expression + | empty + + for_increment : storeormember + | for_increment COMMA for_increment + | empty + + + forstatement : FOR LPAREN for_initialize SEMI for_compare SEMI for_increment RPAREN statement_block + | error + + forinstatement : FOR LPAREN for_initialize IN varorcall RPAREN statement_block + | FOR LPAREN variable IN varorcall RPAREN statement_block + | error + + switch : SWITCH LPAREN condition RPAREN LBRACE case_block_list RBRACE + + optid : ID + | empty + + trycatch : TRY statement_block CATCH LPAREN optid RPAREN statement_block + + empty : + ''' + global input_data + + lexspan = list(token.lexspan(0)) + data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) + if len(lexspan) == 2: + fromline = token.lineno(0) + global endoffile + endoffile = fromline, lexspan, token.slice[0] + #print fromline, lexspan, token.slice[0] + token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } + numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) + + rspan = lexspan[0] + if str(token.slice[0]) == 'empty' or numelems == 0: token[0] = None + else: + rvalues = [] + for n,s in enumerate(token.slice[1:]): + if s.type != 'empty' and s.value is not None: + val = None + if isinstance(s.value,str): + val = token.lexspan(n+1)[0] + len(s.value) - 1 + else: + val = token.lexspan(n+1)[1] + rvalues.append(val) + rspan = max(rvalues) + lexspan[1] = rspan + + if str(token.slice[0]) == 'regexbody': + token[0] = { "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } + + #if str(token.slice[0]) == 'regex': + # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] + # print " " + "\n ".join([ "%s(%r): %r" % (s.type, token.lexspan(n+1), s.value) for n,s in enumerate(token.slice[1:]) ]) + global seen_tokens, last_ok_token + last_ok_token = token + seen_tokens.append((str(token.slice[0]), token.lineno(0),input_data[lexspan[0]:lexspan[1]+1] )) + global ok_count + ok_count += 1 + if lexspan[0] not in tokelines: + tokelines[lexspan[0]] = token.lexer.lineno + global last_lexspan + last_lexspan = lexspan + + + +last_ok_token = None +error_count = 0 +last_error_token = None +last_error_line = -1 +ok_count = 0 + +def p_error(t): + global error_count + global ok_count + global last_error_token + global last_error_line, seen_tokens , last_ok_token + debug = False # Poner a True para toneladas de debug. + # if error_count == 0: print + if t is not None: + if last_error_token is None or t.lexpos != getattr(last_error_token,"lexpos",None): + if abs(last_error_line - t.lineno) > 4 and ok_count > 1 and error_count < 4: + error_count += 1 + try: print_context(t) + except Exception: pass + if debug == True: + for tokname, tokln, tokdata in seen_tokens[-32:]: + if tokln == t.lineno: + print(tokname, tokdata) + print(repr(last_ok_token[0])) + last_error_line = t.lineno + elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: + last_error_line = t.lineno + parser.errok() + ok_count = 0 + return + + ok_count = 0 + if t is None: + if last_error_token != "EOF": + print("ERROR: End of the file reached.") + global endoffile + print("Last data:", endoffile) + + if last_lexspan: + try: + print("HINT: Last lexspan:", last_lexspan) + print("HINT: Last line:", tokelines[last_lexspan[0]]) + except Exception as e: + print("ERROR:", e) + last_error_token = "EOF" + return t + t = parser.token() + parser.restart() + last_error_token = t + return t + + +# Build the grammar + + +parser = yacc.yacc(method='LALR',debug=0, + optimize = 1, write_tables = 1, debugfile = '/tmp/yaccdebug.txt',outputdir='/tmp/') + +#profile.run("yacc.yacc(method='LALR')") + +global input_data + +def print_context(token): + global input_data + if token is None: return + last_cr = input_data.rfind('\n',0,token.lexpos) + next_cr = input_data.find('\n',token.lexpos) + column = (token.lexpos - last_cr) + column1 = (token.lexpos - last_cr) + while column1 < 16: + column1 = (token.lexpos - last_cr) + last_cr = input_data.rfind('\n',0,last_cr-1) + + print(input_data[last_cr:next_cr].replace("\t"," ")) + print((" " * (column-1)) + "^", column, "#ERROR#" , token) + + +def my_tokenfunc(*args, **kwargs): + #print("Call token:" ,args, kwargs) + ret = lex.lexer.token(*args, **kwargs) + #print "Return (",args, kwargs,") = " , ret + return ret + + +def print_tokentree(token, depth = 0): + print(" " * depth, token.__class__ , "=" , token) + + if str(token.__class__) == "ply.yacc.YaccProduction": + print(token.lexer) + for tk in token.slice: + if tk.value == token: continue + print(" " * (depth+1), tk.type, end=' ') + try: + print(tk.lexpos, end=' ') + print(tk.endlexpos, end=' ') + except: + pass + print() + + print_tokentree(tk.value, depth +1) + +def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): + #if depth > 5: return + source_data = [ + 'source', + 'source_element', + 'statement_list', + 'statement', + 'classdeclarationsource', + 'vardecl_list', + ] + final_obj = {} + final_obj['range'] = obj['02-size'] + has_data = 0 + has_objects = 0 + contentlist = [] + + if alias_mode == 0: + ctype_alias = {} + elif alias_mode == 1: + ctype_alias = { + "member_var" : "member", + "member_call" : "member", + "variable_1" : "variable", + "funccall_1" : "funccall", + "flowinstruction" : "instruction", + "storeequalinstruction" : "instruction", + "vardecl" : "vardeclaration", + #"vardecl_list" : "vardeclaration", + + } + else: + raise ValueError("alias_mode unknown") + + if otype in ctype_alias: + otype = ctype_alias[otype] + #print " " * depth , obj['02-size'] + for n,content in enumerate(obj['50-contents']): + ctype = content['01-type'] + value = content['99-value'] + if ctype in ctype_alias: + ctype = ctype_alias[ctype] + #if ctype in source_data: + # if depth == 0: print "--" + # print_tree(value,depth,num) + # continue + #print " " * depth , "%s:" % ".".join(num+[str(n)]), ctype, + + if type(value) is dict: + #print "*" + if depth < 600: + tree_obj = calctree(value,depth+1,num+[str(n)], ctype, alias_mode=alias_mode) + else: + tree_obj = None + if type(tree_obj) is dict: + if (tree_obj['has_data'] or alias_mode == 0) and ctype != otype : + contentlist.append([ctype,tree_obj]) + has_objects += 1 + else: + contentlist+=tree_obj["content"] + has_data += tree_obj["has_data"] + has_objects += tree_obj["has_objects"] + else: + #print "=", repr(value) + contentlist.append([ctype,value]) + has_data += 1 + + final_obj['content'] = contentlist + final_obj['has_data'] = has_data + final_obj['has_objects'] = has_objects + + return final_obj + + +hashes = [] +ranges = [] +def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdout): + global hashes, ranges + if depth == 0: + hashes = [] + ranges = [] + + sep = " " + marginblocks = { + "classdeclaration" : 1, + "funcdeclaration" : 1, + "statement_block" : 1, + #"instruction" : 1, + } + closingtokens = [ + "RBRACE", + "RPAREN", + "RBRACKET", + "SEMI", + ] + nuevalinea = False + name = "" + lines = [] + l = 0 + + + for ctype, value in tree['content']: + if nuevalinea and ctype in closingtokens: + nuevalinea = False + + if nuevalinea: + for i in range(int(math.ceil(l/2.0))): + lines.append(sep * depth) + nuevalinea = False + + if type(value) is dict and ctype == otype: + tname,tlines,trange = printtree(value, depth, ctype) + if name == "" and tname: + name = tname + + lines += tlines + elif type(value) is dict: + l = 0 + if ctype in marginblocks: l = marginblocks[ctype] + + for i in range(int(math.floor(l/2.0))): + lines.append(sep * depth) + tname,tlines,trange = printtree(value, depth+1, ctype) + # lines.append(sep * depth + "" % (len("".join(tlines)))) + + if value['has_data'] > 0 and value['has_objects'] == 0 and False: + # Do it inline! + if value['has_data']==1 and tname: + lines.append(sep * depth + "<%s id=\"%s\" />" % (ctype,tname)) + else: + txt = "".join([ x.strip() for x in tlines]) + lines.append(sep * depth + "<%s>%s" % (ctype,txt,ctype)) + else: + attrs = [] + if tname: + attrs.append(("id",tname)) + + txtinline = "".join([ line.strip() for line in tlines ]) + + #if len(tlines)>1: + txthash = hashlib.sha1(txtinline).hexdigest()[:16] + #hashes.append(("depth:",depth,"hash:",txthash,"element:",ctype+":"+tname)) + hashes.append((txthash,ctype+":"+tname+"(%d)"% len(txtinline))) + ranges.append([depth,txthash]+trange+[ctype+":"+tname,len(txtinline)]) + #,"start:",trange[0],"end:",trange[1])) + #attrs.append(("start",trange[0])) + #attrs.append(("end",trange[1])) + #attrs.append(("hash",txthash)) + + txtattrs="" + for name1, val1 in attrs: + txtattrs+=" %s=\"%s\"" % (name1,cnvrt(val1)) + + lines.append(sep * depth + "<%s%s>" % (ctype,txtattrs)) + if depth > 50: + lines.append(sep * (depth+1) + "...") + else: + if len(txtinline)<80: + lines.append(sep * (depth+1) + txtinline) + else: + lines+=tlines + if txtattrs: + txtattrs = "" % txtattrs + lines.append(sep * depth + "" % (ctype)) + + nuevalinea = True + else: + if ctype == "ID" and name == "": + name = value + if ctype in flex.token_literals: + lines.append(sep * depth + "<%s value=\"%s\" />" % (ctype,cnvrt(value))) + else: + lines.append(sep * depth + "<%s />" % (ctype)) + + + if mode == "hash": + #print "\n".join(lines) + for row in sorted(ranges): + output.write("\t".join([ str(x) for x in row])) + output.write("\n") + output.flush() + if mode == "xml": + for row in lines: + output.write(row) + output.write("\n") + output.flush() + + return name, lines, tree['range'] + + + +def parse(data): + global input_data + global error_count + global seen_tokens + seen_tokens[:] = [] + parser.error = 0 + input_data = data + flex.lexer.lineno = 1 + error_count = 0 + p = parser.parse(data, debug = 0, tracking = 1, tokenfunc = my_tokenfunc) + if error_count > 0: + print("ERRORS (%d)" % error_count) + if p is None: return p + try: + p["error_count"] = error_count + except Exception as e: + print(e) + return None + + if parser.error: + return None + return p + + +def main(): + global start + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-O", "--output", dest="output", default = "none", + help="Set output TYPE: xml|hash", metavar="TYPE") + parser.add_option("--start", dest="start", default = None, + help="Set start block", metavar="STMT") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + if options.start: + start = options.start + print("Start setted to:" , start) + + + + + def do_it(): + if options.output == "none": return + tree_data = calctree(prog) + if options.output == "hash": + printtree(tree_data, mode = "hash") + elif options.output == "xml": + printtree(tree_data, mode = "xml") + elif options.output == "file": + f1_hash = open(filename+".hash","w") + printtree(tree_data, mode = "hash", output = f1_hash) + f1_hash.close() + + f1_xml = open(filename+".xml","w") + printtree(tree_data, mode = "xml", output = f1_xml) + f1_xml.close() + elif options.output == "yaml": + import yaml + print(yaml.dump(tree_data['content'])) + + else: + print("Unknown outputmode", options.output) + + prog = "$$$" + if len(args) > 0 : + for filename in args: + fs = filename.split("/") + sys.stderr.write("Loading %s ..." % fs[-1]) + sys.stderr.flush() + data = open(filename).read() + sys.stderr.write(" parsing ...") + sys.stderr.flush() + prog = parse(data) + sys.stderr.write(" formatting ...") + sys.stderr.flush() + if prog: do_it() + sys.stderr.write(" Done.\n") + sys.stderr.flush() + + else: + + + line = "" + while 1: + try: + line1 = input("flscript> ") + if line1.startswith("#"): + comm = line1[1:].split(" ") + if comm[0] == "setstart": + start = comm[1] + print("Start setted to:" , start) + if comm[0] == "parse": + print() + prog = parse(line) + line = "" + else: + line += line1 + except EOFError: + break; + line += "\n" + print() + prog = parse(line) + do_it() + """ + import yaml + + print yaml.dump(tree_data) + """ + #print_tokentree(prog) + + + + + #for varName in prog.byDefName: + # var = prog.byDefName[varName] + # print "%-15s / %-15s > " % var.type , varName + + + #import tests.ifaceclass + #tests.ifaceclass.do_test(prog) + + + + + + + +if __name__ == "__main__": main() diff --git a/flscriptparser2 b/flscriptparser2 new file mode 100755 index 0000000..99cb558 --- /dev/null +++ b/flscriptparser2 @@ -0,0 +1,4 @@ +#!/usr/bin/python3 +import postparse +if __name__ == "__main__": + postparse.main() diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..0416d58 --- /dev/null +++ b/install.sh @@ -0,0 +1,18 @@ +#!/bin/bash +[[ $(basename $(pwd)) != "flscriptparser" ]] && { + echo "This program MUST be run from its folder!!!" + exit 1 +} +DEPS=$(cat dependencies.debian) +aptitude install $DEPS -y +while read line; do + test -e /usr/local/bin/fldesigner && unlink /usr/local/bin/fldesigner + ln -s "$line" /usr/local/bin/fldesigner + break +done < <(find /usr/local/ -type f -name designer) + +while read line; do + bname=$(basename "$line") + test -h /usr/local/bin/$bname && unlink /usr/local/bin/$bname + ln -s "$line" /usr/local/bin/$bname +done < <(find $(pwd) -executable -type f \! -path "*/.*") \ No newline at end of file diff --git a/postparse.py b/postparse.py new file mode 100644 index 0000000..9e11c40 --- /dev/null +++ b/postparse.py @@ -0,0 +1,585 @@ +#!/usr/bin/python +from __future__ import print_function +from __future__ import absolute_import +from builtins import str +from builtins import object +from optparse import OptionParser +import os, os.path, sys +import imp, traceback +from lxml import etree +try: + from future.utils import with_metaclass +except ImportError: + pass + + +try: + from pineboolib.flparser import flscriptparse +except ImportError: + import flscriptparse + +USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") + +KNOWN_PARSERS = {} +UNKNOWN_PARSERS = {} +def parse_for(*tagnames): + global KNOWN_PARSERS + def decorator(fn): + for n in tagnames: + KNOWN_PARSERS[n] = fn + return fn + return decorator + +def parse(tagname, treedata): + global KNOWN_PARSERS,UNKNOWN_PARSERS + if tagname not in KNOWN_PARSERS: + UNKNOWN_PARSERS[tagname] = 1 + fn = parse_unknown + else: + fn = KNOWN_PARSERS[tagname] + return fn(tagname,treedata) + + +def getxmltagname(tagname): + if tagname == "source": return "Source" + if tagname == "funcdeclaration": return "Function" + if tagname == "classdeclaration": return "Class" + if tagname == "vardeclaration": return "Variable" + return "Unknown.%s" % tagname + +xml_class_types = [] + +class TagObjectFactory(type): + def __init__(cls, name, bases, dct): + global xml_class_types + xml_class_types.append(cls) + super(TagObjectFactory, cls).__init__(name, bases, dct) + +class TagObject(with_metaclass(TagObjectFactory, object)): + tags = [] + set_child_argn = False + name_is_first_id = False + debug_other = True + adopt_childs_tags = [] + omit_tags = ['empty'] + callback_subelem = {} + promote_child_if_alone = False + + @classmethod + def tagname(self, tagname): return self.__name__ + + @classmethod + def can_process_tag(self, tagname): return tagname in self.tags + + def __init__(self, tagname): + self.astname = tagname + self.xml = etree.Element(self.tagname(tagname)) + self.xmlname = None + self.subelems = [] + self.values = [] + if self.name_is_first_id: + self.xml.set("name","") + + def adopt_children(self, argn, subelem): + for child in subelem.xml.iterchildren(): + if self.set_child_argn: child.set("argn",str(argn)) + else: + if 'argn' in child.attrib: del child.attrib['argn'] + self.xml.append(child) + + def omit_subelem(self, argn, subelem): + return + + def is_in(self, listobj): + return self.__class__ in listobj or self.astname in listobj + + def get(self, listobj, default = None): + if self.__class__ in listobj: return listobj[self.__class__] + if self.astname in listobj: return listobj[self.astname] + return default + + + def add_subelem(self, argn, subelem): + if subelem.is_in(self.omit_tags): return self.omit_subelem(argn, subelem) + if subelem.is_in(self.adopt_childs_tags): return self.adopt_children(argn, subelem) + callback = subelem.get(self.callback_subelem) + if callback: return getattr(self,callback)(argn,subelem) + + if self.set_child_argn: subelem.xml.set("argn",str(argn)) + self.xml.append(subelem.xml) + self.subelems.append(subelem) + + def add_value(self, argn, vtype, value): + self.values.append( (vtype, value) ) + if vtype == "ID" and self.name_is_first_id and self.xmlname is None: + self.xmlname = value + self.xml.set("name",value) + return + + self.xml.set("arg%02d" % argn,vtype + ":" + repr(value)) + + def add_other(self, argn, vtype, data): + if self.debug_other: + self.xml.set("arg%02d" % argn,vtype) + + def polish(self): + if self.promote_child_if_alone: + if len(self.values) == 0 and len(self.subelems) == 1: + return self.subelems[0] + return self + + +class ListObject(TagObject): + set_child_argn = False + debug_other = False + +class NamedObject(TagObject): + name_is_first_id = True + debug_other = False + +class ListNamedObject(TagObject): + name_is_first_id = True + set_child_argn = False + debug_other = False + +class TypedObject(ListObject): + type_arg = 0 + def add_other(self, argn, vtype, value): + if argn == self.type_arg: + self.xml.set("type", vtype) + + +class Source(ListObject): + tags = ["source","basicsource","classdeclarationsource","statement_list","statement_block"] + adopt_childs_tags = ['source_element','statement_list','statement',"statement_block"] + +class Identifier(NamedObject): + tags = ["identifier","optid"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + + +class Arguments(ListObject): + tags = ["arglist"] + adopt_childs_tags = ['vardecl_list'] + +class VariableType(NamedObject): + tags = ["optvartype"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + +class ExtendsType(NamedObject): + tags = ["optextends"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + +class Function(ListNamedObject): + tags = ["funcdeclaration"] + callback_subelem = ListNamedObject.callback_subelem.copy() + callback_subelem[VariableType] = "add_vartype" + + def add_vartype(self, argn, subelem): + self.xml.set("returns", str(subelem.xmlname)) + +class Variable(NamedObject): + tags = ["vardecl"] + callback_subelem = NamedObject.callback_subelem.copy() + callback_subelem[VariableType] = "add_vartype" + + def add_vartype(self, argn, subelem): + self.xml.set("type", str(subelem.xmlname)) + +class DeclarationBlock(ListObject): + tags = ["vardeclaration"] + adopt_childs_tags = ['vardecl_list'] + def add_other(self, argn, vtype, value): + if argn == 0: + self.xml.set("mode", vtype) + def polish(self): + #if len(self.values) == 0 and len(self.subelems) == 1: + # self.subelems[0].xml.set("mode",self.xml.get("mode")) + # return self.subelems[0] + return self + +class Class(ListNamedObject): + tags = ["classdeclaration"] + callback_subelem = ListNamedObject.callback_subelem.copy() + callback_subelem[ExtendsType] = "add_exttype" + def add_exttype(self, argn, subelem): + self.xml.set("extends", str(subelem.xmlname)) + +class Member(TagObject): + debug_other = False + set_child_argn = False + tags = ["member_var","member_call"] + adopt_childs_tags = ['varmemcall',"member_var","member_call"] + +class ArrayMember(TagObject): + debug_other = False + set_child_argn = False + tags = ["array_member"] + adopt_childs_tags = ['variable_1',"func_call"] + +class InstructionCall(TagObject): + debug_other = False + tags = ["callinstruction"] + +class InstructionStore(TagObject): + promote_child_if_alone = True + debug_other = False + tags = ["storeinstruction"] + +class InstructionFlow(TypedObject): + debug_other = True + tags = ["flowinstruction"] + +class Instruction(TagObject): + promote_child_if_alone = True + debug_other = False + tags = ["instruction"] + +class OpMath(TypedObject): + debug_other = True + tags = ["mathoperator"] + +class Compare(TypedObject): + debug_other = True + tags = ["cmp_symbol","boolcmp_symbol"] + +class FunctionCall(NamedObject): + tags = ["funccall_1"] + +class CallArguments(ListObject): + tags = ["callargs"] + +class Constant(ListObject): + tags = ["constant"] + def add_value(self, argn, vtype, value): + value = str(value) #str(value,"ISO-8859-15","replace") + if vtype == "SCONST": + vtype = "String" + value = value[1:-1] + self.xml.set("delim",'"') + if vtype == "CCONST": + vtype = "String" + value = value[1:-1] + self.xml.set("delim","'") + if vtype == "RCONST": vtype = "Regex" + if vtype == "ICONST": vtype = "Number" + if vtype == "FCONST": vtype = "Number" + self.const_value = value + self.const_type = vtype + self.xml.set("type",vtype) + self.xml.set("value",value) + +class InlineUpdate(ListObject): + tags = ["inlinestoreinstruction"] + def add_other(self, argn, vtype, value): + self.xml.set("type", vtype) + if argn == 0: + self.xml.set("mode", "update-read") + if argn == 1: + self.xml.set("mode", "read-update") + +class If(ListObject): + tags = ["ifstatement"] + +class Condition(ListObject): + tags = ["condition"] + +class Else(ListObject): + tags = ["optelse"] + adopt_childs_tags = ['statement_block'] + def polish(self): + if len(self.subelems) == 0: + self.astname = "empty" + return self + +class DictObject(ListObject): + tags = ["dictobject_value_elemlist","dictobject_value"] + adopt_childs_tags = ['dictobject_value_elemlist',"dictobject_value"] + +class DictElem(ListObject): + tags = ["dictobject_value_elem"] + + +class ExpressionContainer(ListObject): + tags = ["expression"] + # adopt_childs_tags = ['base_expression'] + + def polish(self): + if len(self.values) == 0 and len(self.subelems) == 1: + #if isinstance(self.subelems[0], Constant): + if self.subelems[0].xml.tag == "base_expression": + self.subelems[0].xml.tag = "Expression" + return self.subelems[0] + else: + self.xml.tag = "Value" + + return self + +class InstructionUpdate(ListObject): + tags = ["updateinstruction"] + +class Switch(ListObject): + tags = ["switch"] + adopt_childs_tags = ['case_cblock_list','case_block_list'] + +class CaseList(ListObject): + tags = ["case_block_list"] + adopt_childs_tags = ['case_cblock_list','case_block_list'] + +class Case(ListObject): + tags = ["case_block"] + +class CaseDefault(ListObject): + tags = ["case_default"] + +class While(ListObject): + tags = ["whilestatement"] + +class For(ListObject): + tags = ["forstatement"] + +class ForInitialize(ListObject): + tags = ["for_initialize"] + +class ForCompare(ListObject): + tags = ["for_compare"] + +class ForIncrement(ListObject): + tags = ["for_increment"] + +class DoWhile(ListObject): + tags = ["dowhilestatement"] + +class ForIn(ListObject): + tags = ["forinstatement"] + +class With(ListObject): + tags = ["withstatement"] + +class TryCatch(ListObject): + tags = ["trycatch"] + +class New(ListObject): + tags = ["new_operator"] + +class Delete(ListObject): + tags = ["deleteinstruction"] + +class Parentheses(ListObject): + tags = ["parentheses"] + adopt_childs_tags = ['base_expression'] + +class OpUnary(TypedObject): + tags = ["unary_operator"] + +class OpTernary(ListObject): + tags = ["ternary_operator"] + +class OpUpdate(TypedObject): + tags = ["updateoperator"] + + + +# ----- keep this one at the end. +class Unknown(TagObject): + promote_child_if_alone = True + set_child_argn = False + @classmethod + def tagname(self, tagname): return tagname + + @classmethod + def can_process_tag(self, tagname): return True +# ----------------- + +def create_xml(tagname): + classobj = None + for cls in xml_class_types: + if cls.can_process_tag(tagname): + classobj = cls + break + if classobj is None: return None + return classobj(tagname) + +def parse_unknown(tagname, treedata): + xmlelem = create_xml(tagname) + i = 0 + for k, v in treedata['content']: + if type(v) is dict: + instruction = parse(k,v) + xmlelem.add_subelem(i, instruction) + elif k in USEFUL_TOKENS: + xmlelem.add_value(i, k, v) + else: + xmlelem.add_other(i, k, v) + + i+=1 + return xmlelem.polish() + + + +def post_parse(treedata): + source = parse("source",treedata) + #print UNKNOWN_PARSERS.keys() + return source.xml + +class Module(object): + def __init__(self, name, path): + self.name = name + self.path = path + def loadModule(self): + fp = None + try: + description = ('.py', 'U', imp.PY_SOURCE) + #description = ('.pyc', 'U', PY_COMPILED) + pathname = os.path.join(self.path, self.name) + fp = open(pathname) + name = self.name[:self.name.find(".")] + # fp, pathname, description = imp.find_module(self.name,[self.path]) + self.module = imp.load_module(name, fp, pathname, description) + result = True + except Exception as e: + print(traceback.format_exc()) + result = False + if fp: + fp.close() + return result + +def parseArgs(argv): + parser = OptionParser() + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("--path", + dest="storepath", default=None, + help="store XML results in PATH") + + parser.add_option("--topython", + action="store_true", dest="topython", default=False, + help="write python file from xml") + + parser.add_option("--exec-py", + action="store_true", dest="exec_python", default=False, + help="try to execute python file") + + parser.add_option("--toxml", + action="store_true", dest="toxml", default=False, + help="write xml file from qs") + + parser.add_option("--full", + action="store_true", dest="full", default=False, + help="write xml file from qs") + + (options, args) = parser.parse_args(argv) + return (options, args) + +def main(): + options, args = parseArgs(sys.argv[1:]) + execute(options,args) + +def pythonify(filelist): + options, args = parseArgs([]) + options.full = True + if isinstance(filelist, str): filelist = [filelist] + execute(options,filelist) + print(filelist) + + + +def execute(options, args): + if options.optdebug: + print(options, args) + if options.full: + options.full = False + options.toxml = True + print("Pass 1 - Parse and write XML file . . .") + execute(options,args) + + options.toxml = False + options.topython = True + print("Pass 2 - Pythonize and write PY file . . .") + execute(options,[ arg+".xml" for arg in args]) + + options.topython = False + options.exec_python = True + #print "Pass 3 - Test PY file load . . ." + #execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) + print("Done.") + + elif options.exec_python: + from . import qsatype + for filename in args: + realpath = os.path.realpath(filename) + path, name = os.path.split(realpath) + mod = Module(name, path) + if mod.loadModule(): + print(mod.module) + print(mod.module.form) + else: + print("Error cargando modulo %s" % name) + + elif options.topython: + from .pytnyzer import pythonize + for filename in args: + bname = os.path.basename(filename) + if options.storepath: + destname = os.path.join(options.storepath,bname+".py") + else: + destname = filename+".py" + destname = destname.replace(".qs.xml.py",".py") + pythonize(filename, destname) + + else: + nfs = len(args) + for nf, filename in enumerate(args): + bname = os.path.basename(filename) + sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) + sys.stdout.flush(); + prog = flscriptparse.parse(open(filename,"r", encoding="latin-1").read()) + sys.stdout.write("\r"); + if not prog: + print("Error: No se pudo abrir %-35s \n" % (repr(filename))) + continue + if prog["error_count"] > 0: + print("Encontramos %d errores parseando: %-35s \n" % (prog["error_count"], repr(filename))) + continue + if options.toxml == False: + # Si no se quiere guardar resultado, no hace falta calcular mas + continue + + tree_data = flscriptparse.calctree(prog, alias_mode = 0) + if not tree_data: + print("No se pudo parsear %-35s \n" % (repr(filename))) + continue + ast = post_parse(tree_data) + if ast is None: + print("No se pudo analizar %-35s \n" % (repr(filename))) + continue + if options.storepath: + destname = os.path.join(options.storepath,bname+".xml") + else: + destname = filename+".xml" + f1 = open(destname,"wb") + f1.write(etree.tostring(ast, pretty_print = True)) + f1.close() + + + +if __name__ == "__main__": main() diff --git a/pytnyzer.py b/pytnyzer.py new file mode 100644 index 0000000..9b9caa2 --- /dev/null +++ b/pytnyzer.py @@ -0,0 +1,1010 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# ------ Pythonyzer ... reads XML AST created by postparse.py and creates an equivalent Python file. +from __future__ import print_function +from __future__ import absolute_import +from builtins import object +from optparse import OptionParser +import os, os.path, random +from . import flscriptparse +from lxml import etree +from future.utils import with_metaclass + +def id_translate(name): + python_keywords = ['and', 'del', 'for', 'is', 'raise', 'assert', 'elif', + 'from', 'lambda', 'return', 'break', 'else', 'global', 'not', 'try', + 'class', 'except', 'if', 'or', 'while', 'continue', + 'exec', 'import', 'pass', 'yield', 'def', 'finally', 'in', 'print'] + if name in python_keywords: return name + "_" + if name == "false": name = "False" + if name == "true": name = "True" + if name == "null": name = "None" + if name == "unknown": name = "None" + if name == "this": name = "self" + + if name == "startsWith": name = "startswith" + return name + +ast_class_types = [] + +class ASTPythonFactory(type): + def __init__(cls, name, bases, dct): + global ast_class_types + ast_class_types.append(cls) + super(ASTPythonFactory, cls).__init__(name, bases, dct) + +class ASTPython(with_metaclass(ASTPythonFactory, object)): + tags = [] + + @classmethod + def can_process_tag(self, tagname): return self.__name__ == tagname or tagname in self.tags + + def __init__(self, elem): + self.elem = elem + + def polish(self): return self + + def generate(self, **kwargs): + yield "debug", "* not-known-seq * " + etree.tounicode(self.elem) + + +class Source(ASTPython): + def generate(self, break_mode = False, include_pass = True, **kwargs): + elems = 0 + after_lines = [] + for child in self.elem: + #yield "debug", "<%s %s>" % (child.tag, repr(child.attrib)) + for dtype, data in parse_ast(child).generate(break_mode = break_mode, plusplus_as_instruction = True): + if dtype == "line+1": + after_lines.append(data) + continue + if dtype == "line": + elems += 1 + yield dtype, data + if dtype == "line" and after_lines: + for line in after_lines: + elems+=1 + yield dtype, line + after_lines = [] + if dtype == "break": + for line in after_lines: + elems+=1 + yield "line", line + + for line in after_lines: + elems+=1 + yield "line", line + if elems == 0 and include_pass: + yield "line", "pass" + +class Class(ASTPython): + def generate(self, **kwargs): + name = self.elem.get("name") + extends = self.elem.get("extends","object") + + yield "line", "class %s(%s):" % (name,extends) + yield "begin", "block-class-%s" % (name) + for source in self.elem.xpath("Source"): + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-class-%s" % (name) + +class Function(ASTPython): + def generate(self, **kwargs): + name = id_translate(self.elem.get("name")) + returns = self.elem.get("returns",None) + parent = self.elem.getparent() + grandparent = None + if parent is not None: grandparent = parent.getparent() + arguments = [] + + if grandparent is not None: + if grandparent.tag == "Class": + arguments.append("self") + if name == grandparent.get("name"): + name = "__init__" + else: + arguments.append("self") + for n,arg in enumerate(self.elem.xpath("Arguments/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(id_translate(data)) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + if len(expr) == 1: expr += ["=","None"] + arguments.append(" ".join(expr)) + + + + yield "line", "def %s(%s):" % (name,", ".join(arguments)) + yield "begin", "block-def-%s" % (name) + # if returns: yield "debug", "Returns: %s" % returns + for source in self.elem.xpath("Source"): + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-def-%s" % (name) + +class FunctionCall(ASTPython): + def generate(self, **kwargs): + name = id_translate(self.elem.get("name")) + parent = self.elem.getparent() + if parent.tag == "InstructionCall": + classes = parent.xpath("ancestor::Class") + if classes: + class_ = classes[-1] + extends = class_.get("extends") + if extends == name: + name = "super(%s, self).__init__" % class_.get("name") + functions = parent.xpath("//Function[@name=\"%s\"]" % name) + for f in functions: + #yield "debug", "Function to:" + etree.tostring(f) + name = "self.%s" % name + break + + arguments = [] + for n,arg in enumerate(self.elem.xpath("CallArguments/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "%s(%s)" % (name,", ".join(arguments)) + +class If(ASTPython): + def generate(self, break_mode = False, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + + yield "line", "if %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-if" + for obj in parse_ast(source).generate(break_mode = break_mode): yield obj + yield "end", "block-if" + + for source in self.elem.xpath("Else/Source"): + yield "line", "else:" + yield "begin", "block-else" + for obj in parse_ast(source).generate(break_mode = break_mode): yield obj + yield "end", "block-else" + +class TryCatch(ASTPython): + def generate(self, **kwargs): + + tryblock, catchblock = self.elem.xpath("Source") + + yield "line", "try:" + yield "begin", "block-try" + for obj in parse_ast(tryblock).generate(): yield obj + yield "end", "block-try" + + identifier = None + for ident in self.elem.xpath("Identifier"): + expr = [] + for dtype, data in parse_ast(ident).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + identifier = " ".join(expr) + if identifier: + yield "line", "except Exception as %s:" % (identifier) + else: + yield "line", "except Exception:" + yield "begin", "block-except" + if identifier: + # yield "line", "%s = str(%s)" % (identifier, identifier) + yield "line", "%s = traceback.format_exc()" % (identifier) + for obj in parse_ast(catchblock).generate(include_pass = identifier is None): yield obj + yield "end", "block-except" + + +class While(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + + yield "line", "while %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-while" + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-while" + +class For(ASTPython): + def generate(self, **kwargs): + init_expr = [] + for n,arg in enumerate(self.elem.xpath("ForInitialize/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) > 0: + init_expr.append(" ".join(expr)) + if init_expr: + yield "line", " ".join(init_expr) + + incr_expr = [] + incr_lines = [] + for n,arg in enumerate(self.elem.xpath("ForIncrement/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + elif dtype in ["line","line+1"]: + incr_lines.append(data) + else: + yield dtype, data + if len(expr) > 0: + incr_expr.append(" ".join(expr)) + + + main_expr = [] + for n,arg in enumerate(self.elem.xpath("ForCompare/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("True") + else: + main_expr.append(" ".join(expr)) + #yield "debug", "WHILE-FROM-QS-FOR: (%r;%r;%r)" % (init_expr,main_expr,incr_lines) + yield "line", "while %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-for" + for obj in parse_ast(source).generate(include_pass=False): yield obj + if incr_lines: + for line in incr_lines: + yield "line", line + yield "end", "block-for" + +class ForIn(ASTPython): + def generate(self, **kwargs): + list_elem, main_list = "None", "None" + myelems = [] + for e in self.elem: + if e.tag == "Source": break + if e.tag == "ForInitialize": e = list(e)[0] + expr = [] + for dtype, data in parse_ast(e).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + myelems.append(" ".join(expr)) + list_elem, main_list = myelems + yield "debug", "FOR-IN: " + repr(myelems) + yield "line", "for %s in %s:" % (list_elem, main_list) + for source in self.elem.xpath("Source"): + yield "begin", "block-for-in" + for obj in parse_ast(source).generate(include_pass=False): yield obj + yield "end", "block-for-in" + +class Switch(ASTPython): + def generate(self, **kwargs): + key = "%02x" % random.randint(0,255) + name = "s%s_when" % key + name_pr = "s%s_do_work" % key + name_pr2 = "s%s_work_done" % key + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + yield "line", "%s = %s" % (name, " ".join(main_expr)) + yield "line", "%s,%s = %s,%s" % (name_pr,name_pr2, "False","False") + for scase in self.elem.xpath("Case"): + value_expr = [] + for n,arg in enumerate(scase.xpath("Value")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + value_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + value_expr.append(" ".join(expr)) + + yield "line", "if %s == %s: %s,%s = %s,%s" % (name," ".join(value_expr),name_pr,name_pr2, "True", "True") + yield "line", "if %s:" % (name_pr) + yield "begin", "block-if" + count = 0 + for source in scase.xpath("Source"): + for obj in parse_ast(source).generate(break_mode = True): + if obj[0] == "break": + yield "line", "%s = %s # BREAK" % (name_pr,"False") + count += 1 + else: + yield obj + count += 1 + if count < 1: yield "line", "pass" + yield "end", "block-if" + + for scasedefault in self.elem.xpath("CaseDefault"): + yield "line", "if not %s: %s,%s = %s,%s" % (name_pr2, name_pr,name_pr2, "True", "True") + yield "line", "if %s:" % (name_pr) + yield "begin", "block-if" + for source in scasedefault.xpath("Source"): + for obj in parse_ast(source).generate(break_mode = True): + if obj[0] == "break": + yield "line", "%s = %s # BREAK" % (name_pr,"False") + else: + yield obj + yield "end", "block-if" + # yield "line", "assert( not %s )" % name_pr + # yield "line", "assert( %s )" % name_pr2 + +class With(ASTPython): + def generate(self, **kwargs): + key = "%02x" % random.randint(0,255) + name = "w%s_obj" % key + #yield "debug", "WITH: %s" % key + variable, source = [ obj for obj in self.elem ] + var_expr = [] + for dtype, data in parse_ast(variable).generate(isolate = False): + if dtype == "expr": + var_expr.append(data) + else: + yield dtype, data + if len(var_expr) == 0: + var_expr.append("None") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + + yield "line", "%s = %s" % (name, " ".join(var_expr)) + + for obj in parse_ast(source).generate(break_mode = True): + yield obj + yield "line", "del %s" % name + +class Variable(ASTPython): + def generate(self, force_value = False, **kwargs): + name = self.elem.get("name") + yield "expr", name + values = 0 + for value in self.elem.xpath("Value|Expression"): + values += 1 + yield "expr", "=" + expr = 0 + for dtype, data in parse_ast(value).generate(isolate = False): + if dtype == "expr": expr += 1 + yield dtype, data + if expr == 0: + yield "expr", "None" + + dtype = self.elem.get("type",None) + + if values == 0 and force_value == True: + yield "expr", "=" + if dtype is None: + yield "expr", "None" + elif dtype == "String": + yield "expr", "\"\"" + elif dtype == "Number": + yield "expr", "0" + else: + parent1 = self.elem.getparent() + parent2 = parent1.getparent() + parent3 = parent2.getparent() + if parent2 == "Source" and parent3 == "Class": + yield "expr", "None" + else: + yield "expr", "qsatype.%s()" % dtype + + #if dtype and force_value == False: yield "debug", "Variable %s:%s" % (name,dtype) + +class InstructionUpdate(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + if data is None: raise ValueError(etree.tostring(arg)) + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "line", " ".join(arguments) + +class InlineUpdate(ASTPython): + def generate(self, plusplus_as_instruction = False, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + ctype = self.elem.get("type") + mode = self.elem.get("mode") + linetype = "line" + if not plusplus_as_instruction: + if mode == "read-update": + linetype = "line+1" + + yield "expr", arguments[0] + if ctype == "PLUSPLUS": + yield linetype, arguments[0] + " += 1" + elif ctype == "MINUSMINUS": + yield linetype, arguments[0] + " -= 1" + else: + yield linetype, arguments[0] + " ?= 1" + +class InstructionCall(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + yield "line", " ".join(arguments) + +class Instruction(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + if arguments: + yield "debug", "Instruction: Maybe parse-error. This class is only for non-understood instructions or empty ones" + yield "line", " ".join(arguments) + +class InstructionFlow(ASTPython): + def generate(self, break_mode = False, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + ctype = self.elem.get("type") + kw = ctype + if ctype == "RETURN": kw = "return" + if ctype == "BREAK": + kw = "break" + if break_mode: + yield "break", kw + " " + ", ".join(arguments) + return + if ctype == "CONTINUE": kw = "continue" + + if ctype == "THROW": + yield "line", "raise Exception(" + ", ".join(arguments) + ")" + return + + yield "line", kw + " " + ", ".join(arguments) + + + +class Member(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + # Lectura del self.iface.__init + if len(arguments) >= 3 and arguments[0:2] == ["self","iface"] and arguments[2].startswith("__"): + # From: self.iface.__function() + # to: super(className, self.iface).function() + funs = self.elem.xpath("ancestor::Function") + if funs: + fun = funs[-1] + name_parts = fun.get("name").split("_") + classname = name_parts[0] + arguments[2] = arguments[2][2:] + arguments[0:2] = ["super(%s, %s)" % (classname,".".join(arguments[0:2]))] + + # Lectura del self.iface.__init() al nuevo estilo yeboyebo + if len(arguments) >= 2 and arguments[0:1] == ["_i"] and arguments[1].startswith("__"): + # From: self.iface.__function() + # to: super(className, self.iface).function() + funs = self.elem.xpath("ancestor::Function") + if funs: + fun = funs[-1] + name_parts = fun.get("name").split("_") + classname = name_parts[0] + arguments[1] = arguments[1][2:] + arguments[0:1] = ["super(%s, %s)" % (classname,".".join(arguments[0:1]))] + + replace_members = [ + "length", + "text", + "join", + "date", + ] + + for member in replace_members: + for idx,arg in enumerate(arguments): + if member == arg or arg.startswith(member+"("): + + part1 = arguments[:idx] + try: + part2 = arguments[idx+1:] + except IndexError: + part2 = [] # Para los que son últimos y no tienen parte adicional + arguments = ["qsa(%s).%s" % (".".join(part1), arg)] + part2 + yield "expr", ".".join(arguments) + +class ArrayMember(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "%s[%s]" % (arguments[0],arguments[1]) + + +class Value(ASTPython): + def generate(self, isolate = True, **kwargs): + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + if data is None: raise ValueError(etree.tostring(child)) + yield dtype, data + if isolate: yield "expr", ")" + +class Expression(ASTPython): + tags = ["base_expression"] + def generate(self, isolate = True, **kwargs): + if isolate: yield "expr", "(" + coerce_string_mode = False + if self.elem.xpath("OpMath[@type=\"PLUS\"]"): + if self.elem.xpath("Constant[@type=\"String\"]"): + coerce_string_mode = True + if coerce_string_mode: + yield "expr", "ustr(" + for child in self.elem: + if coerce_string_mode and child.tag == "OpMath": + if child.get("type") == "PLUS": + yield "expr","," + continue + + for dtype, data in parse_ast(child).generate(): + yield dtype, data + + if coerce_string_mode: + yield "expr", ")" + if isolate: yield "expr", ")" + +class Parentheses(ASTPython): + def generate(self, **kwargs): + yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(isolate = False): + yield dtype, data + yield "expr", ")" + +class Delete(ASTPython): + def generate(self, **kwargs): + yield "expr", "del" + for child in self.elem: + for dtype, data in parse_ast(child).generate(isolate = False): + yield dtype, data + + +class OpTernary(ASTPython): + def generate(self, isolate = False, **kwargs): + """ + Ejemplo op. ternario + + + + + + + + + + """ + if_cond = self.elem[0] + then_val = self.elem[1] + else_val = self.elem[2] + yield "expr", "(" # Por seguridad, unos paréntesis + for dtype, data in parse_ast(then_val).generate(): yield dtype, data + yield "expr", "if" + for dtype, data in parse_ast(if_cond).generate(): yield dtype, data + yield "expr", "else" + for dtype, data in parse_ast(else_val).generate(): yield dtype, data + yield "expr", ")" # Por seguridad, unos paréntesis + + +class DictObject(ASTPython): + def generate(self, isolate = False, **kwargs): + yield "expr", "{" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + yield "expr", "," # Como en Python la coma final la ignora, pues la ponemos. + yield "expr", "}" + +class DictElem(ASTPython): + def generate(self, isolate = False, **kwargs): + # Clave: + for dtype, data in parse_ast(self.elem[0]).generate(): + yield dtype, data + yield "expr", ":" + # Valor: + for dtype, data in parse_ast(self.elem[1]).generate(): + yield dtype, data + + +class OpUnary(ASTPython): + def generate(self, isolate = False, **kwargs): + ctype = self.elem.get("type") + if ctype == "LNOT": yield "expr", "not" + elif ctype == "MINUS": yield "expr", "-" + else: yield "expr", ctype + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + if isolate: yield "expr", ")" + +class New(ASTPython): + def generate(self, **kwargs): + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + if dtype != "expr": + yield dtype, data + continue + if child.tag == "Identifier": data = data+"()" + ident = data[:data.find("(")] + if ident.find(".") == -1: + if len(self.elem.xpath("//Class[@name='%s']" % ident)) == 0: + data = "qsatype." + data + yield dtype, data + + +class Constant(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + value = self.elem.get("value") + if ctype is None or value is None: + for child in self.elem: + if child.tag == "CallArguments": + arguments = [] + for n,arg in enumerate(child): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "qsatype.Array([%s])" % (", ".join(arguments)) + return + if ctype == "String": + delim = self.elem.get("delim") + if delim == "'": + yield "expr", "u'%s'" % value + else: + yield "expr", "u\"%s\"" % value + else: yield "expr", value + +class Identifier(ASTPython): + def generate(self, **kwargs): + name = id_translate(self.elem.get("name")) + yield "expr", name + +class OpUpdate(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "EQUALS": yield "expr", "=" + elif ctype == "PLUSEQUAL": yield "expr", "+=" + elif ctype == "MINUSEQUAL": yield "expr", "-=" + elif ctype == "TIMESEQUAL": yield "expr", "*=" + elif ctype == "DIVEQUAL": yield "expr", "/=" + else: yield "expr", ctype + +class Compare(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "GT": yield "expr", ">" + elif ctype == "LT": yield "expr", "<" + elif ctype == "LE": yield "expr", "<=" + elif ctype == "GE": yield "expr", ">=" + elif ctype == "EQ": yield "expr", "==" + elif ctype == "NE": yield "expr", "!=" + elif ctype == "IN": yield "expr", "in" + elif ctype == "LOR": yield "expr", "or" + elif ctype == "LAND": yield "expr", "and" + else: yield "expr", ctype + +class OpMath(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "PLUS": yield "expr", "+" + elif ctype == "MINUS": yield "expr", "-" + elif ctype == "TIMES": yield "expr", "*" + elif ctype == "DIVIDE": yield "expr", "/" + elif ctype == "MOD": yield "expr", "%" + elif ctype == "XOR": yield "expr", "^" + elif ctype == "LSHIFT": yield "expr", "<<" + elif ctype == "RSHIFT": yield "expr", ">>" + elif ctype == "AND": yield "expr", "&" + else: yield "expr", ctype + + +class DeclarationBlock(ASTPython): + def generate(self, **kwargs): + mode = self.elem.get("mode") + is_constructor = self.elem.get("constructor") + if mode == "CONST": yield "debug", "Const Declaration:" + for var in self.elem: + expr = [] + for dtype, data in parse_ast(var).generate(force_value=True): + if dtype == "expr": + if data is None: raise ValueError(etree.tostring(var)) + expr.append(data) + else: yield dtype,data + if is_constructor: + expr[0] = "self."+expr[0] + yield "line", " ".join(expr) + + +# ----- keep this one at the end. +class Unknown(ASTPython): + @classmethod + def can_process_tag(self, tagname): return True +# ----------------- + +def astparser_for(elem): + classobj = None + for cls in ast_class_types: + if cls.can_process_tag(elem.tag): + classobj = cls + break + if classobj is None: return None + return classobj(elem) + +def parse_ast(elem): + elemparser = astparser_for(elem) + return elemparser.polish() + + +def file_template(ast): + yield "line", "# -*- coding: utf-8 -*-" + yield "line", "from pineboolib import qsatype" + yield "line", "from pineboolib.qsaglobals import *" + yield "line", "import traceback" + yield "line", "" + sourceclasses = etree.Element("Source") + for cls in ast.xpath("Class"): + sourceclasses.append(cls) + + mainclass = etree.SubElement(sourceclasses,"Class",name="FormInternalObj",extends="qsatype.FormDBWidget") + mainsource = etree.SubElement(mainclass,"Source") + + + constructor = etree.SubElement(mainsource,"Function",name="_class_init") + args = etree.SubElement(constructor,"Arguments") + csource = etree.SubElement(constructor,"Source") + + for child in ast: + if child.tag != "Function": + child.set("constructor","1") + csource.append(child) + else: + mainsource.append(child) + + for dtype, data in parse_ast(sourceclasses).generate(): + yield dtype, data + yield "line", "" + yield "line", "form = None" + +def write_python_file(fobj, ast): + indent = [] + indent_text = " " + last_line_for_indent = {} + numline = 0 + last_dtype = None + for dtype, data in file_template(ast): + if isinstance(data, bytes): data = data.decode("UTF-8","replace") + line = None + if dtype == "line": + line = data + numline +=1 + try: lines_since_last_indent = numline - last_line_for_indent[len(indent)] + except KeyError: lines_since_last_indent = 0 + if lines_since_last_indent > 4: + fobj.write((len(indent)*indent_text) + "\n") + last_line_for_indent[len(indent)] = numline + if dtype == "debug": + line = "# DEBUG:: " + data + print(numline, line) + if dtype == "expr": line = "# EXPR??:: " + data + if dtype == "line+1": line = "# LINE+1??:: " + data + if dtype == "begin": + #line = "# BEGIN:: " + data + indent.append(data) + last_line_for_indent[len(indent)] = numline + if dtype == "end": + if last_dtype == "begin": + fobj.write((len(indent)*indent_text) + "pass\n") + last_line_for_indent[len(indent)] = numline + + if data not in ["block-if"]: + #line = "# END:: " + data + pass + endblock = indent.pop() + if endblock != data: + line = "# END-ERROR!! was %s but %s found. (%s)" % (endblock, data,repr(indent)) + + if line is not None: + fobj.write((len(indent)*indent_text) + line + "\n") + + if dtype == "end": + if data.split("-")[1] in ["class","def","else","except"]: + fobj.write((len(indent)*indent_text) + "\n") + last_line_for_indent[len(indent)] = numline + last_dtype = dtype + +def pythonize(filename, destfilename): + bname = os.path.basename(filename) + + parser = etree.XMLParser(remove_blank_text=True) + try: + ast_tree = etree.parse(open(filename), parser) + except Exception: + print("filename:",filename) + raise + ast = ast_tree.getroot() + + + f1 = open(destfilename,"w") + write_python_file(f1,ast) + f1.close() + +def main(): + parser = OptionParser() + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("--path", + dest="storepath", default=None, + help="store PY results in PATH") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + + for filename in args: + if options.storepath: + destname = os.path.join(options.storepath,bname+".py") + else: + destname = filename+".py" + pythonize(filename, destname) + + + +if __name__ == "__main__": main() diff --git a/qsatype.py b/qsatype.py new file mode 100644 index 0000000..3d5cc6a --- /dev/null +++ b/qsatype.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from builtins import object +import os + +class Object(object): + pass + +class Array(object): + pass + +class FLSqlCursor(object): + pass + +class Boolean(object): + pass + + + diff --git a/test/README b/test/README new file mode 100644 index 0000000..b6bea94 --- /dev/null +++ b/test/README @@ -0,0 +1,5 @@ +Use this folder to debugg the parser (this avoids some trash and annoying caches with the source code). + +Anyway; the cache system isn't very useful because recalculating it each time is not something very expensive. + + diff --git a/test/flscriptparser b/test/flscriptparser new file mode 100755 index 0000000..be05494 --- /dev/null +++ b/test/flscriptparser @@ -0,0 +1,82 @@ +#!/bin/bash +PARSE_FILES="$*" + +if [ "$1" = "--openwith" ]; then + OPENWITH="$2" + PARSE_FILES="" +fi +LOGS="" +if [ "$PARSE_FILES" = "" ]; then + PARSE_FILES=`echo *.qs` + if [ "$PARSE_FILES" = '*.qs' ]; then + PARSE_FILES="" + echo "-- No input files given and no .qs files found on this directory. --" + else + LOGS=`echo .parse-*.qs.log` + if [ "$LOGS" = '.parse-*.qs.log' ]; then + LOGS="" + else + echo "INFO: Parsing previously failed files only" + fi + + fi + +fi + +THISFILE="$0" +readlink "$0" >/dev/null && THISFILE=`readlink "$0"` +THISDIR=`dirname "$THISFILE"` +APPDIR=`dirname "$THISDIR"` +#echo "Starting parsers for $PARSE_FILES" +TOTALFILES="" +OKFILES="" + +for file in $PARSE_FILES +do + NAME=`basename $file` + DIR=`dirname $file` + LOGFILE="$DIR/.parse-$NAME.log" + if [ "$LOGS" != '' ]; then + if [ ! -e $LOGFILE ]; then + continue; + fi + fi + echo -n "$DIR/$NAME (parsing) . . . " + ERRORCOUNT="0" + WARNINGCOUNT="0" + python "$APPDIR/flscriptparse.py" $file 2>/tmp/errorlog.txt >$LOGFILE || (cat $LOGFILE /tmp/errorlog.txt; echo "FATAL ERROR: Can't execute the parser."; ERRORCOUNT="1";) + if [ "$ERRORCOUNT" = "0" ]; then + ERRORCOUNT=`grep "#ERROR#" $LOGFILE | wc -l` + fi + if [ "$WARNINGCOUNT" = "0" ]; then + WARNINGCOUNT=`grep "#WARNING#" $LOGFILE | wc -l` + fi + + TOTALFILES+="$file +" + if [ "$ERRORCOUNT" = "0" ]; then + echo "$WARNINGCOUNT Warnings." + grep "#WARNING#" $LOGFILE + OKFILES+="$file +" + if [ "$file" = "$PARSE_FILES" ]; then + cat $LOGFILE + fi + unlink $LOGFILE + else + if [ "$ERRORCOUNT" = "1" ]; then + echo -n "$ERRORCOUNT Error found." + else + echo -n "$ERRORCOUNT Errors found." + fi + grep "#ERROR#" -B10 -m1 $LOGFILE + echo "___" + if [ "$OPENWITH" != '' ]; then + $OPENWITH $file + fi + fi +done +OKFILES="`echo -n "$OKFILES" | wc -l`" +TOTALFILES="`echo -n "$TOTALFILES" | wc -l`" + +echo "$OKFILES of $TOTALFILES files parsed correctly" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/ifaceclass.py b/tests/ifaceclass.py new file mode 100644 index 0000000..8a0b6c5 --- /dev/null +++ b/tests/ifaceclass.py @@ -0,0 +1,102 @@ +import traceback,sys + + + +def do_test(code): + print "Starting iface Class test . . ." + print + print "Is iface declared?", + if "iface" in code.byDefName: + classname = None + iface = code.byDefName["iface"] + print "yes." + print "iface type is %s/%s . . ." % iface.type, + if iface.type != ("Declaration" , "Variable"): + print " which differs from desired value!" + return False + print " ok." + print "Content: ", iface.value + vtype, vsubtype = iface.value.type + if iface.value.type == ("List","Expression"): + expression = iface.value.slice + if len(expression) == 2: + if str(expression[0]) == "new": + if expression[1].type == ("Item", "FuncCall"): + if str(expression[1].arglist) != "this": + print "`this` must be the olny arg passed to the constructor!" + return False + + classname = expression[1].name + + if classname: + if not analyzeClass(code, classname): + print "Test ifaceClass failed!" + return False + else: + print "Cannot extract class name for iface variable!" + return False + + else: + print "no." + print + print "Bypassing test because iface wasn't found." + return True + + print "Test ifaceClass passed!" + return True + +def analyzeClass(code, classname, parentlist = []): + begin = False + if parentlist == []: begin = True + + print "Analyzing classname", classname, " . . ." + print "Is %s declared?" % classname, + if classname not in code.byDefName: + print "No. Can't find the class declaration in the source. Aborting." + return False + else: + print "yes." + + cclass = code.byDefName[classname] + parentlist.append(classname) + if cclass.extends: + print "class %s is a child of %s." % (classname,str(cclass.extends)) + if str(cclass.extends) == classname: + print "%s fails in analysis because extends a class with the same name." % classname + return False + + if str(cclass.extends) in parentlist: + print "%s fails in analysis because extends a class already in the hierarchy." % classname + return False + + if not analyzeClass(code, str(cclass.extends)): + print "%s fails in analysis because its parent failed." % classname + return False + + + cclass.childclasses = [] + for name in parentlist: + if len(cclass.childclasses) or name == classname: + cclass.childclasses = [name] + cclass.childclasses + + ### Search constructor #### + if classname in cclass.source.byDefName: + print "constructor function for %s found" % classname + constructor = cclass.source.byDefName[classname] + cclass.constructor = constructor + else: + print "#WARNING# Constructor function for %s NOT found" % classname + if cclass.extends: + cextends = code.byDefName[str(cclass.extends)] + if cextends.constructor: + print "#ERROR# Parent class has one constructor and constructor function for %s NOT found!" % classname + + + + + print "%s is a valid class." % (".".join(cclass.childclasses)) + + return True + + + diff --git a/xml2json.py b/xml2json.py new file mode 100644 index 0000000..fd267ee --- /dev/null +++ b/xml2json.py @@ -0,0 +1,634 @@ +#!/usr/bin/python + +from __future__ import print_function +from builtins import object +try: + from json import dumps as json_dumps + from json import loads as json_loads +except: + from json import write as json_dumps + from json import read as json_loads + + +import xml.parsers.expat +from optparse import OptionParser + +# json_string = json.dumps(python_variable) +# python_var = json.loads("string_encoded_jsonvar") + +def printr(*args): + return + print(args[0], end=' ') + for arg in args[1:]: + if type(arg) is str: + arg=arg.encode("utf-8") + print(repr(arg), end=' ') + print() + +def entity_rep(txt,entities=""): + entity_list = list("&'\"<>") + entity_dict = { + '"' : """, + "'" : "'", + '<' : "<", + '>' : ">", + "&" : "&", + } + if entities == "": entities = entity_list + entities = list(entities) + if "&" not in entities: entities.append("&") + + for entity in entity_list: + if entity not in entities: continue + #print entity,entity_dict[entity] + txt = txt.replace(entity,entity_dict[entity]) + return txt + +class xmlElement(object): + def __init__(self, parent, tagname, attrs = {}, ttype = "text", tdata = ""): + self.parent = parent + + self.tagname = tagname + self.attrs = attrs + self.ttype = ttype + self.tdata = tdata + + self.children = [] + + if self.parent: + self.depth = parent.depth + 1 + self.path = self.parent.path + [self.tagname] + self.parent.children.append(self) + else: + self.depth = 0 + self.path = [self.tagname] + + def append(self,text): + self.tdata += text + + def export(self,encoding): + depth = self.depth + tagname = self.tagname + attrs = [ [k,v] for k,v in self.attrs.items() ] + attrs.sort() + tdata = self.tdata.strip() + ttype = self.ttype + + if len(tdata) == 0: + tdata = "" + ttype = "" + + #v = [depth,tagname,attrs,ttype,tdata] + #vt1 = json_dumps(v) + vt2 = "%d)%s" % (depth,tagname) + if attrs: vt2 +="\tattrs:" + json_dumps(attrs) + if ttype: vt2 +="\t%s:%s" % ( ttype, json_dumps(tdata)) + + return vt2.encode(encoding) + + def exportXML(self): + if type(self.attrs) is dict: + attrs = [ [k,v] for k,v in self.attrs.items() ] + attrs.sort() + else: + attrs = self.attrs + + txtattrs = "" + + if attrs: + txtattrs=" " + for key,value in attrs: + txtattrs += '%s="%s" ' % (key,entity_rep(value,'&<"')) + + + depthpad = u" " * self.depth + output = u"" + if self.tagname == "#comment": + output += u"%s\n" % (depthpad, self.tdata) + elif self.tagname[0] == "!": + output += u"%s\n" % (depthpad, self.tagname[1:],txtattrs) + elif self.children: + output += u"%s<%s%s>\n" % (depthpad, self.tagname,txtattrs) + for child in self.children: + output += child.exportXML() + + output += u"%s\n" % (depthpad, self.tagname) + else: + if self.tdata == "": + if txtattrs=="": txtattrs = " " + output += u"%s<%s%s/>\n" % (depthpad, self.tagname,txtattrs) + else: + tdata = self.tdata + if tdata.find("\n")>-1: tdata = "\n%s\n" % tdata + if self.ttype=="cdata": tdata = "" % self.tdata + else: tdata = entity_rep(self.tdata) + + output += u"%s<%s%s>%s\n" % (depthpad, self.tagname,txtattrs, tdata, self.tagname) + + + return output + + + + + +class JSON_Base(object): + def __init__(self, finput, foutput, encoding): + self.finput = finput + self.foutput = foutput + self.encoding = encoding + + self.init_vars() + + def process(self): + print("Please define a process function.") + + def init_vars(self): + pass + + + +class JSON_Reverter(JSON_Base): + def init_vars(self): + self.cElement = None + self.rootXML = [] + + def processCmd(self,key,val): + if key == "encoding": + if self.encoding != "auto": + self.encoding = self.encoding.upper() + if val.upper() != self.encoding: + print(" ignoring %s=%s , using specified value '%s' instead" % (key,val,self.encoding)) + return + self.encoding = val.upper() + return + + print("ERROR: unknown key %s='%s'" % (key,val)) + + def newElement(self,depth,tagname,text,ttype,attrs): + parent = self.cElement + if parent: parentdepth = parent.depth + else: parentdepth = -1 + while parent and parentdepth > depth - 1: + parent = parent.parent + if parent: parentdepth = parent.depth + else: parentdepth = -1 + + self.cElement = xmlElement(parent, tagname, attrs, ttype , text) + if parent is None: self.rootXML.append(self.cElement) + + + def process(self): + for line in self.finput: + line = line.strip() + if len(line) == 0: continue + if line[0]=="!": + lstkeys = line[1:].split(":") + key, val = lstkeys + self.processCmd(key.strip(),val.strip()) + continue + + fields = line.split("\t") + + depth, tag = fields[0].split(")") + depth = int(depth) + text = "" + ttype = "text" + attrs = {} + + #self.foutput.write("%d\t%s\n" % (depth, tag)) + for field in fields[1:]: + tpos = field.find(":") + if tpos == -1: + print("unexpected character:", line) + return + ftype = field[:tpos] + try: + fvalue = json_loads(field[tpos+1:]) + except ValueError: + print("ValueError:", field[tpos+1:]) + + #if type(fvalue) is unicode: + # self.foutput.write("%s = %s\n" % (ftype, fvalue.encode(self.encoding))) + #else: + # self.foutput.write("%s = %s\n" % (ftype, repr(fvalue))) + if ftype == "text": + text = fvalue + ttype = "text" + if ftype == "cdata": + text = fvalue + ttype = "cdata" + if ftype == "attrs": attrs = fvalue + + self.newElement(depth,tag,text,ttype,attrs) + + for element in self.rootXML: + self.foutput.write(element.exportXML().encode(self.encoding)) + +""" + Possible Format: + + List-per-tag: + [ depth, tagname, attrs, ttype, tdata ] + + depth: 0,1,2,3,4...N + + tagname: \w+ -> ElementTag + tagname: !\w+ -> DoctypeTag + tagname: ?\w+ -> XmlDeclTag (always: xml) (attrs = version, encoding?, standalone?) + tagname: #\w+ -> CommentTag (always: comment) (attrs = []) (tdata = comment) + + attrs: dict { attr : val , attr2 : val2 } + + ttype: text|cdata|mixed -multiline? + + tdata: raw text + cdata combined. + + + Problems: + * Handling C-DATA + * Handling multiline texts + * Handling tabs and spaces at the start of each line of multiline text + * Handling comments + + Non-treated: + * NameSpaces + 32.18 + + * Entity Declarations + + + * Element Declarations + + + * Notation Declarations + + + * Attribute List Delcarations + + + + + + Example 1: AbanQ UI (UTF-8) + StartDoctypeDeclHandler: 'UI' None None 0 + EndDoctypeDeclHandler: + StartElementHandler: 'UI' {u'version': u'3.3', u'stdsetdef': u'1'} + CharacterDataHandler: '\n' + StartElementHandler: 'class' {} + CharacterDataHandler: 'formLineasAlbaranesCli' + EndElementHandler: 'class' + + Example 2: JasperReports JRXML (UTF-8) + XmlDeclHandler: '1.0' 'UTF-8' -1 + Unhandled data: '\n' + StartElementHandler: 'jasperReport' {u'xmlns': u'http://jasperreports.sourceforge.net/jasperreports', u'name': u'report1', u'language': u'groovy', u'pageWidth': u'842', u'columnWidth': u'802', u'topMargin': u'20', u'rightMargin': u'20', u'bottomMargin': u'20', u'xmlns:xsi': u'http://www.w3.org/2001/XMLSchema-instance', u'leftMargin': u'20', u'xsi:schemaLocation': u'http://jasperreports.sourceforge.net/jasperreports http://jasperreports.sourceforge.net/xsd/jasperreport.xsd', u'pageHeight': u'595', u'orientation': u'Landscape'} + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + StartElementHandler: 'style' {u'fontName': u'Times New Roman', u'name': u'Title', u'isBold': u'false', u'forecolor': u'#FFFFFF', u'fontSize': u'50', u'isDefault': u'false', u'pdfFontName': u'Times-Bold'} + EndElementHandler: 'style' + + Example 3: AbanQ Actions XML (ISO-8859-1) + StartElementHandler: 'ACTIONS' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'action' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'name' {} + CharacterDataHandler: 'albaranescli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'description' {} + CharacterDataHandler: 'QT_TRANSLATE_NOOP("MetaData","Son los documentos que justifican la entrega de una mercancia a un ciente")' + EndElementHandler: 'description' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + Example 4: AbanQ Tables MTD (ISO-8859-1) + StartDoctypeDeclHandler: 'TMD' None None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'TMD' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'name' {} + CharacterDataHandler: 'facturascli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CommentHandler: 'Facturas de cliente' + CharacterDataHandler: ' ' + StartElementHandler: 'alias' {} + CharacterDataHandler: 'QT_TRANSLATE_NOOP("MetaData","Facturas de Clientes")' + EndElementHandler: 'alias' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + Example 5: AbanQ Report QRY (ISO-8859-1) + StartDoctypeDeclHandler: 'QRY' None None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'QRY' {} + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + StartElementHandler: 'name' {} + CharacterDataHandler: 'presupuestoscli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + + Example 6: AbanQ Report KUT (ISO-8859-1) + XmlDeclHandler: '1.0' 'UTF-8' -1 + Unhandled data: '\n' + StartDoctypeDeclHandler: 'KugarTemplate' 'kugartemplate.dtd' None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'KugarTemplate' {u'TopMargin': u'50', u'PageSize': u'0', u'RightMargin': u'30', u'PageOrientation': u'0', u'BottomMargin': u'50', u'LeftMargin': u'30'} + CharacterDataHandler: '\n' + StartElementHandler: 'Detail' {u'Level': u'0', u'Height': u'0'} + EndElementHandler: 'Detail' + CharacterDataHandler: '\n' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + + + +""" + +class JSON_Converter(JSON_Base): + + def init_vars(self): + self.real_encoding = self.getRealEncoding() + self.xmltag = None + self.taglist = [] + self.p = xml.parsers.expat.ParserCreate(self.real_encoding) + + self.p.StartElementHandler = self.StartElementHandler + self.p.EndElementHandler = self.EndElementHandler + self.p.CharacterDataHandler = self.CharacterDataHandler + self.p.XmlDeclHandler = self.XmlDeclHandler + self.p.StartDoctypeDeclHandler = self.StartDoctypeDeclHandler + self.p.EndDoctypeDeclHandler = self.EndDoctypeDeclHandler + self.p.ElementDeclHandler = self.ElementDeclHandler + self.p.AttlistDeclHandler = self.AttlistDeclHandler + self.p.ProcessingInstructionHandler = self.ProcessingInstructionHandler + self.p.CharacterDataHandler = self.CharacterDataHandler + self.p.EntityDeclHandler = self.EntityDeclHandler + self.p.NotationDeclHandler = self.NotationDeclHandler + self.p.StartNamespaceDeclHandler = self.StartNamespaceDeclHandler + self.p.EndNamespaceDeclHandler = self.EndNamespaceDeclHandler + self.p.CommentHandler = self.CommentHandler + self.p.StartCdataSectionHandler = self.StartCdataSectionHandler + self.p.EndCdataSectionHandler = self.EndCdataSectionHandler + self.p.DefaultHandler = self.DefaultHandler + + def getRealEncoding(self): + validEncodings = ["UTF-8", "UTF-16", "ISO-8859-1"] + self.encoding = self.encoding.upper() + + if self.encoding in validEncodings: return self.encoding + if self.encoding.find("UTF")>=0: + if self.encoding.find("8"): + return "UTF-8" + if self.encoding.find("16"): + return "UTF-16" + return "UTF-8" + + if self.encoding.find("ISO")>=0: + return "ISO-8859-1" + + if self.encoding.find("1252")>=0: + return "ISO-8859-1" + + if self.encoding.find("CP")==0: + return "ISO-8859-1" + + if self.encoding.find("WIN")==0: + return "ISO-8859-1" + + return "UTF-8" + + + def process(self): + self.p.ParseFile(self.finput) + self.foutput.write("!encoding: "+ self.real_encoding+"\n") + for tag in self.taglist: + self.foutput.write(tag.export(self.real_encoding)+"\n") + + def startTag(self,*args): + newtag = xmlElement(self.xmltag, *args) + self.xmltag = newtag + self.taglist.append(newtag) + return newtag + + def endTag(self): + self.xmltag = self.xmltag.parent + return self.xmltag + + + + + def StartElementHandler(self, name, attributes): + printr( "StartElementHandler:", name, attributes) + # tagname: \w+ -> ElementTag + self.startTag(name, attributes) + + def CharacterDataHandler(self, data): + printr( "CharacterDataHandler:", data) + self.xmltag.append(data) + + def EndElementHandler(self, name): + printr( "EndElementHandler:", name) + self.endTag() + + def XmlDeclHandler(self, version, encoding, standalone): + printr( "XmlDeclHandler:", version, encoding, standalone) + # tagname: ?\w+ -> XmlDeclTag (always: xml) (attrs = version, encoding?, standalone?) + attrs = { 'version' : version } + if encoding: attrs['encoding'] = encoding + if standalone: attrs['standalone'] = standalone + + self.startTag("?xml", attrs) + + self.endTag() + + def CommentHandler(self, data): + printr( "CommentHandler:", data) + # tagname: #\w+ -> CommentTag (always: comment) (attrs = []) (tdata = comment) + self.startTag("#comment") + self.xmltag.append(data) + self.endTag() + + + def StartCdataSectionHandler(self): + printr( "StartCdataSectionHandler:") + self.xmltag.ttype = "cdata" + + def EndCdataSectionHandler(self): + printr( "EndCdataSectionHandler:") + # se descarta el cierre... + + def StartDoctypeDeclHandler(self, doctypeName, systemId, publicId, has_internal_subset): + printr( "StartDoctypeDeclHandler:", doctypeName, systemId, publicId, has_internal_subset) + # tagname: !\w+ -> DoctypeTag + attrs = {} + if systemId: attrs['systemId'] = systemId + if publicId: attrs['publicId'] = publicId + if has_internal_subset: attrs['has_internal_subset'] = has_internal_subset + + self.startTag("!"+doctypeName,attrs) + + + def EndDoctypeDeclHandler(self): + printr( "EndDoctypeDeclHandler:") + self.endTag() + + def ElementDeclHandler(self, name, model): + printr( "ElementDeclHandler:", name, model) + + def AttlistDeclHandler(self, elname, attname, type, default, required): + printr( "AttlistDeclHandler:", elname, attname, type, default, required) + + def ProcessingInstructionHandler(self, target, data): + printr( "ProcessingInstructionHandler:", target, data) + + def EntityDeclHandler(self, entityName, is_parameter_entity, value, base, systemId, publicId, notationName): + printr( "EntityDeclHandler:" , entityName, is_parameter_entity, value, base, systemId, publicId, notationName) + + def NotationDeclHandler(self, notationName, base, systemId, publicId): + printr( "NotationDeclHandler:", notationName, base, systemId, publicId) + + def StartNamespaceDeclHandler(self, prefix, uri): + printr( "StartNamespaceDeclHandler:", prefix, uri) + + def EndNamespaceDeclHandler(self, prefix): + printr( "EndNamespaceDeclHandler:", prefix) + + def DefaultHandler(self,data): + printr( "Unhandled data:", data) + + + + + + + +def autodetectXmlEncoding(rawtext): + lines = [ line.strip() for line in rawtext.split("\n") if line.strip() ] + if lines[0].find("")>=0: + # File is QtDesigner UI + return "UTF-8" + + if lines[0].find("UTF-8")>=0: + # Unkown, standard xml (like jrxml) + return "UTF-8" + + if lines[0].find("")>=0: + # AbanQ actions XML + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ table MTD + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ report Query + return "ISO-8859-15" + + if lines[0].find('')>=0: + # AbanQ report Kut + return "ISO-8859-15" + + if lines[0].find("")>=0: + # AbanQ translations + return "ISO-8859-15" + + try: + import chardet + dictEncoding = chardet.detect(rawtext) + encoding = dictEncoding["encoding"] + return encoding + except ImportError: + print("python-chardet library is not installed. Assuming input file is UTF-8.") + #encoding= + #UTF-8, UTF-16, ISO-8859-1 + + + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("-E", "--encoding", dest="encoding", default = "auto", + help="Set encoding=ENC: auto,utf-8,iso-8859-15", metavar="ENC") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + if len(args) < 2: + print("xml2json needs at least an action and a file.") + print("xml2json (revert|convert) file1 [file2] [file3]") + return + action = args.pop(0) + + if action == "convert": + for fname in args: + + fhandler = open(fname) + fw = open(fname+".json","w") + rawtext = fhandler.read() + fhandler.seek(0) + if options.encoding == "auto": + encoding = autodetectXmlEncoding(rawtext) + else: + encoding = options.encoding + jconv = JSON_Converter(fhandler, fw, encoding) + jconv.process() + + fhandler.close() + fw.close() + elif action == "revert": + for fname in args: + lExt = fname.split(".") + ext = "xml" + if lExt[-1]=="json" and len(lExt[-2])>=1 and len(lExt[-2])<=6: + ext = lExt[-2] + fhandler = open(fname) + + fw = open(fname+"."+ext,"w") + jrev = JSON_Reverter(fhandler, fw, options.encoding) + jrev.process() + + fw.close() + fhandler.close() + + else: + print("Unkown action '%s'" % action) + + + + + + + + + +if __name__ == "__main__": main() diff --git a/xmlparse.py b/xmlparse.py new file mode 100644 index 0000000..df1dbd5 --- /dev/null +++ b/xmlparse.py @@ -0,0 +1,200 @@ +from __future__ import print_function +from builtins import str +import xml.parsers.expat +import sys +from optparse import OptionParser +import re + +elements = [] +show_end = True +lasttextdata = "" +lstelements = [] + +def reset(): + global elements, show_end, lstelements, lasttextdata + elements = [] + show_end = True + lasttextdata = "" + lstelements = [] + + +# 3 handler functions +def start_element(name, attrs): + global elements, show_end, lstelements, lasttextdata + lstattrs=list(sorted([ "%s=%s" % (k,v) for k,v in attrs.items() ])) + completename=name + if len(lstattrs): + completename+="&"+"&".join(lstattrs) + + show_end = False + elements.append(completename) + #print 'Start element:', name, attrs + lstelements.append("/".join(elements)) + #print "/".join(elements) + lasttextdata = "" + +def end_element(name): + global elements, show_end, lstelements, lasttextdata + lasttextdata = "" + if show_end: + #print "/".join(elements) + "/" + lstelements.append("/".join(elements) + ";") + + show_end = True + elements.pop() + #print 'End element:', name + +def char_data(data): + global elements, show_end, lstelements, lasttextdata + #data = data.strip() + lasttextdata+=data + if lasttextdata.strip(): + #show_end = True + lstelements.pop() + lstelements.append("/".join(elements)+"(%s)" % repr(lasttextdata.strip())) + + + #print "/".join(elements)+ "(%s)" % repr(data) + + + +def unmap(lines): + + runmap = re.compile(r"^(?P/*)(?P\w+)(?P&[^\(]+)*(?P\(.+\))?$") + # depthlevel + # tagname + elementpool = [] + text = [] + for line in lines: + line = line.strip() + if line[-1] == ";": continue + rg1 = runmap.match(line) + if not rg1: + print("error:") + print(line) + break + + depth = len(rg1.group('depth')) + tagname = str(rg1.group('tagname')) + t_attrs = rg1.group('attrs') + attrs = [] + if t_attrs: + lattrs = t_attrs[1:].split("&") + for attr in lattrs: + key, val = attr.split("=") + attrs.append( (key,val) ) + + t_txt = rg1.group('txt') + txt = "" + if t_txt: + txt = eval(t_txt[1:-1]) + + while depth < len(elementpool): + toclose = elementpool.pop() + text.append("" % toclose) + text.append("\n" + " " * len(elementpool)) + + if depth == len(elementpool): + #print depth, tagname, attrs, txt + txtattrs = "" + if attrs: + for k,v in attrs: + txtattrs+=" %s=\"%s\"" % (k,v) + + if txt: + txt = txt.encode("utf-8") + txt = txt.replace("&","&") + txt = txt.replace("<","<") + else: + txt = "" + text.append("<%s%s>%s" % (tagname, txtattrs,txt)) + elementpool.append(tagname) + + + else: + print("error:") + print(depth, len(elementpool)) + break + + while len(elementpool): + toclose = elementpool.pop() + text.append("" % toclose) + text.append("\n" + " " * len(elementpool)) + + return text + + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + if len(args) < 2: + print("Se necesita al menos una accion y un argumento extra.") + print("xmlparse (map|unmap) file1 [file2] [file3]") + return + action = args.pop(0) + + if action == "map": + global lstelements + separators = [ + "hbox", + "vbox", + "grid", + ] + r1 = re.compile("/widget") + for fname in args: + p = xml.parsers.expat.ParserCreate() + + p.StartElementHandler = start_element + p.EndElementHandler = end_element + p.CharacterDataHandler = char_data + fhandler = open(fname) + fw = open(fname+".map","w") + reset() + p.ParseFile(fhandler) + for t in lstelements: + elems = t.split("/") + lbox = [] + for n,e in enumerate(elems): + if e in separators: lbox.append(n) + if len(lbox)>1: + nlbox = lbox[-2] + while len(elems[nlbox:]) < 2: + nlbox -= 1 + else: + nlbox = 0 + fw.write("/"*(len(elems)-1) + "/".join(elems[-1:])+ "\n") + #print "/"*(len(elems)-1) + "/".join(elems[-1:]) + lstelements = [] + fhandler.close() + fw.close() + elif action == "unmap": + for fname in args: + fhandler = open(fname) + fw = open(fname+".ui","w") + for line in unmap(fhandler): + fw.write(line) + fw.close() + fhandler.close() + + else: + print("Unkown action '%s'" % action) + + +if __name__ == "__main__": main() \ No newline at end of file From 803c0a258ab2e64ea35e49e35991b54342404037 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 21 Sep 2016 17:27:53 +0200 Subject: [PATCH 076/100] flscriptparser: test de commit cruzado desde pineboo --- pytnyzer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytnyzer.py b/pytnyzer.py index 9b9caa2..62867ea 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -10,6 +10,7 @@ from lxml import etree from future.utils import with_metaclass + def id_translate(name): python_keywords = ['and', 'del', 'for', 'is', 'raise', 'assert', 'elif', 'from', 'lambda', 'return', 'break', 'else', 'global', 'not', 'try', From 35df2c9891ac206dda3cd80745152c2464ace81e Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 21 Sep 2016 18:10:21 +0200 Subject: [PATCH 077/100] flparser: agregar soporte de do{}while al pythonificador. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recordad borrar las cachés PY. --- pytnyzer.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pytnyzer.py b/pytnyzer.py index 62867ea..608966b 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -246,6 +246,34 @@ def generate(self, **kwargs): for obj in parse_ast(source).generate(): yield obj yield "end", "block-while" +class DoWhile(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + # TODO ..... + key = "%02x" % random.randint(0,255) + name1st = "s%s_dowhile_1stloop" % key + yield "line", "%s = True" % (name1st) + + yield "line", "while %s or %s:" % (name1st, " ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-while" + yield "line", "%s = False" % (name1st) + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-while" + class For(ASTPython): def generate(self, **kwargs): init_expr = [] From 036879c9dc3bd26b16b54c6b14f29ffa4cb22775 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Wed, 21 Sep 2016 19:30:30 +0200 Subject: [PATCH 078/100] pytnyzer: agregar soporte MUY basico de conversion de [] en QS (FIXME) ... ahora mismo convertira cualquier array a [], incluso si tiene contenido dentro. --- pytnyzer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pytnyzer.py b/pytnyzer.py index 608966b..852ae40 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -804,6 +804,12 @@ def generate(self, **kwargs): value = self.elem.get("value") if ctype is None or value is None: for child in self.elem: + if child.tag == "list_constant": + # TODO/FIXME:: list_constant debe ser ELIMINADO o CONVERTIDO por postparse.py + # .... este generador convertirá todos los arrays en vacíos, sin importar + # .... si realmente tienen algo. + yield "expr", "[]" + if child.tag == "CallArguments": arguments = [] for n,arg in enumerate(child): From 2eb6e72d70ec019725b4b2fa17c195c75e044c84 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Tue, 27 Sep 2016 12:54:10 +0200 Subject: [PATCH 079/100] bugfixes --- postparse.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postparse.py b/postparse.py index 27fc8a1..428a3d8 100644 --- a/postparse.py +++ b/postparse.py @@ -553,11 +553,11 @@ def execute(options, args): sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) sys.stdout.flush(); try: - filecontent = open(filename).read() - except Exception: - print "Error: No se pudo abrir fichero %-35s \n" % (repr(filename)) + filecontent = open(filename,"r", encoding="latin-1").read() + except Exception as e: + print("Error: No se pudo abrir fichero %-35s \n" % (repr(filename)), e) continue - prog = flscriptparse.parse(open(filename,"r", encoding="latin-1").read()) + prog = flscriptparse.parse(filecontent) sys.stdout.write("\r"); if not prog: print("Error: No se pudo abrir %-35s \n" % (repr(filename))) From 1ab516f6323903dcb647b68446b56e78e35b7338 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 17 Oct 2016 09:40:01 +0200 Subject: [PATCH 080/100] =?UTF-8?q?agregar=20versi=C3=B3n=20beta=20de=20an?= =?UTF-8?q?alizador=20de=20extensiones=20(copiar=20y=20usar=20desde=20~/gi?= =?UTF-8?q?t/abanq.modules)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- analizar_extensiones.sh | 88 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 analizar_extensiones.sh diff --git a/analizar_extensiones.sh b/analizar_extensiones.sh new file mode 100755 index 0000000..5299ba2 --- /dev/null +++ b/analizar_extensiones.sh @@ -0,0 +1,88 @@ +#!/bin/bash +PRJPATH=$1 +BUILDPATH=$2 + +# Para calcular todas las extensiones: +# for i in ext*-*; do eneboo-assembler build $i fullpatch; eneboo-assembler build $i revfullpatch; done + +# for i in ext*-*; do test -f "$i/build/fullpatch/fullpatch.xml" || { echo $i; eneboo-assembler build $i fullpatch; } done +# for i in ext*-*; do test -f "$i/build/revfullpatch/revfullpatch.xml" || { echo $i; eneboo-assembler build $i revfullpatch; } done + +if [ -z "$BUILDPATH" ]; then + rm .patchtest/$PRJPATH.ext.* + echo /home/gestiweb/git/eneboo-features-fede/*/build | xargs -d ' ' -n1 -P2 $0 $1 + exit 0; +fi +UNPATCHPATH=$BUILDPATH/revfullpatch +REPATCHPATH=$BUILDPATH/fullpatch +MAINPATCH=$BUILDPATH/../patches/ + +PATCHNAME=$(echo $BUILDPATH | cut -f6 -d'/') +#echo ":: $PRJPATH -- $PATCHNAME " + + +test -f "$UNPATCHPATH/revfullpatch.xml" || { echo "NO EXISTE :: $UNPATCHPATH/revfullpatch.xml"; exit 1; } +test -f "$REPATCHPATH/fullpatch.xml" || { echo "NO EXISTE :: $REPATCHPATH/fullpatch.xml"; exit 2; } +test -d "$MAINPATCH" || { echo "NO EXISTE :: $MAINPATCH"; exit 3; } + +mkdir -p ".patchtest/$PRJPATH/" + +DSTFOLDER=.patchtest/$PRJPATH/$PATCHNAME-uninstall +test -d "$DSTFOLDER" || \ +eneboo-mergetool folder-patch "$UNPATCHPATH" "$PRJPATH" "$DSTFOLDER" >/dev/null 2>&1 + + +DSTFOLDER2=.patchtest/$PRJPATH/$PATCHNAME-reinstall +test -d "$DSTFOLDER2" || \ +eneboo-mergetool folder-patch "$REPATCHPATH" "$DSTFOLDER" "$DSTFOLDER2" >/dev/null 2>&1 + + +diff -rN -t -b -d -U1 --exclude=".git" -x '*.png' -x '*.jpg' -x "*.qry" \ + -x "*.kut" -x "*.ui" -x '*~' -x '*.*.*' "$PRJPATH" "$DSTFOLDER" > "$DSTFOLDER.patch" + + diff -rN -t -b -d -U1 --exclude=".git" -x '*.png' -x '*.jpg' -x "*.qry" \ + -x "*.kut" -x "*.ui" -x '*~' -x '*.*.*' "$DSTFOLDER" "$DSTFOLDER2" > "$DSTFOLDER2.patch" + +grep -aE "^-[^-]" "$DSTFOLDER.patch" > "$DSTFOLDER.patch.grep" +grep -aE '^\+[^\+]' "$DSTFOLDER2.patch" > "$DSTFOLDER2.patch.grep" + +#rm "$DSTFOLDER" -R +#rm "$DSTFOLDER2" -R + +BYTES_UNPATCH=$(cat "$DSTFOLDER.patch.grep" | wc -c) +BYTES_REPATCH=$(cat "$DSTFOLDER2.patch.grep" | wc -c ) + +BYTES_MAINPATCH=$(du -bs $MAINPATCH --exclude='*.png' --exclude='*.jpg' --exclude="*.qry" \ + --exclude="*.kut" --exclude="*.ui" --exclude="*~" | cut -f1) + +BYTES_MIN=$(( $BYTES_MAINPATCH / 2)) +BYTES_MIN2=$(( $BYTES_MAINPATCH / 5)) + +DEST=.patchtest/$PRJPATH.ext.others.txt +if [ "$BYTES_UNPATCH" -lt "$BYTES_MIN2" ]; then + exit 0; +fi + +if [ "$BYTES_UNPATCH" -gt "$BYTES_MIN" -a "$BYTES_REPATCH" -gt "$BYTES_MIN" ]; then + DEST=.patchtest/$PRJPATH.ext.installed.txt + echo "EXTENSION :: $PATCHNAME - patch: $BYTES_MAINPATCH reinstall: $BYTES_REPATCH uninstall: $BYTES_UNPATCH" + echo $PATCHNAME >> $DEST + exit 0; +fi + +if [ "$BYTES_UNPATCH" -gt "$BYTES_MIN" ]; then + DEST=.patchtest/$PRJPATH.ext.unsure.txt + echo "UNSURE :: $PATCHNAME - patch: $BYTES_MAINPATCH reinstall: $BYTES_REPATCH uninstall: $BYTES_UNPATCH" + echo $PATCHNAME >> $DEST + exit 0; +fi + +if [ "$BYTES_UNPATCH" -gt "$BYTES_MIN2" -a "$BYTES_REPATCH" -gt "$BYTES_MIN2" ]; then + DEST=.patchtest/$PRJPATH.ext.unsure.txt + echo "UNSURE :: $PATCHNAME - patch: $BYTES_MAINPATCH reinstall: $BYTES_REPATCH uninstall: $BYTES_UNPATCH" +else + echo " :: $PATCHNAME - patch: $BYTES_MAINPATCH reinstall: $BYTES_REPATCH uninstall: $BYTES_UNPATCH" +fi + + +echo $PATCHNAME >> $DEST From 5a64f08c9cb8368c161ecae86cbe57ac09bebc3d Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 6 Mar 2017 08:23:00 +0100 Subject: [PATCH 081/100] convert xml2json to python3 --- xml2json.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/xml2json.py b/xml2json.py index fd267ee..02d0d29 100644 --- a/xml2json.py +++ b/xml2json.py @@ -416,9 +416,9 @@ def getRealEncoding(self): def process(self): self.p.ParseFile(self.finput) - self.foutput.write("!encoding: "+ self.real_encoding+"\n") + self.foutput.write(b"!encoding: "+ bytes(self.real_encoding, "UTF-8")+b"\n") for tag in self.taglist: - self.foutput.write(tag.export(self.real_encoding)+"\n") + self.foutput.write(tag.export(self.real_encoding)+b"\n") def startTag(self,*args): newtag = xmlElement(self.xmltag, *args) @@ -519,32 +519,32 @@ def DefaultHandler(self,data): def autodetectXmlEncoding(rawtext): - lines = [ line.strip() for line in rawtext.split("\n") if line.strip() ] - if lines[0].find("")>=0: + lines = [ line.strip() for line in rawtext.split(b"\n") if line.strip() ] + if lines[0].find(b"")>=0: # File is QtDesigner UI return "UTF-8" - if lines[0].find("UTF-8")>=0: + if lines[0].find(b"UTF-8")>=0: # Unkown, standard xml (like jrxml) return "UTF-8" - if lines[0].find("")>=0: + if lines[0].find(b"")>=0: # AbanQ actions XML return "ISO-8859-15" - if lines[0].find("")>=0: + if lines[0].find(b"")>=0: # AbanQ table MTD return "ISO-8859-15" - if lines[0].find("")>=0: + if lines[0].find(b"")>=0: # AbanQ report Query return "ISO-8859-15" - if lines[0].find('')>=0: + if lines[0].find(b'')>=0: # AbanQ report Kut return "ISO-8859-15" - if lines[0].find("")>=0: + if lines[0].find(b"")>=0: # AbanQ translations return "ISO-8859-15" @@ -592,8 +592,8 @@ def main(): if action == "convert": for fname in args: - fhandler = open(fname) - fw = open(fname+".json","w") + fhandler = open(fname,"rb") + fw = open(fname+".json","wb") rawtext = fhandler.read() fhandler.seek(0) if options.encoding == "auto": From 255ff119c6e0f225f32469c927fc6b394082aea1 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 6 Mar 2017 08:31:44 +0100 Subject: [PATCH 082/100] better regexes --- flscriptparse.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flscriptparse.py b/flscriptparse.py index 1897ff5..6a42bce 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -319,6 +319,9 @@ def p_parse(token): | SQOUTE | DQOUTE | BACKSLASH + | CONDITIONAL1 + | EQUALS + | OR | SCONST | error From 967702c3672a535f90f9a1a47e6d148c009bdf71 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sat, 11 Mar 2017 15:54:29 +0100 Subject: [PATCH 083/100] corregir instalador --- dependencies.debian | 7 ++-- install.sh | 6 +++- test/flscriptparser | 82 --------------------------------------------- 3 files changed, 10 insertions(+), 85 deletions(-) delete mode 100755 test/flscriptparser diff --git a/dependencies.debian b/dependencies.debian index 8fb8480..d2208d7 100644 --- a/dependencies.debian +++ b/dependencies.debian @@ -1,3 +1,6 @@ -python-ply +python3-ply +python3-lxml +python3-six +python3-future git-core -realpath \ No newline at end of file +realpath diff --git a/install.sh b/install.sh index 0416d58..d6bb669 100644 --- a/install.sh +++ b/install.sh @@ -15,4 +15,8 @@ while read line; do bname=$(basename "$line") test -h /usr/local/bin/$bname && unlink /usr/local/bin/$bname ln -s "$line" /usr/local/bin/$bname -done < <(find $(pwd) -executable -type f \! -path "*/.*") \ No newline at end of file +done < <(find $(pwd) -executable -type f \! -path "*/.*") +test -f /usr/local/bin/flscriptparser && unlink /usr/local/bin/flscriptparser + +exit 0 + diff --git a/test/flscriptparser b/test/flscriptparser deleted file mode 100755 index be05494..0000000 --- a/test/flscriptparser +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -PARSE_FILES="$*" - -if [ "$1" = "--openwith" ]; then - OPENWITH="$2" - PARSE_FILES="" -fi -LOGS="" -if [ "$PARSE_FILES" = "" ]; then - PARSE_FILES=`echo *.qs` - if [ "$PARSE_FILES" = '*.qs' ]; then - PARSE_FILES="" - echo "-- No input files given and no .qs files found on this directory. --" - else - LOGS=`echo .parse-*.qs.log` - if [ "$LOGS" = '.parse-*.qs.log' ]; then - LOGS="" - else - echo "INFO: Parsing previously failed files only" - fi - - fi - -fi - -THISFILE="$0" -readlink "$0" >/dev/null && THISFILE=`readlink "$0"` -THISDIR=`dirname "$THISFILE"` -APPDIR=`dirname "$THISDIR"` -#echo "Starting parsers for $PARSE_FILES" -TOTALFILES="" -OKFILES="" - -for file in $PARSE_FILES -do - NAME=`basename $file` - DIR=`dirname $file` - LOGFILE="$DIR/.parse-$NAME.log" - if [ "$LOGS" != '' ]; then - if [ ! -e $LOGFILE ]; then - continue; - fi - fi - echo -n "$DIR/$NAME (parsing) . . . " - ERRORCOUNT="0" - WARNINGCOUNT="0" - python "$APPDIR/flscriptparse.py" $file 2>/tmp/errorlog.txt >$LOGFILE || (cat $LOGFILE /tmp/errorlog.txt; echo "FATAL ERROR: Can't execute the parser."; ERRORCOUNT="1";) - if [ "$ERRORCOUNT" = "0" ]; then - ERRORCOUNT=`grep "#ERROR#" $LOGFILE | wc -l` - fi - if [ "$WARNINGCOUNT" = "0" ]; then - WARNINGCOUNT=`grep "#WARNING#" $LOGFILE | wc -l` - fi - - TOTALFILES+="$file -" - if [ "$ERRORCOUNT" = "0" ]; then - echo "$WARNINGCOUNT Warnings." - grep "#WARNING#" $LOGFILE - OKFILES+="$file -" - if [ "$file" = "$PARSE_FILES" ]; then - cat $LOGFILE - fi - unlink $LOGFILE - else - if [ "$ERRORCOUNT" = "1" ]; then - echo -n "$ERRORCOUNT Error found." - else - echo -n "$ERRORCOUNT Errors found." - fi - grep "#ERROR#" -B10 -m1 $LOGFILE - echo "___" - if [ "$OPENWITH" != '' ]; then - $OPENWITH $file - fi - fi -done -OKFILES="`echo -n "$OKFILES" | wc -l`" -TOTALFILES="`echo -n "$TOTALFILES" | wc -l`" - -echo "$OKFILES of $TOTALFILES files parsed correctly" From 7d7d7e61841eafb79976eb1b434fb619fad0d1d6 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 27 Mar 2017 08:30:33 +0200 Subject: [PATCH 084/100] better support for regex and multiline strings --- flex.py | 2 +- flscriptparse.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/flex.py b/flex.py index dc6f1f7..d5a65ff 100644 --- a/flex.py +++ b/flex.py @@ -168,7 +168,7 @@ def t_ID(t): t_SCONST = r'\"([^\\\n]|(\\.))*?\"' # Character constant 'c' or L'c' -t_CCONST = r'\'([^\\\n]|(\\.))*?\'' +t_CCONST = r'\'([^\'\\\n]|(\\.)|\\\n)*?\'' # REGEX constant #t_RXCONST = r'/[^/ ]+/g?' diff --git a/flscriptparse.py b/flscriptparse.py index 6a42bce..6afbccb 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -318,6 +318,7 @@ def p_parse(token): | DOLLAR | SQOUTE | DQOUTE + | PERIOD | BACKSLASH | CONDITIONAL1 | EQUALS From dff6b9d1430bcd147f3bc6fc4dd89a261ee08228 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sat, 15 Apr 2017 19:08:24 +0200 Subject: [PATCH 085/100] mejoras flscriptparser --- postparse.py | 12 ++++++++---- pytnyzer.py | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/postparse.py b/postparse.py index 428a3d8..06a8cde 100644 --- a/postparse.py +++ b/postparse.py @@ -507,6 +507,8 @@ def execute(options, args): if options.optdebug: print(options, args) if options.full: + execpython = options.exec_python + options.exec_python = False options.full = False options.toxml = True print("Pass 1 - Parse and write XML file . . .") @@ -517,10 +519,12 @@ def execute(options, args): print("Pass 2 - Pythonize and write PY file . . .") execute(options,[ arg+".xml" for arg in args]) - options.topython = False - options.exec_python = True - #print "Pass 3 - Test PY file load . . ." - #execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) + if execpython: + options.exec_python = execpython + print("Pass 3 - Test PY file load . . .") + options.topython = False + execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) + print("Done.") elif options.exec_python: diff --git a/pytnyzer.py b/pytnyzer.py index 852ae40..e9ee49b 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -14,7 +14,7 @@ def id_translate(name): python_keywords = ['and', 'del', 'for', 'is', 'raise', 'assert', 'elif', 'from', 'lambda', 'return', 'break', 'else', 'global', 'not', 'try', - 'class', 'except', 'if', 'or', 'while', 'continue', + 'class', 'except', 'if', 'or', 'while', 'continue', 'from', 'exec', 'import', 'pass', 'yield', 'def', 'finally', 'in', 'print'] if name in python_keywords: return name + "_" if name == "false": name = "False" @@ -441,7 +441,7 @@ def generate(self, **kwargs): class Variable(ASTPython): def generate(self, force_value = False, **kwargs): name = self.elem.get("name") - yield "expr", name + yield "expr", id_translate(name) values = 0 for value in self.elem.xpath("Value|Expression"): values += 1 From f672e55fb278fca3606b2a4d46ad56d02bc19ad3 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 16 Apr 2017 02:26:28 +0200 Subject: [PATCH 086/100] mejoras flparser --- postparse.py | 58 +++++++++++++++++++++++++++++++++++++++++++--------- pytnyzer.py | 2 +- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/postparse.py b/postparse.py index 06a8cde..060d3ce 100644 --- a/postparse.py +++ b/postparse.py @@ -446,6 +446,9 @@ def loadModule(self): # fp, pathname, description = imp.find_module(self.name,[self.path]) self.module = imp.load_module(name, fp, pathname, description) result = True + except FileNotFoundError: + print("Fichero %r no encontrado" % self.name) + result = False except Exception as e: print(traceback.format_exc()) result = False @@ -512,18 +515,30 @@ def execute(options, args): options.full = False options.toxml = True print("Pass 1 - Parse and write XML file . . .") - execute(options,args) + try: + execute(options,args) + except Exception: + print("Error parseando:"); + print(traceback.format_exc()) options.toxml = False options.topython = True print("Pass 2 - Pythonize and write PY file . . .") - execute(options,[ arg+".xml" for arg in args]) + try: + execute(options,[ arg+".xml" for arg in args]) + except Exception: + print("Error convirtiendo:"); + print(traceback.format_exc()) if execpython: options.exec_python = execpython print("Pass 3 - Test PY file load . . .") options.topython = False - execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) + try: + execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) + except Exception: + print("Error al ejecutar Python:"); + print(traceback.format_exc()) print("Done.") @@ -532,15 +547,18 @@ def execute(options, args): for filename in args: realpath = os.path.realpath(filename) path, name = os.path.split(realpath) + if not os.path.exists(realpath): + print("Fichero no existe: %s" % name) + continue + mod = Module(name, path) - if mod.loadModule(): - print(mod.module) - print(mod.module.form) - else: + if not mod.loadModule(): print("Error cargando modulo %s" % name) elif options.topython: from .pytnyzer import pythonize + import io + for filename in args: bname = os.path.basename(filename) if options.storepath: @@ -548,7 +566,22 @@ def execute(options, args): else: destname = filename+".py" destname = destname.replace(".qs.xml.py",".py") - pythonize(filename, destname) + if not os.path.exists(filename): + print("Fichero %r no encontrado" % filename) + continue + old_stderr = sys.stdout + stream = io.StringIO() + sys.stdout = stream + try: + pythonize(filename, destname) + except Exception: + print("Error al pythonificar %r:" % filename) + print(traceback.format_exc()) + sys.stdout = old_stderr + text = stream.getvalue() + if len(text) > 2: + print("%s: " % bname + ("\n%s: " % bname).join(text.splitlines())) + else: nfs = len(args) @@ -572,8 +605,13 @@ def execute(options, args): if options.toxml == False: # Si no se quiere guardar resultado, no hace falta calcular mas continue - - tree_data = flscriptparse.calctree(prog, alias_mode = 0) + + try: + tree_data = flscriptparse.calctree(prog, alias_mode = 0) + except Exception: + print("Error al convertir a XML %r:" % bname) + print("\n".join(traceback.format_exc().splitlines()[-7:])) + if not tree_data: print("No se pudo parsear %-35s \n" % (repr(filename))) continue diff --git a/pytnyzer.py b/pytnyzer.py index e9ee49b..4a9c51e 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -884,7 +884,7 @@ class DeclarationBlock(ASTPython): def generate(self, **kwargs): mode = self.elem.get("mode") is_constructor = self.elem.get("constructor") - if mode == "CONST": yield "debug", "Const Declaration:" + #if mode == "CONST": yield "debug", "Const Declaration:" for var in self.elem: expr = [] for dtype, data in parse_ast(var).generate(force_value=True): From 56297a9226d65d58299f45e04edded44d1e34cc1 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 16 Apr 2017 14:26:55 +0200 Subject: [PATCH 087/100] flparser: agregar soporte para strings de comilla doble cortados con contrabarra --- flex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flex.py b/flex.py index d5a65ff..dcc5f14 100644 --- a/flex.py +++ b/flex.py @@ -165,7 +165,7 @@ def t_ID(t): t_FCONST = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' # String literal -t_SCONST = r'\"([^\\\n]|(\\.))*?\"' +t_SCONST = r'\"([^\"\\\n]|(\\.)|\\\n)*?\"' # Character constant 'c' or L'c' t_CCONST = r'\'([^\'\\\n]|(\\.)|\\\n)*?\'' From ff5f28727b4f9a1177000433511f2c547d00a137 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 16 Apr 2017 14:27:34 +0200 Subject: [PATCH 088/100] flparser: agregar soporte de === y correcciones --- flscriptparse.py | 18 ++++++++++++++---- postparse.py | 1 + pytnyzer.py | 2 ++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 6afbccb..9e801aa 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -402,7 +402,7 @@ def p_parse(token): global endoffile endoffile = fromline, lexspan, token.slice[0] #print fromline, lexspan, token.slice[0] - token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } + token[0] = { "00-toktype": str(token.slice[0]), "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) rspan = lexspan[0] @@ -420,8 +420,8 @@ def p_parse(token): rspan = max(rvalues) lexspan[1] = rspan - if str(token.slice[0]) == 'regexbody': - token[0] = { "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } + #if str(token.slice[0]) == 'regexbody': + # token[0] = { "00-toktype": str(token.slice[0]) , "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } #if str(token.slice[0]) == 'regex': # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] @@ -435,6 +435,8 @@ def p_parse(token): tokelines[lexspan[0]] = token.lexer.lineno global last_lexspan last_lexspan = lexspan + + @@ -580,6 +582,11 @@ def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): otype = ctype_alias[otype] #print " " * depth , obj['02-size'] for n,content in enumerate(obj['50-contents']): + if not isinstance(content,dict): + print("ERROR: content is not a dict!:", repr(content)) + print(".. obj:", repr(obj)) + raise TypeError("content is not a dict") + continue ctype = content['01-type'] value = content['99-value'] if ctype in ctype_alias: @@ -593,7 +600,10 @@ def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): if type(value) is dict: #print "*" if depth < 600: - tree_obj = calctree(value,depth+1,num+[str(n)], ctype, alias_mode=alias_mode) + try: + tree_obj = calctree(value,depth+1,num+[str(n)], ctype, alias_mode=alias_mode) + except Exception: + print("ERROR: trying to calculate member %d on:" % n, repr(obj)) else: tree_obj = None if type(tree_obj) is dict: diff --git a/postparse.py b/postparse.py index 060d3ce..09030e2 100644 --- a/postparse.py +++ b/postparse.py @@ -606,6 +606,7 @@ def execute(options, args): # Si no se quiere guardar resultado, no hace falta calcular mas continue + tree_data = None try: tree_data = flscriptparse.calctree(prog, alias_mode = 0) except Exception: diff --git a/pytnyzer.py b/pytnyzer.py index 4a9c51e..d6c65db 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -860,6 +860,8 @@ def generate(self, **kwargs): elif ctype == "GE": yield "expr", ">=" elif ctype == "EQ": yield "expr", "==" elif ctype == "NE": yield "expr", "!=" + elif ctype == "EQQ": yield "expr", "is" + elif ctype == "NEQ": yield "expr", "not is" elif ctype == "IN": yield "expr", "in" elif ctype == "LOR": yield "expr", "or" elif ctype == "LAND": yield "expr", "and" From cf9b8c5270fff0244c0b9de945d990d68e83ac2a Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 16 Apr 2017 15:41:16 +0200 Subject: [PATCH 089/100] correccion errores y agregar debug --- postparse.py | 2 +- pytnyzer.py | 56 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/postparse.py b/postparse.py index 09030e2..ddb3fbc 100644 --- a/postparse.py +++ b/postparse.py @@ -573,7 +573,7 @@ def execute(options, args): stream = io.StringIO() sys.stdout = stream try: - pythonize(filename, destname) + pythonize(filename, destname, destname + ".debug") except Exception: print("Error al pythonificar %r:" % filename) print(traceback.format_exc()) diff --git a/pytnyzer.py b/pytnyzer.py index d6c65db..ff2417a 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -21,6 +21,7 @@ def id_translate(name): if name == "true": name = "True" if name == "null": name = "None" if name == "unknown": name = "None" + if name == "undefined": name = "None" if name == "this": name = "self" if name == "startsWith": name = "startswith" @@ -36,17 +37,46 @@ def __init__(cls, name, bases, dct): class ASTPython(with_metaclass(ASTPythonFactory, object)): tags = [] - + debug_file = None + generate_depth = 0 + numline = 0 @classmethod def can_process_tag(self, tagname): return self.__name__ == tagname or tagname in self.tags def __init__(self, elem): self.elem = elem - + if self.debug_file: + self.internal_generate = self.generate + self.generate = self._generate + + def debug(self, text): + if self.debug_file is None: return + splen = ASTPython.generate_depth + retlen = 0 + if splen > self.generate_depth: + retlen, splen = splen - self.generate_depth, self.generate_depth + if retlen > 0: + sp = " " * (splen - 1) + "<" + "-" * retlen + else: + sp = " " * splen + cname = self.__class__.__name__ + self.debug_file.write("%04d" % ASTPython.numline + sp + cname + ": " + text + "\n"); + def polish(self): return self def generate(self, **kwargs): yield "debug", "* not-known-seq * " + etree.tounicode(self.elem) + def _generate(self, **kwargs): + self.debug("begin-gen") + ASTPython.generate_depth += 1 + self.generate_depth = ASTPython.generate_depth + for dtype, data in self.internal_generate(**kwargs): + self.debug("%s: %r" % (dtype, data)) + yield dtype, data + ASTPython.generate_depth -= 1 + self.debug("end-gen") + + class Source(ASTPython): @@ -168,7 +198,10 @@ def generate(self, break_mode = False, **kwargs): main_expr = [] for n,arg in enumerate(self.elem.xpath("Condition/*")): expr = [] - for dtype, data in parse_ast(arg).generate(isolate = False): + for dtype, data in parse_ast(arg).generate(isolate = False, ): + if dtype == "line+1": + yield "debug", "Inline update inside IF condition not allowed. Unexpected behavior." + dtype = "line" if dtype == "expr": expr.append(data) else: @@ -441,6 +474,7 @@ def generate(self, **kwargs): class Variable(ASTPython): def generate(self, force_value = False, **kwargs): name = self.elem.get("name") + #if name.startswith("colorFun"): print(name) yield "expr", id_translate(name) values = 0 for value in self.elem.xpath("Value|Expression"): @@ -849,7 +883,7 @@ def generate(self, **kwargs): elif ctype == "MINUSEQUAL": yield "expr", "-=" elif ctype == "TIMESEQUAL": yield "expr", "*=" elif ctype == "DIVEQUAL": yield "expr", "/=" - else: yield "expr", ctype + else: yield "expr", "OpUpdate."+ ctype class Compare(ASTPython): def generate(self, **kwargs): @@ -865,7 +899,7 @@ def generate(self, **kwargs): elif ctype == "IN": yield "expr", "in" elif ctype == "LOR": yield "expr", "or" elif ctype == "LAND": yield "expr", "and" - else: yield "expr", ctype + else: yield "expr", "Compare."+ctype class OpMath(ASTPython): def generate(self, **kwargs): @@ -876,10 +910,11 @@ def generate(self, **kwargs): elif ctype == "DIVIDE": yield "expr", "/" elif ctype == "MOD": yield "expr", "%" elif ctype == "XOR": yield "expr", "^" + elif ctype == "OR": yield "expr", "or" elif ctype == "LSHIFT": yield "expr", "<<" elif ctype == "RSHIFT": yield "expr", ">>" elif ctype == "AND": yield "expr", "&" - else: yield "expr", ctype + else: yield "expr", "Math."+ctype class DeclarationBlock(ASTPython): @@ -954,6 +989,7 @@ def write_python_file(fobj, ast): indent_text = " " last_line_for_indent = {} numline = 0 + ASTPython.numline = 1 last_dtype = None for dtype, data in file_template(ast): if isinstance(data, bytes): data = data.decode("UTF-8","replace") @@ -964,6 +1000,7 @@ def write_python_file(fobj, ast): try: lines_since_last_indent = numline - last_line_for_indent[len(indent)] except KeyError: lines_since_last_indent = 0 if lines_since_last_indent > 4: + ASTPython.numline += 1 fobj.write((len(indent)*indent_text) + "\n") last_line_for_indent[len(indent)] = numline if dtype == "debug": @@ -977,6 +1014,7 @@ def write_python_file(fobj, ast): last_line_for_indent[len(indent)] = numline if dtype == "end": if last_dtype == "begin": + ASTPython.numline += 1 fobj.write((len(indent)*indent_text) + "pass\n") last_line_for_indent[len(indent)] = numline @@ -988,17 +1026,19 @@ def write_python_file(fobj, ast): line = "# END-ERROR!! was %s but %s found. (%s)" % (endblock, data,repr(indent)) if line is not None: + ASTPython.numline += 1 fobj.write((len(indent)*indent_text) + line + "\n") if dtype == "end": if data.split("-")[1] in ["class","def","else","except"]: + ASTPython.numline += 1 fobj.write((len(indent)*indent_text) + "\n") last_line_for_indent[len(indent)] = numline last_dtype = dtype -def pythonize(filename, destfilename): +def pythonize(filename, destfilename, debugname = None): bname = os.path.basename(filename) - + ASTPython.debug_file = open(debugname, "w") if debugname else None parser = etree.XMLParser(remove_blank_text=True) try: ast_tree = etree.parse(open(filename), parser) From 5860ed75b995173c0cce2e00f64cbc02b0d2d5ce Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 16 Apr 2017 15:55:16 +0200 Subject: [PATCH 090/100] =?UTF-8?q?mejoras=20flparser=20y=20documentaci?= =?UTF-8?q?=C3=B3n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pytnyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytnyzer.py b/pytnyzer.py index ff2417a..e453625 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -1005,7 +1005,7 @@ def write_python_file(fobj, ast): last_line_for_indent[len(indent)] = numline if dtype == "debug": line = "# DEBUG:: " + data - print(numline, line) + #print(numline, line) if dtype == "expr": line = "# EXPR??:: " + data if dtype == "line+1": line = "# LINE+1??:: " + data if dtype == "begin": From 55be4dec3617156d6789335624883dac775ea741 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 16 Apr 2017 16:12:17 +0200 Subject: [PATCH 091/100] flparser: agregar opcion --cache --- postparse.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/postparse.py b/postparse.py index ddb3fbc..52a6c2a 100644 --- a/postparse.py +++ b/postparse.py @@ -490,6 +490,10 @@ def parseArgs(argv): action="store_true", dest="full", default=False, help="write xml file from qs") + parser.add_option("--cache", + action="store_true", dest="cache", default=False, + help="If dest file exists, don't regenerate it") + (options, args) = parser.parse_args(argv) return (options, args) @@ -558,7 +562,9 @@ def execute(options, args): elif options.topython: from .pytnyzer import pythonize import io - + if options.cache: + args = [ x for x in args if not os.path.exists((x+".py").replace(".qs.xml.py",".py")) + or os.path.getmtime(x) > os.path.getctime((x+".py").replace(".qs.xml.py",".py")) ] for filename in args: bname = os.path.basename(filename) if options.storepath: @@ -584,6 +590,9 @@ def execute(options, args): else: + if options.cache: + args = [ x for x in args if not os.path.exists(x+".xml") + or os.path.getmtime(x) > os.path.getctime(x+".xml")] nfs = len(args) for nf, filename in enumerate(args): bname = os.path.basename(filename) From bd6b8dee42b9a63e98ba9be6dfafa268f2100d5f Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 16 Apr 2017 22:48:29 +0200 Subject: [PATCH 092/100] flparser: mejoras --- postparse.py | 6 +++++- pytnyzer.py | 6 ++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/postparse.py b/postparse.py index 52a6c2a..c97b5dd 100644 --- a/postparse.py +++ b/postparse.py @@ -565,7 +565,9 @@ def execute(options, args): if options.cache: args = [ x for x in args if not os.path.exists((x+".py").replace(".qs.xml.py",".py")) or os.path.getmtime(x) > os.path.getctime((x+".py").replace(".qs.xml.py",".py")) ] - for filename in args: + + nfs = len(args) + for nf, filename in enumerate(args): bname = os.path.basename(filename) if options.storepath: destname = os.path.join(options.storepath,bname+".py") @@ -575,6 +577,8 @@ def execute(options, args): if not os.path.exists(filename): print("Fichero %r no encontrado" % filename) continue + sys.stdout.write("Pythonizing File: %-35s . . . . (%.1f%%) \r" % (bname,100.0*(nf+1.0)/nfs)) + sys.stdout.flush(); old_stderr = sys.stdout stream = io.StringIO() sys.stdout = stream diff --git a/pytnyzer.py b/pytnyzer.py index e453625..e97702f 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -836,6 +836,7 @@ class Constant(ASTPython): def generate(self, **kwargs): ctype = self.elem.get("type") value = self.elem.get("value") + self.debug("ctype: %r -> %r" % (ctype,value)); if ctype is None or value is None: for child in self.elem: if child.tag == "list_constant": @@ -868,6 +869,10 @@ def generate(self, **kwargs): yield "expr", "u'%s'" % value else: yield "expr", "u\"%s\"" % value + elif ctype == "Number": + value = value.lstrip("0") + if value == "": value = "0" + yield "expr", value else: yield "expr", value class Identifier(ASTPython): @@ -883,6 +888,7 @@ def generate(self, **kwargs): elif ctype == "MINUSEQUAL": yield "expr", "-=" elif ctype == "TIMESEQUAL": yield "expr", "*=" elif ctype == "DIVEQUAL": yield "expr", "/=" + elif ctype == "MODEQUAL": yield "expr", "%=" else: yield "expr", "OpUpdate."+ ctype class Compare(ASTPython): From a79976a87d08aac9a05e470739c226f03344368d Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Mon, 17 Apr 2017 22:10:39 +0200 Subject: [PATCH 093/100] mejoras compatibilidad QS --- postparse.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/postparse.py b/postparse.py index c97b5dd..6f2abf0 100644 --- a/postparse.py +++ b/postparse.py @@ -187,6 +187,12 @@ class Function(ListNamedObject): def add_vartype(self, argn, subelem): self.xml.set("returns", str(subelem.xmlname)) +class FunctionAnon(ListObject): + tags = ["funcdeclaration_anon"] + +class FunctionAnonExec(ListObject): + tags = ["funcdeclaration_anon_exec"] + class Variable(NamedObject): tags = ["vardecl"] callback_subelem = NamedObject.callback_subelem.copy() From 4a7b0c8a2813b3c12ad602d77b39bd2126c93521 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sat, 22 Apr 2017 11:58:15 +0200 Subject: [PATCH 094/100] =?UTF-8?q?correcci=C3=B3n=20bug=20conversor=20pyt?= =?UTF-8?q?hon=20en=20if-else?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- postparse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/postparse.py b/postparse.py index 6f2abf0..af47f93 100644 --- a/postparse.py +++ b/postparse.py @@ -301,7 +301,6 @@ class Condition(ListObject): class Else(ListObject): tags = ["optelse"] - adopt_childs_tags = ['statement_block'] def polish(self): if len(self.subelems) == 0: self.astname = "empty" From 7f825d91946cd1bb4299324b0c3ec54129c10075 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sat, 22 Apr 2017 13:15:40 +0200 Subject: [PATCH 095/100] agregar soporte de pythonificar textos --- flscriptparse.py | 2 ++ pytnyzer.py | 51 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/flscriptparse.py b/flscriptparse.py index 9e801aa..04b24d5 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -123,6 +123,8 @@ def p_parse(token): | vardeclaration | classdeclaration | funcdeclaration + | funcdeclaration_anon + | funcdeclaration_anon_exec source : source_element source : source source_element diff --git a/pytnyzer.py b/pytnyzer.py index e97702f..a088dd4 100644 --- a/pytnyzer.py +++ b/pytnyzer.py @@ -121,20 +121,27 @@ def generate(self, **kwargs): class Function(ASTPython): def generate(self, **kwargs): - name = id_translate(self.elem.get("name")) + _name = self.elem.get("name") + if _name: + name = id_translate(_name) + else: + # Anonima: + name = "_" + withoutself = self.elem.get("withoutself") + returns = self.elem.get("returns",None) parent = self.elem.getparent() grandparent = None if parent is not None: grandparent = parent.getparent() arguments = [] - - if grandparent is not None: - if grandparent.tag == "Class": + if not withoutself: + if grandparent is not None: + if grandparent.tag == "Class": + arguments.append("self") + if name == grandparent.get("name"): + name = "__init__" + else: arguments.append("self") - if name == grandparent.get("name"): - name = "__init__" - else: - arguments.append("self") for n,arg in enumerate(self.elem.xpath("Arguments/*")): expr = [] for dtype, data in parse_ast(arg).generate(): @@ -159,6 +166,9 @@ def generate(self, **kwargs): for obj in parse_ast(source).generate(): yield obj yield "end", "block-def-%s" % (name) +class FunctionAnon(Function): + pass + class FunctionCall(ASTPython): def generate(self, **kwargs): name = id_translate(self.elem.get("name")) @@ -990,14 +1000,25 @@ def file_template(ast): yield "line", "" yield "line", "form = None" -def write_python_file(fobj, ast): + + +def string_template(ast): + sourceclasses = etree.Element("Source") + for child in ast: + child.set("withoutself","1") + sourceclasses.append(child) + + for dtype, data in parse_ast(sourceclasses).generate(): + yield dtype, data + +def write_python_file(fobj, ast, tpl=file_template): indent = [] indent_text = " " last_line_for_indent = {} numline = 0 ASTPython.numline = 1 last_dtype = None - for dtype, data in file_template(ast): + for dtype, data in tpl(ast): if isinstance(data, bytes): data = data.decode("UTF-8","replace") line = None if dtype == "line": @@ -1052,12 +1073,16 @@ def pythonize(filename, destfilename, debugname = None): print("filename:",filename) raise ast = ast_tree.getroot() - + tpl = string_template + for cls in ast.xpath("Class"): + tpl = file_template + break f1 = open(destfilename,"w") - write_python_file(f1,ast) + write_python_file(f1,ast,tpl) f1.close() - + + def main(): parser = OptionParser() parser.add_option("-q", "--quiet", From 5b4dbc2fa708a867df8a098085e76d8183b21dbb Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 27 Apr 2017 16:06:29 +0200 Subject: [PATCH 096/100] permitir carga de .py y .qs.py desde base de datos. mejoras velocidad --- postparse.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postparse.py b/postparse.py index af47f93..12a880d 100644 --- a/postparse.py +++ b/postparse.py @@ -544,7 +544,7 @@ def execute(options, args): print("Pass 3 - Test PY file load . . .") options.topython = False try: - execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".py") for arg in args]) + execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".qs.py") for arg in args]) except Exception: print("Error al ejecutar Python:"); print(traceback.format_exc()) @@ -568,8 +568,8 @@ def execute(options, args): from .pytnyzer import pythonize import io if options.cache: - args = [ x for x in args if not os.path.exists((x+".py").replace(".qs.xml.py",".py")) - or os.path.getmtime(x) > os.path.getctime((x+".py").replace(".qs.xml.py",".py")) ] + args = [ x for x in args if not os.path.exists((x+".py").replace(".qs.xml.py",".qs.py")) + or os.path.getmtime(x) > os.path.getctime((x+".py").replace(".qs.xml.py",".qs.py")) ] nfs = len(args) for nf, filename in enumerate(args): @@ -578,7 +578,7 @@ def execute(options, args): destname = os.path.join(options.storepath,bname+".py") else: destname = filename+".py" - destname = destname.replace(".qs.xml.py",".py") + destname = destname.replace(".qs.xml.py",".qs.py") if not os.path.exists(filename): print("Fichero %r no encontrado" % filename) continue From 192413254660271e790ce27cb2d9c4f88682e150 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Thu, 27 Apr 2017 17:37:10 +0200 Subject: [PATCH 097/100] Acelerar un poco el parseo QS->XML --- flex.py | 20 ++--- flscriptparse.py | 205 ++++++++++++++++++++++++----------------------- lextab.py | 10 +++ 3 files changed, 125 insertions(+), 110 deletions(-) create mode 100644 lextab.py diff --git a/flex.py b/flex.py index dcc5f14..59a7c3d 100644 --- a/flex.py +++ b/flex.py @@ -9,6 +9,8 @@ #sys.path.insert(0,"../..") import ply.lex as lex +from ply.lex import TOKEN + # Reserved words reserved = [ @@ -69,8 +71,8 @@ t_ignore = ' \r\t\x0c' # Newlines +@TOKEN(r'\n+') def t_NEWLINE(t): - r'\n+' t.lexer.lineno += t.value.count("\n") # Operators @@ -150,9 +152,8 @@ def t_NEWLINE(t): +@TOKEN(r'[A-Za-z_]+[\w_]*') def t_ID(t): -# r'[A-Za-z_]+([\.]{0,1}[\w_]*)+' - r'[A-Za-z_]+[\w_]*' t.type = reserved_map.get(t.value,"ID") return t @@ -174,14 +175,13 @@ def t_ID(t): #t_RXCONST = r'/[^/ ]+/g?' # Comments +@TOKEN(r'(/\*( |\*\*)(.|\n)*?\*/)|(//.*)') def t_comment(t): - r'(/\*( |\*\*)(.|\n)*?\*/)|(//.*)' - #r'/\*(.|\n)*?\*/' t.lexer.lineno += t.value.count('\n') +@TOKEN(r'/\*\*[ ]+') def t_DOCSTRINGOPEN(t): - r'/\*\*[ ]+' return t; #t_COMMENTOPEN = r'/\*' @@ -189,8 +189,8 @@ def t_DOCSTRINGOPEN(t): # Preprocessor directive (ignored) +@TOKEN(r'\#(.)*?\n') def t_preprocessor(t): - r'\#(.)*?\n' t.lexer.lineno += 1 @@ -199,8 +199,10 @@ def t_error(t): t.lexer.skip(1) - -lexer = lex.lex(debug=False) +# TODO: Cada vez que se cambia este fichero, se tiene que lanzar sin el "-OO" de python para acelerar. Construye entonces +# ..... el fichero de cache que subimos a git, y se relee desde ahí las siguientes veces con el -OO. +# ..... Si da problemas, hay que volver a optimize=0 y/o eliminar lextab.py +lexer = lex.lex(debug=False,optimize=1) if __name__ == "__main__": lex.runmain(lexer) diff --git a/flscriptparse.py b/flscriptparse.py index 04b24d5..e3a4296 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -54,8 +54,111 @@ def cnvrt(val): seen_tokens = [] tokelines = {} last_lexspan = None + def p_parse(token): - ''' + global input_data + + lexspan = list(token.lexspan(0)) + data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) + if len(lexspan) == 2: + fromline = token.lineno(0) + global endoffile + endoffile = fromline, lexspan, token.slice[0] + #print fromline, lexspan, token.slice[0] + token[0] = { "00-toktype": str(token.slice[0]), "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } + numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) + + rspan = lexspan[0] + if str(token.slice[0]) == 'empty' or numelems == 0: token[0] = None + else: + rvalues = [] + for n,s in enumerate(token.slice[1:]): + if s.type != 'empty' and s.value is not None: + val = None + if isinstance(s.value,str): + val = token.lexspan(n+1)[0] + len(s.value) - 1 + else: + val = token.lexspan(n+1)[1] + rvalues.append(val) + rspan = max(rvalues) + lexspan[1] = rspan + + #if str(token.slice[0]) == 'regexbody': + # token[0] = { "00-toktype": str(token.slice[0]) , "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } + + #if str(token.slice[0]) == 'regex': + # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] + # print " " + "\n ".join([ "%s(%r): %r" % (s.type, token.lexspan(n+1), s.value) for n,s in enumerate(token.slice[1:]) ]) + global seen_tokens, last_ok_token + last_ok_token = token + seen_tokens.append((str(token.slice[0]), token.lineno(0),input_data[lexspan[0]:lexspan[1]+1] )) + global ok_count + ok_count += 1 + if lexspan[0] not in tokelines: + tokelines[lexspan[0]] = token.lexer.lineno + global last_lexspan + last_lexspan = lexspan + + + + + +last_ok_token = None +error_count = 0 +last_error_token = None +last_error_line = -1 +ok_count = 0 + +def p_error(t): + global error_count + global ok_count + global last_error_token + global last_error_line, seen_tokens , last_ok_token + debug = False # Poner a True para toneladas de debug. + # if error_count == 0: print + if t is not None: + if last_error_token is None or t.lexpos != getattr(last_error_token,"lexpos",None): + if abs(last_error_line - t.lineno) > 4 and ok_count > 1 and error_count < 4: + error_count += 1 + try: print_context(t) + except Exception: pass + if debug == True: + error_count += 20 # no imprimir mas de un error en debug. + print + for tokname, tokln, tokdata in seen_tokens[-32:]: + if tokln == t.lineno: + print(tokname, tokdata) + print(repr(last_ok_token[0])) + for s in last_ok_token.slice[:]: + print(">>>" , s.lineno, repr(s), pprint.pformat(s.value,depth=3)) + last_error_line = t.lineno + elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: + last_error_line = t.lineno + parser.errok() + ok_count = 0 + return + + ok_count = 0 + if t is None: + if last_error_token != "EOF": + print("ERROR: End of the file reached.") + global endoffile + print("Last data:", endoffile) + + if last_lexspan: + try: + print("HINT: Last lexspan:", last_lexspan) + print("HINT: Last line:", tokelines[last_lexspan[0]]) + except Exception as e: + print("ERROR:", e) + last_error_token = "EOF" + return t + t = parser.token() + parser.restart() + last_error_token = t + return t + +p_parse.__doc__ = ''' exprval : constant | variable | funccall @@ -395,107 +498,7 @@ def p_parse(token): empty : ''' - global input_data - - lexspan = list(token.lexspan(0)) - data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) - if len(lexspan) == 2: - fromline = token.lineno(0) - global endoffile - endoffile = fromline, lexspan, token.slice[0] - #print fromline, lexspan, token.slice[0] - token[0] = { "00-toktype": str(token.slice[0]), "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } - numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) - - rspan = lexspan[0] - if str(token.slice[0]) == 'empty' or numelems == 0: token[0] = None - else: - rvalues = [] - for n,s in enumerate(token.slice[1:]): - if s.type != 'empty' and s.value is not None: - val = None - if isinstance(s.value,str): - val = token.lexspan(n+1)[0] + len(s.value) - 1 - else: - val = token.lexspan(n+1)[1] - rvalues.append(val) - rspan = max(rvalues) - lexspan[1] = rspan - - #if str(token.slice[0]) == 'regexbody': - # token[0] = { "00-toktype": str(token.slice[0]) , "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } - - #if str(token.slice[0]) == 'regex': - # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] - # print " " + "\n ".join([ "%s(%r): %r" % (s.type, token.lexspan(n+1), s.value) for n,s in enumerate(token.slice[1:]) ]) - global seen_tokens, last_ok_token - last_ok_token = token - seen_tokens.append((str(token.slice[0]), token.lineno(0),input_data[lexspan[0]:lexspan[1]+1] )) - global ok_count - ok_count += 1 - if lexspan[0] not in tokelines: - tokelines[lexspan[0]] = token.lexer.lineno - global last_lexspan - last_lexspan = lexspan - - - - - -last_ok_token = None -error_count = 0 -last_error_token = None -last_error_line = -1 -ok_count = 0 -def p_error(t): - global error_count - global ok_count - global last_error_token - global last_error_line, seen_tokens , last_ok_token - debug = False # Poner a True para toneladas de debug. - # if error_count == 0: print - if t is not None: - if last_error_token is None or t.lexpos != getattr(last_error_token,"lexpos",None): - if abs(last_error_line - t.lineno) > 4 and ok_count > 1 and error_count < 4: - error_count += 1 - try: print_context(t) - except Exception: pass - if debug == True: - error_count += 20 # no imprimir mas de un error en debug. - print - for tokname, tokln, tokdata in seen_tokens[-32:]: - if tokln == t.lineno: - print(tokname, tokdata) - print(repr(last_ok_token[0])) - for s in last_ok_token.slice[:]: - print(">>>" , s.lineno, repr(s), pprint.pformat(s.value,depth=3)) - last_error_line = t.lineno - elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: - last_error_line = t.lineno - parser.errok() - ok_count = 0 - return - - ok_count = 0 - if t is None: - if last_error_token != "EOF": - print("ERROR: End of the file reached.") - global endoffile - print("Last data:", endoffile) - - if last_lexspan: - try: - print("HINT: Last lexspan:", last_lexspan) - print("HINT: Last line:", tokelines[last_lexspan[0]]) - except Exception as e: - print("ERROR:", e) - last_error_token = "EOF" - return t - t = parser.token() - parser.restart() - last_error_token = t - return t # Build the grammar diff --git a/lextab.py b/lextab.py new file mode 100644 index 0000000..417e5d5 --- /dev/null +++ b/lextab.py @@ -0,0 +1,10 @@ +# lextab.py. This file automatically created by PLY (version 3.9). Don't edit! +_tabversion = '3.8' +_lextokens = set(('MINUS', 'XOR', 'LPAREN', 'FOR', 'DIVIDE', 'DOCSTRINGOPEN', 'COMMENTCLOSE', 'WITH', 'CONDITIONAL1', 'COMMA', 'DQOUTE', 'SEMI', 'LSHIFT', 'MINUSEQUAL', 'EQUALS', 'NEW', 'RSHIFT', 'SWITCH', 'LT', 'CASE', 'TIMES', 'OR', 'LOR', 'LE', 'DIVEQUAL', 'WHILE', 'DOLLAR', 'COLON', 'DO', 'THROW', 'LAND', 'SCONST', 'EQ', 'VAR', 'NE', 'PERIOD', 'IN', 'EXTENDS', 'MOD', 'GE', 'IF', 'LBRACKET', 'DELETE', 'PLUSPLUS', 'RETURN', 'LNOT', 'RPAREN', 'BACKSLASH', 'CLASS', 'RBRACE', 'ELSE', 'CCONST', 'CONST', 'CONTINUE', 'TIMESEQUAL', 'STATIC', 'TYPEOF', 'ID', 'EQQ', 'MODEQUAL', 'GT', 'FUNCTION', 'MINUSMINUS', 'DEFAULT', 'CATCH', 'AND', 'PLUSEQUAL', 'LBRACE', 'TRY', 'ICONST', 'SQOUTE', 'AT', 'BREAK', 'FCONST', 'PLUS', 'RBRACKET', 'NEQ')) +_lexreflags = 0 +_lexliterals = '' +_lexstateinfo = {'INITIAL': 'inclusive'} +_lexstatere = {'INITIAL': [('(?P\\n+)|(?P[A-Za-z_]+[\\w_]*)|(?P(/\\*( |\\*\\*)(.|\\n)*?\\*/)|(//.*))|(?P/\\*\\*[ ]+)|(?P\\#(.)*?\\n)|(?P((\\d+)(\\.\\d+)(e(\\+|-)?(\\d+))? | (\\d+)e(\\+|-)?(\\d+))([lL]|[fF])?)|(?P\\d+([uU]|[lL]|[uU][lL]|[lL][uU])?)|(?P\\\'([^\\\'\\\\\\n]|(\\\\.)|\\\\\\n)*?\\\')|(?P\\"([^\\"\\\\\\n]|(\\\\.)|\\\\\\n)*?\\")|(?P\\+\\+)|(?P\\|\\|)|(?P\\*/)|(?P\\*=)|(?P===)|(?P\\+=)|(?P!==)|(?P>>)|(?P\\|)|(?P\\()|(?P>=)|(?P/=)|(?P\\^)|(?P\\*)|(?P!=)|(?P\\))|(?P-=)|(?P--)|(?P\\})|(?P\\?)|(?P\\+)|(?P\\.)|(?P\\])|(?P\\\\)|(?P\\[)|(?P<<)|(?P\\$)|(?P\\{)|(?P<=)|(?P%=)|(?P==)|(?P&&)|(?P")|(?P>)|(?P-)|(?P=)|(?P\')|(?P;)|(?P!)|(?P/)|(?P,)|(?P<)|(?P:)|(?P%)|(?P@)|(?P&)', [None, ('t_NEWLINE', 'NEWLINE'), ('t_ID', 'ID'), ('t_comment', 'comment'), None, None, None, None, ('t_DOCSTRINGOPEN', 'DOCSTRINGOPEN'), ('t_preprocessor', 'preprocessor'), None, (None, 'FCONST'), None, None, None, None, None, None, None, None, None, None, (None, 'ICONST'), None, (None, 'CCONST'), None, None, (None, 'SCONST'), None, None, (None, 'PLUSPLUS'), (None, 'LOR'), (None, 'COMMENTCLOSE'), (None, 'TIMESEQUAL'), (None, 'EQQ'), (None, 'PLUSEQUAL'), (None, 'NEQ'), (None, 'RSHIFT'), (None, 'OR'), (None, 'LPAREN'), (None, 'GE'), (None, 'DIVEQUAL'), (None, 'XOR'), (None, 'TIMES'), (None, 'NE'), (None, 'RPAREN'), (None, 'MINUSEQUAL'), (None, 'MINUSMINUS'), (None, 'RBRACE'), (None, 'CONDITIONAL1'), (None, 'PLUS'), (None, 'PERIOD'), (None, 'RBRACKET'), (None, 'BACKSLASH'), (None, 'LBRACKET'), (None, 'LSHIFT'), (None, 'DOLLAR'), (None, 'LBRACE'), (None, 'LE'), (None, 'MODEQUAL'), (None, 'EQ'), (None, 'LAND'), (None, 'DQOUTE'), (None, 'GT'), (None, 'MINUS'), (None, 'EQUALS'), (None, 'SQOUTE'), (None, 'SEMI'), (None, 'LNOT'), (None, 'DIVIDE'), (None, 'COMMA'), (None, 'LT'), (None, 'COLON'), (None, 'MOD'), (None, 'AT'), (None, 'AND')])]} +_lexstateignore = {'INITIAL': ' \r\t\x0c'} +_lexstateerrorf = {'INITIAL': 't_error'} +_lexstateeoff = {} From c5328370089a2c3fd20ef09f7d405026e3601a29 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sat, 29 Apr 2017 22:51:36 +0200 Subject: [PATCH 098/100] cambios varios --- postparse.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/postparse.py b/postparse.py index 12a880d..47b7148 100644 --- a/postparse.py +++ b/postparse.py @@ -523,7 +523,7 @@ def execute(options, args): options.exec_python = False options.full = False options.toxml = True - print("Pass 1 - Parse and write XML file . . .") + if options.verbose: print("Pass 1 - Parse and write XML file . . .") try: execute(options,args) except Exception: @@ -532,7 +532,7 @@ def execute(options, args): options.toxml = False options.topython = True - print("Pass 2 - Pythonize and write PY file . . .") + if options.verbose: print("Pass 2 - Pythonize and write PY file . . .") try: execute(options,[ arg+".xml" for arg in args]) except Exception: @@ -541,7 +541,7 @@ def execute(options, args): if execpython: options.exec_python = execpython - print("Pass 3 - Test PY file load . . .") + if options.verbose: print("Pass 3 - Test PY file load . . .") options.topython = False try: execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".qs.py") for arg in args]) @@ -582,8 +582,8 @@ def execute(options, args): if not os.path.exists(filename): print("Fichero %r no encontrado" % filename) continue - sys.stdout.write("Pythonizing File: %-35s . . . . (%.1f%%) \r" % (bname,100.0*(nf+1.0)/nfs)) - sys.stdout.flush(); + if options.verbose: sys.stdout.write("Pythonizing File: %-35s . . . . (%.1f%%) \r" % (bname,100.0*(nf+1.0)/nfs)) + if options.verbose: sys.stdout.flush(); old_stderr = sys.stdout stream = io.StringIO() sys.stdout = stream @@ -605,8 +605,8 @@ def execute(options, args): nfs = len(args) for nf, filename in enumerate(args): bname = os.path.basename(filename) - sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) - sys.stdout.flush(); + if options.verbose: sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) + if options.verbose: sys.stdout.flush(); try: filecontent = open(filename,"r", encoding="latin-1").read() except Exception as e: From e260edb478175a7ee314b012c825f779a139e585 Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Fri, 26 May 2017 13:36:19 +0200 Subject: [PATCH 099/100] add semi option for regexes --- flscriptparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flscriptparse.py b/flscriptparse.py index e3a4296..09fedf1 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -429,6 +429,7 @@ def p_error(t): | EQUALS | OR | SCONST + | SEMI | error regexflags : ID From 0b287f204e39ca1b247961c33274377dfdfaa69d Mon Sep 17 00:00:00 2001 From: David Martinez Marti Date: Sun, 29 Apr 2018 08:32:23 +0100 Subject: [PATCH 100/100] Ignore no semicolon for dowhile --- flparser | 1 + flscriptparse.py | 5 +++-- flscriptparser2 | 2 +- lextab.py | 10 +++++----- 4 files changed, 10 insertions(+), 8 deletions(-) create mode 120000 flparser diff --git a/flparser b/flparser new file mode 120000 index 0000000..945c9b4 --- /dev/null +++ b/flparser @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/flscriptparse.py b/flscriptparse.py index 09fedf1..0e8c359 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -98,8 +98,8 @@ def p_parse(token): tokelines[lexspan[0]] = token.lexer.lineno global last_lexspan last_lexspan = lexspan - - + + @@ -463,6 +463,7 @@ def p_error(t): whilestatement : WHILE LPAREN condition RPAREN statement_block dowhilestatement : DO statement_block WHILE LPAREN condition RPAREN SEMI + | DO statement_block WHILE LPAREN condition RPAREN withstatement : WITH LPAREN variable RPAREN statement_block | error diff --git a/flscriptparser2 b/flscriptparser2 index 99cb558..6d660b9 100755 --- a/flscriptparser2 +++ b/flscriptparser2 @@ -1,4 +1,4 @@ #!/usr/bin/python3 -import postparse +from flparser import postparse if __name__ == "__main__": postparse.main() diff --git a/lextab.py b/lextab.py index 417e5d5..342b88d 100644 --- a/lextab.py +++ b/lextab.py @@ -1,10 +1,10 @@ -# lextab.py. This file automatically created by PLY (version 3.9). Don't edit! -_tabversion = '3.8' -_lextokens = set(('MINUS', 'XOR', 'LPAREN', 'FOR', 'DIVIDE', 'DOCSTRINGOPEN', 'COMMENTCLOSE', 'WITH', 'CONDITIONAL1', 'COMMA', 'DQOUTE', 'SEMI', 'LSHIFT', 'MINUSEQUAL', 'EQUALS', 'NEW', 'RSHIFT', 'SWITCH', 'LT', 'CASE', 'TIMES', 'OR', 'LOR', 'LE', 'DIVEQUAL', 'WHILE', 'DOLLAR', 'COLON', 'DO', 'THROW', 'LAND', 'SCONST', 'EQ', 'VAR', 'NE', 'PERIOD', 'IN', 'EXTENDS', 'MOD', 'GE', 'IF', 'LBRACKET', 'DELETE', 'PLUSPLUS', 'RETURN', 'LNOT', 'RPAREN', 'BACKSLASH', 'CLASS', 'RBRACE', 'ELSE', 'CCONST', 'CONST', 'CONTINUE', 'TIMESEQUAL', 'STATIC', 'TYPEOF', 'ID', 'EQQ', 'MODEQUAL', 'GT', 'FUNCTION', 'MINUSMINUS', 'DEFAULT', 'CATCH', 'AND', 'PLUSEQUAL', 'LBRACE', 'TRY', 'ICONST', 'SQOUTE', 'AT', 'BREAK', 'FCONST', 'PLUS', 'RBRACKET', 'NEQ')) -_lexreflags = 0 +# lextab.py. This file automatically created by PLY (version 3.11). Don't edit! +_tabversion = '3.10' +_lextokens = set(('AND', 'AT', 'BACKSLASH', 'BREAK', 'CASE', 'CATCH', 'CCONST', 'CLASS', 'COLON', 'COMMA', 'COMMENTCLOSE', 'CONDITIONAL1', 'CONST', 'CONTINUE', 'DEFAULT', 'DELETE', 'DIVEQUAL', 'DIVIDE', 'DO', 'DOCSTRINGOPEN', 'DOLLAR', 'DQOUTE', 'ELSE', 'EQ', 'EQQ', 'EQUALS', 'EXTENDS', 'FCONST', 'FOR', 'FUNCTION', 'GE', 'GT', 'ICONST', 'ID', 'IF', 'IN', 'LAND', 'LBRACE', 'LBRACKET', 'LE', 'LNOT', 'LOR', 'LPAREN', 'LSHIFT', 'LT', 'MINUS', 'MINUSEQUAL', 'MINUSMINUS', 'MOD', 'MODEQUAL', 'NE', 'NEQ', 'NEW', 'OR', 'PERIOD', 'PLUS', 'PLUSEQUAL', 'PLUSPLUS', 'RBRACE', 'RBRACKET', 'RETURN', 'RPAREN', 'RSHIFT', 'SCONST', 'SEMI', 'SQOUTE', 'STATIC', 'SWITCH', 'THROW', 'TIMES', 'TIMESEQUAL', 'TRY', 'TYPEOF', 'VAR', 'WHILE', 'WITH', 'XOR')) +_lexreflags = 64 _lexliterals = '' _lexstateinfo = {'INITIAL': 'inclusive'} -_lexstatere = {'INITIAL': [('(?P\\n+)|(?P[A-Za-z_]+[\\w_]*)|(?P(/\\*( |\\*\\*)(.|\\n)*?\\*/)|(//.*))|(?P/\\*\\*[ ]+)|(?P\\#(.)*?\\n)|(?P((\\d+)(\\.\\d+)(e(\\+|-)?(\\d+))? | (\\d+)e(\\+|-)?(\\d+))([lL]|[fF])?)|(?P\\d+([uU]|[lL]|[uU][lL]|[lL][uU])?)|(?P\\\'([^\\\'\\\\\\n]|(\\\\.)|\\\\\\n)*?\\\')|(?P\\"([^\\"\\\\\\n]|(\\\\.)|\\\\\\n)*?\\")|(?P\\+\\+)|(?P\\|\\|)|(?P\\*/)|(?P\\*=)|(?P===)|(?P\\+=)|(?P!==)|(?P>>)|(?P\\|)|(?P\\()|(?P>=)|(?P/=)|(?P\\^)|(?P\\*)|(?P!=)|(?P\\))|(?P-=)|(?P--)|(?P\\})|(?P\\?)|(?P\\+)|(?P\\.)|(?P\\])|(?P\\\\)|(?P\\[)|(?P<<)|(?P\\$)|(?P\\{)|(?P<=)|(?P%=)|(?P==)|(?P&&)|(?P")|(?P>)|(?P-)|(?P=)|(?P\')|(?P;)|(?P!)|(?P/)|(?P,)|(?P<)|(?P:)|(?P%)|(?P@)|(?P&)', [None, ('t_NEWLINE', 'NEWLINE'), ('t_ID', 'ID'), ('t_comment', 'comment'), None, None, None, None, ('t_DOCSTRINGOPEN', 'DOCSTRINGOPEN'), ('t_preprocessor', 'preprocessor'), None, (None, 'FCONST'), None, None, None, None, None, None, None, None, None, None, (None, 'ICONST'), None, (None, 'CCONST'), None, None, (None, 'SCONST'), None, None, (None, 'PLUSPLUS'), (None, 'LOR'), (None, 'COMMENTCLOSE'), (None, 'TIMESEQUAL'), (None, 'EQQ'), (None, 'PLUSEQUAL'), (None, 'NEQ'), (None, 'RSHIFT'), (None, 'OR'), (None, 'LPAREN'), (None, 'GE'), (None, 'DIVEQUAL'), (None, 'XOR'), (None, 'TIMES'), (None, 'NE'), (None, 'RPAREN'), (None, 'MINUSEQUAL'), (None, 'MINUSMINUS'), (None, 'RBRACE'), (None, 'CONDITIONAL1'), (None, 'PLUS'), (None, 'PERIOD'), (None, 'RBRACKET'), (None, 'BACKSLASH'), (None, 'LBRACKET'), (None, 'LSHIFT'), (None, 'DOLLAR'), (None, 'LBRACE'), (None, 'LE'), (None, 'MODEQUAL'), (None, 'EQ'), (None, 'LAND'), (None, 'DQOUTE'), (None, 'GT'), (None, 'MINUS'), (None, 'EQUALS'), (None, 'SQOUTE'), (None, 'SEMI'), (None, 'LNOT'), (None, 'DIVIDE'), (None, 'COMMA'), (None, 'LT'), (None, 'COLON'), (None, 'MOD'), (None, 'AT'), (None, 'AND')])]} +_lexstatere = {'INITIAL': [('(?P\\n+)|(?P[A-Za-z_]+[\\w_]*)|(?P(/\\*( |\\*\\*)(.|\\n)*?\\*/)|(//.*))|(?P/\\*\\*[ ]+)|(?P\\#(.)*?\\n)|(?P((\\d+)(\\.\\d+)(e(\\+|-)?(\\d+))? | (\\d+)e(\\+|-)?(\\d+))([lL]|[fF])?)|(?P\\d+([uU]|[lL]|[uU][lL]|[lL][uU])?)|(?P\\"([^\\"\\\\\\n]|(\\\\.)|\\\\\\n)*?\\")|(?P\\\'([^\\\'\\\\\\n]|(\\\\.)|\\\\\\n)*?\\\')|(?P\\|\\|)|(?P\\+\\+)|(?P===)|(?P!==)|(?P\\*=)|(?P\\+=)|(?P\\*/)|(?P\\\\)|(?P\\$)|(?P\\+)|(?P\\*)|(?P\\|)|(?P\\^)|(?P<<)|(?P>>)|(?P&&)|(?P<=)|(?P>=)|(?P==)|(?P!=)|(?P\\?)|(?P/=)|(?P%=)|(?P-=)|(?P--)|(?P\\()|(?P\\))|(?P\\[)|(?P\\])|(?P\\{)|(?P\\})|(?P\\.)|(?P\')|(?P")|(?P-)|(?P/)|(?P%)|(?P&)|(?P!)|(?P<)|(?P>)|(?P=)|(?P,)|(?P;)|(?P:)|(?P@)', [None, ('t_NEWLINE', 'NEWLINE'), ('t_ID', 'ID'), ('t_comment', 'comment'), None, None, None, None, ('t_DOCSTRINGOPEN', 'DOCSTRINGOPEN'), ('t_preprocessor', 'preprocessor'), None, (None, 'FCONST'), None, None, None, None, None, None, None, None, None, None, (None, 'ICONST'), None, (None, 'SCONST'), None, None, (None, 'CCONST'), None, None, (None, 'LOR'), (None, 'PLUSPLUS'), (None, 'EQQ'), (None, 'NEQ'), (None, 'TIMESEQUAL'), (None, 'PLUSEQUAL'), (None, 'COMMENTCLOSE'), (None, 'BACKSLASH'), (None, 'DOLLAR'), (None, 'PLUS'), (None, 'TIMES'), (None, 'OR'), (None, 'XOR'), (None, 'LSHIFT'), (None, 'RSHIFT'), (None, 'LAND'), (None, 'LE'), (None, 'GE'), (None, 'EQ'), (None, 'NE'), (None, 'CONDITIONAL1'), (None, 'DIVEQUAL'), (None, 'MODEQUAL'), (None, 'MINUSEQUAL'), (None, 'MINUSMINUS'), (None, 'LPAREN'), (None, 'RPAREN'), (None, 'LBRACKET'), (None, 'RBRACKET'), (None, 'LBRACE'), (None, 'RBRACE'), (None, 'PERIOD'), (None, 'SQOUTE'), (None, 'DQOUTE'), (None, 'MINUS'), (None, 'DIVIDE'), (None, 'MOD'), (None, 'AND'), (None, 'LNOT'), (None, 'LT'), (None, 'GT'), (None, 'EQUALS'), (None, 'COMMA'), (None, 'SEMI'), (None, 'COLON'), (None, 'AT')])]} _lexstateignore = {'INITIAL': ' \r\t\x0c'} _lexstateerrorf = {'INITIAL': 't_error'} _lexstateeoff = {}