diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..900fe79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +actualiza +*~ +*.orig +*.directory +*.BASE.* +*.REMOTE.* +*.LOCAL.* +*.log +*.rej +*.pyc diff --git a/INSTALL-ES b/INSTALL-ES index d6298b0..88e6e43 100644 --- a/INSTALL-ES +++ b/INSTALL-ES @@ -2,6 +2,9 @@ Parseador de Código fuente tipo Javascript para AbanQ ------------------------------------------------------- DEPENDENCIAS: +- python-ply +- git-core +- realpath flscriptparser depende únicamente del proyecto PLY de Python. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c9cf0d1 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ + +all: + @echo "> run sudo make install to install." + +install: + bash install.sh + diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/analizar_extensiones.sh b/analizar_extensiones.sh new file mode 100755 index 0000000..5299ba2 --- /dev/null +++ b/analizar_extensiones.sh @@ -0,0 +1,88 @@ +#!/bin/bash +PRJPATH=$1 +BUILDPATH=$2 + +# Para calcular todas las extensiones: +# for i in ext*-*; do eneboo-assembler build $i fullpatch; eneboo-assembler build $i revfullpatch; done + +# for i in ext*-*; do test -f "$i/build/fullpatch/fullpatch.xml" || { echo $i; eneboo-assembler build $i fullpatch; } done +# for i in ext*-*; do test -f "$i/build/revfullpatch/revfullpatch.xml" || { echo $i; eneboo-assembler build $i revfullpatch; } done + +if [ -z "$BUILDPATH" ]; then + rm .patchtest/$PRJPATH.ext.* + echo /home/gestiweb/git/eneboo-features-fede/*/build | xargs -d ' ' -n1 -P2 $0 $1 + exit 0; +fi +UNPATCHPATH=$BUILDPATH/revfullpatch +REPATCHPATH=$BUILDPATH/fullpatch +MAINPATCH=$BUILDPATH/../patches/ + +PATCHNAME=$(echo $BUILDPATH | cut -f6 -d'/') +#echo ":: $PRJPATH -- $PATCHNAME " + + +test -f "$UNPATCHPATH/revfullpatch.xml" || { echo "NO EXISTE :: $UNPATCHPATH/revfullpatch.xml"; exit 1; } +test -f "$REPATCHPATH/fullpatch.xml" || { echo "NO EXISTE :: $REPATCHPATH/fullpatch.xml"; exit 2; } +test -d "$MAINPATCH" || { echo "NO EXISTE :: $MAINPATCH"; exit 3; } + +mkdir -p ".patchtest/$PRJPATH/" + +DSTFOLDER=.patchtest/$PRJPATH/$PATCHNAME-uninstall +test -d "$DSTFOLDER" || \ +eneboo-mergetool folder-patch "$UNPATCHPATH" "$PRJPATH" "$DSTFOLDER" >/dev/null 2>&1 + + +DSTFOLDER2=.patchtest/$PRJPATH/$PATCHNAME-reinstall +test -d "$DSTFOLDER2" || \ +eneboo-mergetool folder-patch "$REPATCHPATH" "$DSTFOLDER" "$DSTFOLDER2" >/dev/null 2>&1 + + +diff -rN -t -b -d -U1 --exclude=".git" -x '*.png' -x '*.jpg' -x "*.qry" \ + -x "*.kut" -x "*.ui" -x '*~' -x '*.*.*' "$PRJPATH" "$DSTFOLDER" > "$DSTFOLDER.patch" + + diff -rN -t -b -d -U1 --exclude=".git" -x '*.png' -x '*.jpg' -x "*.qry" \ + -x "*.kut" -x "*.ui" -x '*~' -x '*.*.*' "$DSTFOLDER" "$DSTFOLDER2" > "$DSTFOLDER2.patch" + +grep -aE "^-[^-]" "$DSTFOLDER.patch" > "$DSTFOLDER.patch.grep" +grep -aE '^\+[^\+]' "$DSTFOLDER2.patch" > "$DSTFOLDER2.patch.grep" + +#rm "$DSTFOLDER" -R +#rm "$DSTFOLDER2" -R + +BYTES_UNPATCH=$(cat "$DSTFOLDER.patch.grep" | wc -c) +BYTES_REPATCH=$(cat "$DSTFOLDER2.patch.grep" | wc -c ) + +BYTES_MAINPATCH=$(du -bs $MAINPATCH --exclude='*.png' --exclude='*.jpg' --exclude="*.qry" \ + --exclude="*.kut" --exclude="*.ui" --exclude="*~" | cut -f1) + +BYTES_MIN=$(( $BYTES_MAINPATCH / 2)) +BYTES_MIN2=$(( $BYTES_MAINPATCH / 5)) + +DEST=.patchtest/$PRJPATH.ext.others.txt +if [ "$BYTES_UNPATCH" -lt "$BYTES_MIN2" ]; then + exit 0; +fi + +if [ "$BYTES_UNPATCH" -gt "$BYTES_MIN" -a "$BYTES_REPATCH" -gt "$BYTES_MIN" ]; then + DEST=.patchtest/$PRJPATH.ext.installed.txt + echo "EXTENSION :: $PATCHNAME - patch: $BYTES_MAINPATCH reinstall: $BYTES_REPATCH uninstall: $BYTES_UNPATCH" + echo $PATCHNAME >> $DEST + exit 0; +fi + +if [ "$BYTES_UNPATCH" -gt "$BYTES_MIN" ]; then + DEST=.patchtest/$PRJPATH.ext.unsure.txt + echo "UNSURE :: $PATCHNAME - patch: $BYTES_MAINPATCH reinstall: $BYTES_REPATCH uninstall: $BYTES_UNPATCH" + echo $PATCHNAME >> $DEST + exit 0; +fi + +if [ "$BYTES_UNPATCH" -gt "$BYTES_MIN2" -a "$BYTES_REPATCH" -gt "$BYTES_MIN2" ]; then + DEST=.patchtest/$PRJPATH.ext.unsure.txt + echo "UNSURE :: $PATCHNAME - patch: $BYTES_MAINPATCH reinstall: $BYTES_REPATCH uninstall: $BYTES_UNPATCH" +else + echo " :: $PATCHNAME - patch: $BYTES_MAINPATCH reinstall: $BYTES_REPATCH uninstall: $BYTES_UNPATCH" +fi + + +echo $PATCHNAME >> $DEST diff --git a/analyze_controls.sh b/analyze_controls.sh new file mode 100755 index 0000000..d9419d0 --- /dev/null +++ b/analyze_controls.sh @@ -0,0 +1,13 @@ + +for file_ui in $( git ls-files -- "*.ui" ); do + file_qs1="${file_ui%\.*}.qs" + file_qs="${file_qs1/forms/scripts}" + + + test -f "$file_qs" && { + for control in $(grep -Po "this\.child\(.(\w+).\)" "$file_qs" | sort -u | sed "s/\"/\t/g" | awk '{print $2}'); do + grep -Eq "$control" "$file_ui" || echo $file_qs $control; + done + } || echo "ERROR: $file_qs" + +done diff --git a/dependencies.debian b/dependencies.debian new file mode 100644 index 0000000..d2208d7 --- /dev/null +++ b/dependencies.debian @@ -0,0 +1,6 @@ +python3-ply +python3-lxml +python3-six +python3-future +git-core +realpath diff --git a/flalign.py b/flalign.py new file mode 100644 index 0000000..b16f80a --- /dev/null +++ b/flalign.py @@ -0,0 +1,489 @@ +from __future__ import print_function +from builtins import next +from builtins import object +import os, os.path, sys +import math +from optparse import OptionParser +import difflib +import re + +class ProcessFile(object): + def __init__(self, filename): + self.filename = filename + self.process() + self.indexLines() + self.bnames = set(self.idxnames.keys()) + + def process(self): + self.blocks = [] + self.idxlines = {} + self.idxnames = {} + self.sortednames = [] + + fblocks = open(self.filename + ".blocks") + n = 0 + line = fblocks.readline() + while line: + start, end, name = line.strip().split("\t") + self.blocks.append( (start,end, name) ) + self.idxlines[start] = n + self.sortednames.append(name) + if name in self.idxnames: + self.idxnames[name] = None + else: + self.idxnames[name] = n + n+=1 + line = fblocks.readline() + + def indexLines(self): + fqs = open(self.filename) + self.lines = [] + for line in fqs: + self.lines.append(line) + + + + def diffTo(self,pfile2): + added = pfile2.bnames - self.bnames + deleted = self.bnames - pfile2.bnames + + return added, deleted + +class LineNumber(object): + def __init__(self, letter, lines): + self.nl=0 + if len(letter)>1: + self.diffFrom = letter[0] + else: + self.diffFrom = None + self.error = None + + self.letter = letter[-1] + self.lines = lines + + def line(self): + l = self.lines[self.nl] + if self.diffFrom: + return l[2:] + else: + return l[:] + + def symbol(self): + l = self.lines[self.nl] + if self.diffFrom: + return l[0] + else: + return " " + + def next(self,t=1): + self.nl+=int(t) + + def __iadd__(self,other): + self.nl+=int(other) + return self + + def __int__(self): + return self.nl + + def __index__(self): + return self.nl + + +def appliedDiff(C, A, B, prefer = "C", debug = False, quiet = False, swap = False): + diffAB = list(difflib.ndiff(A.sortednames, B.sortednames)) + diffAC = list(difflib.ndiff(A.sortednames, C.sortednames)) + + + nlA = LineNumber("A",A.sortednames) + nlB = LineNumber("B",B.sortednames) + nlC = LineNumber("C",C.sortednames) + nlAB = LineNumber("AB", diffAB) + nlAC = LineNumber("AC", diffAC) + + maxA = len(A.sortednames) + maxB = len(B.sortednames) + maxC = len(C.sortednames) + patchedResult = [] + + def AddPatchLine(mode): + modefrom = mode[0] + modetype = mode[1] + linefrom = None + linetext = "" + conflict = False + linenumbers = int(nlA),int(nlB),int(nlC) + if modetype == "-": + linetext = lineaA + linefrom = "A" + if modefrom == "B": + DeletedB.append(lineaA) + elif modefrom == "C": + DeletedC.append(lineaA) + elif modetype == "=": + modefrom = prefer + if modefrom == "A": + linetext = lineaA + elif modefrom == "B": + linetext = lineaB + elif modefrom == "C": + linetext = lineaC + linefrom = modefrom + elif modetype == "+": + if modefrom == "A": + linetext = lineaA + elif modefrom == "B": + linetext = lineaB + if lineaB in AddedB: conflict = True + if lineaB in AddedC: conflict = True + AddedB.append(lineaB) + elif modefrom == "C": + linetext = lineaC + if lineaC in AddedB: conflict = True + if lineaC in AddedC: conflict = True + AddedC.append(lineaC) + linefrom = modefrom + line = ( + modefrom, modetype, + linenumbers, + linefrom, + linetext + ) + #ConflictMode = False + if len(ConflictMode): + if linetext[0]!="#": ConflictMode.pop() + else: + #print ">>", len(ConflictMode),linetext + if len(ConflictMode)>1 and linetext.find("separator") == -1: + ConflictMode.pop() + elif linetext.find("separator") >= 0: + ConflictMode.pop() + return + if conflict and linetext[0]!="#": + if debug: + print("WARNING: Omitting previously added block <%s>" % linetext) + + while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") >= 0: + #print "sep//", + patchedResult.pop() + + while patchedResult[-1][4][0]=="#" and patchedResult[-1][4].find("separator") == -1: + #print "comm//", + patchedResult.pop() + #if patchedResult[-1][4][0]=="#": patchedResult.pop() + + ConflictMode.append(1) + ConflictMode.append(1) + else: + patchedResult.append(line) + + AddedB = [] + AddedC = [] + DeletedB = [] + DeletedC = [] + ConflictMode = [] + + def getVars(letter): + if letter=="B": + return nlA, nlB, nlC, nlAB, nlAC + if letter=="C": + return nlA, nlC, nlB, nlAC, nlAB + raise TypeError + + def Patch(code): + letter = code[0] + sign = code[1] + nBase, nLocal, nRemote, nDiffLocal, nDiffRemote = getVars(letter) + + def Minus(): + AddPatchLine(letter + "-") + if debug: + print(letter + "-%04d" % int(nBase), nBase.line()) + if nBase.line() != nDiffLocal.line() and not quiet: + print(letter + "! " , nDiffLocal.line()) + if nDiffRemote.symbol()!=" " and not quiet: + if nDiffRemote.symbol()=="-": + #Removing twice! + nBase.error = "removing-twice" + else: + print("??", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) + else: + next(nRemote) + + next(nDiffLocal) + next(nDiffRemote) + next(nBase) + + def Plus(): + AddPatchLine(letter + "+") + if debug: + print(letter + "+%04d" % int(nLocal), nLocal.line()) + if nLocal.line()!=nDiffLocal.line() and not quiet: + print(letter + "! " , nDiffLocal.line()) + if nBase.error == "removing-twice": + nBase.error = None + while nDiffRemote.symbol() == "?": + next(nDiffRemote) + if nDiffRemote.symbol() == "+": + if not quiet: + print("!~", nRemote.letter, nDiffRemote.symbol(),nDiffRemote.line()) + next(nDiffRemote) + next(nRemote) + + + next(nDiffLocal) + next(nLocal) + + def Info(): + #if debug: print letter + "?> " , nDiffLocal.line() + next(nDiffLocal) + + if sign == "-": Minus() + elif sign == "+": Plus() + elif sign == "?": Info() + + + + while True: + + if ( + int(nlA) >= maxA and + int(nlB) >= maxB and + int(nlC) >= maxC + ): break + lineaA = " " + sAB = cAB = lineaA = " " + sAC = cAC = lineaC = " " + if int(nlA) >= maxA : + if int(nlA) - maxA > 0 : print("A overflow!", int(nlA) - maxA) + lineaA = " " + else: lineaA = A.sortednames[nlA] + + if int(nlB) >= maxB : + sAB = cAB = lineaA = " " + if int(nlB) - maxB > 0 : print("B overflow!", int(nlB) - maxB) + else: + lineaB = B.sortednames[nlB] + + if int(nlAB) < len(diffAB): + sAB = diffAB[nlAB][0] + cAB = diffAB[nlAB][2:] + + if int(nlC) >= maxC : + sAC = cAC = lineaC = " " + if int(nlC) - maxC > 0 : print("C overflow!", int(nlC) - maxC) + else: + lineaC = C.sortednames[nlC] + + if int(nlAC) < len(diffAC): + sAC = diffAC[nlAC][0] + cAC = diffAC[nlAC][2:] + + #print nlA, nlB, nlC + if sAB == " " and sAC == " ": + AddPatchLine("A=") + if debug: + if prefer == "": + print("ABC=%04d%+d%+d" % (nlA,nlB-nlA,nlC-nlA),lineaA) + elif prefer == "A": + print("A=%04d" % nlA,lineaA) + elif prefer == "B": + print("B=%04d" % nlB,lineaB) + elif prefer == "C": + print("C=%04d" % nlC,lineaC) + else: + assert prefer in ["A","B","C"] + if not quiet: + if lineaA!=cAB: + print("A!=B " , cAB) + if lineaA!=cAC: + print("A!=C " , cAC) + if lineaA != lineaB or lineaC != lineaA: + print("wtf!?A:", lineaA) + print("wtf!?B:", lineaB) + print("wtf!?C:", lineaC) + + next(nlAB) + next(nlAC) + next(nlA) + next(nlB) + next(nlC) + elif swap and sAB=="+": Patch("B+") + elif sAC=="+": Patch("C+") + elif sAC=="?": Patch("C?") + elif sAB=="+": Patch("B+") + elif sAB=="?": Patch("B?") + elif swap and sAB=="-": Patch("B-") + elif sAC=="-": Patch("C-") + elif sAB=="-": Patch("B-") + else: + if not quiet: + print(sAB,"*", sAC) + break + + addedB = set([x for x in AddedB if x[0]!="#"]) + addedC = set([x for x in AddedC if x[0]!="#"]) + + deletedB = set([x for x in DeletedB if x[0]!="#"]) + deletedC = set([x for x in DeletedC if x[0]!="#"]) + + movedB = addedB & deletedB + movedC = addedC & deletedC + + conflictsAA = addedB & addedC + conflictsDD = deletedB & deletedC + conflictsAD = addedB & deletedC + conflictsDA = deletedB & addedC + + if movedB: + print("CONFLICTS BLOCK MOVED A(%s)->B(%s):" % (A.filename,B.filename)) + for name in movedB: print("-",name) + + if movedC: + print("CONFLICTS BLOCK MOVED A(%s)->C(%s):" % (A.filename,C.filename)) + for name in movedC: print("-",name) + + if conflictsAA: + print("CONFLICTS SAME BLOCK ADDED B(%s)-C(%s):" % (B.filename,C.filename)) + for name in conflictsAA: print("-",name) + + if conflictsDD: + print("CONFLICTS SAME BLOCK DELETED B(%s)-C(%s):" % (B.filename,C.filename)) + for name in conflictsDD: print("-",name) + + if conflictsAD: + print("CONFLICTS BLOCK ADDED BY %s , DELETED BY %s:" % (B.filename,C.filename)) + for name in conflictsAD: print("-",name) + + if conflictsDA: + print("CONFLICTS BLOCK DELETED BY %s , ADDED BY %s:" % (B.filename,C.filename)) + for name in conflictsDA: print("-",name) + + + + return patchedResult + """ + added, deleted = pfrom.diffTo(pto) + plist = [] + for start,end,name in ptarget.blocks: + if name in added: + n = pto.idxnames[name] + if n is not None: + start,end,name = pto.blocks[n] + bobject = pto.filename + else: + print "Conflict with element", name + elif name in deleted: + bobject = "deleted" + plist.append((bobject,start,end,name)) + else: + bobject = ptarget.filename + plist.append((bobject,start,end,name)) + + return plist +""" + +def writeAlignedFile(C, A, B, prefer = "C", debug = False, quiet = False, swap = False): + patchlist = appliedDiff(C, A, B, prefer , debug, quiet, swap) + F = {"A": A, "B": B, "C": C} + L = ["A", "B", "C"] + + fout = open(F[prefer].filename + ".aligned","w") + classlist = [] + for Fby,action, nlines, Fwhich, line in patchlist: + nlA, nlB, nlC = nlines + if action not in ("+","="): continue + nl = nlines[L.index(Fwhich)] + try: + linebegin, lineend, line = F[Fwhich].blocks[nl] + except IndexError: + print("!!!ERROR MERGING!!") + continue + + text = "".join( + F[Fwhich].lines[int(linebegin):int(lineend)] + ) + #text = text.replace("\t"," ") + sline = line.split(":") + if sline[0]=="classdeclaration": + if len(classlist): + lastclass = classlist[-1] + else: + lastclass = None + + thisclass = sline[1] + classlist.append(thisclass) + if lastclass: + rs1 = re.search("class (\w+) extends (\w+)",text) + if rs1: + if lastclass != rs1.group(2): + #print "INFO: Changing >> class", thisclass, "extends",rs1.group(2), "--> extends", lastclass + text = re.sub(r"class (\w+) extends (\w+)", "class %s extends %s" % (thisclass,lastclass),text) + rs2 = re.search(r"function .*%s\(.*context.*\) { (\w+)" % thisclass,text) + if rs2: + if lastclass != rs2.group(1): + badline = rs2.group(0) + goodline = badline.replace(rs2.group(1),lastclass) + #print "INFO: Changing >>", badline , "-->", goodline + text = text.replace(badline,goodline) + + else: + print(text[:64]) + + #if debug: + # fout.write("<<< %s || %s >>>\n" % (Fwhich, line)) + # fout.write("<<< (%d:%d) >>>\n" % (int(linebegin),int(lineend))) + fout.write(text) + + + fout.close() + +def main(): + parser = OptionParser() + #parser.add_option("-q", "--quiet", + # action="store_false", dest="verbose", default=True, + # help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("-q","--quiet", + action="store_true", dest="quiet", default=False, + help="don't print status messages to stdout") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + + filenames = [x for x in args if os.path.isfile(x)] + not_a_file = set(args) - set(filenames) + if len(not_a_file): + print("WARNING: Not a file:", ", ".join(not_a_file)) + return + + if len(filenames) != 3: + print("MUST have exactly 3 files to align.") + pfiles = [] + for file1 in filenames: + #print "Load File:", file1 + pf = ProcessFile(file1) + pfiles.append(pf) + + A = pfiles[0] + B = pfiles[1] + C = pfiles[2] + + #addedAB, deletedAB = A.diffTo(B) + #addedAC, deletedAC = A.diffTo(C) + is_debug = options.debug + writeAlignedFile(C, A, B, swap = True, debug= is_debug) + writeAlignedFile(B, A, C, debug= is_debug) + writeAlignedFile(B, A, C, prefer = "A", debug= is_debug) + #writeAlignedFile(A, A, C) + + +if __name__ == "__main__": main() diff --git a/flclasses.py b/flclasses.py index 5f3a0ae..2bb0f00 100644 --- a/flclasses.py +++ b/flclasses.py @@ -1,6 +1,9 @@ +from __future__ import print_function +from builtins import str +from builtins import object debug = 0 -class cBase: +class cBase(object): def __init__(self): self.type = ("Unknown","Unknown") self.codedepth = 0 @@ -8,14 +11,14 @@ def __init__(self): def setSubtype(self,newsubtype): x, y = self.type self.type = (x,newsubtype) - + def setType(self,newtype): x, y = self.type self.type = (newtype,y) - + def addCodeDepth(self): self.codedepth += 1 - + def __len__(self): return 1; @@ -40,10 +43,10 @@ def __init__(self,itemList,prefix,suffix,subtype="Unknown"): t,st = itemList.slice[0].type #itemList.setSubtype(st) itemList.setSubtype("OneItem") - - + + if not isinstance(itemList,cBaseList): - raise NameError, "itemList no es un cBaseList: %s" % repr(itemList) + raise NameError("itemList no es un cBaseList: %s" % repr(itemList)) self.itemList = itemList self.prefix = prefix self.suffix = suffix @@ -52,15 +55,15 @@ def __init__(self,itemList,prefix,suffix,subtype="Unknown"): ctype, csubtype = item.type ctype = subtype item.type = (ctype, csubtype) - + def __str__(self): global debug txt = str(self.prefix) + str(self.itemList) + str(self.suffix) txt = txt.strip() if debug>=2: txt = "{%s/%s:" % self.itemList.type + txt + "}" - return txt - + return txt + class cBaseVarSpec(cBase): def __init__(self,name,vartype=None,value=None): @@ -76,8 +79,8 @@ def __str__(self): if self.value: txt+="="+str(self.value) return txt - - + + class cBaseList(cBase): def __init__(self): cBase.__init__(self) @@ -93,18 +96,18 @@ def __len__(self): def includeItem(self,child): if not isinstance(child,cBase): - raise NameError, "Child is not an instance of Base Class!" - + raise NameError("Child is not an instance of Base Class!") + try: ctype, csubtype = child.type except: - raise NameError, "Base Class doesn't have `type` atribute or is incorrect." + raise NameError("Base Class doesn't have `type` atribute or is incorrect.") if not hasattr(self.byType,ctype): self.byType[ctype]=[] if not hasattr(self.bySubtype,ctype): self.bySubtype["%s:%s" % (ctype,csubtype)]=[] - + self.byType[ctype].append(child) self.bySubtype["%s:%s" % (ctype,csubtype)].append(child) @@ -112,31 +115,31 @@ def includeItem(self,child): try: cname = child.name except: - raise NameError, "Declaration Class doesn't have `name` atribute." - + raise NameError("Declaration Class doesn't have `name` atribute.") + if cname in self.byDefName: - print "#WARNING# Variable %s found, but previously defined in this block" % cname + print("#WARNING# Variable %s found, but previously defined in this block" % cname) # self.byDefName[cname]=None else: self.byDefName[cname]=child try: child.addCodeDepth() except: - print repr(child) - raise NameError, "Base Class doesn't have `addCodeDepth` function." + print(repr(child)) + raise NameError("Base Class doesn't have `addCodeDepth` function.") if isinstance(child,cBaseItemList): sslice = child.itemList.slice[:] for e in child.itemList.hidden: sslice.remove(e) - - + + for item in sslice: itype, isubtype = item.type if not isinstance(item,cBaseItemList): self.includeItem(item) #self.hidden.append(item) - + def addCodeDepth(self): cBase.addCodeDepth(self) sslice = self.slice[:] @@ -146,7 +149,7 @@ def addCodeDepth(self): for child in sslice: child.addCodeDepth() - + def addAuto(self,element,subtype=None): @@ -154,14 +157,14 @@ def addAuto(self,element,subtype=None): element = cBaseItem(element) if subtype: element.setSubtype(subtype) - + self.addChild(element) - + def addChild(self,child,hidden=False): self.includeItem(child) self.slice.append(child) if hidden: self.hidden.append(child) - + def __str__(self): global debug sslice = self.slice[:] @@ -172,7 +175,7 @@ def __str__(self): if debug>=1: txt = "\n" - # --------- DEBUG OUTPUT VARDECL --------- + # --------- DEBUG OUTPUT VARDECL --------- if len(self.byDefName)>0: txt += " " * self.codedepth + " /** Declared vars: " for definition in self.byDefName: @@ -187,7 +190,7 @@ def __str__(self): return txt - + if len(sslice) == 1: c = str(sslice[0]) if "\n" not in c: @@ -199,7 +202,7 @@ def __str__(self): # txt += str(child.type) + "\n" # txt += str(child) + "\n" # txt += "____"+ "\n" - + lastmargin = 0 for c in sslice: t1, t2 = c.type @@ -214,9 +217,9 @@ def __str__(self): txt += "\n" * topmargin + line + "\n" * (margin+1) lastmargin = margin return txt - - - + + + class cBaseListInline(cBaseList): def __init__(self,separator=", "): cBaseList.__init__(self) @@ -227,20 +230,20 @@ def __str__(self): txt = "" for c in self.slice: if debug>=3: - txt += "[%s/%s: " % c.type - + txt += "[%s/%s: " % c.type + txt += str(c) if debug>=3: txt += "]" - + txt += self.separator - + if len(self.separator) == 0: return txt else: return txt[:-len(self.separator)] - - + + class cStatementList(cBaseList): def __init__(self): cBaseList.__init__(self) @@ -251,7 +254,7 @@ def __init__(self,name): cBase.__init__(self) self.type = ("Declaration","Unknown") self.name = name - + def __str__(self): return "@unknown declaration %s" % self.name @@ -266,26 +269,26 @@ def __init__(self,name,arglist,rettype,source): iList = cBaseList() iList.addAuto(source) source = iList - - + + if not isinstance(source,cBaseList): - raise NameError, "source no es un cBaseList: %s" % repr(itemList) + raise NameError("source no es un cBaseList: %s" % repr(source)) self.source = source - + def addCodeDepth(self): cBase.addCodeDepth(self) try: self.source.addCodeDepth() except: import traceback,sys - print "Exception in user code:" - print '-'*60 + print("Exception in user code:") + print('-'*60) traceback.print_exc(file=sys.stdout) - print '-'*60 + print('-'*60) + + - - def __str__(self): if self.rettype: @@ -305,28 +308,28 @@ def __init__(self,name,extends,source): iList = cBaseList() iList.addAuto(source) source = iList - - + + if not isinstance(source,cBaseList): - raise NameError, "source no es un cBaseList: %s" % repr(itemList) + raise NameError("source no es un cBaseList: %s" % repr(source)) self.source = source - + def addCodeDepth(self): cBase.addCodeDepth(self) try: self.source.addCodeDepth() except: import traceback,sys - print "Exception in user code:" - print '-'*60 + print("Exception in user code:") + print('-'*60) traceback.print_exc(file=sys.stdout) - print '-'*60 + print('-'*60) def __str__(self): if self.extends: ext = " extends " + self.extends else: ext = "" - + return 'class %s%s {%s}' % (self.name,ext,self.source) diff --git a/flex.py b/flex.py index ee93427..59a7c3d 100644 --- a/flex.py +++ b/flex.py @@ -1,3 +1,4 @@ +from __future__ import print_function # ---------------------------------------------------------------------- # clex.py # @@ -8,32 +9,34 @@ #sys.path.insert(0,"../..") import ply.lex as lex +from ply.lex import TOKEN + # Reserved words reserved = [ - 'BREAK', 'CASE', 'CONST', 'CONTINUE', 'DEFAULT', 'DO', - 'ELSE', 'FOR', 'IF', - 'RETURN', - #'STRUCT', - 'SWITCH', - 'WHILE', 'CLASS', 'VAR', 'FUNCTION', - 'EXTENDS', 'NEW','WITH','TRY','CATCH','THROW', 'DELETE' + 'BREAK', 'CASE', 'CONST', 'STATIC', 'CONTINUE', 'DEFAULT', 'DO', + 'ELSE', 'FOR', 'IF', 'IN', + 'RETURN', + #'STRUCT', + 'SWITCH', + 'WHILE', 'CLASS', 'VAR', 'FUNCTION', + 'EXTENDS', 'NEW','WITH','TRY','CATCH','THROW', 'DELETE', 'TYPEOF' ] token_literals = [ # Literals (identifier, integer constant, float constant, string constant, char const) - 'ID', 'ICONST', 'FCONST', 'SCONST', 'CCONST', 'RXCONST' + 'ID', 'ICONST', 'FCONST', 'SCONST', 'CCONST' #, 'RXCONST' ] tokens = reserved + token_literals + [ # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=) 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', - 'OR', 'AND', + 'OR', 'AND', 'CONDITIONAL1','AT', - #'NOT', + #'NOT', 'XOR', 'LSHIFT', 'RSHIFT', 'LOR', 'LAND', 'LNOT', - 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', - + 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'EQQ', 'NEQ', + # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', # 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL', @@ -46,7 +49,7 @@ # Conditional operator (?) # 'CONDOP', - + # Delimeters ( ) [ ] { } , . ; : 'LPAREN', 'RPAREN', 'LBRACKET', 'RBRACKET', @@ -58,17 +61,25 @@ 'DOCSTRINGOPEN', # 'COMMENTOPEN', 'COMMENTCLOSE', + 'DOLLAR', + 'SQOUTE', + 'DQOUTE', + 'BACKSLASH', ] # Completely ignored characters t_ignore = ' \r\t\x0c' # Newlines +@TOKEN(r'\n+') def t_NEWLINE(t): - r'\n+' t.lexer.lineno += t.value.count("\n") - + # Operators +t_BACKSLASH = '\\\\' +t_DOLLAR = r'\$' +t_SQOUTE = '\'' +t_DQOUTE = '"' t_PLUS = r'\+' t_MINUS = r'-' t_TIMES = r'\*' @@ -89,6 +100,8 @@ def t_NEWLINE(t): t_GE = r'>=' t_EQ = r'==' t_NE = r'!=' +t_EQQ = r'===' +t_NEQ = r'!==' t_CONDITIONAL1 = r'\?' # Assignment operators @@ -139,9 +152,8 @@ def t_NEWLINE(t): +@TOKEN(r'[A-Za-z_]+[\w_]*') def t_ID(t): -# r'[A-Za-z_]+([\.]{0,1}[\w_]*)+' - r'[A-Za-z_]+[\w_]*' t.type = reserved_map.get(t.value,"ID") return t @@ -154,46 +166,47 @@ def t_ID(t): t_FCONST = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' # String literal -t_SCONST = r'\"([^\\\n]|(\\.))*?\"' +t_SCONST = r'\"([^\"\\\n]|(\\.)|\\\n)*?\"' # Character constant 'c' or L'c' -t_CCONST = r'(L)?\'([^\\\n]|(\\.))*?\'' +t_CCONST = r'\'([^\'\\\n]|(\\.)|\\\n)*?\'' -# REGEX constant -t_RXCONST = r'/.+/g' +# REGEX constant +#t_RXCONST = r'/[^/ ]+/g?' # Comments +@TOKEN(r'(/\*( |\*\*)(.|\n)*?\*/)|(//.*)') def t_comment(t): - r'(/\*( |\*\*)(.|\n)*?\*/)|(//.*)' - #r'/\*(.|\n)*?\*/' t.lexer.lineno += t.value.count('\n') +@TOKEN(r'/\*\*[ ]+') def t_DOCSTRINGOPEN(t): - r'/\*\*[ ]+' return t; #t_COMMENTOPEN = r'/\*' t_COMMENTCLOSE = r'\*/' - + # Preprocessor directive (ignored) +@TOKEN(r'\#(.)*?\n') def t_preprocessor(t): - r'\#(.)*?\n' t.lexer.lineno += 1 - + def t_error(t): - print "Illegal character %s" % repr(t.value[0]) + print("Illegal character %s" % repr(t.value[0])) t.lexer.skip(1) - -lexer = lex.lex(debug=False) +# TODO: Cada vez que se cambia este fichero, se tiene que lanzar sin el "-OO" de python para acelerar. Construye entonces +# ..... el fichero de cache que subimos a git, y se relee desde ahí las siguientes veces con el -OO. +# ..... Si da problemas, hay que volver a optimize=0 y/o eliminar lextab.py +lexer = lex.lex(debug=False,optimize=1) if __name__ == "__main__": lex.runmain(lexer) - + diff --git a/flmergetool b/flmergetool new file mode 100755 index 0000000..4d2845a --- /dev/null +++ b/flmergetool @@ -0,0 +1,199 @@ +#!/bin/bash + +# FLMERGETOOL. +# To add this tool to git, do the following: +# flmergetool INSTALL +editors=(medit kate gedit kwrite nano vim vi) +EDITOR_FILEPATH= +for editor in "${editors[@]}"; do + EDITOR_FILEPATH=$(command -v $editor) + [[ $EDITOR_FILEPATH ]] && break +done +echo $EDITOR_FILEPATH + +thisprg=`realpath $0` +thisdir=`dirname $thisprg` +if test "$1" = "INSTALL" +then + sudo ln -s $thisprg /usr/local/bin/flmergetool + git config --global mergetool.flmergetool.cmd \ + "flmergetool \$MERGED \$BASE \$LOCAL \$REMOTE" + git config --global mergetool.flmergetool.trustExitCode true + exit 0 +fi +if [[ $1 == "--debug" ]]; then + DEBUG="--debug" + shift +else + DEBUG="" +fi +MERGE=$1 +BASE=$2 +LOCAL=$4 +REMOTE=$3 +if test -f $LOCAL +then + OK=1 +else + echo "File $LOCAL does not exist." + exit 1 +fi + +if test -f $REMOTE +then + OK=1 +else + echo "File $REMOTE does not exist." + exit 1 +fi + +if test -f $BASE +then + OK=1 +else + if kdiff3 $LOCAL $REMOTE -o $MERGE >/dev/null 2>&1 + then + #echo "KDIFF3 OK" + OK=1 + exit 0 + else + echo "KDIFF3 FAILED" + exit 1 + fi +fi + + +merge_other() { + kdiff3 "$BASE" "$LOCAL" "$REMOTE" -o "$MERGE" --auto >/dev/null 2>&1 + return $? +} + +merge_xml() { + python $thisdir/xml2json.py convert "$BASE" "$LOCAL" "$REMOTE" && \ + kdiff3 "$BASE.json" "$LOCAL.json" "$REMOTE.json" -o "$MERGE.json" --auto >/dev/null 2>&1 && \ + python $thisdir/xml2json.py revert "$MERGE.json" && \ + cp $MERGE.json.* $MERGE + for f in "$BASE" "$LOCAL" "$REMOTE" "$MERGE"; do + rm "$f".json "$f".json.* + done + return $? +} + +merge_qs() { + if echo $MERGE | grep -E '\.qs' + then + OK=1 + else + echo "File $MERGE does not have .qs" + exit 1 + fi + + ( + for file1 in "$BASE" "$LOCAL" "$REMOTE"; do + is_done=1 + until [[ $is_done == 0 ]] + do + if python $thisdir/flscriptparse.py -O file "$file1" && test -f "$file1.xml" + then + #echo "FLScriptParse OK" + is_done=0 + else + patched=0 + for kpatch in /tmp/knownpatch.*; do + [[ $kpatch == "/tmp/knownpatch.*" ]] && break + patch "$file1" "$kpatch" -f -F4 -l && { + patched=1 + } + done + + read -p "Fallo en el parseo del fichero, desea abrir un editor para corregirlo manualmente? [Y/n]:" editor + [[ $editor == "n" ]] && { + echo "FLScriptParse FAILED" + exit 1 + } + bname=$(basename "$file1") + cp $file1 /tmp/$bname.orig + "$EDITOR_FILEPATH" "$file1" + read -p "Pulse Intro si ha terminado." + diff -U6 -p --minimal -d /tmp/$bname.orig $file1 >/tmp/knownpatch.$bname.patch.tmp + if [[ $? == 1 ]] && [[ $(wc -l /tmp/knownpatch.$bname.patch.tmp) > 0 ]] ; then + mv /tmp/knownpatch.$bname.patch.tmp /tmp/knownpatch.$bname.patch + else + unlink /tmp/knownpatch.$bname.patch.tmp + fi + + fi + done + done + + + if python $thisdir/flpremerge.py $DEBUG $BASE $LOCAL $REMOTE + then + #echo "FLPreMerge OK" + OK=1 + else + echo "FLPreMerge FAILED" + exit 1 + fi + + if python $thisdir/flalign.py $DEBUG $BASE $LOCAL $REMOTE + then + #echo "FLAlign OK" + OK=1 + else + echo "FLAlign FAILED" + exit 1 + fi + + if test -f $BASE.aligned && test -f $LOCAL.aligned && test -f $REMOTE.aligned + then + OK=1 + else + echo "FLAlign FAILED" + exit 1 + fi + unlink $MERGE + #echo "kdiff3 $BASE.aligned $LOCAL.aligned $REMOTE.aligned -o $MERGE --auto" + if kdiff3 --auto $BASE.aligned $LOCAL.aligned $REMOTE.aligned -o $MERGE >/dev/null 2>&1 + then + #echo "KDIFF3 OK" + OK=1 + else + echo "KDIFF3 FAILED" + exit 1 + fi + ) + + for file1 in "$BASE" "$LOCAL" "$REMOTE"; do + unlink "$file1.xml" + unlink "$file1.blocks" + unlink "$file1.aligned" + unlink "$file1.hash" + done + + + return $?; +} + + + + + + +if echo $MERGE | grep -E '\.qs' ; then + merge_qs + exit $? +fi +if echo $MERGE | grep -E '\.(xml|mtd|ui)' ; then + merge_xml + exit $? +fi +if echo $MERGE | grep -E '\.(kut|qry)' ; then + merge_other + exit $? +fi + +print "Unknown extension." +exit 3 + + diff --git a/flparser b/flparser new file mode 120000 index 0000000..945c9b4 --- /dev/null +++ b/flparser @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/flpatcher b/flpatcher new file mode 100755 index 0000000..290df58 --- /dev/null +++ b/flpatcher @@ -0,0 +1,356 @@ +#!/bin/bash +APP_DIR=$(dirname $(realpath $0)) + + +SRC_PROJECT="" +SRC_START_COMMIT="" +SRC_END_COMMIT="HEAD" +DST_PROJECT=$(pwd) +DST_APPLY_COMMIT="HEAD" +TYPE_CS="" +TYPE_BL="" + +show_help() { + echo " $(basename $0) [options]" + echo + echo " ## options ##" + echo + echo " --src=FOLDER -- sets FOLDER as the source project" + echo " --dst=FOLDER -- sets FOLDER as the dest project" + echo " --start=COMMIT -- sets COMMIT as the source start commit" + echo " --end=COMMIT -- sets COMMIT as the source end commit" + echo " --apply=COMMIT -- sets COMMIT as the dest apply commit" + echo " --help -- shows this help message" + echo " --cs=(m|l|r) -- sets the default action for Both Created" + echo " m: 2-way merge l: use local r: use remote" + echo " --bl=(b|i) -- sets the default action for Local Deleted" + echo " b: delete i: include remote" + echo + echo " ## examples ## " + echo + echo " Default Invocation:" + echo " $(basename $0) --src=../otherprj --start=origin/base-code --end=dev-code " + echo + echo + + +} + + +process_argument() { + local argname="$1" + local value="$2" + + case $argname in + src) SRC_PROJECT="$value" ;; + dst) DST_PROJECT="$value" ;; + start) SRC_START_COMMIT="$value" ;; + end) SRC_END_COMMIT="$value" ;; + apply) DST_APPLY_COMMIT="$value" ;; + cs) TYPE_CS="$value" ;; + bl) TYPE_BL="$value" ;; + help) show_help; exit 0;; + *) echo "Unexpected argument '$argname'"; return 1 ;; + esac + + #echo "* $argname -> '$value'" + return 0 +} + +let n=0 + +ARGS=() + +for arg in "$@"; do + let n+=1 + if [[ $arg == --* ]] ; then + argname="${arg#--}" + value="" + + if [[ $argname == *"="* ]] ; then + value="${argname#*=}" + argname="${argname%%=*}" + fi + + process_argument "$argname" "$value" || exit 1 + + else + ARGS+=( "$arg" ) + fi +done + +let n=0 +for arg in "${ARGS[@]}"; do + let n+=1 + echo "Unexpected positional argument '$arg'" + exit 1 +done + +is_valid_git_folder() { + local gitfolder="$1" + test -d "$gitfolder" || return 1 + ( + cd "$gitfolder" || exit 1 + test -d ".git" || exit 1 + local line="" + + while read line; do + local status="${line%% *}" + local filename="${line#* }" + case $status in + ??) continue ;; + *) echo "Modified file in working directory '$filename' ($status)"; exit 1;; + esac + # echo "*$status* '$filename'" + done < <(git status --porcelain 2>/dev/null) + + ) || return 1 + +} +LAST_VALID_COMMIT_ID="" +LAST_VALID_COMMIT_MSG="" + +is_valid_git_commit() { + local gitfolder="$1" + local commit="$2" + + local COMMIT_LINE="" + pushd "$gitfolder" || exit 1 + [[ "$commit" ]] || exit 2 + COMMIT_LINE=$(git log "$commit" --pretty=oneline --abbrev=8 --abbrev-commit -1 ) + [[ $? != 0 ]] && { echo "$(pwd) git log salio con estado $? " >&2; exit 3; } + LAST_VALID_COMMIT_ID=${COMMIT_LINE:0:8} + LAST_VALID_COMMIT_MSG=${COMMIT_LINE:9} + popd + + return $? +} + +# VALIDAR -------------- +is_valid_git_folder "$SRC_PROJECT" || { + echo "El proyecto de origen '$SRC_PROJECT' no es valido." >&2; + exit 1; +} + +is_valid_git_folder "$DST_PROJECT" || { + echo "El proyecto de destino '$DST_PROJECT' no es valido." >&2; + exit 1; +} + +is_valid_git_commit "$SRC_PROJECT" "$SRC_START_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; + echo "El commit inicial <$SRC_START_COMMIT> no es valido." >&2; + exit 1; +} +echo "<$LAST_VALID_COMMIT_ID>" + +SRC_START_COMMIT=$LAST_VALID_COMMIT_ID +SRC_START_COMMIT_MSG=$LAST_VALID_COMMIT_MSG + +is_valid_git_commit "$SRC_PROJECT" "$SRC_END_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; + echo "El commit final <$SRC_END_COMMIT> no es valido." >&2; + exit 1; +} +SRC_END_COMMIT=$LAST_VALID_COMMIT_ID +SRC_END_COMMIT_MSG=$LAST_VALID_COMMIT_MSG + +is_valid_git_commit "$DST_PROJECT" "$DST_APPLY_COMMIT" || { + echo "is_valid_git_commit salio con estado $?" >&2; + echo "El commit a aplicar <$DST_APPLY_COMMIT> no es valido." >&2; + exit 1; +} +DST_APPLY_COMMIT=$LAST_VALID_COMMIT_ID +DST_APPLY_COMMIT_MSG=$LAST_VALID_COMMIT_MSG + + +echo " [$SRC_START_COMMIT] - [$SRC_END_COMMIT] -> [$DST_APPLY_COMMIT]" + +# 1.- Calcular lista de ficheros que componen la diferencia en SRC + +FILE_LIST=() +pushd "$SRC_PROJECT" >/dev/null +IFS=$'\t' +while read -a line; do + FILE_LIST+=("${line[2]}") + #echo "${line[2]}" +done < <(git diff --numstat "$SRC_START_COMMIT" "$SRC_END_COMMIT") +unset IFS + +TMP_FILES=() + +git_show() { + local commit="$1" + local src="$2" + local dst="$3" + if git show "$commit":"$src" >"$dst" 2>/dev/null + then + TMP_FILES+=("$dst") + else + unlink "$dst" + fi + +} +# 2.- Para cada fichero en la diferencia, extraer las versiones BASE y REMOTE +for filename in "${FILE_LIST[@]}"; do + basename=$(basename "$filename") + if [[ $basename =~ .+\..+ ]] ; then + fileext=".${basename##*.}" + else + fileext="" + fi + # Ojo! si el fichero ha sido creado, aparece en REMOTE pero NO aparece en BASE + # Ojo^2! si el fichero ha sido borrado, aparece en BASE pero NO aparece en REMOTE + git_show $SRC_START_COMMIT "$filename" "$DST_PROJECT/$filename.BASE$fileext" + git_show $SRC_END_COMMIT "$filename" "$DST_PROJECT/$filename.REMOTE$fileext" +done + + +popd >/dev/null +# 3.- Para cada fichero en la diferencia, extraer la versión LOCAL +pushd "$DST_PROJECT" >/dev/null +for filename in "${FILE_LIST[@]}"; do + basename=$(basename "$filename") + if [[ $basename =~ .+\..+ ]] ; then + fileext=".${basename##*.}" + else + fileext="" + fi + + git_show $DST_APPLY_COMMIT "$filename" "$DST_PROJECT/$filename.LOCAL$fileext" +done + + +# 4.- Para cada fichero en la diferencia, ejecutar el +# sistema de mezcla adecuado ** segun EXTENSION** +for filename in "${FILE_LIST[@]}"; do + basename=$(basename "$filename") + if [[ $basename =~ .+\..+ ]] ; then + fileext=".${basename##*.}" + else + fileext="" + fi + + MERGE="$filename" + BASE="$filename.BASE$fileext" + LOCAL="$filename.LOCAL$fileext" + REMOTE="$filename.REMOTE$fileext" + + if test \( -f "$BASE" \) -a \( -f "$LOCAL" \) -a \( -f "$REMOTE" \) ; then + + # MEZCLA A 3 + #echo "EXT: $fileext;" + case $fileext in + .qs) flmergetool "$MERGE" "$BASE" "$LOCAL" "$REMOTE" ;; + .ui) cp "$LOCAL" "$MERGE"; fldesigner "$BASE" "$REMOTE" "$MERGE" ;; + # mtd) + # xml) + # kut) + # qry) + *) kdiff3 --auto "$BASE" "$LOCAL" "$REMOTE" -o "$MERGE" >/dev/null 2>&1 ;; + esac + if [[ $? == 0 ]] ; then + git add "$MERGE" + else + echo "Fallo en la mezcla" + # aqui podemos preguntar al usuario + fi + elif test \( \! -f "$BASE" \) -a \( \! -f "$LOCAL" \) -a \( -f "$REMOTE" \) ; then + # FICHERO CREADO + cp "$REMOTE" "$MERGE" + git add "$MERGE" + elif test \( -f "$BASE" \) -a \( -f "$LOCAL" \) -a \( \! -f "$REMOTE" \) ; then + # FICHERO BORRADO + git rm "$MERGE" + elif test \( -f "$BASE" \) -a \( \! -f "$LOCAL" \) -a \( \! -f "$REMOTE" \) ; then + # FICHERO BORRADO EN LOS DOS EXTREMOS + rm "$BASE" "$REMOTE" # igoramos realmente el fichero. + elif test \( \! -f "$BASE" \) -a \( -f "$LOCAL" \) -a \( -f "$REMOTE" \) ; then + # CREACION SIMULTANEA + if [[ $TYPE_CS ]] ; then + type=$TYPE_CS + else + echo "El fichero $MERGE ha sido creado simultaneamente en ambos proyectos" + echo "(para recordar esta respuesta las proximas veces escriba: m* l* r*)" + question="Elija una opcion [(m)erge|(l)ocal|(r)emoto]: " + read -p "$question" answer + type="${answer:0:1}" + [[ ${answer:1:1} == "*" ]] && TYPE_CS=$type + fi + case "$type" in + m) kdiff3 "$LOCAL" "$REMOTE" -o "$MERGE" >/dev/null 2>&1 ;; + l) cp "$LOCAL" "$MERGE" ;; + r) cp "$REMOTE" "$MERGE" ;; + *) echo "Unexpected type"; [[ "" ]] ;; + esac + if [[ $? == 0 ]] ; then + git add "$MERGE" + else + echo "Fallo en la mezcla" + # aqui podemos preguntar al usuario + fi + + elif test \( -f "$BASE" \) -a \( \! -f "$LOCAL" \) -a \( -f "$REMOTE" \) ; then + # BORRADO LOCAL + if [[ $TYPE_BL ]] ; then + type=$TYPE_BL + else + echo "El fichero $MERGE no existe en este proyecto pero si fue modificado en el parche" + echo "(para recordar esta respuesta las proximas veces escriba: b* i*)" + question="Elija una opcion [(b)orrar|(i)ncluir]: " + read -p "$question" answer + type="${answer:0:1}" + [[ ${answer:1:1} == "*" ]] && TYPE_BL=$type + fi + + case "${type}" in + b) rm "$BASE" "$REMOTE" && git rm "$MERGE" ;; + i) cp "$REMOTE" "$MERGE" && git add "$MERGE";; + *) echo "Unexpected type" ;; + esac + + else + echo "Ha ocurrido algun error anteriormente" + fi +done + +for f in ${TMP_FILES[@]}; do + unlink "$f" +done + +# 5.- Para cada mezcla finalizada, añadirla al proyecto DST (git add) + + + +# 6.- Si todos los ficheros se mezclaron sin error: +# hacer un commit, proponer un mensaje de commit. +GIT_MSGFILE="$DST_PROJECT/.git/GITGUI_MSG" +echo "patch: $SRC_START_COMMIT..$SRC_END_COMMIT > $DST_APPLY_COMMIT -- $SRC_PROJECT" > $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +echo "project: $SRC_PROJECT" >> $GIT_MSGFILE +echo "start: $SRC_START_COMMIT $SRC_START_COMMIT_MSG" >> $GIT_MSGFILE +echo "end: $SRC_END_COMMIT $SRC_END_COMMIT_MSG" >> $GIT_MSGFILE +echo "applied: $DST_APPLY_COMMIT $DST_APPLY_COMMIT_MSG" >> $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +echo " -- files patched: " >> $GIT_MSGFILE +echo "" >> $GIT_MSGFILE +for filename in "${FILE_LIST[@]}"; do + echo " $filename" >> $GIT_MSGFILE +done +echo "" >> $GIT_MSGFILE + + +popd >/dev/null + + + + + + + + + + + + + diff --git a/flpremerge.py b/flpremerge.py index 0dc7a2e..7339970 100644 --- a/flpremerge.py +++ b/flpremerge.py @@ -1,9 +1,223 @@ +from __future__ import print_function +from __future__ import division +from builtins import zip +from builtins import range +from past.utils import old_div +from builtins import object import os, os.path, sys import math from optparse import OptionParser +import difflib +import re -class processedFile: - pass +class processedFile(object): + def __init__(self, filename, debug = False): + self.debug = debug + self.table = {} # Carga literal de la lista de hashes, pk: (startbyte, endbyte) = csvrow + self.idxdepth = {} + self.idxtree = {} # pk: objnum (1.5.5.1.1) = (startbyte, endbyte) + self.hashes = {} + self.list_hashes = [] + self.filename = filename + self.cacheFullQname = {} + self.processHashFile() + self.indexNames() + self.computeSortedBlocks() + + def processHashFile(self): + self.cacheFullQname = {} + self.table, self.idxdepth,self.idxtree, self.hashes, self.list_hashes = process(self.filename+".hash") + + + def indexNames(self): + self.names = {} + self.sortedNames = [] + for pk in self.idxtree: + if len(pk) > 1: continue + name = self.fullQName(pk) + if name not in self.names: self.names[name] = [] + self.names[name].append(pk) + self.sortedNames.append((self.idxtree[pk],name)) + + self.sNames = set(self.names.keys()) + + def Qname(self,p): + row = self.table[self.idxtree[p]] + return row["name"] + + + def fullQName(self,p): + dcache = self.cacheFullQname + if p not in dcache: + name=[] + + for n in range(len(p)): + ps1 = p[:n+1] + name.append(self.Qname(ps1)) + fullname = "/".join(name) + dcache[p] = fullname + else: + fullname = dcache[p] + + return fullname + + def computeSortedBlocks(self, fout=None): + self.computedBlocks = [] + if fout is None: + fout = open(self.filename + ".blocks","w") + + antdesde, anthasta = 0 , -1 + fB = open(self.filename) + pos = 0 + linePosChar = [pos] + fB.seek(0) + line = fB.readline() + while line: + pos += len(line) + linePosChar.append(pos) + line = fB.readline() + linenum = 0 + self.linePosChar = linePosChar + for pk, bl_name in list(sorted(self.sortedNames))+[((None,None),None)]: + desde, hasta = pk + if desde is None or desde >= anthasta +1: + + bdesde = anthasta + 1 + fB.seek(bdesde) + if desde: + bhasta = desde - 1 + sB = fB.read((bhasta-bdesde)+1) + else: + sB = fB.read() + bhasta = bdesde + len(sB) -1 + + + while linenum < len(linePosChar) and linePosChar[linenum+1]<=bdesde: linenum += 1 + startline = linenum + linesize=linePosChar[startline] - linePosChar[startline-1] + curpos = bdesde - linePosChar[startline-1] + #if curpos != linesize: print "****", linesize, curpos + + + while linenum < len(linePosChar) and linePosChar[linenum+1]<=bhasta: linenum += 1 + endline = linenum + #while startline > 1 and linePosChar[startline-1]-bdesde >=-2: startline-=1 + + #print linePosChar[startline-1]-bdesde, linePosChar[startline]-bdesde, linePosChar[startline+1]-bdesde + + #print (startline, endline), "BLOCK", (linePosChar[startline], linePosChar[endline]), (bdesde , bhasta), (bhasta-bdesde)+1 + + #print "<<<<" + mode = "" + nline = startline + beginline = startline + bblocks = [] + blockdesc = [] + if len(sB.splitlines(1)) != endline-startline+1: + print(startline, endline, repr(sB)) + print(linePosChar[endline-1]-bhasta, linePosChar[endline]-bhasta, linePosChar[endline+1]-bhasta) + + print("Block lines doesnt match:", len(sB.splitlines(1)), endline-startline, startline, endline) + print(linePosChar[startline-2:startline+3],bdesde) + print(linePosChar[endline-2:endline+3],bhasta) + print(repr(sB.splitlines(1))) + for line in sB.splitlines(1): + nline += 1 + if line[-1]!='\n': break + ltype = "junk" + isseparator = re.match(r'[ \t]*\n',line) + iscommentline1 = re.match(r'[ \t]*//.+\n',line) + iscommentline2 = re.match(r'[ \t]*/\*.+\*/\n',line) + iscommentbegin = re.match(r'[ \t]*/\*.+\n',line) + iscommentend = re.match(r'.+\*/[ \t]*\n',line) + + + + if isseparator: ltype = "separator" + elif iscommentline1: ltype = "comment_inline" + elif iscommentline2: ltype = "comment_block_inline" + elif iscommentbegin: ltype = "comment_block" + elif iscommentend: ltype = "comment_blockend" + #else: print "junk?", line, + + if mode == "comment_block" and ltype == "comment_blockend": + mode = "comment_blockend" + + if mode == "comment_block" and ltype != "comment_blockend": + ltype = "comment_block" + + + if mode != ltype: + if mode: + if mode == "comment_blockend": + mode = "comment_block" + thisblock = ( (beginline, nline-1), ""+mode , ".".join(blockdesc)[:32]) + blockdesc = [] + bblocks.append(thisblock) + mode = ltype + beginline = nline - 1 + + words = re.split(r'\W+',line) + if words: + text = " ".join(words) + text = text.strip() + text = text.replace(" ", "-") + if len(text)>1: + blockdesc.append(text) + + if mode: + if mode == "comment_blockend": + mode = "comment_block" + + thisblock = ( (beginline, nline),""+mode, ".".join(blockdesc)[:32] ) + bblocks.append(thisblock) + blockdesc = [] + #print ltype, line, + + for lines, bname, desc in bblocks: + #print lines, " ", "%s:%s" % (bname, desc) + startline, endline = lines + name = "#..%s:%s" % (bname, desc) + + self.computedBlocks.append((startline, endline,name)) + fout.write("%d\t%d\t%s\n" % (startline, endline,name)) + #print "#..%s:%s" % (bname, desc) + #print sB, + #print ">>>>" + if desde is None: break + initline = linenum + while linenum < len(linePosChar) and linePosChar[linenum+1] 2: + print(linePosChar[startline] - desde, linePosChar[endline-1] - hasta, linePosChar[endline] - hasta, bl_name) + name = bl_name + #print (startline, endline), name, (linePosChar[startline], linePosChar[endline]), pk, (hasta-desde)+1 + self.computedBlocks.append((startline, endline,name)) + fout.write("%d\t%d\t%s\n" % (startline, endline,name)) + #print "%s" % name + antdesde, anthasta = pk + if anthasta <= linePosChar[linenum-1] + 1: + anthasta = linePosChar[linenum-1] + 1 + elif anthasta <= linePosChar[linenum] + 1: + anthasta = linePosChar[linenum] + 1 + + fB.close() + fout.close() + + +def linejunk(line): + line = line.strip() + if len(line)<4: return True + if line[2] == "//": return True + return False + +def charjunk(char): + junk = [" ", "\t"] + if char in junk: return True + return False def main(): parser = OptionParser() @@ -17,26 +231,29 @@ def main(): action="store_true", dest="optdebug", default=False, help="debug optparse module") + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + (options, args) = parser.parse_args() if options.optdebug: - print options, args + print(options, args) - filenames = filter(lambda x: os.path.isfile(x) , args) + filenames = [x for x in args if os.path.isfile(x)] not_a_file = set(args) - set(filenames) if len(not_a_file): - print "WARNING: Not a file:", ", ".join(not_a_file) + print("WARNING: Not a file:", ", ".join(not_a_file)) if len(filenames): pfiles = [] for file1 in filenames: - print "File:", file1 - pf = processedFile() - pf.filename = file1 - pf.table, pf.idxdepth,pf.idxtree, pf.hashes, pf.list_hashes = process(file1) + #print "File:", file1 + pf = processedFile(file1, debug = options.debug) pfiles.append(pf) - pfA = pfiles[0] - for pfB in pfiles[1:]: - feq = FindEquivalences(pfA, pfB) + #pfA = pfiles[0] + #for pfB in pfiles[1:]: + # feq = FindEquivalences(pfA, pfB) + def tree_parents(pk): parents = [] while (len(pk)>0): @@ -44,20 +261,113 @@ def tree_parents(pk): pk = pk[:-1] return parents -class FindEquivalences: +class FindEquivalences(object): def __init__(self,pfA, pfB, autoCompute = True): self.equivalences = {} self.parent_equivalences = {} self.max_known_eq = {} self.pfA, self.pfB = pfA, pfB if autoCompute: self.compute() - + def compute(self): - print "Finding equivalences between A (%s) -> B (%s):" % ( - self.pfA.filename, + print("Finding equivalences between A (%s) -> B (%s):" % ( + self.pfA.filename, self.pfB.filename - ) - + )) + print("Modified names:") + commonNames = sorted(list(self.pfA.sNames & self.pfB.sNames)) + for name in commonNames: + if len(self.pfA.names[name]) > 1 or len(self.pfB.names[name]) > 1: + print("-", name,"(%d,%d)" % (len(self.pfA.names[name]),len(self.pfB.names[name]))) + else: + keyA = self.pfA.idxtree[self.pfA.names[name][0]] + rowA = self.pfA.table[keyA] + keyB = self.pfB.idxtree[self.pfB.names[name][0]] + rowB = self.pfB.table[keyB] + if rowA['hash'] != rowB['hash']: + print("###", name,"###") + fileA = self.pfA.filename.replace(".hash","") + fileB = self.pfB.filename.replace(".hash","") + if os.path.isfile(fileA) and os.path.isfile(fileB): + fA = open(fileA) + fA.seek(keyA[0]) + sA = fA.read(keyA[1]-keyA[0]+1) + fA.close() + + fB = open(fileB) + fB.seek(keyB[0]) + sB = fB.read(keyB[1]-keyB[0]+1) + fB.close() + + sA = sA.replace("\t", " ") + sB = sB.replace("\t", " ") + lines = list(difflib.ndiff(sA.splitlines(1), sB.splitlines(1),linejunk,charjunk)) + modifiedlines = [] + for n,line in enumerate(lines): + if line[0] != " ": + modifiedlines.append(n) + ml = 0 + omit = [] + Context = 3 + n = 0 + for line in lines: + d = [] + for m in modifiedlines: + dt = abs(m-n) + d.append(dt) + if dt < Context: break + dmin = min(d) + if dmin < Context: + if len(omit) : + if len(omit)>= Context: + for line in omit: + if line[0] in (' ','+'): n+=1 + print(" ", "(... %d lines ommitted ...)" % len(omit)) + else: + for line in omit: + if line[0] in (' ','+'): n+=1 + print("%03d" % n , line, end=' ') + omit = [] + if line[0] in (' ','+'): + n+=1 + print("%03d" % n , line, end=' ') + else: + print("%03d" % (n+1) , line, end=' ') + else: + omit.append(line) + if len(omit) : + if len(omit)>= Context: + for line in omit: + if line[0] in (' ','+'): n+=1 + print(" ", "(... %d lines ommitted ...)" % len(omit)) + else: + for line in omit: + if line[0] in (' ','+'): n+=1 + print("%03d" % n , line, end=' ') + omit = [] + print() + else: + print("(diff ommitted because we couldn't find original files)") + + print() + print("Deleted names:") + deletedNames = sorted(list(self.pfA.sNames - self.pfB.sNames)) + for name in deletedNames: + print("-", name) + print() + print("Added names:") + addedNames = sorted(list(self.pfB.sNames - self.pfA.sNames)) + for name in addedNames: + print("-", name) + print() + + return + + + + + + for key in self.pfA.list_hashes: if key in self.pfB.hashes: lpkA = self.pfA.hashes[key] @@ -69,36 +379,36 @@ def compute(self): for pkA in self.pfA.idxtree: parentA = pkA[:-1] - if pkA not in self.equivalences: + if pkA not in self.equivalences: self.equivalences[pkA] = {} if parentA: - if parentA not in self.parent_equivalences: + if parentA not in self.parent_equivalences: self.parent_equivalences[parentA] = [] - + for pkA in sorted(self.pfA.idxtree): - parentsA = tree_parents(pkA) - for pkB, punt in self.equivalences[pkA].iteritems(): + parentsA = tree_parents(pkA) + for pkB, punt in self.equivalences[pkA].items(): if len(pkA) != len(pkB): continue parentsB = tree_parents(pkB) - parentsAB = zip(parentsA,parentsB) + parentsAB = list(zip(parentsA,parentsB)) for pA, pB in parentsAB: sz2a, sz2b = self.pfA.idxtree[pkA] sz2 = sz2b - sz2a + 1 sz1a, sz1b = self.pfA.idxtree[pA] sz1 = sz1b - sz1a + 1 - lev2 = 1.0 + sz1 / float(sz2) + lev2 = 1.0 + old_div(sz1, float(sz2)) #lev2 = 2**(len(pkA) - len(pA)) #lev2_list = [ pC for pC in self.pfA.idxtree if len(pC) >= len(pA) and pC[:len(pA)] == pA ] #lev2_plist = set([]) #for l in lev2_list: # lev2_plist |= set(tree_parents(l)[1:]) #lev2 = len(set(lev2_list) - set(lev2_plist)) - - pEq = (pB, float(punt)/lev2) - if pA not in self.parent_equivalences: + + pEq = (pB, old_div(float(punt),lev2)) + if pA not in self.parent_equivalences: self.parent_equivalences[pA] = [] self.parent_equivalences[pA].append(pEq) - + norepeat = (0,) self.parent_equivalences2 = {} for pA in sorted(self.parent_equivalences): @@ -107,35 +417,42 @@ def compute(self): for pB, punt in self.parent_equivalences[pA]: if pB not in count: count[pB] = 0 count[pB] += punt - rcount = [] + rcount = [] ppA = pA[:-1] if ppA in self.parent_equivalences2: ppB = self.parent_equivalences2[ppA] else: ppB = None - + rowA = self.pfA.table[self.pfA.idxtree[pA]] - for key, punt in count.copy().iteritems(): - if ppB and key[:-1] != ppB: continue + for key, punt in count.copy().items(): + if ppB and key[:-1] != ppB: continue rowB = self.pfB.table[self.pfB.idxtree[pB]] - nameA = rowA['name'].split(":") - nameB = rowB['name'].split(":") - if nameA[0] != nameB[0]: punt /=3.0 - if nameA[1] != nameB[1]: punt /=1.0+len(nameA[1]) / 40.0+len(nameB[1]) / 40.0 + nameA = self.pfA.fullQName(pA) #rowA['name'].split(":") + nameB = self.pfB.fullQName(pB) #rowB['name'].split(":") + s = difflib.SequenceMatcher() + s.set_seqs(nameA,nameB) + t = s.quick_ratio() + t -= 0.8 + if t < 0: t = 0 + t *= 10.0 + punt *= t + #if nameA[0] != nameB[0]: punt /=3.0 + #if nameA[1] != nameB[1]: punt /=1.0+len(nameA[1]) / 40.0+len(nameB[1]) / 40.0 if punt >= 0.50: rcount.append((round(punt*100),key)) - + if len(rcount): punt, pB = max(rcount) self.parent_equivalences2[pA] = pB rowB = self.pfB.table[self.pfB.idxtree[pB]] - print "parent:", pA, rowA['name'], "%d%%\t" % punt, pB, len(rcount) , rowB['name'] + print("parent:", pA, self.pfA.fullQName(pA), "%d%%\t" % punt, pB, len(rcount) , self.pfB.fullQName(pB)) if punt > 100: norepeat = pA - else: + else: if len(pA) == 1: - print "parent:", pA, rowA['name'], "0%\t ???" - + print("parent:", pA, self.pfA.fullQName(pA), "0%\t ???") + """ norepeat = (0,) prevprint = None @@ -149,37 +466,39 @@ def compute(self): if len(pkA) > len(prevprint) and prevprint[:-1] == pkA[:len(prevprint)-1]: continue elif pkA < len(prevprint): prevprint = None elif prevprint[:-1] != pkA[:len(prevprint)-1]: prevprint = None - + print pkA,":", - + for pkB, punt in self.equivalences[pkA].iteritems(): - if punt > 0.96: + if punt > 0.96: norepeat = pkA if punt >= 0.1: print pkB, punt, ";", prevprint = pkA - print - """ + print + """ """ for pkB, punt in self.equivalences[pkA].iteritems(): - if punt > 0.96: + if punt > 0.96: norepeat = pkA print ">>", ".".join(["%02d" % x for x in pkB]) prevprint = pkA - """ + """ + + def getMaxKnown(self,pkA): pkA = tuple(pkA) if len(pkA) == 0: return 1.0, None if pkA not in self.max_known_eq: return 0.0, None pkB = self.max_known_eq[pkA] - eq_prob = self.equivalences[pkA][pkB] + eq_prob = self.equivalences[pkA][pkB] return eq_prob, pkB - - + + def addEquivalences(self,lpkA,lpkB): lstEquivalences = self.multiplyEquivalences(lpkA,lpkB) - base_probability = 1.0 / len(lstEquivalences) + base_probability = old_div(1.0, len(lstEquivalences)) if base_probability < 0.01: return for pkA,pkB in lstEquivalences: @@ -187,35 +506,35 @@ def addEquivalences(self,lpkA,lpkB): parentA = pkA[:-1] parent_prob, parentB = self.getMaxKnown(parentA) if parentB: - if parentB != pkB[:-1]: parent_prob = (1-parent_prob) / 2.0 - probability *= parent_prob - + if parentB != pkB[:-1]: parent_prob = old_div((1-parent_prob), 2.0) + probability *= parent_prob + parentB = pkB[:-1] if probability < 0.01: continue if pkA not in self.equivalences: self.equivalences[pkA] = {} if pkB in self.equivalences[pkA]: - print "DUPLICATE", pkA,pkB + print("DUPLICATE", pkA,pkB) self.equivalences[pkA][pkB] = probability previousMax, prevPkB = self.getMaxKnown(pkA) if probability > previousMax: self.max_known_eq[pkA] = pkB - - - + + + def multiplyEquivalences(self,lpkA,lpkB): leq = set([]) for pkA in lpkA: for pkB in lpkB: leq|=set([(pkA,pkB)]) return list(leq) - + def process(filename): table, idxdepth,idxtree = load(filename) #print table.items()[:10] treebydepth = {} - + for k in sorted(idxtree.keys()): td = len(k)-1 if td not in treebydepth: treebydepth[td] = [] @@ -225,19 +544,20 @@ def process(filename): maxd = 0 hashes = {} list_hashes = [] - - for d,idx in treebydepth.iteritems(): + + for d,idx in treebydepth.items(): + if d > 2: break nitems = len(idx) if maxitems < nitems: maxitems = nitems - elif nitems * 30 < maxitems: break - print "Depth:", d, "(%d items)" % nitems + elif nitems * 2 < maxitems: break + #print "Depth:", d, "(%d items)" % nitems maxd = d for k in idx: #print ".".join([str(x) for x in k]) pk = idxtree[k] r = table[pk] rhash = r["hash"] - if rhash not in hashes: + if rhash not in hashes: list_hashes.append(rhash) hashes[rhash] = [] hashes[rhash].append(k) @@ -252,12 +572,12 @@ def isinside(parent, child): if cfrom > pto and cto > pto: return 1 #print "ERROR:", child , " is superior to ", parent return 0 - + def load(filename): file = open(filename) def getpk(row): return (row["start"],row["end"]) - + fields = [ "depth", "hash", @@ -275,15 +595,15 @@ def getpk(row): ] bydepth = {} for line in file: - row = dict(zip(fields,line[:-1].split("\t"))) + row = dict(list(zip(fields,line[:-1].split("\t")))) for f in intfields: row[f]=int(row[f]) pk = getpk(row) depth = row["depth"] if depth not in bydepth: bydepth[depth] = [] bydepth[depth].append(pk) rows[pk]=row - - for dpth, items in bydepth.iteritems(): + + for dpth, items in bydepth.items(): bydepth[dpth] = list(sorted(items)) idxtree = {} @@ -295,7 +615,7 @@ def getpk(row): else: pdepth = currdepth - 1 np = 0 - + for pk in bydepth[currdepth]: n+=1 if currdepth > 0: @@ -310,7 +630,7 @@ def getpk(row): np += offset if offset: n = 1 if offset < 0: - print list(enumerate(bydepth[pdepth])) + print(list(enumerate(bydepth[pdepth]))) assert(offset >= 0) #if it > 100: # print it,n,pk,np, ppk, offset @@ -318,25 +638,25 @@ def getpk(row): # print list(enumerate(bydepth[pdepth])) # assert(offset >= 0) # assert(it < 250) - + prow = rows[ppk] nparent = prow["tree_id"] - - + + row = rows[pk] tree_id = nparent + [n] row["tree_id"] = tree_id if tuple(tree_id) in idxtree: - print "ERROR:", tuple(tree_id), " is duplicated:" - print "previous:", idxtree[tuple(tree_id)] - print "new:" , pk + print("ERROR:", tuple(tree_id), " is duplicated:") + print("previous:", idxtree[tuple(tree_id)]) + print("new:" , pk) else: - idxtree[tuple(tree_id)] = pk - - - - + idxtree[tuple(tree_id)] = pk + + + + return rows, bydepth, idxtree - -if __name__ == "__main__": main() \ No newline at end of file + +if __name__ == "__main__": main() diff --git a/flscriptparse.py b/flscriptparse.py index 75e7f55..0e8c359 100644 --- a/flscriptparse.py +++ b/flscriptparse.py @@ -1,17 +1,27 @@ +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from builtins import input +from builtins import str +from builtins import range # ----------------------------------------------------------------------------- # flscriptparse.py # -# Simple parser for FacturaLUX SCripting Language (QSA). +# Simple parser for FacturaLUX SCripting Language (QSA). # ----------------------------------------------------------------------------- from optparse import OptionParser - +import pprint import sys, math import hashlib -import flex import ply.yacc as yacc import ply.lex as lex - -from flclasses import * + +try: + from pineboolib.flparser import flex + from pineboolib.flparser.flclasses import * +except ImportError: + import flex + from flclasses import * # Get the token map tokens = flex.tokens @@ -20,6 +30,8 @@ reserv=['nonassoc'] reserv+=list(flex.reserved) +endoffile = None + def cnvrt(val): val = str(val) val = val.replace('&','&') @@ -32,57 +44,195 @@ def cnvrt(val): precedence = ( ('nonassoc', 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL'), ('left','LOR', 'LAND'), + ('left', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'EQQ', 'NEQ'), ('right', 'LNOT'), - ('left', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE'), ('left', 'PLUS', 'MINUS'), ('left', 'TIMES', 'DIVIDE', 'MOD'), ('left', 'OR', 'AND', 'XOR', 'LSHIFT', 'RSHIFT'), ) +seen_tokens = [] +tokelines = {} +last_lexspan = None def p_parse(token): - ''' + global input_data + + lexspan = list(token.lexspan(0)) + data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) + if len(lexspan) == 2: + fromline = token.lineno(0) + global endoffile + endoffile = fromline, lexspan, token.slice[0] + #print fromline, lexspan, token.slice[0] + token[0] = { "00-toktype": str(token.slice[0]), "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } + numelems = len([ s for s in token.slice[1:] if s.type != 'empty' and s.value is not None ]) + + rspan = lexspan[0] + if str(token.slice[0]) == 'empty' or numelems == 0: token[0] = None + else: + rvalues = [] + for n,s in enumerate(token.slice[1:]): + if s.type != 'empty' and s.value is not None: + val = None + if isinstance(s.value,str): + val = token.lexspan(n+1)[0] + len(s.value) - 1 + else: + val = token.lexspan(n+1)[1] + rvalues.append(val) + rspan = max(rvalues) + lexspan[1] = rspan + + #if str(token.slice[0]) == 'regexbody': + # token[0] = { "00-toktype": str(token.slice[0]) , "02-size" : lexspan, "50-contents" : input_data[lexspan[0]:lexspan[1]+1] } + + #if str(token.slice[0]) == 'regex': + # print "\r\n",str(token.slice[0]) ,":" , input_data[lexspan[0]:lexspan[1]+1] + # print " " + "\n ".join([ "%s(%r): %r" % (s.type, token.lexspan(n+1), s.value) for n,s in enumerate(token.slice[1:]) ]) + global seen_tokens, last_ok_token + last_ok_token = token + seen_tokens.append((str(token.slice[0]), token.lineno(0),input_data[lexspan[0]:lexspan[1]+1] )) + global ok_count + ok_count += 1 + if lexspan[0] not in tokelines: + tokelines[lexspan[0]] = token.lexer.lineno + global last_lexspan + last_lexspan = lexspan + + + + + +last_ok_token = None +error_count = 0 +last_error_token = None +last_error_line = -1 +ok_count = 0 + +def p_error(t): + global error_count + global ok_count + global last_error_token + global last_error_line, seen_tokens , last_ok_token + debug = False # Poner a True para toneladas de debug. + # if error_count == 0: print + if t is not None: + if last_error_token is None or t.lexpos != getattr(last_error_token,"lexpos",None): + if abs(last_error_line - t.lineno) > 4 and ok_count > 1 and error_count < 4: + error_count += 1 + try: print_context(t) + except Exception: pass + if debug == True: + error_count += 20 # no imprimir mas de un error en debug. + print + for tokname, tokln, tokdata in seen_tokens[-32:]: + if tokln == t.lineno: + print(tokname, tokdata) + print(repr(last_ok_token[0])) + for s in last_ok_token.slice[:]: + print(">>>" , s.lineno, repr(s), pprint.pformat(s.value,depth=3)) + last_error_line = t.lineno + elif abs(last_error_line - t.lineno) > 1 and ok_count > 1: + last_error_line = t.lineno + parser.errok() + ok_count = 0 + return + + ok_count = 0 + if t is None: + if last_error_token != "EOF": + print("ERROR: End of the file reached.") + global endoffile + print("Last data:", endoffile) + + if last_lexspan: + try: + print("HINT: Last lexspan:", last_lexspan) + print("HINT: Last line:", tokelines[last_lexspan[0]]) + except Exception as e: + print("ERROR:", e) + last_error_token = "EOF" + return t + t = parser.token() + parser.restart() + last_error_token = t + return t + +p_parse.__doc__ = ''' exprval : constant | variable | funccall | error + identifier : ID + + dictobject_value : LBRACE RBRACE + | LBRACE dictobject_value_elemlist RBRACE + + dictobject_value_elemlist : dictobject_value_elem + | dictobject_value_elemlist COMMA dictobject_value_elem + + dictobject_value_elem : exprval COLON expression + base_expression : exprval + | inlinestoreinstruction | base_expression mathoperator base_expression | base_expression cmp_symbol base_expression | base_expression boolcmp_symbol base_expression - | LPAREN base_expression RPAREN - | LNOT base_expression + | parentheses + | unary_operator + | new_operator + | ternary_operator + | dictobject_value + | typeof_operator + + + + + parentheses : LPAREN base_expression RPAREN + | LPAREN variable_1 RPAREN + + unary_operator : LNOT base_expression | MINUS base_expression | PLUS base_expression - | NEW funccall_1 - | NEW ID - | base_expression CONDITIONAL1 base_expression COLON base_expression + + new_operator : NEW funccall_1 + | NEW identifier + + typeof_operator : TYPEOF variable + | TYPEOF base_expression + + ternary_operator : base_expression CONDITIONAL1 base_expression COLON base_expression expression : base_expression + | funcdeclaration_anon + | funcdeclaration_anon_exec + | LPAREN expression RPAREN | error - case_cblock_list : case_block - case_cblock_list : case_cblock_list case_block + case_cblock_list : case_block + case_cblock_list : case_cblock_list case_block - case_block : CASE base_expression COLON statement_list + case_block : CASE expression COLON statement_list case_default : DEFAULT COLON statement_list case_block_list : empty case_block_list : case_default - case_block_list : case_cblock_list + case_block_list : case_cblock_list case_block_list : case_cblock_list case_default source_element : docstring | vardeclaration | classdeclaration | funcdeclaration + | funcdeclaration_anon + | funcdeclaration_anon_exec source : source_element source : source source_element | statement_list - + basicsource : statement_list | empty @@ -91,14 +241,16 @@ def p_parse(token): | vardeclaration | ifstatement | whilestatement + | dowhilestatement | withstatement | forstatement + | forinstatement | switch | trycatch statement_list : statement_list statement - statement_list : statement + statement_list : statement statement_list : LBRACE statement_list RBRACE @@ -110,10 +262,11 @@ def p_parse(token): vardeclaration : VAR vardecl_list SEMI | CONST vardecl_list SEMI - vardeclaration : VAR vardecl_list - | CONST vardecl_list + vardeclaration : VAR vardecl_list + | CONST vardecl_list + | STATIC VAR vardecl_list - vardecl : ID optvartype EQUALS expression + vardecl : ID optvartype EQUALS expression vardecl : ID optvartype vardecl_list : vardecl @@ -123,6 +276,11 @@ def p_parse(token): | funcdeclaration : FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE + funcdeclaration : STATIC FUNCTION ID LPAREN arglist RPAREN optvartype LBRACE basicsource RBRACE + funcdeclaration_anon : FUNCTION LPAREN arglist RPAREN LBRACE basicsource RBRACE + | FUNCTION LPAREN RPAREN LBRACE basicsource RBRACE + funcdeclaration_anon_exec : funcdeclaration_anon LPAREN RPAREN + | funcdeclaration_anon LPAREN arglist RPAREN callarg : expression @@ -134,7 +292,7 @@ def p_parse(token): | funccall_1 | member_call | member_var - | base_expression + | base_expression member_var : varmemcall PERIOD variable_1 member_call : LPAREN member_var RPAREN PERIOD funccall_1 @@ -146,9 +304,11 @@ def p_parse(token): | member_call | LPAREN member_call RPAREN | LPAREN funccall_1 RPAREN + | LPAREN error RPAREN funccall_1 : ID LPAREN callargs RPAREN | ID LPAREN RPAREN + | TYPEOF LPAREN callargs RPAREN mathoperator : PLUS | MINUS @@ -163,50 +323,59 @@ def p_parse(token): variable : variable_1 | member_var - | LPAREN variable_1 RPAREN + | LPAREN variable_1 RPAREN | LPAREN member_var RPAREN - variable_1 : ID - | inlinestoreinstruction - | variable_1 LBRACKET base_expression RBRACKET - | funccall_1 LBRACKET base_expression RBRACKET + variable_1 : identifier + | array_member - inlinestoreinstruction : PLUSPLUS ID - | MINUSMINUS ID - | ID PLUSPLUS - | ID MINUSMINUS + array_member : variable_1 LBRACKET expression RBRACKET + | funccall_1 LBRACKET expression RBRACKET - storeequalinstruction : variable EQUALS expression - | variable EQUALS storeequalinstruction + inlinestoreinstruction : PLUSPLUS variable + | MINUSMINUS variable + | variable PLUSPLUS + | variable MINUSMINUS + updateoperator : EQUALS + | PLUSEQUAL + | MINUSEQUAL + | MODEQUAL + | DIVEQUAL + | TIMESEQUAL - + updateinstruction : variable updateoperator expression + | variable updateoperator updateinstruction + + deleteinstruction : DELETE variable storeinstruction : inlinestoreinstruction - | storeequalinstruction - | variable PLUSEQUAL expression - | variable MINUSEQUAL expression - | variable MODEQUAL expression - | variable DIVEQUAL expression - | variable TIMESEQUAL expression - | DELETE variable - - - flowinstruction : RETURN expression - | THROW expression - | RETURN - | BREAK - | CONTINUE + | updateinstruction + | deleteinstruction + + + flowinstruction : RETURN expression + | THROW expression + | RETURN + | BREAK + | CONTINUE instruction : base_instruction SEMI | SEMI - | base_instruction + | base_instruction + | funcdeclaration + | error SEMI + + callinstruction : funccall + | variable + base_instruction : storeinstruction - | funccall - | flowinstruction - | variable - | variable PLUSPLUS - | variable MINUSMINUS + | callinstruction + | flowinstruction + + varorcall : variable + | funccall + | base_expression optextends : EXTENDS ID | empty @@ -230,13 +399,46 @@ def p_parse(token): | FCONST | CCONST | SCONST - | RXCONST + | regex | list_constant - | error - + + regex : DIVIDE regexbody DIVIDE regexflags + | DIVIDE regexbody COMMENTCLOSE regexflags + + regexbody : regexchar + | regexbody regexchar + + regexchar : LPAREN + | RPAREN + | ID + | COMMA + | XOR + | LBRACKET + | RBRACKET + | ICONST + | PLUS + | MINUS + | LBRACE + | RBRACE + | DOLLAR + | SQOUTE + | DQOUTE + | PERIOD + | BACKSLASH + | CONDITIONAL1 + | EQUALS + | OR + | SCONST + | SEMI + | error + + regexflags : ID + | empty + statement_block : statement | LBRACE statement_list RBRACE + | LBRACE RBRACE optelse : ELSE statement_block | empty @@ -247,86 +449,65 @@ def p_parse(token): | GE | EQ | NE + | EQQ + | NEQ + | IN boolcmp_symbol : LOR | LAND - condition : expression + condition : expression + | error ifstatement : IF LPAREN condition RPAREN statement_block optelse - whilestatement : WHILE LPAREN condition RPAREN statement_block - whilestatement : DO statement_block WHILE LPAREN condition RPAREN SEMI + whilestatement : WHILE LPAREN condition RPAREN statement_block + dowhilestatement : DO statement_block WHILE LPAREN condition RPAREN SEMI + | DO statement_block WHILE LPAREN condition RPAREN + + withstatement : WITH LPAREN variable RPAREN statement_block | error - withstatement : WITH LPAREN variable RPAREN statement_block + storeormember : storeinstruction + | member_var + + for_initialize : storeinstruction + | VAR vardecl + | for_initialize COMMA for_initialize + | empty + + for_compare : expression + | empty + + for_increment : storeormember + | for_increment COMMA for_increment + | empty + + + forstatement : FOR LPAREN for_initialize SEMI for_compare SEMI for_increment RPAREN statement_block | error - forstatement : FOR LPAREN storeinstruction SEMI base_expression SEMI storeinstruction RPAREN statement_block - forstatement : FOR LPAREN VAR vardecl SEMI base_expression SEMI storeinstruction RPAREN statement_block - | FOR LPAREN SEMI base_expression SEMI storeinstruction RPAREN statement_block + forinstatement : FOR LPAREN for_initialize IN varorcall RPAREN statement_block + | FOR LPAREN variable IN varorcall RPAREN statement_block | error - switch : SWITCH LPAREN expression RPAREN LBRACE case_block_list RBRACE + switch : SWITCH LPAREN condition RPAREN LBRACE case_block_list RBRACE optid : ID | empty - trycatch : TRY LBRACE statement_list RBRACE CATCH LPAREN optid RPAREN LBRACE statement_list RBRACE - trycatch : TRY LBRACE statement_list RBRACE CATCH LPAREN optid RPAREN LBRACE RBRACE + trycatch : TRY statement_block CATCH LPAREN optid RPAREN statement_block - empty : + empty : ''' - lexspan = list(token.lexspan(0)) - data = str(token.lexer.lexdata[lexspan[0]:lexspan[1]]) - token[0] = { "02-size" : lexspan, "50-contents" : [ { "01-type": s.type, "99-value" : s.value} for s in token.slice[1:] ] } - -error_count = 0 -last_error_token = None -def p_error(t): - global error_count - global last_error_token - - if repr(t) != repr(last_error_token): - error_count += 1 - if error_count>100 or t is None: - yacc.errok() - return - try: - print_context(t) - except: - pass - print "ERROR" - import sys - sys.exit() - #while 1: - # if t is None: - # print "*** Se encontro EOF intentando resolver el error *** " - # break - # if t.type == 'RBRACE': break - # if t.type == 'SEMI': break - # t = yacc.token() # Get the next token - - #if t is not None: - # yacc.errok() - #else: - if t is None: - yacc.errok() - return - t = yacc.token() - yacc.restart() - last_error_token = t - return t - - # Build the grammar -parser = yacc.yacc(method='LALR',debug=0, - optimize = 0, write_tables = 1, debugfile = '/tmp/yaccdebug.txt',outputdir='/tmp/') +parser = yacc.yacc(method='LALR',debug=0, + optimize = 1, write_tables = 1, debugfile = '/tmp/yaccdebug.txt',outputdir='/tmp/') #profile.run("yacc.yacc(method='LALR')") @@ -334,7 +515,7 @@ def p_error(t): def print_context(token): global input_data - if not token: return + if token is None: return last_cr = input_data.rfind('\n',0,token.lexpos) next_cr = input_data.find('\n',token.lexpos) column = (token.lexpos - last_cr) @@ -343,36 +524,35 @@ def print_context(token): column1 = (token.lexpos - last_cr) last_cr = input_data.rfind('\n',0,last_cr-1) - print input_data[last_cr:next_cr].replace("\t"," ") - print (" " * (column-1)) + "^", column, "#ERROR#" , token - print - - + print(input_data[last_cr:next_cr].replace("\t"," ")) + print((" " * (column-1)) + "^", column, "#ERROR#" , token) + + def my_tokenfunc(*args, **kwargs): - #print "Call token:" ,args, kwargs - ret = lex.lexer.token(*args, **kwargs) + #print("Call token:" ,args, kwargs) + ret = lex.lexer.token(*args, **kwargs) #print "Return (",args, kwargs,") = " , ret return ret def print_tokentree(token, depth = 0): - print " " * depth, token.__class__ , "=" , token - + print(" " * depth, token.__class__ , "=" , token) + if str(token.__class__) == "ply.yacc.YaccProduction": - print token.lexer - for tk in token.slice: + print(token.lexer) + for tk in token.slice: if tk.value == token: continue - print " " * (depth+1), tk.type, + print(" " * (depth+1), tk.type, end=' ') try: - print tk.lexpos, - print tk.endlexpos, + print(tk.lexpos, end=' ') + print(tk.endlexpos, end=' ') except: pass - print - + print() + print_tokentree(tk.value, depth +1) -def calctree(obj, depth = 0, num = [], otype = "source"): +def calctree(obj, depth = 0, num = [], otype = "source", alias_mode = 1): #if depth > 5: return source_data = [ 'source', @@ -383,25 +563,37 @@ def calctree(obj, depth = 0, num = [], otype = "source"): 'vardecl_list', ] final_obj = {} - final_obj['range'] = obj['02-size'] + final_obj['range'] = obj['02-size'] has_data = 0 has_objects = 0 contentlist = [] - ctype_alias = { - "member_var" : "member", - "member_call" : "member", - "variable_1" : "variable", - "funccall_1" : "funccall", - "flowinstruction" : "instruction", - "storeequalinstruction" : "instruction", - "vardecl" : "vardeclaration", - #"vardecl_list" : "vardeclaration", - - } + + if alias_mode == 0: + ctype_alias = {} + elif alias_mode == 1: + ctype_alias = { + "member_var" : "member", + "member_call" : "member", + "variable_1" : "variable", + "funccall_1" : "funccall", + "flowinstruction" : "instruction", + "storeequalinstruction" : "instruction", + "vardecl" : "vardeclaration", + #"vardecl_list" : "vardeclaration", + + } + else: + raise ValueError("alias_mode unknown") + if otype in ctype_alias: otype = ctype_alias[otype] #print " " * depth , obj['02-size'] for n,content in enumerate(obj['50-contents']): + if not isinstance(content,dict): + print("ERROR: content is not a dict!:", repr(content)) + print(".. obj:", repr(obj)) + raise TypeError("content is not a dict") + continue ctype = content['01-type'] value = content['99-value'] if ctype in ctype_alias: @@ -411,12 +603,18 @@ def calctree(obj, depth = 0, num = [], otype = "source"): # print_tree(value,depth,num) # continue #print " " * depth , "%s:" % ".".join(num+[str(n)]), ctype, - + if type(value) is dict: #print "*" - tree_obj = calctree(value,depth+1,num+[str(n)], ctype) + if depth < 600: + try: + tree_obj = calctree(value,depth+1,num+[str(n)], ctype, alias_mode=alias_mode) + except Exception: + print("ERROR: trying to calculate member %d on:" % n, repr(obj)) + else: + tree_obj = None if type(tree_obj) is dict: - if tree_obj['has_data'] and ctype != otype: + if (tree_obj['has_data'] or alias_mode == 0) and ctype != otype : contentlist.append([ctype,tree_obj]) has_objects += 1 else: @@ -431,9 +629,9 @@ def calctree(obj, depth = 0, num = [], otype = "source"): final_obj['content'] = contentlist final_obj['has_data'] = has_data final_obj['has_objects'] = has_objects - + return final_obj - + hashes = [] ranges = [] @@ -442,7 +640,7 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou if depth == 0: hashes = [] ranges = [] - + sep = " " marginblocks = { "classdeclaration" : 1, @@ -459,31 +657,33 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou nuevalinea = False name = "" lines = [] - - - + l = 0 + + for ctype, value in tree['content']: if nuevalinea and ctype in closingtokens: nuevalinea = False - - if nuevalinea: - for i in range(int(math.ceil(l/2.0))): lines.append(sep * depth) + + if nuevalinea: + for i in range(int(math.ceil(l/2.0))): + lines.append(sep * depth) nuevalinea = False - + if type(value) is dict and ctype == otype: tname,tlines,trange = printtree(value, depth, ctype) if name == "" and tname: name = tname - + lines += tlines elif type(value) is dict: l = 0 if ctype in marginblocks: l = marginblocks[ctype] - - for i in range(int(math.floor(l/2.0))): lines.append(sep * depth) + + for i in range(int(math.floor(l/2.0))): + lines.append(sep * depth) tname,tlines,trange = printtree(value, depth+1, ctype) # lines.append(sep * depth + "" % (len("".join(tlines)))) - + if value['has_data'] > 0 and value['has_objects'] == 0 and False: # Do it inline! if value['has_data']==1 and tname: @@ -495,23 +695,23 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou attrs = [] if tname: attrs.append(("id",tname)) - + txtinline = "".join([ line.strip() for line in tlines ]) - + #if len(tlines)>1: txthash = hashlib.sha1(txtinline).hexdigest()[:16] - #hashes.append(("depth:",depth,"hash:",txthash,"element:",ctype+":"+tname)) - hashes.append((txthash,ctype+":"+tname+"(%d)"% len(txtinline))) + #hashes.append(("depth:",depth,"hash:",txthash,"element:",ctype+":"+tname)) + hashes.append((txthash,ctype+":"+tname+"(%d)"% len(txtinline))) ranges.append([depth,txthash]+trange+[ctype+":"+tname,len(txtinline)]) #,"start:",trange[0],"end:",trange[1])) #attrs.append(("start",trange[0])) #attrs.append(("end",trange[1])) #attrs.append(("hash",txthash)) - + txtattrs="" for name1, val1 in attrs: txtattrs+=" %s=\"%s\"" % (name1,cnvrt(val1)) - + lines.append(sep * depth + "<%s%s>" % (ctype,txtattrs)) if depth > 50: lines.append(sep * (depth+1) + "...") @@ -523,7 +723,7 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou if txtattrs: txtattrs = "" % txtattrs lines.append(sep * depth + "" % (ctype)) - + nuevalinea = True else: if ctype == "ID" and name == "": @@ -532,8 +732,8 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou lines.append(sep * depth + "<%s value=\"%s\" />" % (ctype,cnvrt(value))) else: lines.append(sep * depth + "<%s />" % (ctype)) - - + + if mode == "hash": #print "\n".join(lines) for row in sorted(ranges): @@ -545,26 +745,44 @@ def printtree(tree, depth = 0, otype = "source", mode = None, output = sys.stdou output.write(row) output.write("\n") output.flush() - + return name, lines, tree['range'] - + def parse(data): global input_data + global error_count + global seen_tokens + seen_tokens[:] = [] parser.error = 0 input_data = data + flex.lexer.lineno = 1 + error_count = 0 p = parser.parse(data, debug = 0, tracking = 1, tokenfunc = my_tokenfunc) - if parser.error: return None + if error_count > 0: + print("ERRORS (%d)" % error_count) + if p is None: return p + try: + p["error_count"] = error_count + except Exception as e: + print(e) + return None + + if parser.error: + return None return p def main(): + global start parser = OptionParser() #parser.add_option("-f", "--file", dest="filename", # help="write report to FILE", metavar="FILE") - parser.add_option("-O", "--output", dest="output", default = "xml", + parser.add_option("-O", "--output", dest="output", default = "none", help="Set output TYPE: xml|hash", metavar="TYPE") + parser.add_option("--start", dest="start", default = None, + help="Set start block", metavar="STMT") parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True, help="don't print status messages to stdout") @@ -573,16 +791,25 @@ def main(): action="store_true", dest="optdebug", default=False, help="debug optparse module") + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + (options, args) = parser.parse_args() if options.optdebug: - print options, args + print(options, args) + if options.start: + start = options.start + print("Start setted to:" , start) def do_it(): + if options.output == "none": return tree_data = calctree(prog) - if options.output == "hash": + if options.output == "hash": printtree(tree_data, mode = "hash") elif options.output == "xml": printtree(tree_data, mode = "xml") @@ -590,41 +817,56 @@ def do_it(): f1_hash = open(filename+".hash","w") printtree(tree_data, mode = "hash", output = f1_hash) f1_hash.close() - + f1_xml = open(filename+".xml","w") printtree(tree_data, mode = "xml", output = f1_xml) f1_xml.close() - + elif options.output == "yaml": + import yaml + print(yaml.dump(tree_data['content'])) + else: - print "Unknown outputmode", options.output + print("Unknown outputmode", options.output) prog = "$$$" - if len(args) > 0 : + if len(args) > 0 : for filename in args: - sys.stderr.write("Processing %s ..." % filename) + fs = filename.split("/") + sys.stderr.write("Loading %s ..." % fs[-1]) sys.stderr.flush() - data = open(filename).read() + data = open(filename).read() sys.stderr.write(" parsing ...") sys.stderr.flush() - prog = parse(data) + prog = parse(data) sys.stderr.write(" formatting ...") sys.stderr.flush() if prog: do_it() sys.stderr.write(" Done.\n") sys.stderr.flush() - + else: line = "" while 1: try: - line += raw_input("flscript> ") - except EOFError: + line1 = input("flscript> ") + if line1.startswith("#"): + comm = line1[1:].split(" ") + if comm[0] == "setstart": + start = comm[1] + print("Start setted to:" , start) + if comm[0] == "parse": + print() + prog = parse(line) + line = "" + else: + line += line1 + except EOFError: break; - line += "\n" - print - prog = parse(line) + line += "\n" + print() + prog = parse(line) do_it() """ import yaml @@ -639,9 +881,9 @@ def do_it(): #for varName in prog.byDefName: # var = prog.byDefName[varName] # print "%-15s / %-15s > " % var.type , varName - - #import tests.ifaceclass + + #import tests.ifaceclass #tests.ifaceclass.do_test(prog) @@ -650,4 +892,4 @@ def do_it(): -if __name__ == "__main__": main() \ No newline at end of file +if __name__ == "__main__": main() diff --git a/flscriptparser2 b/flscriptparser2 new file mode 100755 index 0000000..6d660b9 --- /dev/null +++ b/flscriptparser2 @@ -0,0 +1,4 @@ +#!/usr/bin/python3 +from flparser import postparse +if __name__ == "__main__": + postparse.main() diff --git a/flscriptparser_automatic.txt b/flscriptparser_automatic.txt new file mode 100644 index 0000000..99a4d14 --- /dev/null +++ b/flscriptparser_automatic.txt @@ -0,0 +1,16 @@ +analisis completo: +git ls-files -- *.qs | xargs -t -P8 -IARG -n1 bash -c "eneboo-mergetool file-check qs-classes ARG >ARG.err 2>&1"; find * -iname "*err" -empty -delete + +analisis de los antiguos errores: +git ls-files -- *.qs | xargs -t -P8 -IARG -n1 bash -c "test -f ARG.err && eneboo-mergetool file-check qs-classes ARG >ARG.err 2>&1"; find * -iname "*err" -empty -delete + +lectura del analisis resumido: +grep -vE "(no corresponde|>>function |clase ifaceCtx|no heredó iface|para diferentes clases|más abajo en el código)" $(find * -iname "*err" ) + + +limpiar: +git clean -fx -- "*.qs.err" + + +auto-corregir "asumiendo class_definition": (PELIGROSO!!) +grep -h "asum" $(find * -iname "*err" ) | awk 'BEGIN { FS = " " } ; { print $16 "\t" $7 " " $8 "\t" $20 " " $21 }' | sed 's/[@\)]/ /g' | sed 's/:/\t/g' | awk '{ print "sed -i '"'"'" $2-3 "," $2+3 "s/" $3 " " $4 "/" $5 " " $6 "/g'"'"' " $1 }' diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..d6bb669 --- /dev/null +++ b/install.sh @@ -0,0 +1,22 @@ +#!/bin/bash +[[ $(basename $(pwd)) != "flscriptparser" ]] && { + echo "This program MUST be run from its folder!!!" + exit 1 +} +DEPS=$(cat dependencies.debian) +aptitude install $DEPS -y +while read line; do + test -e /usr/local/bin/fldesigner && unlink /usr/local/bin/fldesigner + ln -s "$line" /usr/local/bin/fldesigner + break +done < <(find /usr/local/ -type f -name designer) + +while read line; do + bname=$(basename "$line") + test -h /usr/local/bin/$bname && unlink /usr/local/bin/$bname + ln -s "$line" /usr/local/bin/$bname +done < <(find $(pwd) -executable -type f \! -path "*/.*") +test -f /usr/local/bin/flscriptparser && unlink /usr/local/bin/flscriptparser + +exit 0 + diff --git a/lextab.py b/lextab.py new file mode 100644 index 0000000..342b88d --- /dev/null +++ b/lextab.py @@ -0,0 +1,10 @@ +# lextab.py. This file automatically created by PLY (version 3.11). Don't edit! +_tabversion = '3.10' +_lextokens = set(('AND', 'AT', 'BACKSLASH', 'BREAK', 'CASE', 'CATCH', 'CCONST', 'CLASS', 'COLON', 'COMMA', 'COMMENTCLOSE', 'CONDITIONAL1', 'CONST', 'CONTINUE', 'DEFAULT', 'DELETE', 'DIVEQUAL', 'DIVIDE', 'DO', 'DOCSTRINGOPEN', 'DOLLAR', 'DQOUTE', 'ELSE', 'EQ', 'EQQ', 'EQUALS', 'EXTENDS', 'FCONST', 'FOR', 'FUNCTION', 'GE', 'GT', 'ICONST', 'ID', 'IF', 'IN', 'LAND', 'LBRACE', 'LBRACKET', 'LE', 'LNOT', 'LOR', 'LPAREN', 'LSHIFT', 'LT', 'MINUS', 'MINUSEQUAL', 'MINUSMINUS', 'MOD', 'MODEQUAL', 'NE', 'NEQ', 'NEW', 'OR', 'PERIOD', 'PLUS', 'PLUSEQUAL', 'PLUSPLUS', 'RBRACE', 'RBRACKET', 'RETURN', 'RPAREN', 'RSHIFT', 'SCONST', 'SEMI', 'SQOUTE', 'STATIC', 'SWITCH', 'THROW', 'TIMES', 'TIMESEQUAL', 'TRY', 'TYPEOF', 'VAR', 'WHILE', 'WITH', 'XOR')) +_lexreflags = 64 +_lexliterals = '' +_lexstateinfo = {'INITIAL': 'inclusive'} +_lexstatere = {'INITIAL': [('(?P\\n+)|(?P[A-Za-z_]+[\\w_]*)|(?P(/\\*( |\\*\\*)(.|\\n)*?\\*/)|(//.*))|(?P/\\*\\*[ ]+)|(?P\\#(.)*?\\n)|(?P((\\d+)(\\.\\d+)(e(\\+|-)?(\\d+))? | (\\d+)e(\\+|-)?(\\d+))([lL]|[fF])?)|(?P\\d+([uU]|[lL]|[uU][lL]|[lL][uU])?)|(?P\\"([^\\"\\\\\\n]|(\\\\.)|\\\\\\n)*?\\")|(?P\\\'([^\\\'\\\\\\n]|(\\\\.)|\\\\\\n)*?\\\')|(?P\\|\\|)|(?P\\+\\+)|(?P===)|(?P!==)|(?P\\*=)|(?P\\+=)|(?P\\*/)|(?P\\\\)|(?P\\$)|(?P\\+)|(?P\\*)|(?P\\|)|(?P\\^)|(?P<<)|(?P>>)|(?P&&)|(?P<=)|(?P>=)|(?P==)|(?P!=)|(?P\\?)|(?P/=)|(?P%=)|(?P-=)|(?P--)|(?P\\()|(?P\\))|(?P\\[)|(?P\\])|(?P\\{)|(?P\\})|(?P\\.)|(?P\')|(?P")|(?P-)|(?P/)|(?P%)|(?P&)|(?P!)|(?P<)|(?P>)|(?P=)|(?P,)|(?P;)|(?P:)|(?P@)', [None, ('t_NEWLINE', 'NEWLINE'), ('t_ID', 'ID'), ('t_comment', 'comment'), None, None, None, None, ('t_DOCSTRINGOPEN', 'DOCSTRINGOPEN'), ('t_preprocessor', 'preprocessor'), None, (None, 'FCONST'), None, None, None, None, None, None, None, None, None, None, (None, 'ICONST'), None, (None, 'SCONST'), None, None, (None, 'CCONST'), None, None, (None, 'LOR'), (None, 'PLUSPLUS'), (None, 'EQQ'), (None, 'NEQ'), (None, 'TIMESEQUAL'), (None, 'PLUSEQUAL'), (None, 'COMMENTCLOSE'), (None, 'BACKSLASH'), (None, 'DOLLAR'), (None, 'PLUS'), (None, 'TIMES'), (None, 'OR'), (None, 'XOR'), (None, 'LSHIFT'), (None, 'RSHIFT'), (None, 'LAND'), (None, 'LE'), (None, 'GE'), (None, 'EQ'), (None, 'NE'), (None, 'CONDITIONAL1'), (None, 'DIVEQUAL'), (None, 'MODEQUAL'), (None, 'MINUSEQUAL'), (None, 'MINUSMINUS'), (None, 'LPAREN'), (None, 'RPAREN'), (None, 'LBRACKET'), (None, 'RBRACKET'), (None, 'LBRACE'), (None, 'RBRACE'), (None, 'PERIOD'), (None, 'SQOUTE'), (None, 'DQOUTE'), (None, 'MINUS'), (None, 'DIVIDE'), (None, 'MOD'), (None, 'AND'), (None, 'LNOT'), (None, 'LT'), (None, 'GT'), (None, 'EQUALS'), (None, 'COMMA'), (None, 'SEMI'), (None, 'COLON'), (None, 'AT')])]} +_lexstateignore = {'INITIAL': ' \r\t\x0c'} +_lexstateerrorf = {'INITIAL': 't_error'} +_lexstateeoff = {} diff --git a/postparse.py b/postparse.py new file mode 100644 index 0000000..47b7148 --- /dev/null +++ b/postparse.py @@ -0,0 +1,651 @@ +#!/usr/bin/python +from __future__ import print_function +from __future__ import absolute_import +from builtins import str +from builtins import object +from optparse import OptionParser +import os, os.path, sys +import imp, traceback +from lxml import etree +try: + from future.utils import with_metaclass +except ImportError: + pass + + +try: + from pineboolib.flparser import flscriptparse +except ImportError: + import flscriptparse + +USEFUL_TOKENS="ID,ICONST,FCONST,SCONST,CCONST,RXCONST".split(",") + +KNOWN_PARSERS = {} +UNKNOWN_PARSERS = {} +def parse_for(*tagnames): + global KNOWN_PARSERS + def decorator(fn): + for n in tagnames: + KNOWN_PARSERS[n] = fn + return fn + return decorator + +def parse(tagname, treedata): + global KNOWN_PARSERS,UNKNOWN_PARSERS + if tagname not in KNOWN_PARSERS: + UNKNOWN_PARSERS[tagname] = 1 + fn = parse_unknown + else: + fn = KNOWN_PARSERS[tagname] + return fn(tagname,treedata) + + +def getxmltagname(tagname): + if tagname == "source": return "Source" + if tagname == "funcdeclaration": return "Function" + if tagname == "classdeclaration": return "Class" + if tagname == "vardeclaration": return "Variable" + return "Unknown.%s" % tagname + +xml_class_types = [] + +class TagObjectFactory(type): + def __init__(cls, name, bases, dct): + global xml_class_types + xml_class_types.append(cls) + super(TagObjectFactory, cls).__init__(name, bases, dct) + +class TagObject(with_metaclass(TagObjectFactory, object)): + tags = [] + set_child_argn = False + name_is_first_id = False + debug_other = True + adopt_childs_tags = [] + omit_tags = ['empty'] + callback_subelem = {} + promote_child_if_alone = False + + @classmethod + def tagname(self, tagname): return self.__name__ + + @classmethod + def can_process_tag(self, tagname): return tagname in self.tags + + def __init__(self, tagname): + self.astname = tagname + self.xml = etree.Element(self.tagname(tagname)) + self.xmlname = None + self.subelems = [] + self.values = [] + if self.name_is_first_id: + self.xml.set("name","") + + def adopt_children(self, argn, subelem): + for child in subelem.xml.iterchildren(): + if self.set_child_argn: child.set("argn",str(argn)) + else: + if 'argn' in child.attrib: del child.attrib['argn'] + self.xml.append(child) + + def omit_subelem(self, argn, subelem): + return + + def is_in(self, listobj): + return self.__class__ in listobj or self.astname in listobj + + def get(self, listobj, default = None): + if self.__class__ in listobj: return listobj[self.__class__] + if self.astname in listobj: return listobj[self.astname] + return default + + + def add_subelem(self, argn, subelem): + if subelem.is_in(self.omit_tags): return self.omit_subelem(argn, subelem) + if subelem.is_in(self.adopt_childs_tags): return self.adopt_children(argn, subelem) + callback = subelem.get(self.callback_subelem) + if callback: return getattr(self,callback)(argn,subelem) + + if self.set_child_argn: subelem.xml.set("argn",str(argn)) + self.xml.append(subelem.xml) + self.subelems.append(subelem) + + def add_value(self, argn, vtype, value): + self.values.append( (vtype, value) ) + if vtype == "ID" and self.name_is_first_id and self.xmlname is None: + self.xmlname = value + self.xml.set("name",value) + return + + self.xml.set("arg%02d" % argn,vtype + ":" + repr(value)) + + def add_other(self, argn, vtype, data): + if self.debug_other: + self.xml.set("arg%02d" % argn,vtype) + + def polish(self): + if self.promote_child_if_alone: + if len(self.values) == 0 and len(self.subelems) == 1: + return self.subelems[0] + return self + + +class ListObject(TagObject): + set_child_argn = False + debug_other = False + +class NamedObject(TagObject): + name_is_first_id = True + debug_other = False + +class ListNamedObject(TagObject): + name_is_first_id = True + set_child_argn = False + debug_other = False + +class TypedObject(ListObject): + type_arg = 0 + def add_other(self, argn, vtype, value): + if argn == self.type_arg: + self.xml.set("type", vtype) + + +class Source(ListObject): + tags = ["source","basicsource","classdeclarationsource","statement_list","statement_block"] + adopt_childs_tags = ['source_element','statement_list','statement',"statement_block"] + +class Identifier(NamedObject): + tags = ["identifier","optid"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + + +class Arguments(ListObject): + tags = ["arglist"] + adopt_childs_tags = ['vardecl_list'] + +class VariableType(NamedObject): + tags = ["optvartype"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + +class ExtendsType(NamedObject): + tags = ["optextends"] + def polish(self): + if self.xmlname is None: + self.astname = "empty" + return self + +class Function(ListNamedObject): + tags = ["funcdeclaration"] + callback_subelem = ListNamedObject.callback_subelem.copy() + callback_subelem[VariableType] = "add_vartype" + + def add_vartype(self, argn, subelem): + self.xml.set("returns", str(subelem.xmlname)) + +class FunctionAnon(ListObject): + tags = ["funcdeclaration_anon"] + +class FunctionAnonExec(ListObject): + tags = ["funcdeclaration_anon_exec"] + +class Variable(NamedObject): + tags = ["vardecl"] + callback_subelem = NamedObject.callback_subelem.copy() + callback_subelem[VariableType] = "add_vartype" + + def add_vartype(self, argn, subelem): + self.xml.set("type", str(subelem.xmlname)) + +class DeclarationBlock(ListObject): + tags = ["vardeclaration"] + adopt_childs_tags = ['vardecl_list'] + def add_other(self, argn, vtype, value): + if argn == 0: + self.xml.set("mode", vtype) + def polish(self): + #if len(self.values) == 0 and len(self.subelems) == 1: + # self.subelems[0].xml.set("mode",self.xml.get("mode")) + # return self.subelems[0] + return self + +class Class(ListNamedObject): + tags = ["classdeclaration"] + callback_subelem = ListNamedObject.callback_subelem.copy() + callback_subelem[ExtendsType] = "add_exttype" + def add_exttype(self, argn, subelem): + self.xml.set("extends", str(subelem.xmlname)) + +class Member(TagObject): + debug_other = False + set_child_argn = False + tags = ["member_var","member_call"] + adopt_childs_tags = ['varmemcall',"member_var","member_call"] + +class ArrayMember(TagObject): + debug_other = False + set_child_argn = False + tags = ["array_member"] + adopt_childs_tags = ['variable_1',"func_call"] + +class InstructionCall(TagObject): + debug_other = False + tags = ["callinstruction"] + +class InstructionStore(TagObject): + promote_child_if_alone = True + debug_other = False + tags = ["storeinstruction"] + +class InstructionFlow(TypedObject): + debug_other = True + tags = ["flowinstruction"] + +class Instruction(TagObject): + promote_child_if_alone = True + debug_other = False + tags = ["instruction"] + +class OpMath(TypedObject): + debug_other = True + tags = ["mathoperator"] + +class Compare(TypedObject): + debug_other = True + tags = ["cmp_symbol","boolcmp_symbol"] + +class FunctionCall(NamedObject): + tags = ["funccall_1"] + +class CallArguments(ListObject): + tags = ["callargs"] + +class Constant(ListObject): + tags = ["constant"] + def add_value(self, argn, vtype, value): + value = str(value) #str(value,"ISO-8859-15","replace") + if vtype == "SCONST": + vtype = "String" + value = value[1:-1] + self.xml.set("delim",'"') + if vtype == "CCONST": + vtype = "String" + value = value[1:-1] + self.xml.set("delim","'") + if vtype == "RCONST": vtype = "Regex" + if vtype == "ICONST": vtype = "Number" + if vtype == "FCONST": vtype = "Number" + self.const_value = value + self.const_type = vtype + self.xml.set("type",vtype) + self.xml.set("value",value) + +class InlineUpdate(ListObject): + tags = ["inlinestoreinstruction"] + def add_other(self, argn, vtype, value): + self.xml.set("type", vtype) + if argn == 0: + self.xml.set("mode", "update-read") + if argn == 1: + self.xml.set("mode", "read-update") + +class If(ListObject): + tags = ["ifstatement"] + +class Condition(ListObject): + tags = ["condition"] + +class Else(ListObject): + tags = ["optelse"] + def polish(self): + if len(self.subelems) == 0: + self.astname = "empty" + return self + +class DictObject(ListObject): + tags = ["dictobject_value_elemlist","dictobject_value"] + adopt_childs_tags = ['dictobject_value_elemlist',"dictobject_value"] + +class DictElem(ListObject): + tags = ["dictobject_value_elem"] + + +class ExpressionContainer(ListObject): + tags = ["expression"] + # adopt_childs_tags = ['base_expression'] + + def polish(self): + if len(self.values) == 0 and len(self.subelems) == 1: + #if isinstance(self.subelems[0], Constant): + if self.subelems[0].xml.tag == "base_expression": + self.subelems[0].xml.tag = "Expression" + return self.subelems[0] + else: + self.xml.tag = "Value" + + return self + +class InstructionUpdate(ListObject): + tags = ["updateinstruction"] + +class Switch(ListObject): + tags = ["switch"] + adopt_childs_tags = ['case_cblock_list','case_block_list'] + +class CaseList(ListObject): + tags = ["case_block_list"] + adopt_childs_tags = ['case_cblock_list','case_block_list'] + +class Case(ListObject): + tags = ["case_block"] + +class CaseDefault(ListObject): + tags = ["case_default"] + +class While(ListObject): + tags = ["whilestatement"] + +class For(ListObject): + tags = ["forstatement"] + +class ForInitialize(ListObject): + tags = ["for_initialize"] + +class ForCompare(ListObject): + tags = ["for_compare"] + +class ForIncrement(ListObject): + tags = ["for_increment"] + +class DoWhile(ListObject): + tags = ["dowhilestatement"] + +class ForIn(ListObject): + tags = ["forinstatement"] + +class With(ListObject): + tags = ["withstatement"] + +class TryCatch(ListObject): + tags = ["trycatch"] + +class New(ListObject): + tags = ["new_operator"] + +class Delete(ListObject): + tags = ["deleteinstruction"] + +class Parentheses(ListObject): + tags = ["parentheses"] + adopt_childs_tags = ['base_expression'] + +class OpUnary(TypedObject): + tags = ["unary_operator"] + +class OpTernary(ListObject): + tags = ["ternary_operator"] + +class OpUpdate(TypedObject): + tags = ["updateoperator"] + + + +# ----- keep this one at the end. +class Unknown(TagObject): + promote_child_if_alone = True + set_child_argn = False + @classmethod + def tagname(self, tagname): return tagname + + @classmethod + def can_process_tag(self, tagname): return True +# ----------------- + +def create_xml(tagname): + classobj = None + for cls in xml_class_types: + if cls.can_process_tag(tagname): + classobj = cls + break + if classobj is None: return None + return classobj(tagname) + +def parse_unknown(tagname, treedata): + xmlelem = create_xml(tagname) + i = 0 + for k, v in treedata['content']: + if type(v) is dict: + instruction = parse(k,v) + xmlelem.add_subelem(i, instruction) + elif k in USEFUL_TOKENS: + xmlelem.add_value(i, k, v) + else: + xmlelem.add_other(i, k, v) + + i+=1 + return xmlelem.polish() + + + +def post_parse(treedata): + source = parse("source",treedata) + #print UNKNOWN_PARSERS.keys() + return source.xml + +class Module(object): + def __init__(self, name, path): + self.name = name + self.path = path + def loadModule(self): + fp = None + try: + description = ('.py', 'U', imp.PY_SOURCE) + #description = ('.pyc', 'U', PY_COMPILED) + pathname = os.path.join(self.path, self.name) + fp = open(pathname) + name = self.name[:self.name.find(".")] + # fp, pathname, description = imp.find_module(self.name,[self.path]) + self.module = imp.load_module(name, fp, pathname, description) + result = True + except FileNotFoundError: + print("Fichero %r no encontrado" % self.name) + result = False + except Exception as e: + print(traceback.format_exc()) + result = False + if fp: + fp.close() + return result + +def parseArgs(argv): + parser = OptionParser() + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("--path", + dest="storepath", default=None, + help="store XML results in PATH") + + parser.add_option("--topython", + action="store_true", dest="topython", default=False, + help="write python file from xml") + + parser.add_option("--exec-py", + action="store_true", dest="exec_python", default=False, + help="try to execute python file") + + parser.add_option("--toxml", + action="store_true", dest="toxml", default=False, + help="write xml file from qs") + + parser.add_option("--full", + action="store_true", dest="full", default=False, + help="write xml file from qs") + + parser.add_option("--cache", + action="store_true", dest="cache", default=False, + help="If dest file exists, don't regenerate it") + + (options, args) = parser.parse_args(argv) + return (options, args) + +def main(): + options, args = parseArgs(sys.argv[1:]) + execute(options,args) + +def pythonify(filelist): + options, args = parseArgs([]) + options.full = True + if isinstance(filelist, str): filelist = [filelist] + execute(options,filelist) + print(filelist) + + + +def execute(options, args): + if options.optdebug: + print(options, args) + if options.full: + execpython = options.exec_python + options.exec_python = False + options.full = False + options.toxml = True + if options.verbose: print("Pass 1 - Parse and write XML file . . .") + try: + execute(options,args) + except Exception: + print("Error parseando:"); + print(traceback.format_exc()) + + options.toxml = False + options.topython = True + if options.verbose: print("Pass 2 - Pythonize and write PY file . . .") + try: + execute(options,[ arg+".xml" for arg in args]) + except Exception: + print("Error convirtiendo:"); + print(traceback.format_exc()) + + if execpython: + options.exec_python = execpython + if options.verbose: print("Pass 3 - Test PY file load . . .") + options.topython = False + try: + execute(options,[ (arg+".xml.py").replace(".qs.xml.py",".qs.py") for arg in args]) + except Exception: + print("Error al ejecutar Python:"); + print(traceback.format_exc()) + + print("Done.") + + elif options.exec_python: + from . import qsatype + for filename in args: + realpath = os.path.realpath(filename) + path, name = os.path.split(realpath) + if not os.path.exists(realpath): + print("Fichero no existe: %s" % name) + continue + + mod = Module(name, path) + if not mod.loadModule(): + print("Error cargando modulo %s" % name) + + elif options.topython: + from .pytnyzer import pythonize + import io + if options.cache: + args = [ x for x in args if not os.path.exists((x+".py").replace(".qs.xml.py",".qs.py")) + or os.path.getmtime(x) > os.path.getctime((x+".py").replace(".qs.xml.py",".qs.py")) ] + + nfs = len(args) + for nf, filename in enumerate(args): + bname = os.path.basename(filename) + if options.storepath: + destname = os.path.join(options.storepath,bname+".py") + else: + destname = filename+".py" + destname = destname.replace(".qs.xml.py",".qs.py") + if not os.path.exists(filename): + print("Fichero %r no encontrado" % filename) + continue + if options.verbose: sys.stdout.write("Pythonizing File: %-35s . . . . (%.1f%%) \r" % (bname,100.0*(nf+1.0)/nfs)) + if options.verbose: sys.stdout.flush(); + old_stderr = sys.stdout + stream = io.StringIO() + sys.stdout = stream + try: + pythonize(filename, destname, destname + ".debug") + except Exception: + print("Error al pythonificar %r:" % filename) + print(traceback.format_exc()) + sys.stdout = old_stderr + text = stream.getvalue() + if len(text) > 2: + print("%s: " % bname + ("\n%s: " % bname).join(text.splitlines())) + + + else: + if options.cache: + args = [ x for x in args if not os.path.exists(x+".xml") + or os.path.getmtime(x) > os.path.getctime(x+".xml")] + nfs = len(args) + for nf, filename in enumerate(args): + bname = os.path.basename(filename) + if options.verbose: sys.stdout.write("Parsing File: %-35s . . . . (%.1f%%) " % (bname,100.0*(nf+1.0)/nfs)) + if options.verbose: sys.stdout.flush(); + try: + filecontent = open(filename,"r", encoding="latin-1").read() + except Exception as e: + print("Error: No se pudo abrir fichero %-35s \n" % (repr(filename)), e) + continue + prog = flscriptparse.parse(filecontent) + sys.stdout.write("\r"); + if not prog: + print("Error: No se pudo abrir %-35s \n" % (repr(filename))) + continue + if prog["error_count"] > 0: + print("Encontramos %d errores parseando: %-35s \n" % (prog["error_count"], repr(filename))) + continue + if options.toxml == False: + # Si no se quiere guardar resultado, no hace falta calcular mas + continue + + tree_data = None + try: + tree_data = flscriptparse.calctree(prog, alias_mode = 0) + except Exception: + print("Error al convertir a XML %r:" % bname) + print("\n".join(traceback.format_exc().splitlines()[-7:])) + + if not tree_data: + print("No se pudo parsear %-35s \n" % (repr(filename))) + continue + ast = post_parse(tree_data) + if ast is None: + print("No se pudo analizar %-35s \n" % (repr(filename))) + continue + if options.storepath: + destname = os.path.join(options.storepath,bname+".xml") + else: + destname = filename+".xml" + f1 = open(destname,"wb") + f1.write(etree.tostring(ast, pretty_print = True)) + f1.close() + + + +if __name__ == "__main__": main() diff --git a/pytnyzer.py b/pytnyzer.py new file mode 100644 index 0000000..a088dd4 --- /dev/null +++ b/pytnyzer.py @@ -0,0 +1,1118 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# ------ Pythonyzer ... reads XML AST created by postparse.py and creates an equivalent Python file. +from __future__ import print_function +from __future__ import absolute_import +from builtins import object +from optparse import OptionParser +import os, os.path, random +from . import flscriptparse +from lxml import etree +from future.utils import with_metaclass + + +def id_translate(name): + python_keywords = ['and', 'del', 'for', 'is', 'raise', 'assert', 'elif', + 'from', 'lambda', 'return', 'break', 'else', 'global', 'not', 'try', + 'class', 'except', 'if', 'or', 'while', 'continue', 'from', + 'exec', 'import', 'pass', 'yield', 'def', 'finally', 'in', 'print'] + if name in python_keywords: return name + "_" + if name == "false": name = "False" + if name == "true": name = "True" + if name == "null": name = "None" + if name == "unknown": name = "None" + if name == "undefined": name = "None" + if name == "this": name = "self" + + if name == "startsWith": name = "startswith" + return name + +ast_class_types = [] + +class ASTPythonFactory(type): + def __init__(cls, name, bases, dct): + global ast_class_types + ast_class_types.append(cls) + super(ASTPythonFactory, cls).__init__(name, bases, dct) + +class ASTPython(with_metaclass(ASTPythonFactory, object)): + tags = [] + debug_file = None + generate_depth = 0 + numline = 0 + @classmethod + def can_process_tag(self, tagname): return self.__name__ == tagname or tagname in self.tags + + def __init__(self, elem): + self.elem = elem + if self.debug_file: + self.internal_generate = self.generate + self.generate = self._generate + + def debug(self, text): + if self.debug_file is None: return + splen = ASTPython.generate_depth + retlen = 0 + if splen > self.generate_depth: + retlen, splen = splen - self.generate_depth, self.generate_depth + if retlen > 0: + sp = " " * (splen - 1) + "<" + "-" * retlen + else: + sp = " " * splen + cname = self.__class__.__name__ + self.debug_file.write("%04d" % ASTPython.numline + sp + cname + ": " + text + "\n"); + + def polish(self): return self + + def generate(self, **kwargs): + yield "debug", "* not-known-seq * " + etree.tounicode(self.elem) + def _generate(self, **kwargs): + self.debug("begin-gen") + ASTPython.generate_depth += 1 + self.generate_depth = ASTPython.generate_depth + for dtype, data in self.internal_generate(**kwargs): + self.debug("%s: %r" % (dtype, data)) + yield dtype, data + ASTPython.generate_depth -= 1 + self.debug("end-gen") + + + + +class Source(ASTPython): + def generate(self, break_mode = False, include_pass = True, **kwargs): + elems = 0 + after_lines = [] + for child in self.elem: + #yield "debug", "<%s %s>" % (child.tag, repr(child.attrib)) + for dtype, data in parse_ast(child).generate(break_mode = break_mode, plusplus_as_instruction = True): + if dtype == "line+1": + after_lines.append(data) + continue + if dtype == "line": + elems += 1 + yield dtype, data + if dtype == "line" and after_lines: + for line in after_lines: + elems+=1 + yield dtype, line + after_lines = [] + if dtype == "break": + for line in after_lines: + elems+=1 + yield "line", line + + for line in after_lines: + elems+=1 + yield "line", line + if elems == 0 and include_pass: + yield "line", "pass" + +class Class(ASTPython): + def generate(self, **kwargs): + name = self.elem.get("name") + extends = self.elem.get("extends","object") + + yield "line", "class %s(%s):" % (name,extends) + yield "begin", "block-class-%s" % (name) + for source in self.elem.xpath("Source"): + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-class-%s" % (name) + +class Function(ASTPython): + def generate(self, **kwargs): + _name = self.elem.get("name") + if _name: + name = id_translate(_name) + else: + # Anonima: + name = "_" + withoutself = self.elem.get("withoutself") + + returns = self.elem.get("returns",None) + parent = self.elem.getparent() + grandparent = None + if parent is not None: grandparent = parent.getparent() + arguments = [] + if not withoutself: + if grandparent is not None: + if grandparent.tag == "Class": + arguments.append("self") + if name == grandparent.get("name"): + name = "__init__" + else: + arguments.append("self") + for n,arg in enumerate(self.elem.xpath("Arguments/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(id_translate(data)) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + if len(expr) == 1: expr += ["=","None"] + arguments.append(" ".join(expr)) + + + + yield "line", "def %s(%s):" % (name,", ".join(arguments)) + yield "begin", "block-def-%s" % (name) + # if returns: yield "debug", "Returns: %s" % returns + for source in self.elem.xpath("Source"): + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-def-%s" % (name) + +class FunctionAnon(Function): + pass + +class FunctionCall(ASTPython): + def generate(self, **kwargs): + name = id_translate(self.elem.get("name")) + parent = self.elem.getparent() + if parent.tag == "InstructionCall": + classes = parent.xpath("ancestor::Class") + if classes: + class_ = classes[-1] + extends = class_.get("extends") + if extends == name: + name = "super(%s, self).__init__" % class_.get("name") + functions = parent.xpath("//Function[@name=\"%s\"]" % name) + for f in functions: + #yield "debug", "Function to:" + etree.tostring(f) + name = "self.%s" % name + break + + arguments = [] + for n,arg in enumerate(self.elem.xpath("CallArguments/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "%s(%s)" % (name,", ".join(arguments)) + +class If(ASTPython): + def generate(self, break_mode = False, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False, ): + if dtype == "line+1": + yield "debug", "Inline update inside IF condition not allowed. Unexpected behavior." + dtype = "line" + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + + yield "line", "if %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-if" + for obj in parse_ast(source).generate(break_mode = break_mode): yield obj + yield "end", "block-if" + + for source in self.elem.xpath("Else/Source"): + yield "line", "else:" + yield "begin", "block-else" + for obj in parse_ast(source).generate(break_mode = break_mode): yield obj + yield "end", "block-else" + +class TryCatch(ASTPython): + def generate(self, **kwargs): + + tryblock, catchblock = self.elem.xpath("Source") + + yield "line", "try:" + yield "begin", "block-try" + for obj in parse_ast(tryblock).generate(): yield obj + yield "end", "block-try" + + identifier = None + for ident in self.elem.xpath("Identifier"): + expr = [] + for dtype, data in parse_ast(ident).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + identifier = " ".join(expr) + if identifier: + yield "line", "except Exception as %s:" % (identifier) + else: + yield "line", "except Exception:" + yield "begin", "block-except" + if identifier: + # yield "line", "%s = str(%s)" % (identifier, identifier) + yield "line", "%s = traceback.format_exc()" % (identifier) + for obj in parse_ast(catchblock).generate(include_pass = identifier is None): yield obj + yield "end", "block-except" + + +class While(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + + yield "line", "while %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-while" + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-while" + +class DoWhile(ASTPython): + def generate(self, **kwargs): + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + # TODO ..... + key = "%02x" % random.randint(0,255) + name1st = "s%s_dowhile_1stloop" % key + yield "line", "%s = True" % (name1st) + + yield "line", "while %s or %s:" % (name1st, " ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-while" + yield "line", "%s = False" % (name1st) + for obj in parse_ast(source).generate(): yield obj + yield "end", "block-while" + +class For(ASTPython): + def generate(self, **kwargs): + init_expr = [] + for n,arg in enumerate(self.elem.xpath("ForInitialize/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) > 0: + init_expr.append(" ".join(expr)) + if init_expr: + yield "line", " ".join(init_expr) + + incr_expr = [] + incr_lines = [] + for n,arg in enumerate(self.elem.xpath("ForIncrement/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + elif dtype in ["line","line+1"]: + incr_lines.append(data) + else: + yield dtype, data + if len(expr) > 0: + incr_expr.append(" ".join(expr)) + + + main_expr = [] + for n,arg in enumerate(self.elem.xpath("ForCompare/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("True") + else: + main_expr.append(" ".join(expr)) + #yield "debug", "WHILE-FROM-QS-FOR: (%r;%r;%r)" % (init_expr,main_expr,incr_lines) + yield "line", "while %s:" % (" ".join(main_expr)) + for source in self.elem.xpath("Source"): + yield "begin", "block-for" + for obj in parse_ast(source).generate(include_pass=False): yield obj + if incr_lines: + for line in incr_lines: + yield "line", line + yield "end", "block-for" + +class ForIn(ASTPython): + def generate(self, **kwargs): + list_elem, main_list = "None", "None" + myelems = [] + for e in self.elem: + if e.tag == "Source": break + if e.tag == "ForInitialize": e = list(e)[0] + expr = [] + for dtype, data in parse_ast(e).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + myelems.append(" ".join(expr)) + list_elem, main_list = myelems + yield "debug", "FOR-IN: " + repr(myelems) + yield "line", "for %s in %s:" % (list_elem, main_list) + for source in self.elem.xpath("Source"): + yield "begin", "block-for-in" + for obj in parse_ast(source).generate(include_pass=False): yield obj + yield "end", "block-for-in" + +class Switch(ASTPython): + def generate(self, **kwargs): + key = "%02x" % random.randint(0,255) + name = "s%s_when" % key + name_pr = "s%s_do_work" % key + name_pr2 = "s%s_work_done" % key + main_expr = [] + for n,arg in enumerate(self.elem.xpath("Condition/*")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + main_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + main_expr.append(" ".join(expr)) + yield "line", "%s = %s" % (name, " ".join(main_expr)) + yield "line", "%s,%s = %s,%s" % (name_pr,name_pr2, "False","False") + for scase in self.elem.xpath("Case"): + value_expr = [] + for n,arg in enumerate(scase.xpath("Value")): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + value_expr.append("False") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + else: + value_expr.append(" ".join(expr)) + + yield "line", "if %s == %s: %s,%s = %s,%s" % (name," ".join(value_expr),name_pr,name_pr2, "True", "True") + yield "line", "if %s:" % (name_pr) + yield "begin", "block-if" + count = 0 + for source in scase.xpath("Source"): + for obj in parse_ast(source).generate(break_mode = True): + if obj[0] == "break": + yield "line", "%s = %s # BREAK" % (name_pr,"False") + count += 1 + else: + yield obj + count += 1 + if count < 1: yield "line", "pass" + yield "end", "block-if" + + for scasedefault in self.elem.xpath("CaseDefault"): + yield "line", "if not %s: %s,%s = %s,%s" % (name_pr2, name_pr,name_pr2, "True", "True") + yield "line", "if %s:" % (name_pr) + yield "begin", "block-if" + for source in scasedefault.xpath("Source"): + for obj in parse_ast(source).generate(break_mode = True): + if obj[0] == "break": + yield "line", "%s = %s # BREAK" % (name_pr,"False") + else: + yield obj + yield "end", "block-if" + # yield "line", "assert( not %s )" % name_pr + # yield "line", "assert( %s )" % name_pr2 + +class With(ASTPython): + def generate(self, **kwargs): + key = "%02x" % random.randint(0,255) + name = "w%s_obj" % key + #yield "debug", "WITH: %s" % key + variable, source = [ obj for obj in self.elem ] + var_expr = [] + for dtype, data in parse_ast(variable).generate(isolate = False): + if dtype == "expr": + var_expr.append(data) + else: + yield dtype, data + if len(var_expr) == 0: + var_expr.append("None") + yield "debug", "Expression %d not understood" % n + yield "debug", etree.tostring(arg) + + yield "line", "%s = %s" % (name, " ".join(var_expr)) + + for obj in parse_ast(source).generate(break_mode = True): + yield obj + yield "line", "del %s" % name + +class Variable(ASTPython): + def generate(self, force_value = False, **kwargs): + name = self.elem.get("name") + #if name.startswith("colorFun"): print(name) + yield "expr", id_translate(name) + values = 0 + for value in self.elem.xpath("Value|Expression"): + values += 1 + yield "expr", "=" + expr = 0 + for dtype, data in parse_ast(value).generate(isolate = False): + if dtype == "expr": expr += 1 + yield dtype, data + if expr == 0: + yield "expr", "None" + + dtype = self.elem.get("type",None) + + if values == 0 and force_value == True: + yield "expr", "=" + if dtype is None: + yield "expr", "None" + elif dtype == "String": + yield "expr", "\"\"" + elif dtype == "Number": + yield "expr", "0" + else: + parent1 = self.elem.getparent() + parent2 = parent1.getparent() + parent3 = parent2.getparent() + if parent2 == "Source" and parent3 == "Class": + yield "expr", "None" + else: + yield "expr", "qsatype.%s()" % dtype + + #if dtype and force_value == False: yield "debug", "Variable %s:%s" % (name,dtype) + +class InstructionUpdate(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + if data is None: raise ValueError(etree.tostring(arg)) + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "line", " ".join(arguments) + +class InlineUpdate(ASTPython): + def generate(self, plusplus_as_instruction = False, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + ctype = self.elem.get("type") + mode = self.elem.get("mode") + linetype = "line" + if not plusplus_as_instruction: + if mode == "read-update": + linetype = "line+1" + + yield "expr", arguments[0] + if ctype == "PLUSPLUS": + yield linetype, arguments[0] + " += 1" + elif ctype == "MINUSMINUS": + yield linetype, arguments[0] + " -= 1" + else: + yield linetype, arguments[0] + " ?= 1" + +class InstructionCall(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + yield "line", " ".join(arguments) + +class Instruction(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + if arguments: + yield "debug", "Instruction: Maybe parse-error. This class is only for non-understood instructions or empty ones" + yield "line", " ".join(arguments) + +class InstructionFlow(ASTPython): + def generate(self, break_mode = False, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + ctype = self.elem.get("type") + kw = ctype + if ctype == "RETURN": kw = "return" + if ctype == "BREAK": + kw = "break" + if break_mode: + yield "break", kw + " " + ", ".join(arguments) + return + if ctype == "CONTINUE": kw = "continue" + + if ctype == "THROW": + yield "line", "raise Exception(" + ", ".join(arguments) + ")" + return + + yield "line", kw + " " + ", ".join(arguments) + + + +class Member(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + # Lectura del self.iface.__init + if len(arguments) >= 3 and arguments[0:2] == ["self","iface"] and arguments[2].startswith("__"): + # From: self.iface.__function() + # to: super(className, self.iface).function() + funs = self.elem.xpath("ancestor::Function") + if funs: + fun = funs[-1] + name_parts = fun.get("name").split("_") + classname = name_parts[0] + arguments[2] = arguments[2][2:] + arguments[0:2] = ["super(%s, %s)" % (classname,".".join(arguments[0:2]))] + + # Lectura del self.iface.__init() al nuevo estilo yeboyebo + if len(arguments) >= 2 and arguments[0:1] == ["_i"] and arguments[1].startswith("__"): + # From: self.iface.__function() + # to: super(className, self.iface).function() + funs = self.elem.xpath("ancestor::Function") + if funs: + fun = funs[-1] + name_parts = fun.get("name").split("_") + classname = name_parts[0] + arguments[1] = arguments[1][2:] + arguments[0:1] = ["super(%s, %s)" % (classname,".".join(arguments[0:1]))] + + replace_members = [ + "length", + "text", + "join", + "date", + ] + + for member in replace_members: + for idx,arg in enumerate(arguments): + if member == arg or arg.startswith(member+"("): + + part1 = arguments[:idx] + try: + part2 = arguments[idx+1:] + except IndexError: + part2 = [] # Para los que son últimos y no tienen parte adicional + arguments = ["qsa(%s).%s" % (".".join(part1), arg)] + part2 + yield "expr", ".".join(arguments) + +class ArrayMember(ASTPython): + def generate(self, **kwargs): + arguments = [] + for n,arg in enumerate(self.elem): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate=False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "%s[%s]" % (arguments[0],arguments[1]) + + +class Value(ASTPython): + def generate(self, isolate = True, **kwargs): + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + if data is None: raise ValueError(etree.tostring(child)) + yield dtype, data + if isolate: yield "expr", ")" + +class Expression(ASTPython): + tags = ["base_expression"] + def generate(self, isolate = True, **kwargs): + if isolate: yield "expr", "(" + coerce_string_mode = False + if self.elem.xpath("OpMath[@type=\"PLUS\"]"): + if self.elem.xpath("Constant[@type=\"String\"]"): + coerce_string_mode = True + if coerce_string_mode: + yield "expr", "ustr(" + for child in self.elem: + if coerce_string_mode and child.tag == "OpMath": + if child.get("type") == "PLUS": + yield "expr","," + continue + + for dtype, data in parse_ast(child).generate(): + yield dtype, data + + if coerce_string_mode: + yield "expr", ")" + if isolate: yield "expr", ")" + +class Parentheses(ASTPython): + def generate(self, **kwargs): + yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(isolate = False): + yield dtype, data + yield "expr", ")" + +class Delete(ASTPython): + def generate(self, **kwargs): + yield "expr", "del" + for child in self.elem: + for dtype, data in parse_ast(child).generate(isolate = False): + yield dtype, data + + +class OpTernary(ASTPython): + def generate(self, isolate = False, **kwargs): + """ + Ejemplo op. ternario + + + + + + + + + + """ + if_cond = self.elem[0] + then_val = self.elem[1] + else_val = self.elem[2] + yield "expr", "(" # Por seguridad, unos paréntesis + for dtype, data in parse_ast(then_val).generate(): yield dtype, data + yield "expr", "if" + for dtype, data in parse_ast(if_cond).generate(): yield dtype, data + yield "expr", "else" + for dtype, data in parse_ast(else_val).generate(): yield dtype, data + yield "expr", ")" # Por seguridad, unos paréntesis + + +class DictObject(ASTPython): + def generate(self, isolate = False, **kwargs): + yield "expr", "{" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + yield "expr", "," # Como en Python la coma final la ignora, pues la ponemos. + yield "expr", "}" + +class DictElem(ASTPython): + def generate(self, isolate = False, **kwargs): + # Clave: + for dtype, data in parse_ast(self.elem[0]).generate(): + yield dtype, data + yield "expr", ":" + # Valor: + for dtype, data in parse_ast(self.elem[1]).generate(): + yield dtype, data + + +class OpUnary(ASTPython): + def generate(self, isolate = False, **kwargs): + ctype = self.elem.get("type") + if ctype == "LNOT": yield "expr", "not" + elif ctype == "MINUS": yield "expr", "-" + else: yield "expr", ctype + if isolate: yield "expr", "(" + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + yield dtype, data + if isolate: yield "expr", ")" + +class New(ASTPython): + def generate(self, **kwargs): + for child in self.elem: + for dtype, data in parse_ast(child).generate(): + if dtype != "expr": + yield dtype, data + continue + if child.tag == "Identifier": data = data+"()" + ident = data[:data.find("(")] + if ident.find(".") == -1: + if len(self.elem.xpath("//Class[@name='%s']" % ident)) == 0: + data = "qsatype." + data + yield dtype, data + + +class Constant(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + value = self.elem.get("value") + self.debug("ctype: %r -> %r" % (ctype,value)); + if ctype is None or value is None: + for child in self.elem: + if child.tag == "list_constant": + # TODO/FIXME:: list_constant debe ser ELIMINADO o CONVERTIDO por postparse.py + # .... este generador convertirá todos los arrays en vacíos, sin importar + # .... si realmente tienen algo. + yield "expr", "[]" + + if child.tag == "CallArguments": + arguments = [] + for n,arg in enumerate(child): + expr = [] + for dtype, data in parse_ast(arg).generate(isolate = False): + if dtype == "expr": + expr.append(data) + else: + yield dtype, data + if len(expr) == 0: + arguments.append("unknownarg") + yield "debug", "Argument %d not understood" % n + yield "debug", etree.tostring(arg) + else: + arguments.append(" ".join(expr)) + + yield "expr", "qsatype.Array([%s])" % (", ".join(arguments)) + return + if ctype == "String": + delim = self.elem.get("delim") + if delim == "'": + yield "expr", "u'%s'" % value + else: + yield "expr", "u\"%s\"" % value + elif ctype == "Number": + value = value.lstrip("0") + if value == "": value = "0" + yield "expr", value + else: yield "expr", value + +class Identifier(ASTPython): + def generate(self, **kwargs): + name = id_translate(self.elem.get("name")) + yield "expr", name + +class OpUpdate(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "EQUALS": yield "expr", "=" + elif ctype == "PLUSEQUAL": yield "expr", "+=" + elif ctype == "MINUSEQUAL": yield "expr", "-=" + elif ctype == "TIMESEQUAL": yield "expr", "*=" + elif ctype == "DIVEQUAL": yield "expr", "/=" + elif ctype == "MODEQUAL": yield "expr", "%=" + else: yield "expr", "OpUpdate."+ ctype + +class Compare(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "GT": yield "expr", ">" + elif ctype == "LT": yield "expr", "<" + elif ctype == "LE": yield "expr", "<=" + elif ctype == "GE": yield "expr", ">=" + elif ctype == "EQ": yield "expr", "==" + elif ctype == "NE": yield "expr", "!=" + elif ctype == "EQQ": yield "expr", "is" + elif ctype == "NEQ": yield "expr", "not is" + elif ctype == "IN": yield "expr", "in" + elif ctype == "LOR": yield "expr", "or" + elif ctype == "LAND": yield "expr", "and" + else: yield "expr", "Compare."+ctype + +class OpMath(ASTPython): + def generate(self, **kwargs): + ctype = self.elem.get("type") + if ctype == "PLUS": yield "expr", "+" + elif ctype == "MINUS": yield "expr", "-" + elif ctype == "TIMES": yield "expr", "*" + elif ctype == "DIVIDE": yield "expr", "/" + elif ctype == "MOD": yield "expr", "%" + elif ctype == "XOR": yield "expr", "^" + elif ctype == "OR": yield "expr", "or" + elif ctype == "LSHIFT": yield "expr", "<<" + elif ctype == "RSHIFT": yield "expr", ">>" + elif ctype == "AND": yield "expr", "&" + else: yield "expr", "Math."+ctype + + +class DeclarationBlock(ASTPython): + def generate(self, **kwargs): + mode = self.elem.get("mode") + is_constructor = self.elem.get("constructor") + #if mode == "CONST": yield "debug", "Const Declaration:" + for var in self.elem: + expr = [] + for dtype, data in parse_ast(var).generate(force_value=True): + if dtype == "expr": + if data is None: raise ValueError(etree.tostring(var)) + expr.append(data) + else: yield dtype,data + if is_constructor: + expr[0] = "self."+expr[0] + yield "line", " ".join(expr) + + +# ----- keep this one at the end. +class Unknown(ASTPython): + @classmethod + def can_process_tag(self, tagname): return True +# ----------------- + +def astparser_for(elem): + classobj = None + for cls in ast_class_types: + if cls.can_process_tag(elem.tag): + classobj = cls + break + if classobj is None: return None + return classobj(elem) + +def parse_ast(elem): + elemparser = astparser_for(elem) + return elemparser.polish() + + +def file_template(ast): + yield "line", "# -*- coding: utf-8 -*-" + yield "line", "from pineboolib import qsatype" + yield "line", "from pineboolib.qsaglobals import *" + yield "line", "import traceback" + yield "line", "" + sourceclasses = etree.Element("Source") + for cls in ast.xpath("Class"): + sourceclasses.append(cls) + + mainclass = etree.SubElement(sourceclasses,"Class",name="FormInternalObj",extends="qsatype.FormDBWidget") + mainsource = etree.SubElement(mainclass,"Source") + + + constructor = etree.SubElement(mainsource,"Function",name="_class_init") + args = etree.SubElement(constructor,"Arguments") + csource = etree.SubElement(constructor,"Source") + + for child in ast: + if child.tag != "Function": + child.set("constructor","1") + csource.append(child) + else: + mainsource.append(child) + + for dtype, data in parse_ast(sourceclasses).generate(): + yield dtype, data + yield "line", "" + yield "line", "form = None" + + + +def string_template(ast): + sourceclasses = etree.Element("Source") + for child in ast: + child.set("withoutself","1") + sourceclasses.append(child) + + for dtype, data in parse_ast(sourceclasses).generate(): + yield dtype, data + +def write_python_file(fobj, ast, tpl=file_template): + indent = [] + indent_text = " " + last_line_for_indent = {} + numline = 0 + ASTPython.numline = 1 + last_dtype = None + for dtype, data in tpl(ast): + if isinstance(data, bytes): data = data.decode("UTF-8","replace") + line = None + if dtype == "line": + line = data + numline +=1 + try: lines_since_last_indent = numline - last_line_for_indent[len(indent)] + except KeyError: lines_since_last_indent = 0 + if lines_since_last_indent > 4: + ASTPython.numline += 1 + fobj.write((len(indent)*indent_text) + "\n") + last_line_for_indent[len(indent)] = numline + if dtype == "debug": + line = "# DEBUG:: " + data + #print(numline, line) + if dtype == "expr": line = "# EXPR??:: " + data + if dtype == "line+1": line = "# LINE+1??:: " + data + if dtype == "begin": + #line = "# BEGIN:: " + data + indent.append(data) + last_line_for_indent[len(indent)] = numline + if dtype == "end": + if last_dtype == "begin": + ASTPython.numline += 1 + fobj.write((len(indent)*indent_text) + "pass\n") + last_line_for_indent[len(indent)] = numline + + if data not in ["block-if"]: + #line = "# END:: " + data + pass + endblock = indent.pop() + if endblock != data: + line = "# END-ERROR!! was %s but %s found. (%s)" % (endblock, data,repr(indent)) + + if line is not None: + ASTPython.numline += 1 + fobj.write((len(indent)*indent_text) + line + "\n") + + if dtype == "end": + if data.split("-")[1] in ["class","def","else","except"]: + ASTPython.numline += 1 + fobj.write((len(indent)*indent_text) + "\n") + last_line_for_indent[len(indent)] = numline + last_dtype = dtype + +def pythonize(filename, destfilename, debugname = None): + bname = os.path.basename(filename) + ASTPython.debug_file = open(debugname, "w") if debugname else None + parser = etree.XMLParser(remove_blank_text=True) + try: + ast_tree = etree.parse(open(filename), parser) + except Exception: + print("filename:",filename) + raise + ast = ast_tree.getroot() + tpl = string_template + for cls in ast.xpath("Class"): + tpl = file_template + break + + f1 = open(destfilename,"w") + write_python_file(f1,ast,tpl) + f1.close() + + +def main(): + parser = OptionParser() + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("--path", + dest="storepath", default=None, + help="store PY results in PATH") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + + for filename in args: + if options.storepath: + destname = os.path.join(options.storepath,bname+".py") + else: + destname = filename+".py" + pythonize(filename, destname) + + + +if __name__ == "__main__": main() diff --git a/qsatype.py b/qsatype.py new file mode 100644 index 0000000..3d5cc6a --- /dev/null +++ b/qsatype.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +from builtins import object +import os + +class Object(object): + pass + +class Array(object): + pass + +class FLSqlCursor(object): + pass + +class Boolean(object): + pass + + + diff --git a/test/flscriptparser b/test/flscriptparser deleted file mode 100755 index b8cab0c..0000000 --- a/test/flscriptparser +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/sh -PARSE_FILES="$*" - -if [ "$1" = "--openwith" ]; then - OPENWITH="$2" - PARSE_FILES="" -fi -LOGS="" -if [ "$PARSE_FILES" = "" ]; then - PARSE_FILES=`echo *.qs` - if [ "$PARSE_FILES" = '*.qs' ]; then - PARSE_FILES="" - echo "-- No input files given and no .qs files found on this directory. --" - else - LOGS=`echo .parse-*.qs.log` - if [ "$LOGS" = '.parse-*.qs.log' ]; then - LOGS="" - else - echo "INFO: Parsing previously failed files only" - fi - - fi - -fi - -THISFILE="$0" -readlink "$0" >/dev/null && THISFILE=`readlink "$0"` -THISDIR=`dirname "$THISFILE"` -APPDIR=`dirname "$THISDIR"` -#echo "Starting parsers for $PARSE_FILES" -TOTALFILES="" -OKFILES="" - -for file in $PARSE_FILES -do - NAME=`basename $file` - DIR=`dirname $file` - LOGFILE="$DIR/.parse-$NAME.log" - if [ "$LOGS" != '' ]; then - if [ ! -e $LOGFILE ]; then - continue; - fi - fi - echo -n "$DIR/$NAME (parsing) . . . " - ERRORCOUNT="0" - WARNINGCOUNT="0" - python "$APPDIR/flscriptparse.py" $file 2>/tmp/errorlog.txt >$LOGFILE || (cat $LOGFILE /tmp/errorlog.txt; echo "FATAL ERROR: Can't execute the parser."; ERRORCOUNT="1";) - if [ "$ERRORCOUNT" = "0" ]; then - ERRORCOUNT=`grep "#ERROR#" $LOGFILE | wc -l` - fi - if [ "$WARNINGCOUNT" = "0" ]; then - WARNINGCOUNT=`grep "#WARNING#" $LOGFILE | wc -l` - fi - - TOTALFILES+="$file -" - if [ "$ERRORCOUNT" = "0" ]; then - echo "$WARNINGCOUNT Warnings." - grep "#WARNING#" $LOGFILE - OKFILES+="$file -" - if [ "$file" = "$PARSE_FILES" ]; then - cat $LOGFILE - fi - unlink $LOGFILE - else - if [ "$ERRORCOUNT" = "1" ]; then - echo -n "$ERRORCOUNT Error found." - else - echo -n "$ERRORCOUNT Errors found." - fi - grep "#ERROR#" -B10 -m1 $LOGFILE - echo "___" - if [ "$OPENWITH" != '' ]; then - $OPENWITH $file - fi - fi -done -OKFILES="`echo -n "$OKFILES" | wc -l`" -TOTALFILES="`echo -n "$TOTALFILES" | wc -l`" - -echo "$OKFILES of $TOTALFILES files parsed correctly" diff --git a/xml2json.py b/xml2json.py new file mode 100644 index 0000000..02d0d29 --- /dev/null +++ b/xml2json.py @@ -0,0 +1,634 @@ +#!/usr/bin/python + +from __future__ import print_function +from builtins import object +try: + from json import dumps as json_dumps + from json import loads as json_loads +except: + from json import write as json_dumps + from json import read as json_loads + + +import xml.parsers.expat +from optparse import OptionParser + +# json_string = json.dumps(python_variable) +# python_var = json.loads("string_encoded_jsonvar") + +def printr(*args): + return + print(args[0], end=' ') + for arg in args[1:]: + if type(arg) is str: + arg=arg.encode("utf-8") + print(repr(arg), end=' ') + print() + +def entity_rep(txt,entities=""): + entity_list = list("&'\"<>") + entity_dict = { + '"' : """, + "'" : "'", + '<' : "<", + '>' : ">", + "&" : "&", + } + if entities == "": entities = entity_list + entities = list(entities) + if "&" not in entities: entities.append("&") + + for entity in entity_list: + if entity not in entities: continue + #print entity,entity_dict[entity] + txt = txt.replace(entity,entity_dict[entity]) + return txt + +class xmlElement(object): + def __init__(self, parent, tagname, attrs = {}, ttype = "text", tdata = ""): + self.parent = parent + + self.tagname = tagname + self.attrs = attrs + self.ttype = ttype + self.tdata = tdata + + self.children = [] + + if self.parent: + self.depth = parent.depth + 1 + self.path = self.parent.path + [self.tagname] + self.parent.children.append(self) + else: + self.depth = 0 + self.path = [self.tagname] + + def append(self,text): + self.tdata += text + + def export(self,encoding): + depth = self.depth + tagname = self.tagname + attrs = [ [k,v] for k,v in self.attrs.items() ] + attrs.sort() + tdata = self.tdata.strip() + ttype = self.ttype + + if len(tdata) == 0: + tdata = "" + ttype = "" + + #v = [depth,tagname,attrs,ttype,tdata] + #vt1 = json_dumps(v) + vt2 = "%d)%s" % (depth,tagname) + if attrs: vt2 +="\tattrs:" + json_dumps(attrs) + if ttype: vt2 +="\t%s:%s" % ( ttype, json_dumps(tdata)) + + return vt2.encode(encoding) + + def exportXML(self): + if type(self.attrs) is dict: + attrs = [ [k,v] for k,v in self.attrs.items() ] + attrs.sort() + else: + attrs = self.attrs + + txtattrs = "" + + if attrs: + txtattrs=" " + for key,value in attrs: + txtattrs += '%s="%s" ' % (key,entity_rep(value,'&<"')) + + + depthpad = u" " * self.depth + output = u"" + if self.tagname == "#comment": + output += u"%s\n" % (depthpad, self.tdata) + elif self.tagname[0] == "!": + output += u"%s\n" % (depthpad, self.tagname[1:],txtattrs) + elif self.children: + output += u"%s<%s%s>\n" % (depthpad, self.tagname,txtattrs) + for child in self.children: + output += child.exportXML() + + output += u"%s\n" % (depthpad, self.tagname) + else: + if self.tdata == "": + if txtattrs=="": txtattrs = " " + output += u"%s<%s%s/>\n" % (depthpad, self.tagname,txtattrs) + else: + tdata = self.tdata + if tdata.find("\n")>-1: tdata = "\n%s\n" % tdata + if self.ttype=="cdata": tdata = "" % self.tdata + else: tdata = entity_rep(self.tdata) + + output += u"%s<%s%s>%s\n" % (depthpad, self.tagname,txtattrs, tdata, self.tagname) + + + return output + + + + + +class JSON_Base(object): + def __init__(self, finput, foutput, encoding): + self.finput = finput + self.foutput = foutput + self.encoding = encoding + + self.init_vars() + + def process(self): + print("Please define a process function.") + + def init_vars(self): + pass + + + +class JSON_Reverter(JSON_Base): + def init_vars(self): + self.cElement = None + self.rootXML = [] + + def processCmd(self,key,val): + if key == "encoding": + if self.encoding != "auto": + self.encoding = self.encoding.upper() + if val.upper() != self.encoding: + print(" ignoring %s=%s , using specified value '%s' instead" % (key,val,self.encoding)) + return + self.encoding = val.upper() + return + + print("ERROR: unknown key %s='%s'" % (key,val)) + + def newElement(self,depth,tagname,text,ttype,attrs): + parent = self.cElement + if parent: parentdepth = parent.depth + else: parentdepth = -1 + while parent and parentdepth > depth - 1: + parent = parent.parent + if parent: parentdepth = parent.depth + else: parentdepth = -1 + + self.cElement = xmlElement(parent, tagname, attrs, ttype , text) + if parent is None: self.rootXML.append(self.cElement) + + + def process(self): + for line in self.finput: + line = line.strip() + if len(line) == 0: continue + if line[0]=="!": + lstkeys = line[1:].split(":") + key, val = lstkeys + self.processCmd(key.strip(),val.strip()) + continue + + fields = line.split("\t") + + depth, tag = fields[0].split(")") + depth = int(depth) + text = "" + ttype = "text" + attrs = {} + + #self.foutput.write("%d\t%s\n" % (depth, tag)) + for field in fields[1:]: + tpos = field.find(":") + if tpos == -1: + print("unexpected character:", line) + return + ftype = field[:tpos] + try: + fvalue = json_loads(field[tpos+1:]) + except ValueError: + print("ValueError:", field[tpos+1:]) + + #if type(fvalue) is unicode: + # self.foutput.write("%s = %s\n" % (ftype, fvalue.encode(self.encoding))) + #else: + # self.foutput.write("%s = %s\n" % (ftype, repr(fvalue))) + if ftype == "text": + text = fvalue + ttype = "text" + if ftype == "cdata": + text = fvalue + ttype = "cdata" + if ftype == "attrs": attrs = fvalue + + self.newElement(depth,tag,text,ttype,attrs) + + for element in self.rootXML: + self.foutput.write(element.exportXML().encode(self.encoding)) + +""" + Possible Format: + + List-per-tag: + [ depth, tagname, attrs, ttype, tdata ] + + depth: 0,1,2,3,4...N + + tagname: \w+ -> ElementTag + tagname: !\w+ -> DoctypeTag + tagname: ?\w+ -> XmlDeclTag (always: xml) (attrs = version, encoding?, standalone?) + tagname: #\w+ -> CommentTag (always: comment) (attrs = []) (tdata = comment) + + attrs: dict { attr : val , attr2 : val2 } + + ttype: text|cdata|mixed -multiline? + + tdata: raw text + cdata combined. + + + Problems: + * Handling C-DATA + * Handling multiline texts + * Handling tabs and spaces at the start of each line of multiline text + * Handling comments + + Non-treated: + * NameSpaces + 32.18 + + * Entity Declarations + + + * Element Declarations + + + * Notation Declarations + + + * Attribute List Delcarations + + + + + + Example 1: AbanQ UI (UTF-8) + StartDoctypeDeclHandler: 'UI' None None 0 + EndDoctypeDeclHandler: + StartElementHandler: 'UI' {u'version': u'3.3', u'stdsetdef': u'1'} + CharacterDataHandler: '\n' + StartElementHandler: 'class' {} + CharacterDataHandler: 'formLineasAlbaranesCli' + EndElementHandler: 'class' + + Example 2: JasperReports JRXML (UTF-8) + XmlDeclHandler: '1.0' 'UTF-8' -1 + Unhandled data: '\n' + StartElementHandler: 'jasperReport' {u'xmlns': u'http://jasperreports.sourceforge.net/jasperreports', u'name': u'report1', u'language': u'groovy', u'pageWidth': u'842', u'columnWidth': u'802', u'topMargin': u'20', u'rightMargin': u'20', u'bottomMargin': u'20', u'xmlns:xsi': u'http://www.w3.org/2001/XMLSchema-instance', u'leftMargin': u'20', u'xsi:schemaLocation': u'http://jasperreports.sourceforge.net/jasperreports http://jasperreports.sourceforge.net/xsd/jasperreport.xsd', u'pageHeight': u'595', u'orientation': u'Landscape'} + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + StartElementHandler: 'style' {u'fontName': u'Times New Roman', u'name': u'Title', u'isBold': u'false', u'forecolor': u'#FFFFFF', u'fontSize': u'50', u'isDefault': u'false', u'pdfFontName': u'Times-Bold'} + EndElementHandler: 'style' + + Example 3: AbanQ Actions XML (ISO-8859-1) + StartElementHandler: 'ACTIONS' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'action' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'name' {} + CharacterDataHandler: 'albaranescli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'description' {} + CharacterDataHandler: 'QT_TRANSLATE_NOOP("MetaData","Son los documentos que justifican la entrega de una mercancia a un ciente")' + EndElementHandler: 'description' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + Example 4: AbanQ Tables MTD (ISO-8859-1) + StartDoctypeDeclHandler: 'TMD' None None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'TMD' {} + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + StartElementHandler: 'name' {} + CharacterDataHandler: 'facturascli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CommentHandler: 'Facturas de cliente' + CharacterDataHandler: ' ' + StartElementHandler: 'alias' {} + CharacterDataHandler: 'QT_TRANSLATE_NOOP("MetaData","Facturas de Clientes")' + EndElementHandler: 'alias' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + Example 5: AbanQ Report QRY (ISO-8859-1) + StartDoctypeDeclHandler: 'QRY' None None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'QRY' {} + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + StartElementHandler: 'name' {} + CharacterDataHandler: 'presupuestoscli' + EndElementHandler: 'name' + CharacterDataHandler: '\n' + CharacterDataHandler: '\n' + CharacterDataHandler: '\t' + + Example 6: AbanQ Report KUT (ISO-8859-1) + XmlDeclHandler: '1.0' 'UTF-8' -1 + Unhandled data: '\n' + StartDoctypeDeclHandler: 'KugarTemplate' 'kugartemplate.dtd' None 0 + EndDoctypeDeclHandler: + Unhandled data: '\n' + StartElementHandler: 'KugarTemplate' {u'TopMargin': u'50', u'PageSize': u'0', u'RightMargin': u'30', u'PageOrientation': u'0', u'BottomMargin': u'50', u'LeftMargin': u'30'} + CharacterDataHandler: '\n' + StartElementHandler: 'Detail' {u'Level': u'0', u'Height': u'0'} + EndElementHandler: 'Detail' + CharacterDataHandler: '\n' + CharacterDataHandler: '\n' + CharacterDataHandler: ' ' + + + + +""" + +class JSON_Converter(JSON_Base): + + def init_vars(self): + self.real_encoding = self.getRealEncoding() + self.xmltag = None + self.taglist = [] + self.p = xml.parsers.expat.ParserCreate(self.real_encoding) + + self.p.StartElementHandler = self.StartElementHandler + self.p.EndElementHandler = self.EndElementHandler + self.p.CharacterDataHandler = self.CharacterDataHandler + self.p.XmlDeclHandler = self.XmlDeclHandler + self.p.StartDoctypeDeclHandler = self.StartDoctypeDeclHandler + self.p.EndDoctypeDeclHandler = self.EndDoctypeDeclHandler + self.p.ElementDeclHandler = self.ElementDeclHandler + self.p.AttlistDeclHandler = self.AttlistDeclHandler + self.p.ProcessingInstructionHandler = self.ProcessingInstructionHandler + self.p.CharacterDataHandler = self.CharacterDataHandler + self.p.EntityDeclHandler = self.EntityDeclHandler + self.p.NotationDeclHandler = self.NotationDeclHandler + self.p.StartNamespaceDeclHandler = self.StartNamespaceDeclHandler + self.p.EndNamespaceDeclHandler = self.EndNamespaceDeclHandler + self.p.CommentHandler = self.CommentHandler + self.p.StartCdataSectionHandler = self.StartCdataSectionHandler + self.p.EndCdataSectionHandler = self.EndCdataSectionHandler + self.p.DefaultHandler = self.DefaultHandler + + def getRealEncoding(self): + validEncodings = ["UTF-8", "UTF-16", "ISO-8859-1"] + self.encoding = self.encoding.upper() + + if self.encoding in validEncodings: return self.encoding + if self.encoding.find("UTF")>=0: + if self.encoding.find("8"): + return "UTF-8" + if self.encoding.find("16"): + return "UTF-16" + return "UTF-8" + + if self.encoding.find("ISO")>=0: + return "ISO-8859-1" + + if self.encoding.find("1252")>=0: + return "ISO-8859-1" + + if self.encoding.find("CP")==0: + return "ISO-8859-1" + + if self.encoding.find("WIN")==0: + return "ISO-8859-1" + + return "UTF-8" + + + def process(self): + self.p.ParseFile(self.finput) + self.foutput.write(b"!encoding: "+ bytes(self.real_encoding, "UTF-8")+b"\n") + for tag in self.taglist: + self.foutput.write(tag.export(self.real_encoding)+b"\n") + + def startTag(self,*args): + newtag = xmlElement(self.xmltag, *args) + self.xmltag = newtag + self.taglist.append(newtag) + return newtag + + def endTag(self): + self.xmltag = self.xmltag.parent + return self.xmltag + + + + + def StartElementHandler(self, name, attributes): + printr( "StartElementHandler:", name, attributes) + # tagname: \w+ -> ElementTag + self.startTag(name, attributes) + + def CharacterDataHandler(self, data): + printr( "CharacterDataHandler:", data) + self.xmltag.append(data) + + def EndElementHandler(self, name): + printr( "EndElementHandler:", name) + self.endTag() + + def XmlDeclHandler(self, version, encoding, standalone): + printr( "XmlDeclHandler:", version, encoding, standalone) + # tagname: ?\w+ -> XmlDeclTag (always: xml) (attrs = version, encoding?, standalone?) + attrs = { 'version' : version } + if encoding: attrs['encoding'] = encoding + if standalone: attrs['standalone'] = standalone + + self.startTag("?xml", attrs) + + self.endTag() + + def CommentHandler(self, data): + printr( "CommentHandler:", data) + # tagname: #\w+ -> CommentTag (always: comment) (attrs = []) (tdata = comment) + self.startTag("#comment") + self.xmltag.append(data) + self.endTag() + + + def StartCdataSectionHandler(self): + printr( "StartCdataSectionHandler:") + self.xmltag.ttype = "cdata" + + def EndCdataSectionHandler(self): + printr( "EndCdataSectionHandler:") + # se descarta el cierre... + + def StartDoctypeDeclHandler(self, doctypeName, systemId, publicId, has_internal_subset): + printr( "StartDoctypeDeclHandler:", doctypeName, systemId, publicId, has_internal_subset) + # tagname: !\w+ -> DoctypeTag + attrs = {} + if systemId: attrs['systemId'] = systemId + if publicId: attrs['publicId'] = publicId + if has_internal_subset: attrs['has_internal_subset'] = has_internal_subset + + self.startTag("!"+doctypeName,attrs) + + + def EndDoctypeDeclHandler(self): + printr( "EndDoctypeDeclHandler:") + self.endTag() + + def ElementDeclHandler(self, name, model): + printr( "ElementDeclHandler:", name, model) + + def AttlistDeclHandler(self, elname, attname, type, default, required): + printr( "AttlistDeclHandler:", elname, attname, type, default, required) + + def ProcessingInstructionHandler(self, target, data): + printr( "ProcessingInstructionHandler:", target, data) + + def EntityDeclHandler(self, entityName, is_parameter_entity, value, base, systemId, publicId, notationName): + printr( "EntityDeclHandler:" , entityName, is_parameter_entity, value, base, systemId, publicId, notationName) + + def NotationDeclHandler(self, notationName, base, systemId, publicId): + printr( "NotationDeclHandler:", notationName, base, systemId, publicId) + + def StartNamespaceDeclHandler(self, prefix, uri): + printr( "StartNamespaceDeclHandler:", prefix, uri) + + def EndNamespaceDeclHandler(self, prefix): + printr( "EndNamespaceDeclHandler:", prefix) + + def DefaultHandler(self,data): + printr( "Unhandled data:", data) + + + + + + + +def autodetectXmlEncoding(rawtext): + lines = [ line.strip() for line in rawtext.split(b"\n") if line.strip() ] + if lines[0].find(b"")>=0: + # File is QtDesigner UI + return "UTF-8" + + if lines[0].find(b"UTF-8")>=0: + # Unkown, standard xml (like jrxml) + return "UTF-8" + + if lines[0].find(b"")>=0: + # AbanQ actions XML + return "ISO-8859-15" + + if lines[0].find(b"")>=0: + # AbanQ table MTD + return "ISO-8859-15" + + if lines[0].find(b"")>=0: + # AbanQ report Query + return "ISO-8859-15" + + if lines[0].find(b'')>=0: + # AbanQ report Kut + return "ISO-8859-15" + + if lines[0].find(b"")>=0: + # AbanQ translations + return "ISO-8859-15" + + try: + import chardet + dictEncoding = chardet.detect(rawtext) + encoding = dictEncoding["encoding"] + return encoding + except ImportError: + print("python-chardet library is not installed. Assuming input file is UTF-8.") + #encoding= + #UTF-8, UTF-16, ISO-8859-1 + + + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + parser.add_option("-E", "--encoding", dest="encoding", default = "auto", + help="Set encoding=ENC: auto,utf-8,iso-8859-15", metavar="ENC") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + if len(args) < 2: + print("xml2json needs at least an action and a file.") + print("xml2json (revert|convert) file1 [file2] [file3]") + return + action = args.pop(0) + + if action == "convert": + for fname in args: + + fhandler = open(fname,"rb") + fw = open(fname+".json","wb") + rawtext = fhandler.read() + fhandler.seek(0) + if options.encoding == "auto": + encoding = autodetectXmlEncoding(rawtext) + else: + encoding = options.encoding + jconv = JSON_Converter(fhandler, fw, encoding) + jconv.process() + + fhandler.close() + fw.close() + elif action == "revert": + for fname in args: + lExt = fname.split(".") + ext = "xml" + if lExt[-1]=="json" and len(lExt[-2])>=1 and len(lExt[-2])<=6: + ext = lExt[-2] + fhandler = open(fname) + + fw = open(fname+"."+ext,"w") + jrev = JSON_Reverter(fhandler, fw, options.encoding) + jrev.process() + + fw.close() + fhandler.close() + + else: + print("Unkown action '%s'" % action) + + + + + + + + + +if __name__ == "__main__": main() diff --git a/xmlparse.py b/xmlparse.py new file mode 100644 index 0000000..df1dbd5 --- /dev/null +++ b/xmlparse.py @@ -0,0 +1,200 @@ +from __future__ import print_function +from builtins import str +import xml.parsers.expat +import sys +from optparse import OptionParser +import re + +elements = [] +show_end = True +lasttextdata = "" +lstelements = [] + +def reset(): + global elements, show_end, lstelements, lasttextdata + elements = [] + show_end = True + lasttextdata = "" + lstelements = [] + + +# 3 handler functions +def start_element(name, attrs): + global elements, show_end, lstelements, lasttextdata + lstattrs=list(sorted([ "%s=%s" % (k,v) for k,v in attrs.items() ])) + completename=name + if len(lstattrs): + completename+="&"+"&".join(lstattrs) + + show_end = False + elements.append(completename) + #print 'Start element:', name, attrs + lstelements.append("/".join(elements)) + #print "/".join(elements) + lasttextdata = "" + +def end_element(name): + global elements, show_end, lstelements, lasttextdata + lasttextdata = "" + if show_end: + #print "/".join(elements) + "/" + lstelements.append("/".join(elements) + ";") + + show_end = True + elements.pop() + #print 'End element:', name + +def char_data(data): + global elements, show_end, lstelements, lasttextdata + #data = data.strip() + lasttextdata+=data + if lasttextdata.strip(): + #show_end = True + lstelements.pop() + lstelements.append("/".join(elements)+"(%s)" % repr(lasttextdata.strip())) + + + #print "/".join(elements)+ "(%s)" % repr(data) + + + +def unmap(lines): + + runmap = re.compile(r"^(?P/*)(?P\w+)(?P&[^\(]+)*(?P\(.+\))?$") + # depthlevel + # tagname + elementpool = [] + text = [] + for line in lines: + line = line.strip() + if line[-1] == ";": continue + rg1 = runmap.match(line) + if not rg1: + print("error:") + print(line) + break + + depth = len(rg1.group('depth')) + tagname = str(rg1.group('tagname')) + t_attrs = rg1.group('attrs') + attrs = [] + if t_attrs: + lattrs = t_attrs[1:].split("&") + for attr in lattrs: + key, val = attr.split("=") + attrs.append( (key,val) ) + + t_txt = rg1.group('txt') + txt = "" + if t_txt: + txt = eval(t_txt[1:-1]) + + while depth < len(elementpool): + toclose = elementpool.pop() + text.append("" % toclose) + text.append("\n" + " " * len(elementpool)) + + if depth == len(elementpool): + #print depth, tagname, attrs, txt + txtattrs = "" + if attrs: + for k,v in attrs: + txtattrs+=" %s=\"%s\"" % (k,v) + + if txt: + txt = txt.encode("utf-8") + txt = txt.replace("&","&") + txt = txt.replace("<","<") + else: + txt = "" + text.append("<%s%s>%s" % (tagname, txtattrs,txt)) + elementpool.append(tagname) + + + else: + print("error:") + print(depth, len(elementpool)) + break + + while len(elementpool): + toclose = elementpool.pop() + text.append("" % toclose) + text.append("\n" + " " * len(elementpool)) + + return text + + +def main(): + parser = OptionParser() + #parser.add_option("-f", "--file", dest="filename", + # help="write report to FILE", metavar="FILE") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + parser.add_option("--optdebug", + action="store_true", dest="optdebug", default=False, + help="debug optparse module") + + parser.add_option("--debug", + action="store_true", dest="debug", default=False, + help="prints lots of useless messages") + + + (options, args) = parser.parse_args() + if options.optdebug: + print(options, args) + if len(args) < 2: + print("Se necesita al menos una accion y un argumento extra.") + print("xmlparse (map|unmap) file1 [file2] [file3]") + return + action = args.pop(0) + + if action == "map": + global lstelements + separators = [ + "hbox", + "vbox", + "grid", + ] + r1 = re.compile("/widget") + for fname in args: + p = xml.parsers.expat.ParserCreate() + + p.StartElementHandler = start_element + p.EndElementHandler = end_element + p.CharacterDataHandler = char_data + fhandler = open(fname) + fw = open(fname+".map","w") + reset() + p.ParseFile(fhandler) + for t in lstelements: + elems = t.split("/") + lbox = [] + for n,e in enumerate(elems): + if e in separators: lbox.append(n) + if len(lbox)>1: + nlbox = lbox[-2] + while len(elems[nlbox:]) < 2: + nlbox -= 1 + else: + nlbox = 0 + fw.write("/"*(len(elems)-1) + "/".join(elems[-1:])+ "\n") + #print "/"*(len(elems)-1) + "/".join(elems[-1:]) + lstelements = [] + fhandler.close() + fw.close() + elif action == "unmap": + for fname in args: + fhandler = open(fname) + fw = open(fname+".ui","w") + for line in unmap(fhandler): + fw.write(line) + fw.close() + fhandler.close() + + else: + print("Unkown action '%s'" % action) + + +if __name__ == "__main__": main() \ No newline at end of file