3 ^^:@sdZddlZddlmZddlmZddlmZddlZddlm Z m Z ddl m Z ddl mZdd lmZGd d d Zd d ZGdddZddiZddZGddde eZddZd ddZd!ddZedkredS)"zGa similarities / code duplication command line tool and pylint checker N) defaultdict)getopt)groupby) BaseCheckertable_lines_from_stats) IRawChecker)Table)decoding_streamc@sLeZdZdZdddZdddZd d Zd d Zd dZddZ ddZ dS)Similarz,finds copy-pasted lines of code in a projectFcCs"||_||_||_||_g|_dS)N) min_linesignore_commentsignore_docstringsignore_importslinesets)selfr r rrr9/tmp/pip-build-8app2_gc/pylint/pylint/checkers/similar.py__init__$s zSimilar.__init__Nc CsZ|dkr|j}n t||j}y$|jjt|||j|j|jWntk rTYnXdS)z(append a file to search for similaritiesN) readlinesr rappendLineSetr rrUnicodeDecodeError)rZstreamidstreamencodingrrrr append_stream1s zSimilar.append_streamcCs|j|jdS)zjsz)Similar._display_sims..z)TOTAL lines=%s duplicates=%s percent=%.2fgY@)printr,sortedname _real_linesrstripsumr) rr)Znb_lignes_dupliqueesr%r(r.idxlineZnb_total_lignesrrrr\s$zSimilar._display_simsccs|j}|j}|j}d}|j}x|t|krd}d} x|||D]} d} xtt|||| D]L\} \\} } \} }| |kr| |kr| |||| fVt|| }P| r`| d7} q`W| d7} | |kr| |||| fVt|| }q@W||7}qWdS)z+find similarities in the two given linesetsrr+N)enumerate_strippedfindr r, enumeratezipmax)rr&r'Zlines1Zlines2r9Zindex1r skipr%Zindex2Z non_blank_Zline1Zline2rrr _find_commonts0(  zSimilar._find_commonccs\xVt|jddD]@\}}x6|j|ddD] }x|j||D] }|VqBWq0WqWdS)zWiterate on similarities among all files, by making a cartesian product Nr+)r:rr?)rr6r.r'simrrrr szSimilar._iter_sims)r FFF)N) __name__ __module__ __qualname____doc__rrrrrr?r rrrrr !s  r c s|rBtjdj|}dd|jD}ddt|dddD}d }g}d } xt|d d D]\} j|r| rtfd ddDrd d} dd | rj| rd } d|r|j | |}|rd|rj dd dj|j qXW|S)z\return lines with leading/trailing whitespace and any ignored code features removed css&|]}|jt|tjtjffVqdS)N)lineno isinstanceastroidImport ImportFrom)r-noderrr sz!stripped_lines..cSs$i|]\}}tdd|D|qS)css|]\}}|VqdS)Nr)r-r>Z is_importrrrrMsz,stripped_lines...)all)r-rGZnode_is_import_grouprrr sz"stripped_lines..cSs|dS)Nrr)xrrrsz stripped_lines..)keyFNr+)startc3s|]}j|VqdS)N) startswith)r-i)r7rrrMs"""'''r"""r'''#r)rVrWrXrY) rIparsejoinbodyrr:stripanyendswithgetsplitr) linesr rrtreeZnode_is_import_by_linenoZline_begins_importZcurrent_line_is_importZ strippedlinesZ docstringrGr)r7rstripped_liness<    rfc@s\eZdZdZdddZddZddZd d Zd d Zd dZ dddZ ddZ ddZ dS)rz7Holds and indexes all the lines of a single source fileFcCs*||_||_t|||||_|j|_dS)N)r2r3rf_stripped_lines _mk_index_index)rr2rdr rrrrrrs zLineSet.__init__cCs d|jS)Nz)r2)rrrr__str__szLineSet.__str__cCs t|jS)N)r,r3)rrrr__len__szLineSet.__len__cCs |j|S)N)rg)rindexrrr __getitem__szLineSet.__getitem__cCs |j|jkS)N)r2)rotherrrr__lt__szLineSet.__lt__cCst|S)N)id)rrrr__hash__szLineSet.__hash__rccsB|}|r|j|d}n|j}x|D]}||fV|d7}q$WdS)zgreturn an iterator on stripped lines, starting from a given index if specified, else 0 Nr+)rg)rZstart_atr6rdr7rrrr8s  zLineSet.enumerate_strippedcCs|jj|fS)z7return positions of the given stripped line in this set)rirb)rZ stripped_linerrrr9sz LineSet.findcCs6tt}x(t|jD]\}}|r||j|qW|S)zcreate the index for this set)rrr:rgr)rrlZline_nor7rrrrhs zLineSet._mk_indexN)FFF)r) rBrCrDrErrjrkrmrorqr8r9rhrrrrrs  rR0801Similar lines in %s files %sduplicate-codeIndicates that a set of similar lines has been detected among multiple file. This usually means that the code should be refactored to avoid this duplication.cCs6ddddg}|t||d 7}|jt|dddd d S) z/make a layout with some stats about duplicationrFnowprevious differencenb_duplicated_linespercent_duplicated_linesr r+)childrencolsZrheadersZcheadersN)ryrz)rrr)sectstatsZ old_statsrdrrrreport_similaritiess  rc @seZdZdZefZdZeZddddddfd d d d d dfdd d d ddfddd d ddffZ dde ffZ d ddZ d!ddZ ddZddZddZdS)"SimilarCheckerzchecks for similarities and duplicated code. This computation may be memory / CPU intensive, so you should disable it if you experiment some problems. Z similaritieszmin-similarity-linesr intzz%Minimum lines number of a similarity.)defaulttypemetavarhelpzignore-commentsTZynzz,Ignore comments when computing similarities.zignore-docstringsz.Ignore docstrings when computing similarities.zignore-importsFz+Ignore imports when computing similarities.ZRP0801Z DuplicationNcCs(tj||tj|ddddd|_dS)Nr T)r r r)rrr r~)rlinterrrrrOs zSimilarChecker.__init__cCsdtj||||||dkr&|jj|_n:|dkr:|jj|_n&|dkrN|jj|_n|dkr`|jj|_dS)zmethod called to set an option (registered in the options list) overridden to report options setting to Similar zmin-similarity-lineszignore-commentszignore-docstringszignore-importsN)r set_optionconfigZmin_similarity_linesr r rr)roptnamevalueactionoptdictrrrrVs   zSimilarChecker.set_optioncCsg|_|jjddd|_dS)z.rNz==%s:%srr )argsr+rygY@rz) r5rr~rrr2r#r3r4Z add_messager,r]) rtotalZ duplicatedr~r%r(msgr.r6r7rrrclosevs zSimilarChecker.close)N)NN)rBrCrDrErZ__implements__r2MSGSZmsgsoptionsrZreportsrrrrrrrrrrs<     rcCs|jt|dS)z.required method to auto register this checker N)Zregister_checkerr)rrrrregistersrcCs$tdttdtj|dS)z&display command line usage informationz*finds copy pasted blocks in a set of filesz~Usage: symilar [-d|--duplicates min_duplicated_lines] [-i|--ignore-comments] [--ignore-docstrings] [--ignore-imports] file1...N)r0sysexit)statusrrrusages rc Cs|dkrtjdd}d}d}d }d }d }d }t|||\}}xZ|D]R\} } | dkr^t| }qD| dkrntqD| dkr|d}qD| dkrd}qD| dkrDd}qDW|stdt||||} x,|D]$} t| } | j| | WdQRXqW| jtj ddS)z$standalone command line access pointNr+Zhdir duplicates=ignore-commentsignore-importsignore-docstringsr F-d --duplicates-h--help-i--ignore-commentsT--ignore-docstrings--ignore-importsr)rrrrr)rr)rr)rr)r)r) rargvrrrr rrrr)rZs_optsZl_optsr r rroptsroptvalrAfilenamerrrrRuns@   r__main__)rsrtru)r)N)rEr collectionsrr itertoolsrrIZpylint.checkersrrZpylint.interfacesrZpylint.reporters.ureports.nodesrZ pylint.utilsr r rfrrrrrrrrBrrrrs.      {,= s '