diff --git a/_build/postprocessor.py b/_build/postprocessor.py index d3465b6da..a90e25a89 100644 --- a/_build/postprocessor.py +++ b/_build/postprocessor.py @@ -54,8 +54,8 @@ def test_fix_latex_macros(): burble burble burble ''' result = fix_latex_macros(test) - print result - print expect + print(result) + print(expect) assert (result == expect) if __name__ == '__main__': diff --git a/_build/postprocessor.py.bak b/_build/postprocessor.py.bak new file mode 100644 index 000000000..d3465b6da --- /dev/null +++ b/_build/postprocessor.py.bak @@ -0,0 +1,76 @@ +import re + +def fix_latex_macros(text): + '''Use the AOSA style macros for headers, etc.''' + rs = ( + (r'\\section\{', r'\\aosasecti{'), + (r'\\subsection\{', r'\\aosasectii{'), + (r'\\subsubsection\{', r'\\aosasectiii{'), + (r'\\begin\{itemize\}', r'\\begin{aosaitemize}'), + (r'\\end\{itemize\}', r'\\end{aosaitemize}'), + (r'\\begin\{enumerate\}', r'\\begin{aosaenumerate}'), + (r'\\end\{enumerate\}', r'\\end{aosaenumerate}'), + (r'\\begin\{description\}', r'\\begin{aosadescription}'), + (r'\\end\{description\}', r'\\end{aosadescription}'), + (r'\\itemsep1pt\\parskip0pt\\parsep0pt', ''), + ) + for (old, new) in rs: + text = re.sub(old, new, text) + return text + +def test_fix_latex_macros(): + test = r''' +burble burble burble + +section this is a section or a subsection{ + +\begin{itemize} + +\begin{enumerate} + +\section{This is a header First} + +\subsection{This is a header First} + +\subsubsection{This is a header} + +burble burble burble +''' + expect = r''' +burble burble burble + +section this is a section or a subsection{ + +\begin{aosaitemize} + +\begin{aosaenumerate} + +\aosasecti{This is a header First} + +\aosasectii{This is a header First} + +\aosasectiii{This is a header} + +burble burble burble +''' + result = fix_latex_macros(test) + print result + print expect + assert (result == expect) + +if __name__ == '__main__': + import sys + import argparse + parser = argparse.ArgumentParser(description="Fix output for latex") + parser.add_argument('doc', nargs='*') + parser.add_argument('--output', dest='output', required=True) + args = parser.parse_args() + destination_file = open(args.output, 'w') + if len(args.doc) > 0: + input_file = open(args.doc[0]) + else: + input_file = sys.stdin + input_document = input_file.read() + input_document = fix_latex_macros(input_document) + out = input_document + destination_file.write(out) diff --git a/_build/preprocessor.py b/_build/preprocessor.py index a796e7b28..6da840b8b 100644 --- a/_build/preprocessor.py +++ b/_build/preprocessor.py @@ -34,10 +34,10 @@ def test_parse_aosafigures(): for input_text, expected in test: aosafigs = parse_aosafigures(input_text) - print aosafigs - print - print 'expected' - print expected + print(aosafigs) + print() + print('expected') + print(expected) assert aosafigs == expected def fix_figure_index(text, chapter): @@ -85,7 +85,7 @@ def fix_table_references(text, chapter): table_count = 1 for table in tables: - print '\\aosatblref{'+table+'}' + print(('\\aosatblref{'+table+'}')) table_id = "{}.{}".format(chapter, table_count) text = text.replace('\\label{'+table+'}', 'Table ' + table_id + ' - ') text = text.replace('\\aosatblref{'+table+'}', 'Table ' + table_id) @@ -177,7 +177,7 @@ def test_htmlify_refs(): expected = ''' dsaf asd [[@bla-99]] fas [[@o:o_f23]] df [[@bla-99]]. ''' - print htmlify_refs(test) + print((htmlify_refs(test))) assert htmlify_refs(test) == expected diff --git a/_build/preprocessor.py.bak b/_build/preprocessor.py.bak new file mode 100644 index 000000000..ac42433c6 --- /dev/null +++ b/_build/preprocessor.py.bak @@ -0,0 +1,219 @@ +import re + +def remove_tags(tagname, text): + """Use a regular expression + to remove parts of a chapter that shouldn't be in the target + format. + """ + regex = re.compile(r'<' + tagname + r'>(.|\s+)*?', flags=re.MULTILINE) + return regex.sub('', text) + +def strip_tags(tagname, text): + """Use a regular expression to remove tags from + the parts of the chapter that *should* be in the target format. + """ + regex = re.compile(r'<' + tagname + r'>((.|\s+)*?)', flags=re.MULTILINE) + return regex.sub(r'\1', text) + +def root_paths(text): + return re.sub(r'\(([^()]*?g)\)', r'(/\1)', text) + +def parse_aosafigures(text): + '''parse_aosafigures(text) -> [(fullmatch, width, location, caption, id)]''' + # \aosafigure[233pt]{warp-images/2.png}{Event driven}{fig.warp.f2} + regex = re.compile("(\\\\aosafigure\[?([^\]]+)?\]?\{(.*)\}\{(.*)\}\{(.*)\})",re.MULTILINE) + figures_list = regex.findall(text) + return figures_list + +def test_parse_aosafigures(): + test = ( + ('''\\aosafigure[240pt]{bla}{bla}{bla} + +\\aosafigure{bla}{bla}{bla} +''', [('\\aosafigure[240pt]{bla}{bla}{bla}', '240pt', 'bla', 'bla', 'bla'), ('\\aosafigure{bla}{bla}{bla}', '', 'bla', 'bla', 'bla')]),) + + for input_text, expected in test: + aosafigs = parse_aosafigures(input_text) + print(aosafigs) + print() + print('expected') + print(expected) + assert aosafigs == expected + +def fix_figure_index(text, chapter): + """ + Creates figures references. + """ + figures_list = parse_aosafigures(text) + figures = [] + index = 1 + for figure in figures_list: + # regex tokens + figures.append(figure) + width = figure[1] + location = figure[2] + label = figure[3] + ref = figure[4] + + # the original line, to be replaced by the image in the HTML version + # plus the label + original_aosafigure = figure[0] + fignum = "{chapter}.{index}".format(chapter=chapter, index=str(index)) + html_figure_text = '
Figure ' + fignum + ' - ' + label + '

Figure ' + fignum + ' - ' + label + '

' + text = text.replace(original_aosafigure, html_figure_text) + + # we want to replace the reference + to_replace = '\\aosafigref{'+ref+'}' + # with a nice phrase + new_value = 'Figure ' + fignum + '' + text = text.replace(to_replace, new_value) + + index += 1 + return text + +def fix_table_references(text, chapter): + """ + Creates table references. + """ + + # Find table captions with :\s\label{$VAR} + tables = list() + table_caption_regex = re.compile('^:\s\\\\label\{([a-zA-Z0-9\.\~\!\?]*)\}', re.MULTILINE) + tables_found = table_caption_regex.findall(text) + for caption in tables_found: + tables.append(caption) + + table_count = 1 + for table in tables: + print('\\aosatblref{'+table+'}') + table_id = "{}.{}".format(chapter, table_count) + text = text.replace('\\label{'+table+'}', 'Table ' + table_id + ' - ') + text = text.replace('\\aosatblref{'+table+'}', 'Table ' + table_id) + table_count += 1 + + return text + +def fix_section_references(text): + """ + Creates section references. + """ + + # Find titles before \label + titles = {} + text = text.replace('\r\n', '\n') + lines = text.split('\n') + last_line = None + title_regex = re.compile('^#+(.*)$') + label_regex = re.compile('\\\\label\{([a-zA-Z0-9\.\~\!\?]*)\}') + for line in lines: + line = line.strip(' \t\n\r') + found_label = label_regex.findall(line) + if (len(found_label) > 0 and last_line != None): + le_label = found_label[0].strip(' \t\n\r') + found_titles = title_regex.findall(last_line) + if (len(found_titles) > 0): + le_title = found_titles[0].strip(' \t\n\r') + titles[le_label] = le_title + if line != '': + last_line = line + + #print titles + + # the text is in \aosasecref{sec.warp.arch} and ... + # \label{sec.warp.arch} + # Create an associative array with each reference and its index + regex = re.compile("(\\\\aosasecref\{([a-zA-Z0-9\.\~\!\?]*)\})",re.MULTILINE) + references_list = regex.findall(text) + references = [] + index = 1 + for reference in references_list: + # regex tokens + references.append(reference) + label = reference[1] + + # the reference text to be replaced by a link + # print 'label: ' + titles[label] + html_reference_text = ''+titles[label]+'' + text = text.replace(reference[0], html_reference_text) + + # we want to replace the reference + to_replace = '\\label{'+label+'}' + # with the link + new_value = ' ' + text = text.replace(to_replace, new_value) + + index += 1 + return text + +def test_root_paths(): + test = ''' +burble burble burble + +[hey (there)](ethercalc-images/ethercalc.png) + +burble burble burble some more''' + expected = ''' +burble burble burble + +[hey (there)](/ethercalc-images/ethercalc.png) + +burble burble burble some more''' + result = root_paths(test) + assert result == expected + + +def htmlify_refs(text): + # Double square brackets because pandoc renders only the number. + pat = r'\\cite\{([a-zA-Z0-9+\-:_]+)\}' + regex = re.compile(pat) + return regex.sub('[[@\\1]]', text) + + +def test_htmlify_refs(): + test = ''' +dsaf asd \\cite{bla-99} fas \\cite{o:o_f23} df \\cite{bla-99}. +''' + # Double square brackets because pandoc renders only the number. + expected = ''' +dsaf asd [[@bla-99]] fas [[@o:o_f23]] df [[@bla-99]]. +''' + print(htmlify_refs(test)) + assert htmlify_refs(test) == expected + + +if __name__ == '__main__': + import sys + import argparse + parser = argparse.ArgumentParser(description="Remove the specified kind of tag from the input document") + parser.add_argument('doc', nargs='*') + parser.add_argument('--output', dest='output', required=True) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--latex', action='store_true', default=False) + group.add_argument('--markdown', action='store_true', default=False) + parser.add_argument('--chapter', type=int, default=0) + parser.add_argument('--html-paths', action='store_true', default=False) + parser.add_argument('--html-refs', action='store_true', default=False) + args = parser.parse_args() + destination_file = open(args.output, 'w') + if len(args.doc) > 0: + input_file = open(args.doc[0]) + else: + input_file = sys.stdin + input_document = input_file.read() + if args.markdown: + tag_name = 'markdown' + other_tag_name = 'latex' + else: + tag_name = 'latex' + other_tag_name = 'markdown' + input_document = fix_figure_index(input_document, args.chapter) + input_document = fix_section_references(input_document) + input_document = fix_table_references(input_document, args.chapter) + if args.html_refs: + input_document = htmlify_refs(input_document) + out = input_document + out = remove_tags(tag_name, out) + out = strip_tags(other_tag_name, out) + if args.html_paths: + out = root_paths(out) + destination_file.write(out) diff --git a/build.py b/build.py index 3e68d85de..ace01cce1 100644 --- a/build.py +++ b/build.py @@ -121,8 +121,8 @@ def main(chapters=[], epub=False, pdf=False, html=False, mobi=False, pandoc_epub build_epub(process_chapters, pandoc_epub) if mobi and not epub: - print 'Cannot build .mobi; depends on .epub.' - print 'Use --epub --mobi to build .mobi file.' + print('Cannot build .mobi; depends on .epub.') + print('Use --epub --mobi to build .mobi file.') elif mobi: build_mobi() @@ -161,7 +161,7 @@ def build_epub(chapter_markdowns, pandoc_epub): cmd = '{pandoc} --chapters -S -f markdown+mmd_title_block --highlight-style=kate -o 500L.epub epubtitle.txt introduction.markdown {markdowns}' if pandoc_epub: run(cmd.format(pandoc=pandoc_path, markdowns=' '.join(basenames))) - print cmd.format(pandoc=pandoc_path, markdowns=' '.join(basenames)) + print((cmd.format(pandoc=pandoc_path, markdowns=' '.join(basenames)))) # import subprocess as sp # output = ' '.join(open('image-list.txt').read().splitlines()) # print 'zip 500L.epub META-INF mimetype nav.xhtml toc.ncx stylesheet.css content.opf ' + output @@ -224,7 +224,7 @@ def preprocessor_command(chapter_markdown): temp = 'python _build/preprocessor.py --output=tex/{basename}.markdown --markdown {md}' basename = getbasename(chapter_markdown) result = temp.format(basename=basename, md=chapter_markdown) - print result + print(result) return (result, basename) @@ -238,23 +238,23 @@ def pandoc_cmd(chapter_markdown): result = envoy.run(cmd) new_chapter_markdown = basename + '.markdown' if result.status_code != 0: - print result.std_err + print((result.std_err)) else: - print result.std_out + print((result.std_out)) result = envoy.run(_pandoc_cmd(new_chapter_markdown)) if result.status_code != 0: - print result.std_err + print((result.std_err)) else: - print result.std_out + print((result.std_out)) result2 = envoy.run(postprocessor_command(basename)) return result2 def run(cmd): - print cmd + print(cmd) result = envoy.run(cmd) - print result.std_out - print result.std_err + print((result.std_out)) + print((result.std_err)) return result diff --git a/build.py.bak b/build.py.bak new file mode 100644 index 000000000..4c0d7e6df --- /dev/null +++ b/build.py.bak @@ -0,0 +1,271 @@ +#!/usr/bin/env python + +import envoy +import glob +import os + + +def main(chapters=[], epub=False, pdf=False, html=False, mobi=False, pandoc_epub=False): + if not os.path.isdir('output'): + os.mkdir('output') + else: + output_files = glob.glob('output/*') + for f in output_files: + run('rm {}'.format(f)) + + chapter_dirs = [ + 'blockcode', + 'ci', + 'cluster', + 'contingent', + 'crawler', + 'dagoba', + 'data-store', + 'event-web-framework', + 'flow-shop', + 'functionalDB', + 'image-filters', + 'interpreter', + 'modeller', + 'objmodel', + 'ocr', + 'pedometer', + 'same-origin-policy', + 'sampler', + 'spreadsheet', + 'static-analysis', + 'template-engine', + 'web-server', + ] + if len(chapters) > 0: + chapter_dirs = [ + chapter_dir + for chapter_dir in + chapter_dirs + if chapter_dir in chapters + ] + + chapter_markdowns = [ + './' + chapter_dir + '/' + chapter_dir + '.markdown' + for chapter_dir in + chapter_dirs + ] + + chapter_markdowns_exist = [ + envoy.run('test -f ' + chapter_markdown).status_code + for chapter_markdown in + chapter_markdowns + ] + + process_chapters = [ + chapter_markdown + for chapter_markdown, process in + zip(chapter_markdowns, chapter_markdowns_exist) + if process == 0 + ] + + chapter_names = [ + getbasename(chapter) + for chapter in + chapter_dirs + ] + + image_paths = [ + './blockcode/blockcode-images', + './ci/ci-images', + './cluster/cluster-images', + './contingent/contingent-images', + './crawler/crawler-images', + './data-store/data-store-images', + './flow-shop/flow-shop-images', + './functionalDB/functionalDB-images', + './image-filters/image-filters-images', + './interpreter/interpreter-images', + './modeller/modeller-images', + './objmodel/objmodel-images', + './ocr/ocr-images', + './pedometer/pedometer-images', + './same-origin-policy/same-origin-policy-images', + './sampler/sampler-images', + './spreadsheet/spreadsheet-images', + './web-server/web-server-images', + ] + + run('cp -r minutiae/pdf/ tex') + + with open('tex/500L.tex', 'w') as out: + with open('tex/500L.template.tex') as template: + lines = template.readlines() + for line in lines: + if 'chapterchapterchapter' in line: + out.write( + '\n'.join( + '\include{%s}\n' % (chapter_name) + for chapter_name in chapter_names + ) + ) + else: + out.write(line) + + if pdf: + for imgpath in image_paths: + run('cp -a {imgpath} tex/'.format(imgpath=imgpath)) + for chapter_markdown in process_chapters: + pandoc_cmd(chapter_markdown) + build_pdf() + + if epub: + for imgpath in image_paths: + run('cp -a {imgpath} epub/'.format(imgpath=imgpath)) + run('cp minutiae/html/introduction.md epub/introduction.markdown') + build_epub(process_chapters, pandoc_epub) + + if mobi and not epub: + print('Cannot build .mobi; depends on .epub.') + print('Use --epub --mobi to build .mobi file.') + elif mobi: + build_mobi() + + if html: + for imgpath in image_paths: + run('cp -a {imgpath} html/content/pages/'.format(imgpath=imgpath)) + run('cp minutiae/html/introduction.md html/content/pages/.') + build_html(process_chapters) + for imgpath in image_paths: + run('cp -a {imgpath} html/output/pages/'.format(imgpath=imgpath)) + +def build_pdf(): + os.chdir('tex') + run('pdflatex -interaction nonstopmode 500L') + os.chdir('..') + run('mv tex/500L.pdf output/') + + +def build_epub(chapter_markdowns, pandoc_epub): + basenames = [ + os.path.splitext( + os.path.split(chapter_markdown)[1] + )[0] + '.markdown' + for chapter_markdown in chapter_markdowns + ] + temp = 'python _build/preprocessor.py --chapter {chapnum} --output=epub/{basename}.markdown.1 --latex {md}' + for i, markdown in enumerate(chapter_markdowns): + basename = os.path.splitext(os.path.split(markdown)[1])[0] + run(temp.format(md=markdown, basename=basename, chapnum=i+1)) + os.chdir('epub') + temp = '../_build/increaseheaders.sh {basename}.markdown.1 {basename}.markdown {chapnum}' + for i, markdown in enumerate(chapter_markdowns): + basename = os.path.splitext(os.path.split(markdown)[1])[0] + run(temp.format(md=markdown, basename=basename, chapnum=i+1)) + pandoc_path = 'pandoc' + cmd = '{pandoc} --chapters -S -f markdown+mmd_title_block --highlight-style=kate -o 500L.epub epubtitle.txt introduction.markdown {markdowns}' + if pandoc_epub: + run(cmd.format(pandoc=pandoc_path, markdowns=' '.join(basenames))) + print(cmd.format(pandoc=pandoc_path, markdowns=' '.join(basenames))) +# import subprocess as sp +# output = ' '.join(open('image-list.txt').read().splitlines()) +# print 'zip 500L.epub META-INF mimetype nav.xhtml toc.ncx stylesheet.css content.opf ' + output +# sp.check_output( +# 'zip 500L.epub META-INF mimetype nav.xhtml toc.ncx stylesheet.css content.opf ' + output, +# shell=True) +# if os.path.isdir('tmp-epub-contents'): +# run('rm -r tmp-epub-contents') +# os.mkdir('tmp-epub-contents') +# sp.check_output( +# 'unzip 500L.epub -d tmp-epub-contents/', +# shell=True, +# ) +# sp.check_output( +# 'rsync -a tmp-epub-contents/* ./', +# shell=True +# ) +# run('rm -r tmp-epub-contents') + run('cp 500L.epub ../output/500L.epub') + os.chdir('..') + + +def build_mobi(): + run('ebook-convert output/500L.epub output/500L.mobi') + + +def build_html(chapter_markdowns): + run('mkdir -p html/content/pages') + temp = 'python _build/preprocessor.py --chapter {chap} --html-refs --output={md}.1 --latex {md}' + temp2 = 'pandoc --csl=minutiae/pdf/ieee.csl --mathjax -t html -f markdown+citations -o html/content/pages/{basename}.md {md}.1' + temp3 = './_build/fix_html_title.sh html/content/pages/{basename}.md' + for i, markdown in enumerate(chapter_markdowns): + basename = os.path.splitext(os.path.split(markdown)[1])[0] + run(temp.format(chap=i+1, md=markdown, basename=basename)) + run(temp2.format(md=markdown, basename=basename)) + run(temp3.format(md=markdown, basename=basename)) + os.chdir('html') + run('make html') + os.chdir('..') + + +def getbasename(chapter_markdown): + import os + basename = os.path.splitext( + os.path.split(chapter_markdown)[1] + )[0] + return basename + + +def _pandoc_cmd(chapter_markdown): + pandoc_path = 'pandoc' + # tex/md because that's where the preprocessed markdowns end up + temp = '{pandoc} -V chaptertoken={chaptertoken} -t latex --chapters -S -f markdown+mmd_title_block+tex_math_dollars --template=tex/chaptertemplate.tex --no-highlight -o tex/{basename}.tex.1 tex/{md}' + basename = getbasename(chapter_markdown) + result = temp.format(pandoc=pandoc_path, basename=basename, md=chapter_markdown, chaptertoken='s:' + basename) + return result + + +def preprocessor_command(chapter_markdown): + temp = 'python _build/preprocessor.py --output=tex/{basename}.markdown --markdown {md}' + basename = getbasename(chapter_markdown) + result = temp.format(basename=basename, md=chapter_markdown) + print(result) + return (result, basename) + + +def postprocessor_command(basename): + temp = 'python _build/postprocessor.py --output=tex/{basename}.tex tex/{basename}.tex.1' + return temp.format(basename=basename) + + +def pandoc_cmd(chapter_markdown): + cmd, basename = preprocessor_command(chapter_markdown) + result = envoy.run(cmd) + new_chapter_markdown = basename + '.markdown' + if result.status_code != 0: + print(result.std_err) + else: + print(result.std_out) + result = envoy.run(_pandoc_cmd(new_chapter_markdown)) + if result.status_code != 0: + print(result.std_err) + else: + print(result.std_out) + result2 = envoy.run(postprocessor_command(basename)) + return result2 + + +def run(cmd): + print(cmd) + result = envoy.run(cmd) + print(result.std_out) + print(result.std_err) + return result + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('chapters', nargs='*') + parser.add_argument('--epub', action='store_true', default=False) + parser.add_argument('--mobi', action='store_true', default=False) + parser.add_argument('--pdf', action='store_true', default=False) + parser.add_argument('--html', action='store_true', default=False) + parser.add_argument('--pandoc-epub', action='store_true', default=False) + args = parser.parse_args() + main(chapters=args.chapters, epub=args.epub, pdf=args.pdf, html=args.html, mobi=args.mobi, pandoc_epub=args.pandoc_epub) diff --git a/ci/code/dispatcher.py b/ci/code/dispatcher.py index 57a2550fc..380e6915d 100644 --- a/ci/code/dispatcher.py +++ b/ci/code/dispatcher.py @@ -13,7 +13,7 @@ import os import re import socket -import SocketServer +import socketserver import time import threading @@ -24,13 +24,13 @@ def dispatch_tests(server, commit_id): # NOTE: usually we don't run this forever while True: - print "trying to dispatch to runners" + print("trying to dispatch to runners") for runner in server.runners: response = helpers.communicate(runner["host"], int(runner["port"]), "runtest:%s" % commit_id) if response == "OK": - print "adding id %s" % commit_id + print(("adding id %s" % commit_id)) server.dispatched_commits[commit_id] = runner if commit_id in server.pending_commits: server.pending_commits.remove(commit_id) @@ -38,14 +38,14 @@ def dispatch_tests(server, commit_id): time.sleep(2) -class ThreadingTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): runners = [] # Keeps track of test runner pool dead = False # Indicate to other threads that we are no longer running dispatched_commits = {} # Keeps track of commits we dispatched pending_commits = [] # Keeps track of commits we have yet to dispatch -class DispatcherHandler(SocketServer.BaseRequestHandler): +class DispatcherHandler(socketserver.BaseRequestHandler): """ The RequestHandler class for our dispatcher. This will dispatch test runners against the incoming commit @@ -64,18 +64,18 @@ def handle(self): return command = command_groups.group(1) if command == "status": - print "in status" + print("in status") self.request.sendall("OK") elif command == "register": # Add this test runner to our pool - print "register" + print("register") address = command_groups.group(2) host, port = re.findall(r":(\w*)", address) runner = {"host": host, "port":port} self.server.runners.append(runner) self.request.sendall("OK") elif command == "dispatch": - print "going to dispatch" + print("going to dispatch") commit_id = command_groups.group(2)[1:] if not self.server.runners: self.request.sendall("No runners are registered") @@ -84,7 +84,7 @@ def handle(self): self.request.sendall("OK") dispatch_tests(self.server, commit_id) elif command == "results": - print "got test results" + print("got test results") results = command_groups.group(2)[1:] results = results.split(":") commit_id = results[0] @@ -119,11 +119,11 @@ def serve(): # Create the server server = ThreadingTCPServer((args.host, int(args.port)), DispatcherHandler) - print 'serving on %s:%s' % (args.host, int(args.port)) + print(('serving on %s:%s' % (args.host, int(args.port)))) # Create a thread to check the runner pool def runner_checker(server): def manage_commit_lists(runner): - for commit, assigned_runner in server.dispatched_commits.iteritems(): + for commit, assigned_runner in list(server.dispatched_commits.items()): if assigned_runner == runner: del server.dispatched_commits[commit] server.pending_commits.append(commit) @@ -139,7 +139,7 @@ def manage_commit_lists(runner): int(runner["port"]), "ping") if response != "pong": - print "removing runner %s" % runner + print(("removing runner %s" % runner)) manage_commit_lists(runner) except socket.error as e: manage_commit_lists(runner) @@ -148,8 +148,8 @@ def manage_commit_lists(runner): def redistribute(server): while not server.dead: for commit in server.pending_commits: - print "running redistribute" - print server.pending_commits + print("running redistribute") + print((server.pending_commits)) dispatch_tests(server, commit) time.sleep(5) diff --git a/ci/code/dispatcher.py.bak b/ci/code/dispatcher.py.bak new file mode 100644 index 000000000..ac1c14a28 --- /dev/null +++ b/ci/code/dispatcher.py.bak @@ -0,0 +1,172 @@ +""" +This is the test dispatcher. + +It will dispatch tests against any registered test runners when the repo +observer sends it a 'dispatch' message with the commit ID to be used. It +will store results when the test runners have completed running the tests and +send back the results in a 'results' messagee + +It can register as many test runners as you like. To register a test runner, +be sure the dispatcher is started, then start the test runner. +""" +import argparse +import os +import re +import socket +import socketserver +import time +import threading + +import helpers + + +# Shared dispatcher code +def dispatch_tests(server, commit_id): + # NOTE: usually we don't run this forever + while True: + print("trying to dispatch to runners") + for runner in server.runners: + response = helpers.communicate(runner["host"], + int(runner["port"]), + "runtest:%s" % commit_id) + if response == "OK": + print("adding id %s" % commit_id) + server.dispatched_commits[commit_id] = runner + if commit_id in server.pending_commits: + server.pending_commits.remove(commit_id) + return + time.sleep(2) + + +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + runners = [] # Keeps track of test runner pool + dead = False # Indicate to other threads that we are no longer running + dispatched_commits = {} # Keeps track of commits we dispatched + pending_commits = [] # Keeps track of commits we have yet to dispatch + + +class DispatcherHandler(socketserver.BaseRequestHandler): + """ + The RequestHandler class for our dispatcher. + This will dispatch test runners against the incoming commit + and handle their requests and test results + """ + + command_re = re.compile(r"(\w+)(:.+)*") + BUF_SIZE = 1024 + + def handle(self): + # self.request is the TCP socket connected to the client + self.data = self.request.recv(self.BUF_SIZE).strip() + command_groups = self.command_re.match(self.data) + if not command_groups: + self.request.sendall("Invalid command") + return + command = command_groups.group(1) + if command == "status": + print("in status") + self.request.sendall("OK") + elif command == "register": + # Add this test runner to our pool + print("register") + address = command_groups.group(2) + host, port = re.findall(r":(\w*)", address) + runner = {"host": host, "port":port} + self.server.runners.append(runner) + self.request.sendall("OK") + elif command == "dispatch": + print("going to dispatch") + commit_id = command_groups.group(2)[1:] + if not self.server.runners: + self.request.sendall("No runners are registered") + else: + # The coordinator can trust us to dispatch the test + self.request.sendall("OK") + dispatch_tests(self.server, commit_id) + elif command == "results": + print("got test results") + results = command_groups.group(2)[1:] + results = results.split(":") + commit_id = results[0] + length_msg = int(results[1]) + # 3 is the number of ":" in the sent command + remaining_buffer = self.BUF_SIZE - (len(command) + len(commit_id) + len(results[1]) + 3) + if length_msg > remaining_buffer: + self.data += self.request.recv(length_msg - remaining_buffer).strip() + del self.server.dispatched_commits[commit_id] + if not os.path.exists("test_results"): + os.makedirs("test_results") + with open("test_results/%s" % commit_id, "w") as f: + data = self.data.split(":")[3:] + data = "\n".join(data) + f.write(data) + self.request.sendall("OK") + else: + self.request.sendall("Invalid command") + + +def serve(): + parser = argparse.ArgumentParser() + parser.add_argument("--host", + help="dispatcher's host, by default it uses localhost", + default="localhost", + action="store") + parser.add_argument("--port", + help="dispatcher's port, by default it uses 8888", + default=8888, + action="store") + args = parser.parse_args() + + # Create the server + server = ThreadingTCPServer((args.host, int(args.port)), DispatcherHandler) + print('serving on %s:%s' % (args.host, int(args.port))) + # Create a thread to check the runner pool + def runner_checker(server): + def manage_commit_lists(runner): + for commit, assigned_runner in server.dispatched_commits.items(): + if assigned_runner == runner: + del server.dispatched_commits[commit] + server.pending_commits.append(commit) + break + server.runners.remove(runner) + + while not server.dead: + time.sleep(1) + for runner in server.runners: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + response = helpers.communicate(runner["host"], + int(runner["port"]), + "ping") + if response != "pong": + print("removing runner %s" % runner) + manage_commit_lists(runner) + except socket.error as e: + manage_commit_lists(runner) + + # this will kick off tests that failed + def redistribute(server): + while not server.dead: + for commit in server.pending_commits: + print("running redistribute") + print(server.pending_commits) + dispatch_tests(server, commit) + time.sleep(5) + + runner_heartbeat = threading.Thread(target=runner_checker, args=(server,)) + redistributor = threading.Thread(target=redistribute, args=(server,)) + try: + runner_heartbeat.start() + redistributor.start() + # Activate the server; this will keep running until you + # interrupt the program with Ctrl+C or Cmd+C + server.serve_forever() + except (KeyboardInterrupt, Exception): + # if any exception occurs, kill the thread + server.dead = True + runner_heartbeat.join() + redistributor.join() + + +if __name__ == "__main__": + serve() diff --git a/ci/code/repo_observer.py b/ci/code/repo_observer.py index 290582160..3bbe6ca4a 100644 --- a/ci/code/repo_observer.py +++ b/ci/code/repo_observer.py @@ -9,7 +9,7 @@ import os import re import socket -import SocketServer +import socketserver import subprocess import sys import time @@ -58,7 +58,7 @@ def poll(): if response != "OK": raise Exception("Could not dispatch the test: %s" % response) - print "dispatched!" + print("dispatched!") else: # Something wrong happened to the dispatcher raise Exception("Could not dispatch the test: %s" % diff --git a/ci/code/repo_observer.py.bak b/ci/code/repo_observer.py.bak new file mode 100644 index 000000000..290582160 --- /dev/null +++ b/ci/code/repo_observer.py.bak @@ -0,0 +1,70 @@ +""" +This is the repo observer. + +It checks for new commits to the master repo, and will notify the dispatcher of +the latest commit ID, so the dispatcher can dispatch the tests against this +commit ID. +""" +import argparse +import os +import re +import socket +import SocketServer +import subprocess +import sys +import time + +import helpers + + +def poll(): + parser = argparse.ArgumentParser() + parser.add_argument("--dispatcher-server", + help="dispatcher host:port, " \ + "by default it uses localhost:8888", + default="localhost:8888", + action="store") + parser.add_argument("repo", metavar="REPO", type=str, + help="path to the repository this will observe") + args = parser.parse_args() + dispatcher_host, dispatcher_port = args.dispatcher_server.split(":") + while True: + try: + # call the bash script that will update the repo and check + # for changes. If there's a change, it will drop a .commit_id file + # with the latest commit in the current working directory + subprocess.check_output(["./update_repo.sh", args.repo]) + except subprocess.CalledProcessError as e: + raise Exception("Could not update and check repository. Reason: %s" % e.output) + + if os.path.isfile(".commit_id"): + # great, we have a change! let's execute the tests + # First, check the status of the dispatcher server to see + # if we can send the tests + try: + response = helpers.communicate(dispatcher_host, + int(dispatcher_port), + "status") + except socket.error as e: + raise Exception("Could not communicate with dispatcher server: %s" % e) + if response == "OK": + # Dispatcher is present, let's send it a test + commit = "" + with open(".commit_id", "r") as f: + commit = f.readline() + response = helpers.communicate(dispatcher_host, + int(dispatcher_port), + "dispatch:%s" % commit) + if response != "OK": + raise Exception("Could not dispatch the test: %s" % + response) + print "dispatched!" + else: + # Something wrong happened to the dispatcher + raise Exception("Could not dispatch the test: %s" % + response) + time.sleep(5) + + +if __name__ == "__main__": + poll() diff --git a/ci/code/test_runner.py b/ci/code/test_runner.py index b0779f84f..31ceb689f 100644 --- a/ci/code/test_runner.py +++ b/ci/code/test_runner.py @@ -13,7 +13,7 @@ import os import re import socket -import SocketServer +import socketserver import subprocess import time import threading @@ -22,14 +22,14 @@ import helpers -class ThreadingTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): dispatcher_server = None # Holds the dispatcher server host/port information last_communication = None # Keeps track of last communication from dispatcher busy = False # Status flag dead = False # Status flag -class TestHandler(SocketServer.BaseRequestHandler): +class TestHandler(socketserver.BaseRequestHandler): """ The RequestHandler class for our server. """ @@ -45,16 +45,16 @@ def handle(self): self.request.sendall("Invalid command") return if command == "ping": - print "pinged" + print("pinged") self.server.last_communication = time.time() self.request.sendall("pong") elif command == "runtest": - print "got runtest command: am I busy? %s" % self.server.busy + print(("got runtest command: am I busy? %s" % self.server.busy)) if self.server.busy: self.request.sendall("BUSY") else: self.request.sendall("OK") - print "running" + print("running") commit_id = command_groups.group(2)[1:] self.server.busy = True self.run_tests(commit_id, @@ -67,7 +67,7 @@ def run_tests(self, commit_id, repo_folder): # update repo output = subprocess.check_output(["./test_runner_script.sh", repo_folder, commit_id]) - print output + print(output) # run the tests test_folder = os.path.join(repo_folder, "tests") suite = unittest.TestLoader().discover(test_folder) @@ -110,8 +110,8 @@ def serve(): try: server = ThreadingTCPServer((runner_host, runner_port), TestHandler) - print server - print runner_port + print(server) + print(runner_port) break except socket.error as e: if e.errno == errno.EADDRINUSE: @@ -149,11 +149,11 @@ def dispatcher_checker(server): int(server.dispatcher_server["port"]), "status") if response != "OK": - print "Dispatcher is no longer functional" + print("Dispatcher is no longer functional") server.shutdown() return except socket.error as e: - print "Can't communicate with dispatcher: %s" % e + print(("Can't communicate with dispatcher: %s" % e)) server.shutdown() return diff --git a/ci/code/test_runner.py.bak b/ci/code/test_runner.py.bak new file mode 100644 index 000000000..c1e02eb68 --- /dev/null +++ b/ci/code/test_runner.py.bak @@ -0,0 +1,173 @@ +""" +This is the test runner. + +It registers itself with the dispatcher when it first starts up, and then waits +for notification from the dispatcher. When the dispatcher sends it a 'runtest' +command with a commit id, it updates its repository clone and checks out the +given commit. It will then run tests against this version and will send back the +results to the dispatcher. It will then wait for further instruction from the +dispatcher. +""" +import argparse +import errno +import os +import re +import socket +import socketserver +import subprocess +import time +import threading +import unittest + +import helpers + + +class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): + dispatcher_server = None # Holds the dispatcher server host/port information + last_communication = None # Keeps track of last communication from dispatcher + busy = False # Status flag + dead = False # Status flag + + +class TestHandler(socketserver.BaseRequestHandler): + """ + The RequestHandler class for our server. + """ + + command_re = re.compile(r"(\w+)(:.+)*") + + def handle(self): + # self.request is the TCP socket connected to the client + self.data = self.request.recv(1024).strip() + command_groups = self.command_re.match(self.data) + command = command_groups.group(1) + if not command: + self.request.sendall("Invalid command") + return + if command == "ping": + print("pinged") + self.server.last_communication = time.time() + self.request.sendall("pong") + elif command == "runtest": + print("got runtest command: am I busy? %s" % self.server.busy) + if self.server.busy: + self.request.sendall("BUSY") + else: + self.request.sendall("OK") + print("running") + commit_id = command_groups.group(2)[1:] + self.server.busy = True + self.run_tests(commit_id, + self.server.repo_folder) + self.server.busy = False + else: + self.request.sendall("Invalid command") + + def run_tests(self, commit_id, repo_folder): + # update repo + output = subprocess.check_output(["./test_runner_script.sh", + repo_folder, commit_id]) + print(output) + # run the tests + test_folder = os.path.join(repo_folder, "tests") + suite = unittest.TestLoader().discover(test_folder) + result_file = open("results", "w") + unittest.TextTestRunner(result_file).run(suite) + result_file.close() + result_file = open("results", "r") + # give the dispatcher the results + output = result_file.read() + helpers.communicate(self.server.dispatcher_server["host"], + int(self.server.dispatcher_server["port"]), + "results:%s:%s:%s" % (commit_id, len(output), output)) + + +def serve(): + range_start = 8900 + parser = argparse.ArgumentParser() + parser.add_argument("--host", + help="runner's host, by default it uses localhost", + default="localhost", + action="store") + parser.add_argument("--port", + help="runner's port, by default it uses values >=%s" % range_start, + action="store") + parser.add_argument("--dispatcher-server", + help="dispatcher host:port, by default it uses " \ + "localhost:8888", + default="localhost:8888", + action="store") + parser.add_argument("repo", metavar="REPO", type=str, + help="path to the repository this will observe") + args = parser.parse_args() + + runner_host = args.host + runner_port = None + tries = 0 + if not args.port: + runner_port = range_start + while tries < 100: + try: + server = ThreadingTCPServer((runner_host, runner_port), + TestHandler) + print(server) + print(runner_port) + break + except socket.error as e: + if e.errno == errno.EADDRINUSE: + tries += 1 + runner_port = runner_port + tries + continue + else: + raise e + else: + raise Exception("Could not bind to ports in range %s-%s" % (range_start, range_start+tries)) + else: + runner_port = int(args.port) + server = ThreadingTCPServer((runner_host, runner_port), TestHandler) + server.repo_folder = args.repo + + dispatcher_host, dispatcher_port = args.dispatcher_server.split(":") + server.dispatcher_server = {"host":dispatcher_host, "port":dispatcher_port} + response = helpers.communicate(server.dispatcher_server["host"], + int(server.dispatcher_server["port"]), + "register:%s:%s" % + (runner_host, runner_port)) + if response != "OK": + raise Exception("Can't register with dispatcher!") + + def dispatcher_checker(server): + # Checks if the dispatcher went down. If it is down, we will shut down + # if since the dispatcher may not have the same host/port + # when it comes back up. + while not server.dead: + time.sleep(5) + if (time.time() - server.last_communication) > 10: + try: + response = helpers.communicate( + server.dispatcher_server["host"], + int(server.dispatcher_server["port"]), + "status") + if response != "OK": + print("Dispatcher is no longer functional") + server.shutdown() + return + except socket.error as e: + print("Can't communicate with dispatcher: %s" % e) + server.shutdown() + return + + t = threading.Thread(target=dispatcher_checker, args=(server,)) + try: + t.start() + # Activate the server; this will keep running until you + # interrupt the program with Ctrl-C + server.serve_forever() + except (KeyboardInterrupt, Exception): + # if any exception occurs, kill the thread + server.dead = True + t.join() + + +if __name__ == "__main__": + serve() diff --git a/cluster/code/cluster.py b/cluster/code/cluster.py index 6da90b5d0..39412c233 100644 --- a/cluster/code/cluster.py +++ b/cluster/code/cluster.py @@ -4,7 +4,7 @@ import heapq import itertools import logging -import Queue +import queue import random import threading @@ -44,7 +44,7 @@ class Node(object): def __init__(self, network, address): self.network = network - self.address = address or 'N%d' % self.unique_ids.next() + self.address = address or 'N%d' % next(self.unique_ids) self.logger = SimTimeLogger(logging.getLogger(self.address), {'network': self.network}) self.logger.info('starting') self.roles = [] @@ -198,7 +198,7 @@ def __init__(self, node, execute_fn, state, slot, decisions, peers): def do_Invoke(self, sender, caller, client_id, input_value): proposal = Proposal(caller, client_id, input_value) - slot = next((s for s, p in self.proposals.iteritems() if p == proposal), None) + slot = next((s for s, p in list(self.proposals.items()) if p == proposal), None) # propose, or re-propose if this proposal already has a slot self.propose(proposal, slot) @@ -241,7 +241,7 @@ def do_Decision(self, sender, slot, proposal): def commit(self, slot, proposal): """Actually commit a proposal that is decided and in sequence""" - decided_proposals = [p for s, p in self.decisions.iteritems() if s < slot] + decided_proposals = [p for s, p in list(self.decisions.items()) if s < slot] if proposal in decided_proposals: self.logger.info("not committing duplicate proposal %r at slot %d", proposal, slot) return # duplicate @@ -340,7 +340,7 @@ def send_prepare(self): def update_accepted(self, accepted_proposals): acc = self.accepted_proposals - for slot, (ballot_num, proposal) in accepted_proposals.iteritems(): + for slot, (ballot_num, proposal) in list(accepted_proposals.items()): if slot not in acc or acc[slot][0] < ballot_num: acc[slot] = (ballot_num, proposal) @@ -352,7 +352,7 @@ def do_Promise(self, sender, ballot_num, accepted_proposals): if len(self.acceptors) >= self.quorum: # strip the ballot numbers from self.accepted_proposals, now that it # represents a majority - accepted_proposals = dict((s, p) for s, (b, p) in self.accepted_proposals.iteritems()) + accepted_proposals = dict((s, p) for s, (b, p) in list(self.accepted_proposals.items())) # We're adopted; note that this does *not* mean that no other leader is active. # Any such conflicts will be handled by the commanders. self.node.send([self.node.address], @@ -490,7 +490,7 @@ class Requester(Role): def __init__(self, node, n, callback): super(Requester, self).__init__(node) - self.client_id = self.client_ids.next() + self.client_id = next(self.client_ids) self.n = n self.output = None self.callback = callback @@ -528,7 +528,7 @@ def start(self): def invoke(self, input_value, request_cls=Requester): assert self.requester is None - q = Queue.Queue() + q = queue.Queue() self.requester = request_cls(self.node, input_value, q.put) self.requester.start() output = q.get() diff --git a/cluster/code/cluster.py.bak b/cluster/code/cluster.py.bak new file mode 100644 index 000000000..65871f412 --- /dev/null +++ b/cluster/code/cluster.py.bak @@ -0,0 +1,536 @@ +from collections import namedtuple +import copy +import functools +import heapq +import itertools +import logging +import queue +import random +import threading + +# data types +Proposal = namedtuple('Proposal', ['caller', 'client_id', 'input']) +Ballot = namedtuple('Ballot', ['n', 'leader']) + +# message types +Accepted = namedtuple('Accepted', ['slot', 'ballot_num']) +Accept = namedtuple('Accept', ['slot', 'ballot_num', 'proposal']) +Decision = namedtuple('Decision', ['slot', 'proposal']) +Invoked = namedtuple('Invoked', ['client_id', 'output']) +Invoke = namedtuple('Invoke', ['caller', 'client_id', 'input_value']) +Join = namedtuple('Join', []) +Active = namedtuple('Active', []) +Prepare = namedtuple('Prepare', ['ballot_num']) +Promise = namedtuple('Promise', ['ballot_num', 'accepted_proposals']) +Propose = namedtuple('Propose', ['slot', 'proposal']) +Welcome = namedtuple('Welcome', ['state', 'slot', 'decisions']) +Decided = namedtuple('Decided', ['slot']) +Preempted = namedtuple('Preempted', ['slot', 'preempted_by']) +Adopted = namedtuple('Adopted', ['ballot_num', 'accepted_proposals']) +Accepting = namedtuple('Accepting', ['leader']) + +# constants - these times should really be in terms of average round-trip time +JOIN_RETRANSMIT = 0.7 +CATCHUP_INTERVAL = 0.6 +ACCEPT_RETRANSMIT = 1.0 +PREPARE_RETRANSMIT = 1.0 +INVOKE_RETRANSMIT = 0.5 +LEADER_TIMEOUT = 1.0 +NULL_BALLOT = Ballot(-1, -1) # sorts before all real ballots +NOOP_PROPOSAL = Proposal(None, None, None) # no-op to fill otherwise empty slots + +class Node(object): + unique_ids = itertools.count() + + def __init__(self, network, address): + self.network = network + self.address = address or 'N%d' % next(self.unique_ids) + self.logger = SimTimeLogger(logging.getLogger(self.address), {'network': self.network}) + self.logger.info('starting') + self.roles = [] + self.send = functools.partial(self.network.send, self) + + def register(self, roles): + self.roles.append(roles) + + def unregister(self, roles): + self.roles.remove(roles) + + def receive(self, sender, message): + handler_name = 'do_%s' % type(message).__name__ + + for comp in self.roles[:]: + if not hasattr(comp, handler_name): + continue + comp.logger.debug("received %s from %s", message, sender) + fn = getattr(comp, handler_name) + fn(sender=sender, **message._asdict()) + +class Timer(object): + + def __init__(self, expires, address, callback): + self.expires = expires + self.address = address + self.callback = callback + self.cancelled = False + + def __cmp__(self, other): + return cmp(self.expires, other.expires) + + def cancel(self): + self.cancelled = True + +class Network(object): + PROP_DELAY = 0.03 + PROP_JITTER = 0.02 + DROP_PROB = 0.05 + + def __init__(self, seed): + self.nodes = {} + self.rnd = random.Random(seed) + self.timers = [] + self.now = 1000.0 + + def new_node(self, address=None): + node = Node(self, address=address) + self.nodes[node.address] = node + return node + + def run(self): + while self.timers: + next_timer = self.timers[0] + if next_timer.expires > self.now: + self.now = next_timer.expires + heapq.heappop(self.timers) + if next_timer.cancelled: + continue + if not next_timer.address or next_timer.address in self.nodes: + next_timer.callback() + + def stop(self): + self.timers = [] + + def set_timer(self, address, seconds, callback): + timer = Timer(self.now + seconds, address, callback) + heapq.heappush(self.timers, timer) + return timer + + def send(self, sender, destinations, message): + sender.logger.debug("sending %s to %s", message, destinations) + # avoid aliasing by making a closure containing distinct deep copy of message for each dest + def sendto(dest, message): + if dest == sender.address: + # reliably deliver local messages with no delay + self.set_timer(sender.address, 0, lambda: sender.receive(sender.address, message)) + elif self.rnd.uniform(0, 1.0) > self.DROP_PROB: + delay = self.PROP_DELAY + self.rnd.uniform(-self.PROP_JITTER, self.PROP_JITTER) + self.set_timer(dest, delay, functools.partial(self.nodes[dest].receive, + sender.address, message)) + for dest in (d for d in destinations if d in self.nodes): + sendto(dest, copy.deepcopy(message)) + +class SimTimeLogger(logging.LoggerAdapter): + + def process(self, msg, kwargs): + return "T=%.3f %s" % (self.extra['network'].now, msg), kwargs + + def getChild(self, name): + return self.__class__(self.logger.getChild(name), + {'network': self.extra['network']}) + +class Role(object): + + def __init__(self, node): + self.node = node + self.node.register(self) + self.running = True + self.logger = node.logger.getChild(type(self).__name__) + + def set_timer(self, seconds, callback): + return self.node.network.set_timer(self.node.address, seconds, + lambda: self.running and callback()) + + def stop(self): + self.running = False + self.node.unregister(self) + +class Acceptor(Role): + + def __init__(self, node): + super(Acceptor, self).__init__(node) + self.ballot_num = NULL_BALLOT + self.accepted_proposals = {} # {slot: (ballot_num, proposal)} + + def do_Prepare(self, sender, ballot_num): + if ballot_num > self.ballot_num: + self.ballot_num = ballot_num + # we've heard from a scout, so it might be the next leader + self.node.send([self.node.address], Accepting(leader=sender)) + + self.node.send([sender], Promise(ballot_num=self.ballot_num, accepted_proposals=self.accepted_proposals)) + + def do_Accept(self, sender, ballot_num, slot, proposal): + if ballot_num >= self.ballot_num: + self.ballot_num = ballot_num + acc = self.accepted_proposals + if slot not in acc or acc[slot][0] < ballot_num: + acc[slot] = (ballot_num, proposal) + + self.node.send([sender], Accepted( + slot=slot, ballot_num=self.ballot_num)) + +class Replica(Role): + + def __init__(self, node, execute_fn, state, slot, decisions, peers): + super(Replica, self).__init__(node) + self.execute_fn = execute_fn + self.state = state + self.slot = slot + self.decisions = decisions + self.peers = peers + self.proposals = {} + # next slot num for a proposal (may lead slot) + self.next_slot = slot + self.latest_leader = None + self.latest_leader_timeout = None + + # making proposals + + def do_Invoke(self, sender, caller, client_id, input_value): + proposal = Proposal(caller, client_id, input_value) + slot = next((s for s, p in self.proposals.items() if p == proposal), None) + # propose, or re-propose if this proposal already has a slot + self.propose(proposal, slot) + + def propose(self, proposal, slot=None): + """Send (or resend, if slot is specified) a proposal to the leader""" + if not slot: + slot, self.next_slot = self.next_slot, self.next_slot + 1 + self.proposals[slot] = proposal + # find a leader we think is working - either the latest we know of, or + # ourselves (which may trigger a scout to make us the leader) + leader = self.latest_leader or self.node.address + self.logger.info("proposing %s at slot %d to leader %s" % (proposal, slot, leader)) + self.node.send([leader], Propose(slot=slot, proposal=proposal)) + + # handling decided proposals + + def do_Decision(self, sender, slot, proposal): + assert not self.decisions.get(self.slot, None), \ + "next slot to commit is already decided" + if slot in self.decisions: + assert self.decisions[slot] == proposal, \ + "slot %d already decided with %r!" % (slot, self.decisions[slot]) + return + self.decisions[slot] = proposal + self.next_slot = max(self.next_slot, slot + 1) + + # re-propose our proposal in a new slot if it lost its slot and wasn't a no-op + our_proposal = self.proposals.get(slot) + if our_proposal is not None and our_proposal != proposal and our_proposal.caller: + self.propose(our_proposal) + + # execute any pending, decided proposals + while True: + commit_proposal = self.decisions.get(self.slot) + if not commit_proposal: + break # not decided yet + commit_slot, self.slot = self.slot, self.slot + 1 + + self.commit(commit_slot, commit_proposal) + + def commit(self, slot, proposal): + """Actually commit a proposal that is decided and in sequence""" + decided_proposals = [p for s, p in self.decisions.items() if s < slot] + if proposal in decided_proposals: + self.logger.info("not committing duplicate proposal %r at slot %d", proposal, slot) + return # duplicate + + self.logger.info("committing %r at slot %d" % (proposal, slot)) + if proposal.caller is not None: + # perform a client operation + self.state, output = self.execute_fn(self.state, proposal.input) + self.node.send([proposal.caller], Invoked(client_id=proposal.client_id, output=output)) + + # tracking the leader + + def do_Adopted(self, sender, ballot_num, accepted_proposals): + self.latest_leader = self.node.address + self.leader_alive() + + def do_Accepting(self, sender, leader): + self.latest_leader = leader + self.leader_alive() + + def do_Active(self, sender): + if sender != self.latest_leader: + return + self.leader_alive() + + def leader_alive(self): + if self.latest_leader_timeout: + self.latest_leader_timeout.cancel() + + def reset_leader(): + idx = self.peers.index(self.latest_leader) + self.latest_leader = self.peers[(idx + 1) % len(self.peers)] + self.logger.debug("leader timed out; tring the next one, %s", self.latest_leader) + self.latest_leader_timeout = self.set_timer(LEADER_TIMEOUT, reset_leader) + + # adding new cluster members + + def do_Join(self, sender): + if sender in self.peers: + self.node.send([sender], Welcome( + state=self.state, slot=self.slot, decisions=self.decisions)) + +class Commander(Role): + + def __init__(self, node, ballot_num, slot, proposal, peers): + super(Commander, self).__init__(node) + self.ballot_num = ballot_num + self.slot = slot + self.proposal = proposal + self.acceptors = set([]) + self.peers = peers + self.quorum = len(peers) / 2 + 1 + + def start(self): + self.node.send(set(self.peers) - self.acceptors, Accept( + slot=self.slot, ballot_num=self.ballot_num, proposal=self.proposal)) + self.set_timer(ACCEPT_RETRANSMIT, self.start) + + def finished(self, ballot_num, preempted): + if preempted: + self.node.send([self.node.address], Preempted(slot=self.slot, preempted_by=ballot_num)) + else: + self.node.send([self.node.address], Decided(slot=self.slot)) + self.stop() + + def do_Accepted(self, sender, slot, ballot_num): + if slot != self.slot: + return + if ballot_num == self.ballot_num: + self.acceptors.add(sender) + if len(self.acceptors) < self.quorum: + return + self.node.send(self.peers, Decision(slot=self.slot, proposal=self.proposal)) + self.finished(ballot_num, False) + else: + self.finished(ballot_num, True) + +class Scout(Role): + + def __init__(self, node, ballot_num, peers): + super(Scout, self).__init__(node) + self.ballot_num = ballot_num + self.accepted_proposals = {} + self.acceptors = set([]) + self.peers = peers + self.quorum = len(peers) / 2 + 1 + self.retransmit_timer = None + + def start(self): + self.logger.info("scout starting") + self.send_prepare() + + def send_prepare(self): + self.node.send(self.peers, Prepare(ballot_num=self.ballot_num)) + self.retransmit_timer = self.set_timer(PREPARE_RETRANSMIT, self.send_prepare) + + def update_accepted(self, accepted_proposals): + acc = self.accepted_proposals + for slot, (ballot_num, proposal) in accepted_proposals.items(): + if slot not in acc or acc[slot][0] < ballot_num: + acc[slot] = (ballot_num, proposal) + + def do_Promise(self, sender, ballot_num, accepted_proposals): + if ballot_num == self.ballot_num: + self.logger.info("got matching promise; need %d" % self.quorum) + self.update_accepted(accepted_proposals) + self.acceptors.add(sender) + if len(self.acceptors) >= self.quorum: + # strip the ballot numbers from self.accepted_proposals, now that it + # represents a majority + accepted_proposals = dict((s, p) for s, (b, p) in self.accepted_proposals.items()) + # We're adopted; note that this does *not* mean that no other leader is active. + # Any such conflicts will be handled by the commanders. + self.node.send([self.node.address], + Adopted(ballot_num=ballot_num, accepted_proposals=accepted_proposals)) + self.stop() + else: + # this acceptor has promised another leader a higher ballot number, so we've lost + self.node.send([self.node.address], Preempted(slot=None, preempted_by=ballot_num)) + self.stop() + +class Leader(Role): + + def __init__(self, node, peers, commander_cls=Commander, scout_cls=Scout): + super(Leader, self).__init__(node) + self.ballot_num = Ballot(0, node.address) + self.active = False + self.proposals = {} + self.commander_cls = commander_cls + self.scout_cls = scout_cls + self.scouting = False + self.peers = peers + + def start(self): + # reminder others we're active before LEADER_TIMEOUT expires + def active(): + if self.active: + self.node.send(self.peers, Active()) + self.set_timer(LEADER_TIMEOUT / 2.0, active) + active() + + def spawn_scout(self): + assert not self.scouting + self.scouting = True + self.scout_cls(self.node, self.ballot_num, self.peers).start() + + def do_Adopted(self, sender, ballot_num, accepted_proposals): + self.scouting = False + self.proposals.update(accepted_proposals) + # note that we don't re-spawn commanders here; if there are undecided + # proposals, the replicas will re-propose + self.logger.info("leader becoming active") + self.active = True + + def spawn_commander(self, ballot_num, slot): + proposal = self.proposals[slot] + self.commander_cls(self.node, ballot_num, slot, proposal, self.peers).start() + + def do_Preempted(self, sender, slot, preempted_by): + if not slot: # from the scout + self.scouting = False + self.logger.info("leader preempted by %s", preempted_by.leader) + self.active = False + self.ballot_num = Ballot((preempted_by or self.ballot_num).n + 1, self.ballot_num.leader) + + def do_Propose(self, sender, slot, proposal): + if slot not in self.proposals: + if self.active: + self.proposals[slot] = proposal + self.logger.info("spawning commander for slot %d" % (slot,)) + self.spawn_commander(self.ballot_num, slot) + else: + if not self.scouting: + self.logger.info("got PROPOSE when not active - scouting") + self.spawn_scout() + else: + self.logger.info("got PROPOSE while scouting; ignored") + else: + self.logger.info("got PROPOSE for a slot already being proposed") + +class Bootstrap(Role): + + def __init__(self, node, peers, execute_fn, + replica_cls=Replica, acceptor_cls=Acceptor, leader_cls=Leader, + commander_cls=Commander, scout_cls=Scout): + super(Bootstrap, self).__init__(node) + self.execute_fn = execute_fn + self.peers = peers + self.peers_cycle = itertools.cycle(peers) + self.replica_cls = replica_cls + self.acceptor_cls = acceptor_cls + self.leader_cls = leader_cls + self.commander_cls = commander_cls + self.scout_cls = scout_cls + + def start(self): + self.join() + + def join(self): + self.node.send([next(self.peers_cycle)], Join()) + self.set_timer(JOIN_RETRANSMIT, self.join) + + def do_Welcome(self, sender, state, slot, decisions): + self.acceptor_cls(self.node) + self.replica_cls(self.node, execute_fn=self.execute_fn, peers=self.peers, + state=state, slot=slot, decisions=decisions) + self.leader_cls(self.node, peers=self.peers, commander_cls=self.commander_cls, + scout_cls=self.scout_cls).start() + self.stop() + +class Seed(Role): + + def __init__(self, node, initial_state, execute_fn, peers, bootstrap_cls=Bootstrap): + super(Seed, self).__init__(node) + self.initial_state = initial_state + self.execute_fn = execute_fn + self.peers = peers + self.bootstrap_cls = bootstrap_cls + self.seen_peers = set([]) + self.exit_timer = None + + def do_Join(self, sender): + self.seen_peers.add(sender) + if len(self.seen_peers) <= len(self.peers) / 2: + return + + # cluster is ready - welcome everyone + self.node.send(list(self.seen_peers), Welcome( + state=self.initial_state, slot=1, decisions={})) + + # stick around for long enough that we don't hear any new JOINs from + # the newly formed cluster + if self.exit_timer: + self.exit_timer.cancel() + self.exit_timer = self.set_timer(JOIN_RETRANSMIT * 2, self.finish) + + def finish(self): + # bootstrap this node into the cluster we just seeded + bs = self.bootstrap_cls(self.node, peers=self.peers, execute_fn=self.execute_fn) + bs.start() + self.stop() + +class Requester(Role): + + client_ids = itertools.count(start=100000) + + def __init__(self, node, n, callback): + super(Requester, self).__init__(node) + self.client_id = next(self.client_ids) + self.n = n + self.output = None + self.callback = callback + + def start(self): + self.node.send([self.node.address], Invoke(caller=self.node.address, + client_id=self.client_id, input_value=self.n)) + self.invoke_timer = self.set_timer(INVOKE_RETRANSMIT, self.start) + + def do_Invoked(self, sender, client_id, output): + if client_id != self.client_id: + return + self.logger.debug("received output %r" % (output,)) + self.invoke_timer.cancel() + self.callback(output) + self.stop() + +class Member(object): + + def __init__(self, state_machine, network, peers, seed=None, + seed_cls=Seed, bootstrap_cls=Bootstrap): + self.network = network + self.node = network.new_node() + if seed is not None: + self.startup_role = seed_cls(self.node, initial_state=seed, peers=peers, + execute_fn=state_machine) + else: + self.startup_role = bootstrap_cls(self.node, execute_fn=state_machine, peers=peers) + self.requester = None + + def start(self): + self.startup_role.start() + self.thread = threading.Thread(target=self.network.run) + self.thread.start() + + def invoke(self, input_value, request_cls=Requester): + assert self.requester is None + q = queue.Queue() + self.requester = request_cls(self.node, input_value, q.put) + self.requester.start() + output = q.get() + self.requester = None + return output diff --git a/cluster/code/extract_code.py b/cluster/code/extract_code.py index 42661182a..d504b8796 100644 --- a/cluster/code/extract_code.py +++ b/cluster/code/extract_code.py @@ -17,8 +17,8 @@ def __init__(self, filename): def __iter__(self): return self - def next(self): - self.linenum, rv = self.enum_iter.next() + def __next__(self): + self.linenum, rv = next(self.enum_iter) return rv def use_line(self): @@ -48,7 +48,7 @@ def code_block(filename, args): lines.use_line() break else: - print >>sys.stderr, "no match found for %r" % arg + print("no match found for %r" % arg, file=sys.stderr) sys.exit(1) space = re.match('^ *', line).group(0) @@ -72,7 +72,7 @@ def code_block(filename, args): extractors['code_block'] = code_block def from_to(filename, args): - from_re, to_re = map(re.compile, args) + from_re, to_re = list(map(re.compile, args)) lines = LineIterator(filename) result = [] @@ -80,7 +80,7 @@ def from_to(filename, args): if from_re.match(line): break else: - print >>sys.stderr, "no match found for %r" % from_re + print("no match found for %r" % from_re, file=sys.stderr) sys.exit(1) for line in lines: diff --git a/cluster/code/extract_code.py.bak b/cluster/code/extract_code.py.bak new file mode 100644 index 000000000..42661182a --- /dev/null +++ b/cluster/code/extract_code.py.bak @@ -0,0 +1,119 @@ +import sys +import re +import shlex +import textwrap + +extractors = {} + +class LineIterator(object): + + used_lines_per_file = {} + + def __init__(self, filename): + self.enum_iter = enumerate(open(filename)) + self.linenum = -1 + self.used_lines = self.used_lines_per_file.setdefault(filename, set()) + + def __iter__(self): + return self + + def next(self): + self.linenum, rv = self.enum_iter.next() + return rv + + def use_line(self): + self.used_lines.add(self.linenum) + + @classmethod + def print_unused_lines(cls, filename): + sys.stderr.write("*** unused lines in %r ***\n" % filename) + used_lines = cls.used_lines_per_file[filename] + for l, line in enumerate(open(filename)): + if line.strip() and l not in used_lines: + sys.stderr.write("%4d %s" % (l+1, line)) + +file_lines_used = {} + +def code_block(filename, args): + assert args, "args is empty" + lines = LineIterator(filename) + + result = [] + + for arg in args: + beginning_re = re.compile(' *' + arg) + for line in lines: + if beginning_re.match(line): + result.append(line) + lines.use_line() + break + else: + print >>sys.stderr, "no match found for %r" % arg + sys.exit(1) + + space = re.match('^ *', line).group(0) + space_re = re.compile('^%s ' % space) + whitespace = False + for line in lines: + if not line.strip(): + whitespace = True + continue + if not space_re.match(line): + break + if whitespace: + result.append('\n') + whitespace = False + result.append(line) + lines.use_line() + + # left-justify with a four-space leader + return [' ' + line + '\n' + for line in textwrap.dedent(''.join(result)).split('\n')] +extractors['code_block'] = code_block + +def from_to(filename, args): + from_re, to_re = map(re.compile, args) + lines = LineIterator(filename) + + result = [] + for line in lines: + if from_re.match(line): + break + else: + print >>sys.stderr, "no match found for %r" % from_re + sys.exit(1) + + for line in lines: + if to_re.match(line): + break + result.append(line) + lines.use_line() + + # left-justify with a four-space leader + return [' ' + line + '\n' + for line in textwrap.dedent(''.join(result)).split('\n')] +extractors['from_to'] = from_to + +def scan_chapter(): + lines = open('CHAPTER.rst') + result = [] + in_braces = False + for line in lines: + if in_braces: + if not line.startswith('}}}'): + continue + else: + in_braces = False + if not in_braces: + result.append(line) + if line.startswith('{{{'): + result.extend(['.. code-block:: python\n', '\n']) + in_braces = True + args = shlex.split(line[4:]) + extractor = extractors[args[0]] + filename = args[1] + result.extend(extractor(filename, args[2:])) + open("CHAPTER.rst", "w").write(''.join(result)) + +scan_chapter() +LineIterator.print_unused_lines('cluster.py') diff --git a/cluster/code/test/test_integration.py b/cluster/code/test/test_integration.py index a5853617f..369892e87 100644 --- a/cluster/code/test/test_integration.py +++ b/cluster/code/test/test_integration.py @@ -139,4 +139,4 @@ def kill_leader(): self.network.set_timer(None, 15, self.network.stop) self.network.run() - self.assertEqual(set(results), set(xrange(1, N+1))) + self.assertEqual(set(results), set(range(1, N+1))) diff --git a/cluster/code/test/test_integration.py.bak b/cluster/code/test/test_integration.py.bak new file mode 100644 index 000000000..a5853617f --- /dev/null +++ b/cluster/code/test/test_integration.py.bak @@ -0,0 +1,142 @@ +from cluster import * +import unittest +import itertools + + +class Tests(unittest.TestCase): + + def setUp(self): + self.network = Network(1234) + self.addresses = ('node-%d' % d for d in itertools.count()) + self.nodes = [] + self.events = [] + + def tearDown(self): + if self.events: + self.fail("unhandled events: %r" % (self.events,)) + + def event(self, name): + self.events.append((self.network.now, name)) + + def addNode(self, address): + node = self.network.new_node(address=address) + self.nodes.append(node) + return node + + def assertEvent(self, time, name, fuzz=0): + for i, e in enumerate(self.events): + if e[1] == name and time - fuzz <= e[0] <= time + fuzz: + self.events.pop(i) + return + self.fail("event %r not found at or around time %f; events: %r" % (name, time, self.events)) + + def setupNetwork(self, count, execute_fn=None): + def add(state, input): + state += input + return state, state + execute_fn = execute_fn or add + peers = ['N%d' % n for n in range(count)] + nodes = [self.addNode(p) for p in peers] + Seed(nodes[0], initial_state=0, peers=peers, execute_fn=execute_fn) + + for node in nodes[1:]: + bs = Bootstrap(node, execute_fn=execute_fn, peers=peers) + bs.start() + return nodes + + def kill(self, node): + node.logger.warning("KILLED BY TESTS") + del self.network.nodes[node.address] + + def test_two_requests(self): + """Full run with non-overlapping requests succeeds.""" + nodes = self.setupNetwork(5) + # set up some timers for various events + def request_done(output): + self.event("request done: %s" % output) + def make_request(n, node): + self.event("request: %s" % n) + req = Requester(node, n, request_done) + req.start() + for time, callback in [ + (1.0, lambda: make_request(5, nodes[1])), + (5.0, lambda: make_request(6, nodes[2])), + (10.0, self.network.stop), + ]: + self.network.set_timer(None, time, callback) + + self.network.run() + self.assertEvent(1001.0, 'request: 5') + self.assertEvent(1002.0, 'request done: 5', fuzz=1) + self.assertEvent(1005.0, 'request: 6') + self.assertEvent(1005.0, 'request done: 11', fuzz=1) + + def test_parallel_requests(self): + """Full run with parallel request succeeds.""" + N = 10 + nodes = self.setupNetwork(5) + results = [] + for n in range(1, N+1): + req = Requester(nodes[n % 4], n, results.append) + self.network.set_timer(None, 1.0, req.start) + + self.network.set_timer(None, 10.0, self.network.stop) + self.network.run() + self.assertEqual((len(results), results and max(results)), (N, N*(N+1)/2), + "got %r" % (results,)) + + def test_failed_nodes(self): + """Full run with requests and some nodes dying midway through succeeds""" + N = 10 + nodes = self.setupNetwork(7) + results = [] + for n in range(1, N+1): + req = Requester(nodes[n % 3], n, results.append) + self.network.set_timer(None, n+1, req.start) + + # kill nodes 3 and 4 at N/2 seconds + self.network.set_timer(None, N/2-1, lambda: self.kill(nodes[3])) + self.network.set_timer(None, N/2, lambda: self.kill(nodes[4])) + + self.network.set_timer(None, N * 3.0, self.network.stop) + self.network.run() + self.assertEqual((len(results), results and max(results)), (N, N*(N+1)/2), + "got %r" % (results,)) + + def test_failed_leader(self): + """Full run with requests and a dying leader succeeds.""" + N = 10 + # use a bit-setting function so that we can easily ignore requests made + # by the failed node + def identity(state, input): + return state, input + nodes = self.setupNetwork(7, execute_fn=identity) + results = [] + for n in range(1, N+1): + req = Requester(nodes[n % 6], n, results.append) + self.network.set_timer(None, n+1, req.start) + + # kill the leader node at N/2 seconds (it should be stable by then). Some of the + # Requester roles were attached to this node, so we fake success of those requests + # since we don't know what state they're in right now. + def is_leader(n): + try: + leader_role = [c for c in n.roles if isinstance(c, Leader)][0] + return leader_role.active + except IndexError: + return False + def kill_leader(): + active_leader_nodes = [n for n in nodes if is_leader(n)] + if active_leader_nodes: + active_leader = active_leader_nodes[0] + active_idx = nodes.index(active_leader) + # append the N's that this node was requesting + for n in range(1, N+1): + if n % 6 == active_idx: + results.append(n) + self.kill(active_leader) + self.network.set_timer(None, N/2, kill_leader) + + self.network.set_timer(None, 15, self.network.stop) + self.network.run() + self.assertEqual(set(results), set(xrange(1, N+1))) diff --git a/cluster/code/test/utils.py b/cluster/code/test/utils.py index 53e3a3e25..fdac1acad 100644 --- a/cluster/code/test/utils.py +++ b/cluster/code/test/utils.py @@ -31,7 +31,7 @@ def verifyAcceptedProposals(self, accepted_proposals): """Verify that the ``accepted_proposals`` field of a promise is formatted as a dictionary mapping slots to (ballot, proposal) tuples.""" self.assertIsInstance(accepted_proposals, dict) - for k, v in accepted_proposals.iteritems(): + for k, v in list(accepted_proposals.items()): self.assertIsInstance(k, int) self.assertIsInstance(v, tuple) self.assertEqual(len(v), 2) diff --git a/cluster/code/test/utils.py.bak b/cluster/code/test/utils.py.bak new file mode 100644 index 000000000..86d0c16c7 --- /dev/null +++ b/cluster/code/test/utils.py.bak @@ -0,0 +1,39 @@ +from cluster import * +from . import fake_network +import unittest + + +class ComponentTestCase(unittest.TestCase): + + def setUp(self): + self.network = fake_network.FakeNetwork() + self.node = fake_network.FakeNode(self.network) + + def tearDown(self): + if self.node.sent: + self.fail("extra messages from node: %r" % (self.node.sent,)) + + def assertMessage(self, destinations, message): + got = self.node.sent.pop(0) + self.assertEqual((sorted(got[0]), got[1]), + (sorted(destinations), message)) + + def assertNoMessages(self): + self.assertEqual(self.node.sent, []) + + def assertTimers(self, times): + self.assertEqual(self.node.network.get_times(), times) + + def assertUnregistered(self): + self.assertEqual(self.node.roles, []) + + def verifyAcceptedProposals(self, accepted_proposals): + """Verify that the ``accepted_proposals`` field of a promise is formatted + as a dictionary mapping slots to (ballot, proposal) tuples.""" + self.assertIsInstance(accepted_proposals, dict) + for k, v in accepted_proposals.items(): + self.assertIsInstance(k, int) + self.assertIsInstance(v, tuple) + self.assertEqual(len(v), 2) + self.assertIsInstance(v[0], Ballot) + self.assertIsInstance(v[1], Proposal) diff --git a/contingent/code/example/blog_project.py b/contingent/code/example/blog_project.py index c8056335b..129d50e83 100644 --- a/contingent/code/example/blog_project.py +++ b/contingent/code/example/blog_project.py @@ -89,7 +89,7 @@ def render(paths, path): text = '

{}

\n

Date: {}

\n

Previous post: {}

\n{}'.format( title_of(path), date_of(path), previous_title, body_of(path)) - print('-' * 72) + print(('-' * 72)) print(text) return text @@ -108,11 +108,11 @@ def main(): project.verbose = True while True: - print('=' * 72) + print(('=' * 72)) print('Watching for files to change') changed_paths = looping_wait_on(paths) - print('=' * 72) - print('Reloading:', ' '.join(changed_paths)) + print(('=' * 72)) + print(('Reloading:', ' '.join(changed_paths))) with project.cache_off(): for path in changed_paths: read_text_file(path) diff --git a/contingent/code/example/blog_project.py.bak b/contingent/code/example/blog_project.py.bak new file mode 100644 index 000000000..c8056335b --- /dev/null +++ b/contingent/code/example/blog_project.py.bak @@ -0,0 +1,122 @@ +"""Render a directory of cross-referenced blog posts as HTML.""" + +import os +import re +from IPython.nbconvert import HTMLExporter +from IPython.nbformat import current as nbformat +from docutils.core import publish_doctree, publish_parts +from docutils import nodes +from glob import glob +from jinja2 import DictLoader + +from contingent.projectlib import Project +from contingent.io import looping_wait_on + +dl = DictLoader({'full.tpl': """\ +{%- extends 'display_priority.tpl' -%} +{% block input scoped %}
{{ cell.input }}
+{% endblock %} +{% block pyout scoped %}
{{ output.text | ansi2html }}
+{% endblock %} +{% block markdowncell scoped %}{{ cell.source | markdown2html }} +{% endblock %} +"""}) + +project = Project() +task = project.task + +@task +def read_text_file(path): + with open(path) as f: + return f.read() + +@task +def parse(path): + source = read_text_file(path) + if path.endswith('.rst'): + doctree = publish_doctree(source) + docinfos = doctree.traverse(nodes.docinfo) + docinfo = {c.tagname: str(c.children[0]) + for i in docinfos for c in i.children} + parts = publish_parts(source, writer_name='html') + return {'body': parts['body'], + 'date': docinfo.get('date'), + 'title': parts['title']} + elif path.endswith('.ipynb'): + notebook = nbformat.reads_json(source) + exporter = HTMLExporter(config=None, extra_loaders=[dl]) + body, resources = exporter.from_notebook_node(notebook) + return {'body': body, + 'date': notebook['metadata']['date'], + 'title': notebook['metadata']['name']} + +@task +def title_of(path): + info = parse(path) + return info['title'] + +@task +def date_of(path): + info = parse(path) + return info['date'] + +@task +def body_of(path): + info = parse(path) + dirname = os.path.dirname(path) + body = info['body'] + def format_title_reference(match): + filename = match.group(1) + title = title_of(os.path.join(dirname, filename)) + return '{}'.format(title) + body = re.sub(r'title_of\(([^)]+)\)', format_title_reference, body) + return body + +@task +def sorted_posts(paths): + return sorted(paths, key=date_of) + +@task +def previous_post(paths, path): + paths = sorted_posts(paths) + i = paths.index(path) + return paths[i - 1] if i else None + +@task +def render(paths, path): + previous = previous_post(paths, path) + previous_title = 'NONE' if previous is None else title_of(previous) + text = '

{}

\n

Date: {}

\n

Previous post: {}

\n{}'.format( + title_of(path), date_of(path), + previous_title, body_of(path)) + print('-' * 72) + print(text) + return text + +def main(): + thisdir = os.path.dirname(__file__) + indir = os.path.normpath(os.path.join(thisdir, '..', 'posts')) + outdir = os.path.normpath(os.path.join(thisdir, '..', 'output')) + if not os.path.exists(outdir): + os.mkdir(outdir) + + paths = tuple(glob(os.path.join(indir, '*.rst')) + + glob(os.path.join(indir, '*.ipynb'))) + + for path in sorted_posts(paths): + render(paths, path) + + project.verbose = True + while True: + print('=' * 72) + print('Watching for files to change') + changed_paths = looping_wait_on(paths) + print('=' * 72) + print('Reloading:', ' '.join(changed_paths)) + with project.cache_off(): + for path in changed_paths: + read_text_file(path) + project.rebuild() + +if __name__ == '__main__': + main() diff --git a/contingent/code/make.py b/contingent/code/make.py index a092f73bc..39a53fe77 100644 --- a/contingent/code/make.py +++ b/contingent/code/make.py @@ -107,22 +107,22 @@ def main(): project.start_tracing() for path in get_paths(): render(path) - print(project.stop_tracing(True)) + print((project.stop_tracing(True))) open('chapter.dot', 'w').write(as_graphviz(project._graph)) while True: - print('=' * 72) + print(('=' * 72)) print('Watching for files to change') changed_paths = looping_wait_on(get_paths()) - print('=' * 72) - print('Reloading:', ' '.join(changed_paths)) + print(('=' * 72)) + print(('Reloading:', ' '.join(changed_paths))) with project.cache_off(): for path in changed_paths: read_text_file(path) project.start_tracing() project.rebuild() - print(project.stop_tracing(True)) + print((project.stop_tracing(True))) if __name__ == '__main__': diff --git a/contingent/code/make.py.bak b/contingent/code/make.py.bak new file mode 100644 index 000000000..a092f73bc --- /dev/null +++ b/contingent/code/make.py.bak @@ -0,0 +1,129 @@ +import doctest +from glob import glob +import subprocess + +from docutils.core import publish_doctree +from docutils.parsers.rst.directives import register_directive +from docutils.parsers.rst.directives.misc import Include + +from contingent.io import looping_wait_on +from contingent.projectlib import Project +from contingent.rendering import as_graphviz + + +project = Project() +task = project.task + + +class RSTIncludeSpy(Include): + """Tracing reStructuredText include directive + + Determine the exact content included by a docutils publishing run. + + Include directive that tracks the contents included into a published + reStructuredText document. As include directives are processed, the + spy saves the output of each ``run`` call before handing them back + to the directive caller. + + Calling ``get_include_contents`` retrieves the included contents as + a concatenated string and clears its cache for the next run. + + The chapter builder below uses the spy to determine the exact + content included from various external files into the chapter, which + allows it to detect when a change to an included file will *not* + impact the output and halt the build. + + """ + + include_contents = [] + + @classmethod + def get_include_contents(cls): + val = ''.join(cls.include_contents) + cls.include_contents = [] + return val + + def run(self): + # docutils doesn't provide a way for our subclass to replace the + # file reading routine, so we are forced to do our own read here + # to maintain the task graph. Alternatives would be to + # reimplement the entire long superclass method or to + # monkeypatch docutils. This redundancy seems like the best of + # the three choices. + read_text_file(self.arguments[0]) + val = super().run() + self.include_contents.append(str(val[0])) + return val + +register_directive('include', RSTIncludeSpy) + + +@task +def read_text_file(path): + with open(path) as f: + return f.read() + + +@task +def check_rst_includes(path): + publish_doctree(read_text_file(path)) + return RSTIncludeSpy.get_include_contents() + + +@task +def chapter_doctests(path): + read_text_file(path) + doctest.testfile( + path, + module_relative=False, + optionflags=doctest.ELLIPSIS, + ) + + with project.cache_off(): + for dot in glob('*.dot'): + read_text_file(dot) + + +@task +def render(path): + if path.endswith('.dot'): + read_text_file(path) + png = path[:-3] + 'png' + subprocess.call(['dot', '-Tpng', '-o', png, path]) + elif path.endswith('.rst'): + read_text_file(path) + chapter_doctests(path) + check_rst_includes(path) + subprocess.call(['rst2html.py', 'chapter.rst', 'chapter.html']) + + +def get_paths(): + return tuple(glob('*.rst') + glob('contingent/*.py') + glob('*.dot')) + + +def main(): + project.verbose = True + + project.start_tracing() + for path in get_paths(): + render(path) + print(project.stop_tracing(True)) + + open('chapter.dot', 'w').write(as_graphviz(project._graph)) + + while True: + print('=' * 72) + print('Watching for files to change') + changed_paths = looping_wait_on(get_paths()) + print('=' * 72) + print('Reloading:', ' '.join(changed_paths)) + with project.cache_off(): + for path in changed_paths: + read_text_file(path) + project.start_tracing() + project.rebuild() + print(project.stop_tracing(True)) + + +if __name__ == '__main__': + main() diff --git a/crawler/code/supplemental/loop-with-callbacks.py b/crawler/code/supplemental/loop-with-callbacks.py index 14165ee10..2f8d3a08d 100644 --- a/crawler/code/supplemental/loop-with-callbacks.py +++ b/crawler/code/supplemental/loop-with-callbacks.py @@ -57,7 +57,7 @@ def read_response(self, key, mask): urls_todo.remove(self.url) if not urls_todo: stopped = True - print(self.url) + print((self.url)) def body(self): body = self.response.split(b'\r\n\r\n', 1)[1] @@ -65,7 +65,7 @@ def body(self): def parse_links(self): if not self.response: - print('error: {}'.format(self.url)) + print(('error: {}'.format(self.url))) return set() if not self._is_html(): return set() @@ -102,5 +102,5 @@ def _is_html(self): callback = event_key.data callback(event_key, event_mask) -print('{} URLs fetched in {:.1f} seconds, achieved concurrency = {}'.format( - len(seen_urls), time.time() - start, concurrency_achieved)) +print(('{} URLs fetched in {:.1f} seconds, achieved concurrency = {}'.format( + len(seen_urls), time.time() - start, concurrency_achieved))) diff --git a/crawler/code/supplemental/loop-with-callbacks.py.bak b/crawler/code/supplemental/loop-with-callbacks.py.bak new file mode 100644 index 000000000..14165ee10 --- /dev/null +++ b/crawler/code/supplemental/loop-with-callbacks.py.bak @@ -0,0 +1,106 @@ +#!/usr/bin/env python3.4 + +"""Sloppy little crawler, demonstrates a hand-made event loop and callbacks.""" + +from selectors import * +import socket +import re +import urllib.parse +import time + + +urls_todo = set(['/']) +seen_urls = set(['/']) +concurrency_achieved = 0 +selector = DefaultSelector() +stopped = False + + +class Fetcher: + def __init__(self, url): + self.response = b'' + self.url = url + self.sock = None + + def fetch(self): + global concurrency_achieved + concurrency_achieved = max(concurrency_achieved, len(urls_todo)) + + self.sock = socket.socket() + self.sock.setblocking(False) + try: + self.sock.connect(('xkcd.com', 80)) + except BlockingIOError: + pass + selector.register(self.sock.fileno(), EVENT_WRITE, self.connected) + + def connected(self, key, mask): + selector.unregister(key.fd) + get = 'GET {} HTTP/1.0\r\nHost: xkcd.com\r\n\r\n'.format(self.url) + self.sock.send(get.encode('ascii')) + selector.register(key.fd, EVENT_READ, self.read_response) + + def read_response(self, key, mask): + global stopped + + chunk = self.sock.recv(4096) # 4k chunk size. + if chunk: + self.response += chunk + else: + selector.unregister(key.fd) # Done reading. + links = self.parse_links() + for link in links.difference(seen_urls): + urls_todo.add(link) + Fetcher(link).fetch() + + seen_urls.update(links) + urls_todo.remove(self.url) + if not urls_todo: + stopped = True + print(self.url) + + def body(self): + body = self.response.split(b'\r\n\r\n', 1)[1] + return body.decode('utf-8') + + def parse_links(self): + if not self.response: + print('error: {}'.format(self.url)) + return set() + if not self._is_html(): + return set() + urls = set(re.findall(r'''(?i)href=["']?([^\s"'<>]+)''', + self.body())) + + links = set() + for url in urls: + normalized = urllib.parse.urljoin(self.url, url) + parts = urllib.parse.urlparse(normalized) + if parts.scheme not in ('', 'http', 'https'): + continue + host, port = urllib.parse.splitport(parts.netloc) + if host and host.lower() not in ('xkcd.com', 'www.xkcd.com'): + continue + defragmented, frag = urllib.parse.urldefrag(parts.path) + links.add(defragmented) + + return links + + def _is_html(self): + head, body = self.response.split(b'\r\n\r\n', 1) + headers = dict(h.split(': ') for h in head.decode().split('\r\n')[1:]) + return headers.get('Content-Type', '').startswith('text/html') + + +start = time.time() +fetcher = Fetcher('/') +fetcher.fetch() + +while not stopped: + events = selector.select() + for event_key, event_mask in events: + callback = event_key.data + callback(event_key, event_mask) + +print('{} URLs fetched in {:.1f} seconds, achieved concurrency = {}'.format( + len(seen_urls), time.time() - start, concurrency_achieved)) diff --git a/data-store/code/dbdb/tool.py b/data-store/code/dbdb/tool.py index 24b869ec2..51ebd0a20 100644 --- a/data-store/code/dbdb/tool.py +++ b/data-store/code/dbdb/tool.py @@ -1,4 +1,4 @@ -from __future__ import print_function + import sys import dbdb diff --git a/data-store/code/dbdb/tool.py.bak b/data-store/code/dbdb/tool.py.bak new file mode 100644 index 000000000..24b869ec2 --- /dev/null +++ b/data-store/code/dbdb/tool.py.bak @@ -0,0 +1,45 @@ +from __future__ import print_function +import sys + +import dbdb + + +OK = 0 +BAD_ARGS = 1 +BAD_VERB = 2 +BAD_KEY = 3 + + +def usage(): + print("Usage:", file=sys.stderr) + print("\tpython -m dbdb.tool DBNAME get KEY", file=sys.stderr) + print("\tpython -m dbdb.tool DBNAME set KEY VALUE", file=sys.stderr) + print("\tpython -m dbdb.tool DBNAME delete KEY", file=sys.stderr) + + +def main(argv): + if not (4 <= len(argv) <= 5): + usage() + return BAD_ARGS + dbname, verb, key, value = (argv[1:] + [None])[:4] + if verb not in {'get', 'set', 'delete'}: + usage() + return BAD_VERB + db = dbdb.connect(dbname) + try: + if verb == 'get': + sys.stdout.write(db[key]) + elif verb == 'set': + db[key] = value + db.commit() + else: + del db[key] + db.commit() + except KeyError: + print("Key not found", file=sys.stderr) + return BAD_KEY + return OK + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/flow-shop/code/flow.py b/flow-shop/code/flow.py index 328ae6031..cc9229c76 100644 --- a/flow-shop/code/flow.py +++ b/flow-shop/code/flow.py @@ -77,7 +77,7 @@ def solve(data): strat_usage = {strategy: 0 for strategy in STRATEGIES} # Start with a random permutation of the jobs - perm = range(len(data)) + perm = list(range(len(data))) random.shuffle(perm) # Keep track of the best solution @@ -94,12 +94,12 @@ def solve(data): checkpoint = time.time() + time_delta percent_complete = 10 - print "\nSolving..." + print("\nSolving...") while time.time() < time_limit: if time.time() > checkpoint: - print " %d %%" % percent_complete + print((" %d %%" % percent_complete)) percent_complete += 10 checkpoint += time_delta @@ -136,9 +136,9 @@ def solve(data): for s in STRATEGIES]) if DEBUG_SWITCH: - print "\nComputing another switch..." - print "Best performer: %s (%d)" % (results[0][1].name, results[0][0]) - print "Worst performer: %s (%d)" % (results[-1][1].name, results[-1][0]) + print("\nComputing another switch...") + print(("Best performer: %s (%d)" % (results[0][1].name, results[0][0]))) + print(("Worst performer: %s (%d)" % (results[-1][1].name, results[-1][0]))) # Boost the weight for the successful strategies for i in range(len(STRATEGIES)): @@ -151,21 +151,21 @@ def solve(data): time_last_switch = time.time() if DEBUG_SWITCH: - print results - print sorted([strat_weights[STRATEGIES[i]] for i in range(len(STRATEGIES))]) + print(results) + print((sorted([strat_weights[STRATEGIES[i]] for i in range(len(STRATEGIES))]))) strat_improvements = {strategy: 0 for strategy in STRATEGIES} strat_time_spent = {strategy: 0 for strategy in STRATEGIES} - print " %d %%\n" % percent_complete - print "\nWent through %d iterations." % iteration + print((" %d %%\n" % percent_complete)) + print(("\nWent through %d iterations." % iteration)) - print "\n(usage) Strategy:" + print("\n(usage) Strategy:") results = sorted([(strat_weights[STRATEGIES[i]], i) for i in range(len(STRATEGIES))], reverse=True) for (w, i) in results: - print "(%d) \t%s" % (strat_usage[STRATEGIES[i]], STRATEGIES[i].name) + print(("(%d) \t%s" % (strat_usage[STRATEGIES[i]], STRATEGIES[i].name))) return (best_perm, best_make) @@ -177,14 +177,14 @@ def parse_problem(filename, k=1): of flow shop scheduling. They can be found online at the following address: - http://mistic.heig-vd.ch/taillard/problemes.dir/ordonnancement.dir/ordonnancement.html""" - print "\nParsing..." + print("\nParsing...") with open(filename, 'r') as f: # Identify the string that separates instances problem_line = '/number of jobs, number of machines, initial seed, upper bound and lower bound :/' # Strip spaces and newline characters from every line - lines = map(str.strip, f.readlines()) + lines = list(map(str.strip, f.readlines())) # We prep the first line for later lines[0] = '/' + lines[0] @@ -195,15 +195,15 @@ def parse_problem(filename, k=1): lines = '/'.join(lines).split(problem_line)[k].split('/')[2:] except IndexError: max_instances = len('/'.join(lines).split(problem_line)) - 1 - print "\nError: Instance must be within 1 and %d\n" % max_instances + print(("\nError: Instance must be within 1 and %d\n" % max_instances)) sys.exit(0) # Split every line based on spaces and convert each item to an int - data = [map(int, line.split()) for line in lines] + data = [list(map(int, line.split())) for line in lines] # We return the zipped data to rotate the rows and columns, making each # item in data the durations of tasks for a particular job - return zip(*data) + return list(zip(*data)) def pick_strategy(strategies, weights): @@ -272,16 +272,16 @@ def print_solution(data, perm): sol = compile_solution(data, perm) - print "\nPermutation: %s\n" % str([i+1 for i in perm]) + print(("\nPermutation: %s\n" % str([i+1 for i in perm]))) - print "Makespan: %d\n" % makespan(data, perm) + print(("Makespan: %d\n" % makespan(data, perm))) row_format ="{:>15}" * 4 - print row_format.format('Machine', 'Start Time', 'Finish Time', 'Idle Time') + print((row_format.format('Machine', 'Start Time', 'Finish Time', 'Idle Time'))) for mach in range(len(data[0])): finish_time = sol[mach][-1] + data[perm[-1]][mach] idle_time = (finish_time - sol[mach][0]) - sum([job[mach] for job in data]) - print row_format.format(mach+1, sol[mach][0], finish_time, idle_time) + print((row_format.format(mach+1, sol[mach][0], finish_time, idle_time))) results = [] for i in range(len(data)): @@ -289,12 +289,12 @@ def print_solution(data, perm): idle_time = (finish_time - sol[0][i]) - sum([time for time in data[perm[i]]]) results.append((perm[i]+1, sol[0][i], finish_time, idle_time)) - print "\n" - print row_format.format('Job', 'Start Time', 'Finish Time', 'Idle Time') + print("\n") + print((row_format.format('Job', 'Start Time', 'Finish Time', 'Idle Time'))) for r in sorted(results): - print row_format.format(*r) + print((row_format.format(*r))) - print "\n\nNote: Idle time does not include initial or final wait time.\n" + print("\n\nNote: Idle time does not include initial or final wait time.\n") if __name__ == '__main__': @@ -304,7 +304,7 @@ def print_solution(data, perm): elif len(sys.argv) == 3: data = parse_problem(sys.argv[1], int(sys.argv[2])) else: - print "\nUsage: python flow.py []\n" + print("\nUsage: python flow.py []\n") sys.exit(0) (perm, ms) = solve(data) diff --git a/flow-shop/code/flow.py.bak b/flow-shop/code/flow.py.bak new file mode 100644 index 000000000..b838fb710 --- /dev/null +++ b/flow-shop/code/flow.py.bak @@ -0,0 +1,311 @@ +import sys, os, time, random + +from functools import partial +from collections import namedtuple +from itertools import product + +import neighbourhood as neigh +import heuristics as heur + +############## +## Settings ## +############## +TIME_LIMIT = 300.0 # Time (in seconds) to run the solver +TIME_INCREMENT = 13.0 # Time (in seconds) in between heuristic measurements +DEBUG_SWITCH = False # Displays intermediate heuristic info when True +MAX_LNS_NEIGHBOURHOODS = 1000 # Maximum number of neighbours to explore in LNS + + +################ +## Strategies ## +################################################# +## A strategy is a particular configuration +## of neighbourhood generator (to compute +## the next set of candidates) and heuristic +## computation (to select the best candidate). +## + +STRATEGIES = [] + +# Using a namedtuple is a little cleaner than dictionaries. +# E.g., strategy['name'] versus strategy.name +Strategy = namedtuple('Strategy', ['name', 'neighbourhood', 'heuristic']) + +def initialize_strategies(): + + global STRATEGIES + + # Define the neighbourhoods (and parameters) we would like to use + neighbourhoods = [ + ('Random Permutation', partial(neigh.neighbours_random, num=100)), + ('Swapped Pairs', neigh.neighbours_swap), + ('Large Neighbourhood Search (2)', partial(neigh.neighbours_LNS, size=2)), + ('Large Neighbourhood Search (3)', partial(neigh.neighbours_LNS, size=3)), + ('Idle Neighbourhood (3)', partial(neigh.neighbours_idle, size=3)), + ('Idle Neighbourhood (4)', partial(neigh.neighbours_idle, size=4)), + ('Idle Neighbourhood (5)', partial(neigh.neighbours_idle, size=5)) + ] + + # Define the heuristics we would like to use + heuristics = [ + ('Hill Climbing', heur.heur_hillclimbing), + ('Random Selection', heur.heur_random), + ('Biased Random Selection', heur.heur_random_hillclimbing) + ] + + # Combine every neighbourhood and heuristic strategy + for (n, h) in product(neighbourhoods, heuristics): + STRATEGIES.append(Strategy("%s / %s" % (n[0], h[0]), n[1], h[1])) + + + +def solve(data): + """Solves an instance of the flow shop scheduling problem""" + + # We initialize the strategies here to avoid cyclic import issues + initialize_strategies() + global STRATEGIES + + # Record the following for each strategy: + # improvements: The amount a solution was improved by this strategy + # time_spent: The amount of time spent on the strategy + # weights: The weights that correspond to how good a strategy is + # usage: The number of times we use a strategy + strat_improvements = {strategy: 0 for strategy in STRATEGIES} + strat_time_spent = {strategy: 0 for strategy in STRATEGIES} + strat_weights = {strategy: 1 for strategy in STRATEGIES} + strat_usage = {strategy: 0 for strategy in STRATEGIES} + + # Start with a random permutation of the jobs + perm = list(range(len(data))) + random.shuffle(perm) + + # Keep track of the best solution + best_make = makespan(data, perm) + best_perm = perm + res = best_make + + # Maintain statistics and timing for the iterations + iteration = 0 + time_limit = time.time() + TIME_LIMIT + time_last_switch = time.time() + + time_delta = TIME_LIMIT / 10 + checkpoint = time.time() + time_delta + percent_complete = 10 + + print("\nSolving...") + + while time.time() < time_limit: + + if time.time() > checkpoint: + print(" %d %%" % percent_complete) + percent_complete += 10 + checkpoint += time_delta + + iteration += 1 + + # Heuristically choose the best strategy + strategy = pick_strategy(STRATEGIES, strat_weights) + + old_val = res + old_time = time.time() + + # Use the current strategy's heuristic to pick the next permutation from + # the set of candidates generated by the strategy's neighbourhood + candidates = strategy.neighbourhood(data, perm) + perm = strategy.heuristic(data, candidates) + res = makespan(data, perm) + + # Record the statistics on how the strategy did + strat_improvements[strategy] += res - old_val + strat_time_spent[strategy] += time.time() - old_time + strat_usage[strategy] += 1 + + if res < best_make: + best_make = res + best_perm = perm[:] + + # At regular intervals, switch the weighting on the strategies available. + # This way, the search can dynamically shift towards strategies that have + # proven more effective recently. + if time.time() > time_last_switch + TIME_INCREMENT: + + # Normalize the improvements made by the time it takes to make them + results = sorted([(float(strat_improvements[s]) / max(0.001, strat_time_spent[s]), s) + for s in STRATEGIES]) + + if DEBUG_SWITCH: + print("\nComputing another switch...") + print("Best performer: %s (%d)" % (results[0][1].name, results[0][0])) + print("Worst performer: %s (%d)" % (results[-1][1].name, results[-1][0])) + + # Boost the weight for the successful strategies + for i in range(len(STRATEGIES)): + strat_weights[results[i][1]] += len(STRATEGIES) - i + + # Additionally boost the unused strategies to avoid starvation + if results[i][0] == 0: + strat_weights[results[i][1]] += len(STRATEGIES) + + time_last_switch = time.time() + + if DEBUG_SWITCH: + print(results) + print(sorted([strat_weights[STRATEGIES[i]] for i in range(len(STRATEGIES))])) + + strat_improvements = {strategy: 0 for strategy in STRATEGIES} + strat_time_spent = {strategy: 0 for strategy in STRATEGIES} + + + print(" %d %%\n" % percent_complete) + print("\nWent through %d iterations." % iteration) + + print("\n(usage) Strategy:") + results = sorted([(strat_weights[STRATEGIES[i]], i) + for i in range(len(STRATEGIES))], reverse=True) + for (w, i) in results: + print("(%d) \t%s" % (strat_usage[STRATEGIES[i]], STRATEGIES[i].name)) + + return (best_perm, best_make) + + +def parse_problem(filename, k=1): + """Parse the kth instance of a Taillard problem file + + The Taillard problem files are a standard benchmark set for the problem + of flow shop scheduling. They can be found online at the following address: + - http://mistic.heig-vd.ch/taillard/problemes.dir/ordonnancement.dir/ordonnancement.html""" + + print("\nParsing...") + + with open(filename, 'r') as f: + # Identify the string that separates instances + problem_line = '/number of jobs, number of machines, initial seed, upper bound and lower bound :/' + + # Strip spaces and newline characters from every line + lines = list(map(str.strip, f.readlines())) + + # We prep the first line for later + lines[0] = '/' + lines[0] + + # We also know '/' does not appear in the files, so we can use it as + # a separator to find the right lines for the kth problem instance + try: + lines = '/'.join(lines).split(problem_line)[k].split('/')[2:] + except IndexError: + max_instances = len('/'.join(lines).split(problem_line)) - 1 + print("\nError: Instance must be within 1 and %d\n" % max_instances) + sys.exit(0) + + # Split every line based on spaces and convert each item to an int + data = [list(map(int, line.split())) for line in lines] + + # We return the zipped data to rotate the rows and columns, making each + # item in data the durations of tasks for a particular job + return list(zip(*data)) + + +def pick_strategy(strategies, weights): + # Picks a random strategy based on its weight: roulette wheel selection + # Rather than selecting a strategy entirely at random, we bias the + # random selection towards strategies that have worked well in the + # past (according to the weight value). + total = sum([weights[strategy] for strategy in strategies]) + pick = random.uniform(0, total) + count = weights[strategies[0]] + + i = 0 + while pick > count: + count += weights[strategies[i+1]] + i += 1 + + return strategies[i] + + +def makespan(data, perm): + """Computes the makespan of the provided solution + + For scheduling problems, the makespan refers to the difference between + the earliest start time of any job and the latest completion time of + any job. Minimizing the makespan amounts to minimizing the total time + it takes to process all jobs from start to finish.""" + return compile_solution(data, perm)[-1][-1] + data[perm[-1]][-1] + + +def compile_solution(data, perm): + """Compiles a scheduling on the machines given a permutation of jobs""" + + num_machines = len(data[0]) + + # Note that using [[]] * m would be incorrect, as it would simply + # copy the same list m times (as opposed to creating m distinct lists). + machine_times = [[] for _ in range(num_machines)] + + # Assign the initial job to the machines + machine_times[0].append(0) + for mach in range(1,num_machines): + # Start the next task in the job when the previous finishes + machine_times[mach].append(machine_times[mach-1][0] + + data[perm[0]][mach-1]) + + # Assign the remaining jobs + for i in range(1, len(perm)): + + # The first machine never contains any idle time + job = perm[i] + machine_times[0].append(machine_times[0][-1] + data[perm[i-1]][0]) + + # For the remaining machines, the start time is the max of when the + # previous task in the job completed, or when the current machine + # completes the task for the previous job. + for mach in range(1, num_machines): + machine_times[mach].append(max(machine_times[mach-1][i] + data[perm[i]][mach-1], + machine_times[mach][i-1] + data[perm[i-1]][mach])) + + return machine_times + + + +def print_solution(data, perm): + """Prints statistics on the computed solution""" + + sol = compile_solution(data, perm) + + print("\nPermutation: %s\n" % str([i+1 for i in perm])) + + print("Makespan: %d\n" % makespan(data, perm)) + + row_format ="{:>15}" * 4 + print(row_format.format('Machine', 'Start Time', 'Finish Time', 'Idle Time')) + for mach in range(len(data[0])): + finish_time = sol[mach][-1] + data[perm[-1]][mach] + idle_time = (finish_time - sol[mach][0]) - sum([job[mach] for job in data]) + print(row_format.format(mach+1, sol[mach][0], finish_time, idle_time)) + + results = [] + for i in range(len(data)): + finish_time = sol[-1][i] + data[perm[i]][-1] + idle_time = (finish_time - sol[0][i]) - sum([time for time in data[perm[i]]]) + results.append((perm[i]+1, sol[0][i], finish_time, idle_time)) + + print("\n") + print(row_format.format('Job', 'Start Time', 'Finish Time', 'Idle Time')) + for r in sorted(results): + print(row_format.format(*r)) + + print("\n\nNote: Idle time does not include initial or final wait time.\n") + + +if __name__ == '__main__': + + if len(sys.argv) == 2: + data = parse_problem(sys.argv[1]) + elif len(sys.argv) == 3: + data = parse_problem(sys.argv[1], int(sys.argv[2])) + else: + print("\nUsage: python flow.py []\n") + sys.exit(0) + + (perm, ms) = solve(data) + print_solution(data, perm) diff --git a/flow-shop/code/neighbourhood.py b/flow-shop/code/neighbourhood.py index 4e9000081..2bc4aa8ec 100644 --- a/flow-shop/code/neighbourhood.py +++ b/flow-shop/code/neighbourhood.py @@ -20,7 +20,7 @@ def neighbours_random(data, perm, num = 1): def neighbours_swap(data, perm): # Returns the permutations corresponding to swapping every pair of jobs candidates = [perm] - for (i,j) in combinations(range(len(perm)), 2): + for (i,j) in combinations(list(range(len(perm))), 2): candidate = perm[:] candidate[i], candidate[j] = candidate[j], candidate[i] candidates.append(candidate) @@ -31,7 +31,7 @@ def neighbours_LNS(data, perm, size = 2): candidates = [perm] # Bound the number of neighbourhoods in case there are too many jobs - neighbourhoods = list(combinations(range(len(perm)), size)) + neighbourhoods = list(combinations(list(range(len(perm))), size)) random.shuffle(neighbourhoods) for subset in neighbourhoods[:flow.MAX_LNS_NEIGHBOURHOODS]: diff --git a/flow-shop/code/neighbourhood.py.bak b/flow-shop/code/neighbourhood.py.bak new file mode 100644 index 000000000..4e9000081 --- /dev/null +++ b/flow-shop/code/neighbourhood.py.bak @@ -0,0 +1,81 @@ + +import random +from itertools import combinations, permutations + +import flow + +############################## +## Neighbourhood Generators ## +############################## + +def neighbours_random(data, perm, num = 1): + # Returns random job permutations, including the current one + candidates = [perm] + for i in range(num): + candidate = perm[:] + random.shuffle(candidate) + candidates.append(candidate) + return candidates + +def neighbours_swap(data, perm): + # Returns the permutations corresponding to swapping every pair of jobs + candidates = [perm] + for (i,j) in combinations(range(len(perm)), 2): + candidate = perm[:] + candidate[i], candidate[j] = candidate[j], candidate[i] + candidates.append(candidate) + return candidates + +def neighbours_LNS(data, perm, size = 2): + # Returns the Large Neighbourhood Search neighbours + candidates = [perm] + + # Bound the number of neighbourhoods in case there are too many jobs + neighbourhoods = list(combinations(range(len(perm)), size)) + random.shuffle(neighbourhoods) + + for subset in neighbourhoods[:flow.MAX_LNS_NEIGHBOURHOODS]: + + # Keep track of the best candidate for each neighbourhood + best_make = flow.makespan(data, perm) + best_perm = perm + + # Enumerate every permutation of the selected neighbourhood + for ordering in permutations(subset): + candidate = perm[:] + for i in range(len(ordering)): + candidate[subset[i]] = perm[ordering[i]] + res = flow.makespan(data, candidate) + if res < best_make: + best_make = res + best_perm = candidate + + # Record the best candidate as part of the larger neighbourhood + candidates.append(best_perm) + + return candidates + +def neighbours_idle(data, perm, size=4): + # Returns the permutations of the most idle jobs + candidates = [perm] + + # Compute the idle time for each job + sol = flow.compile_solution(data, perm) + results = [] + + for i in range(len(data)): + finish_time = sol[-1][i] + data[perm[i]][-1] + idle_time = (finish_time - sol[0][i]) - sum([t for t in data[perm[i]]]) + results.append((idle_time, i)) + + # Take the most idle jobs + subset = [job_ind for (idle, job_ind) in reversed(sorted(results))][:size] + + # Enumerate the permutations of the idle jobs + for ordering in permutations(subset): + candidate = perm[:] + for i in range(len(ordering)): + candidate[subset[i]] = perm[ordering[i]] + candidates.append(candidate) + + return candidates diff --git a/html/pelicanconf.py b/html/pelicanconf.py index f128bfba1..51acf08be 100644 --- a/html/pelicanconf.py +++ b/html/pelicanconf.py @@ -1,14 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -from __future__ import unicode_literals -AUTHOR = u'Michael DiBernardo' -SITENAME = u'500 Lines or Less' + +AUTHOR = 'Michael DiBernardo' +SITENAME = '500 Lines or Less' SITEURL = '' TIMEZONE = 'Europe/Paris' -DEFAULT_LANG = u'en' +DEFAULT_LANG = 'en' # Feed generation is usually not desired when developing FEED_ALL_ATOM = None diff --git a/html/pelicanconf.py.bak b/html/pelicanconf.py.bak new file mode 100644 index 000000000..f128bfba1 --- /dev/null +++ b/html/pelicanconf.py.bak @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- # +from __future__ import unicode_literals + +AUTHOR = u'Michael DiBernardo' +SITENAME = u'500 Lines or Less' +SITEURL = '' + +TIMEZONE = 'Europe/Paris' + +DEFAULT_LANG = u'en' + +# Feed generation is usually not desired when developing +FEED_ALL_ATOM = None +CATEGORY_FEED_ATOM = None +TRANSLATION_FEED_ATOM = None + +STATIC_PATHS = [ +] + + +DEFAULT_PAGINATION = False + +THEME = '500L' + +TYPOGRIFY = False + +EXTENSIONS = ('latex',) + +# Uncomment following line if you want document-relative URLs when developing +#RELATIVE_URLS = True diff --git a/html/publishconf.py b/html/publishconf.py index 801380172..7881d52f9 100644 --- a/html/publishconf.py +++ b/html/publishconf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -from __future__ import unicode_literals + # This file is only used if you use `make publish` or # explicitly specify it as your config file. diff --git a/html/publishconf.py.bak b/html/publishconf.py.bak new file mode 100644 index 000000000..801380172 --- /dev/null +++ b/html/publishconf.py.bak @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- # +from __future__ import unicode_literals + +# This file is only used if you use `make publish` or +# explicitly specify it as your config file. + +import os +import sys +sys.path.append(os.curdir) +from pelicanconf import * + +SITEURL = 'http://aosabook.org/en/500L' +RELATIVE_URLS = False + +FEED_ALL_ATOM = None +CATEGORY_FEED_ATOM = None +TRANSLATION_FEED_ATOM = None +AUTHOR_FEED_ATOM = None +AUTHOR_FEED_RSS = None + +DELETE_OUTPUT_DIRECTORY = True + +# Following items are often useful when publishing + +#DISQUS_SITENAME = "" +#GOOGLE_ANALYTICS = "" diff --git a/incomplete/bytecode-compiler/extractcode.py b/incomplete/bytecode-compiler/extractcode.py index 73674496b..4a8624266 100644 --- a/incomplete/bytecode-compiler/extractcode.py +++ b/incomplete/bytecode-compiler/extractcode.py @@ -26,11 +26,11 @@ parts = 'top.py assembler.py default'.split() with open('compiler.py', 'w') as f: - f.write('\n'.join(sum(map(outputs.get, parts), []))) + f.write('\n'.join(sum(list(map(outputs.get, parts)), []))) for label in parts: del outputs[label] with open('non-compiler', 'w') as f: - for label, lines in outputs.items(): + for label, lines in list(outputs.items()): for line in lines: f.write(label+': '+line+'\n') diff --git a/incomplete/bytecode-compiler/extractcode.py.bak b/incomplete/bytecode-compiler/extractcode.py.bak new file mode 100644 index 000000000..73674496b --- /dev/null +++ b/incomplete/bytecode-compiler/extractcode.py.bak @@ -0,0 +1,36 @@ +""" +Take Markdown and extract the code sections. +""" + +import sys + +outputs = {} + +eg = sys.stdin.read() +paragraph_break = True +label = 'default' +lines = iter(eg.splitlines()) +for line in lines: + assert '\t' not in line + if paragraph_break and line.startswith(' '): + if line.strip().startswith('!!'): + label = line.strip().strip('!') + line = next(lines) + while line.startswith(' '): + outputs.setdefault(label, []).append(line[4:]) + line = next(lines) + outputs.setdefault(label, []).append('') + paragraph_break = (line.strip() == '') + if not paragraph_break: + label = 'default' + +parts = 'top.py assembler.py default'.split() +with open('compiler.py', 'w') as f: + f.write('\n'.join(sum(map(outputs.get, parts), []))) +for label in parts: + del outputs[label] + +with open('non-compiler', 'w') as f: + for label, lines in outputs.items(): + for line in lines: + f.write(label+': '+line+'\n') diff --git a/incomplete/bytecode-compiler/silly.py b/incomplete/bytecode-compiler/silly.py index 6a3f28c80..e42ecc2a0 100644 --- a/incomplete/bytecode-compiler/silly.py +++ b/incomplete/bytecode-compiler/silly.py @@ -3,7 +3,7 @@ """ #import math -print(['m', 'n'][0]) +print((['m', 'n'][0])) (pow, len) {'a': 42, 'b': 55} {} @@ -14,15 +14,15 @@ def f(a): while a and True: pass if False or a != 1 or False: - print(a, 137) + print((a, 137)) a = a - 1 return pow(2, 16) return -print(f(ga)) +print((f(ga))) t = True while t: break -for j, i in enumerate(range(3)): +for j, i in enumerate(list(range(3))): print(i) #print(-math.sqrt(2)) raise Exception('hi') diff --git a/incomplete/bytecode-compiler/silly.py.bak b/incomplete/bytecode-compiler/silly.py.bak new file mode 100644 index 000000000..6a3f28c80 --- /dev/null +++ b/incomplete/bytecode-compiler/silly.py.bak @@ -0,0 +1,28 @@ +""" +Silly example code to try the compiler on. +""" + +#import math +print(['m', 'n'][0]) +(pow, len) +{'a': 42, 'b': 55} +{} +None +ga = 2+3 +def f(a): + "doc comment" + while a and True: + pass + if False or a != 1 or False: + print(a, 137) + a = a - 1 + return pow(2, 16) + return +print(f(ga)) +t = True +while t: + break +for j, i in enumerate(range(3)): + print(i) +#print(-math.sqrt(2)) +raise Exception('hi') diff --git a/incomplete/rasterizer/rasterizer/__init__.py b/incomplete/rasterizer/rasterizer/__init__.py index 2c142778f..9d854aa00 100644 --- a/incomplete/rasterizer/rasterizer/__init__.py +++ b/incomplete/rasterizer/rasterizer/__init__.py @@ -1,8 +1,8 @@ -from shape import * -from geometry import * -from poly import * -from color import * -from ellipse import * -from image import * -from scene import * -from csg import * +from .shape import * +from .geometry import * +from .poly import * +from .color import * +from .ellipse import * +from .image import * +from .scene import * +from .csg import * diff --git a/incomplete/rasterizer/rasterizer/__init__.py.bak b/incomplete/rasterizer/rasterizer/__init__.py.bak new file mode 100644 index 000000000..2c142778f --- /dev/null +++ b/incomplete/rasterizer/rasterizer/__init__.py.bak @@ -0,0 +1,8 @@ +from shape import * +from geometry import * +from poly import * +from color import * +from ellipse import * +from image import * +from scene import * +from csg import * diff --git a/incomplete/rasterizer/rasterizer/csg.py b/incomplete/rasterizer/rasterizer/csg.py index 1117b70f5..eca4e45f2 100644 --- a/incomplete/rasterizer/rasterizer/csg.py +++ b/incomplete/rasterizer/rasterizer/csg.py @@ -1,5 +1,5 @@ -from shape import Shape -from geometry import * +from .shape import Shape +from .geometry import * class CSG(Shape): def __init__(self, v1, v2, color=None): diff --git a/incomplete/rasterizer/rasterizer/csg.py.bak b/incomplete/rasterizer/rasterizer/csg.py.bak new file mode 100644 index 000000000..1117b70f5 --- /dev/null +++ b/incomplete/rasterizer/rasterizer/csg.py.bak @@ -0,0 +1,45 @@ +from shape import Shape +from geometry import * + +class CSG(Shape): + def __init__(self, v1, v2, color=None): + Shape.__init__(self, color or v1.color or v2.color) + self.v1 = v1 + self.v2 = v2 + def transform(self, t): + return self.__class__(self.v1.transform(t), self.v2.transform(t), + color=self.color) + +class Union(CSG): + def __init__(self, v1, v2, color=None): + CSG.__init__(self, v1, v2, color=color) + self.bound = AABox.from_vectors(v1.bound.low, v1.bound.high, + v2.bound.low, v2.bound.high) + def contains(self, p): + return self.v1.contains(p) or self.v2.contains(p) + def signed_distance_bound(self, p): + b1 = self.v1.signed_distance_bound(p) + b2 = self.v2.signed_distance_bound(p) + return b1 if b1 > b2 else b2 + +class Intersection(CSG): + def __init__(self, v1, v2, color=None): + CSG.__init__(self, v1, v2, color=color) + self.bound = v1.bound.intersection(v2.bound) + def contains(self, p): + return self.v1.contains(p) and self.v2.contains(p) + def signed_distance_bound(self, p): + b1 = self.v1.signed_distance_bound(p) + b2 = self.v2.signed_distance_bound(p) + return b1 if b1 < b2 else b2 + +class Subtraction(CSG): + def __init__(self, v1, v2, color=None): + CSG.__init__(self, v1, v2, color=color) + self.bound = self.v1.bound + def contains(self, p): + return self.v1.contains(p) and not self.v2.contains(p) + def signed_distance_bound(self, p): + b1 = self.v1.signed_distance_bound(p) + b2 = -self.v2.signed_distance_bound(p) + return b1 if b1 < b2 else b2 diff --git a/incomplete/rasterizer/rasterizer/ellipse.py b/incomplete/rasterizer/rasterizer/ellipse.py index 0518ffd55..f1bd84607 100644 --- a/incomplete/rasterizer/rasterizer/ellipse.py +++ b/incomplete/rasterizer/rasterizer/ellipse.py @@ -1,7 +1,7 @@ -from shape import Shape -from poly import ConvexPoly -from geometry import Vector, Transform, quadratic, scale, translate, AABox, HalfPlane -from color import Color +from .shape import Shape +from .poly import ConvexPoly +from .geometry import Vector, Transform, quadratic, scale, translate, AABox, HalfPlane +from .color import Color import sys class Ellipse(Shape): diff --git a/incomplete/rasterizer/rasterizer/ellipse.py.bak b/incomplete/rasterizer/rasterizer/ellipse.py.bak new file mode 100644 index 000000000..0518ffd55 --- /dev/null +++ b/incomplete/rasterizer/rasterizer/ellipse.py.bak @@ -0,0 +1,95 @@ +from shape import Shape +from poly import ConvexPoly +from geometry import Vector, Transform, quadratic, scale, translate, AABox, HalfPlane +from color import Color +import sys + +class Ellipse(Shape): + def __init__(self, a=1.0, b=1.0, c=0.0, d=0.0, e=0.0, f=-1.0, color=None): + Shape.__init__(self, color) + if c*c - 4*a*b >= 0: + raise Exception("Not an ellipse") + self.a = a + self.b = b + self.c = c + self.d = d + self.e = e + self.f = f + self.gradient = Transform(2*a, c, d, c, 2*b, e) + self.center = self.gradient.inverse() * Vector(0, 0) + y1, y2 = quadratic(b-c*c/4*a, e-c*d/2*a, f-d*d/4*a) + x1, x2 = quadratic(a-c*c/4*b, d-c*e/2*b, f-e*e/4*b) + self.bound = AABox.from_vectors(Vector(-(d + c*y1)/2*a, y1), + Vector(-(d + c*y2)/2*a, y2), + Vector(x1, -(e + c*x1)/2*b), + Vector(x2, -(e + c*x2)/2*b)) + if not self.contains(self.center): + raise Exception("Internal error, center not inside ellipse") + def value(self, p): + return self.a*p.x*p.x + self.b*p.y*p.y + self.c*p.x*p.y \ + + self.d*p.x + self.e*p.y + self.f + def contains(self, p): + return self.value(p) < 0 + def transform(self, transform): + i = transform.inverse() + ((m00, m01, m02), (m10, m11, m12),_) = i.m + aa = self.a*m00*m00 + self.b*m10*m10 + self.c*m00*m10 + bb = self.a*m01*m01 + self.b*m11*m11 + self.c*m01*m11 + cc = 2*self.a*m00*m01 + 2*self.b*m10*m11 \ + + self.c*(m00*m11 + m01*m10) + dd = 2*self.a*m00*m02 + 2*self.b*m10*m12 \ + + self.c*(m00*m12 + m02*m10) + self.d*m00 + self.e*m10 + ee = 2*self.a*m01*m02 + 2*self.b*m11*m12 \ + + self.c*(m01*m12 + m02*m11) + self.d*m01 + self.e*m11 + ff = self.a*m02*m02 + self.b*m12*m12 + self.c*m02*m12 \ + + self.d*m02 + self.e*m12 + self.f + return Ellipse(aa, bb, cc, dd, ee, ff, color=self.color) + def intersections(self, c, p): + # returns the two intersections of the line through c and p + # and the ellipse. Defining a line as a function of a single + # parameter u, x(u) = c.x + u * (p.x - c.x), (and same for y) + # this simply solves the quadratic equation f(x(u), y(u)) = 0 + pc = p - c + u2 = self.a*pc.x**2 + self.b*pc.y**2 + self.c*pc.x*pc.y + u1 = 2*self.a*c.x*pc.x + 2*self.b*c.y*pc.y \ + + self.c*c.y*pc.x + self.c*c.x*pc.y + self.d*pc.x \ + + self.e*pc.y + u0 = self.a*c.x**2 + self.b*c.y**2 + self.c*c.x*c.y \ + + self.d*c.x + self.e*c.y + self.f + try: + sols = quadratic(u2, u1, u0) + except ValueError: + raise Exception("Internal error, solutions be real numbers") + return c+pc*sols[0], c+pc*sols[1] + def signed_distance_bound(self, p): + v = self.value(p) + if v == 0: + return 0 + elif v < 0: + # if inside the ellipse, create an inscribed quadrilateral + # that contains the given point and use the minimum distance + # from the point to the quadrilateral as a bound. Since + # the quadrilateral lies entirely inside the ellipse, the + # distance from the point to the ellipse must be smaller. + v0, v2 = self.intersections(p, p + Vector(1, 0)) + v1, v3 = self.intersections(p, p + Vector(0, 1)) + return abs(ConvexPoly([v0,v1,v2,v3]).signed_distance_bound(p)) + else: + c = self.center + crossings = self.intersections(c, p) + # the surface point we want is the one closest to p + if (p - crossings[0]).length() < (p - crossings[1]).length(): + surface_pt = crossings[0] + else: + surface_pt = crossings[1] + # n is the normal at surface_pt + n = self.gradient * surface_pt + n = n * (1.0 / n.length()) + # returns the length of the projection of p - surface_pt + # along the normal + return -abs(n.dot(p - surface_pt)) + +def Circle(center, radius, color=None): + return Ellipse(color=color).transform( + scale(radius, radius)).transform( + translate(center.x, center.y)) diff --git a/incomplete/rasterizer/rasterizer/examples/__init__.py b/incomplete/rasterizer/rasterizer/examples/__init__.py index 19f529fa8..6ed0cba3c 100644 --- a/incomplete/rasterizer/rasterizer/examples/__init__.py +++ b/incomplete/rasterizer/rasterizer/examples/__init__.py @@ -1,4 +1,4 @@ -import e1 -import e2 -import e3 -import destijl +from . import e1 +from . import e2 +from . import e3 +from . import destijl diff --git a/incomplete/rasterizer/rasterizer/examples/__init__.py.bak b/incomplete/rasterizer/rasterizer/examples/__init__.py.bak new file mode 100644 index 000000000..19f529fa8 --- /dev/null +++ b/incomplete/rasterizer/rasterizer/examples/__init__.py.bak @@ -0,0 +1,4 @@ +import e1 +import e2 +import e3 +import destijl diff --git a/incomplete/rasterizer/rasterizer/examples/e2.py b/incomplete/rasterizer/rasterizer/examples/e2.py index 7dde53f48..fe953e39b 100644 --- a/incomplete/rasterizer/rasterizer/examples/e2.py +++ b/incomplete/rasterizer/rasterizer/examples/e2.py @@ -1,4 +1,4 @@ -import destijl +from . import destijl from .. import * def run(image): diff --git a/incomplete/rasterizer/rasterizer/examples/e2.py.bak b/incomplete/rasterizer/rasterizer/examples/e2.py.bak new file mode 100644 index 000000000..7dde53f48 --- /dev/null +++ b/incomplete/rasterizer/rasterizer/examples/e2.py.bak @@ -0,0 +1,15 @@ +import destijl +from .. import * + +def run(image): + painting = destijl.painting() + frame = Scene([Rectangle(Vector(-0.025,-0.025), Vector(1.025, 1.025), Color(0,0,0,1)), + Rectangle(Vector(0,0), Vector(1, 1), Color(1,1,1,1))]) + small_painting = Scene([frame, painting], scale(0.45, 0.45)) + p1 = Scene([small_painting], translate(0.025, 0.025)) + p2 = Scene([small_painting], translate(0.025, 0.525)) + p3 = Scene([small_painting], translate(0.525, 0.025)) + p4 = Scene([small_painting], translate(0.525, 0.525)) + s = Scene([p1, p2, p3, p4]) + s.draw(image) + diff --git a/incomplete/rasterizer/rasterizer/examples/e3.py b/incomplete/rasterizer/rasterizer/examples/e3.py index c2b22e191..71641442b 100644 --- a/incomplete/rasterizer/rasterizer/examples/e3.py +++ b/incomplete/rasterizer/rasterizer/examples/e3.py @@ -8,7 +8,7 @@ def run(image): distance = 0.02 colors = [Color.hex("#1f77b4"), Color.hex("#ff7f0e"), Color.hex("#2ca02c"), Color.hex("#d62728"), Color.hex("#9467bd"), Color.hex("#8c564b")] - for i in xrange(500): + for i in range(500): obj = Circle(Vector(0.5, 0.5), radius, colors[i % len(colors)]) obj = obj.transform(translate(distance, 0)) obj = obj.transform(around(Vector(0.5, 0.5), rotate(angle))) diff --git a/incomplete/rasterizer/rasterizer/examples/e3.py.bak b/incomplete/rasterizer/rasterizer/examples/e3.py.bak new file mode 100644 index 000000000..c2b22e191 --- /dev/null +++ b/incomplete/rasterizer/rasterizer/examples/e3.py.bak @@ -0,0 +1,19 @@ +from .. import * +import math + +def run(image): + s = Scene() + radius = 0.02 + angle = 1 + distance = 0.02 + colors = [Color.hex("#1f77b4"), Color.hex("#ff7f0e"), Color.hex("#2ca02c"), + Color.hex("#d62728"), Color.hex("#9467bd"), Color.hex("#8c564b")] + for i in xrange(500): + obj = Circle(Vector(0.5, 0.5), radius, colors[i % len(colors)]) + obj = obj.transform(translate(distance, 0)) + obj = obj.transform(around(Vector(0.5, 0.5), rotate(angle))) + s.add(obj) + step = 3 * radius / distance + distance = distance + 0.009 * step + angle = angle + step + s.draw(image) diff --git a/incomplete/rasterizer/rasterizer/geometry.py b/incomplete/rasterizer/rasterizer/geometry.py index b3bce9e9c..567dbc707 100644 --- a/incomplete/rasterizer/rasterizer/geometry.py +++ b/incomplete/rasterizer/rasterizer/geometry.py @@ -1,6 +1,7 @@ import random import math from itertools import product +from functools import reduce # Solves the quadratic equation ax^2 + bx + c = 0 # using a variant of the standard quadratic formula @@ -76,8 +77,8 @@ def __mul__(self, other): # if the other element is also a transform, # then return a transform corresponding to the # composition of the two transforms - t = [[0.0] * 3 for i in xrange(3)] - for i, j, k in product(xrange(3), repeat=3): + t = [[0.0] * 3 for i in range(3)] + for i, j, k in product(list(range(3)), repeat=3): t[i][j] += self.m[i][k] * other.m[k][j] return Transform(t[0][0], t[0][1], t[0][2], t[1][0], t[1][1], t[1][2]) diff --git a/incomplete/rasterizer/rasterizer/geometry.py.bak b/incomplete/rasterizer/rasterizer/geometry.py.bak new file mode 100644 index 000000000..d2c5759be --- /dev/null +++ b/incomplete/rasterizer/rasterizer/geometry.py.bak @@ -0,0 +1,121 @@ +import random +import math +from itertools import product +from functools import reduce + +# Solves the quadratic equation ax^2 + bx + c = 0 +# using a variant of the standard quadratic formula +# that behaves better numerically +def quadratic(a, b, c): + if a == 0: + return -c/b, -c/b + d = (b * b - 4 * a * c) ** 0.5 + if b >= 0: + return (-b - d) / (2 * a), (2 * c) / (-b - d) + else: + return (2 * c) / (-b + d), (-b + d) / (2 * a) + +class Vector: + def __init__(self, *args): + self.x, self.y = args + def __add__(self, o): + return Vector(self.x + o.x, self.y + o.y) + def __sub__(self, o): + return Vector(self.x - o.x, self.y - o.y) + def __mul__(self, k): + return Vector(self.x * k, self.y * k) + def dot(self, o): + return self.x * o.x + self.y * o.y + def min(self, o): + return Vector(min(self.x, o.x), min(self.y, o.y)) + def max(self, o): + return Vector(max(self.x, o.x), max(self.y, o.y)) + def length(self): + return (self.x * self.x + self.y * self.y) ** 0.5 + def __repr__(self): + return "[%.3f %.3f]" % (self.x, self.y) + +class AABox: + def __init__(self, p1, p2): + self.low = p1.min(p2) + self.high = p1.max(p2) + def midpoint(self): + return (self.low + self.high) * 0.5 + def size(self): + return self.high - self.low + def contains(self, p): + return self.low.x <= p.x <= self.high.x and \ + self.low.y <= p.y <= self.high.y + def overlaps(self, r): + return not (r.low.x >= self.high.x or r.high.x <= self.low.x or + r.low.y >= self.high.y or r.high.y <= self.low.y) + def intersection(self, other): + return AABox(self.low.max(other.low), self.high.min(other.high)) + @staticmethod + def from_vectors(*args): + return AABox(reduce(Vector.min, args), reduce(Vector.max, args)) + +class HalfPlane: + def __init__(self, p1, p2): + self.v = Vector(-p2.y + p1.y, p2.x - p1.x) + l = self.v.length() + self.c = -self.v.dot(p1) / l + self.v = self.v * (1.0 / l) + def signed_distance(self, p): + return self.v.dot(p) + self.c + +# Transform represents an affine transformation +# of a 2D vector ("affine" because it's a linear transformation +# combined with a translation) +class Transform: + def __init__(self, m11, m12, tx, m21, m22, ty): + self.m = [[m11, m12, tx], + [m21, m22, ty], + [0, 0, 1]] + def __mul__(self, other): + if isinstance(other, Transform): + # if the other element is also a transform, + # then return a transform corresponding to the + # composition of the two transforms + t = [[0.0] * 3 for i in range(3)] + for i, j, k in product(range(3), repeat=3): + t[i][j] += self.m[i][k] * other.m[k][j] + return Transform(t[0][0], t[0][1], t[0][2], + t[1][0], t[1][1], t[1][2]) + else: + # if the other element is a vector, then + # apply the transformation to the vector via + # a matrix-vector multiplication + nx = self.m[0][0] * other.x + self.m[0][1] * other.y + self.m[0][2] + ny = self.m[1][0] * other.x + self.m[1][1] * other.y + self.m[1][2] + return Vector(nx, ny) + def det(s): + return s.m[0][0] * s.m[1][1] - s.m[0][1] * s.m[1][0] + def inverse(self): + d = 1.0 / self.det() + t = Transform(d * self.m[1][1], -d * self.m[0][1], 0, + -d * self.m[1][0], d * self.m[0][0], 0) + v = t * Vector(self.m[0][2], self.m[1][2]) + t.m[0][2] = -v.x + t.m[1][2] = -v.y + return t + +def identity(): + return Transform(1, 0, 0, 0, 1, 0) + +def rotate(theta): + s = math.sin(theta) + c = math.cos(theta) + return Transform(c, -s, 0, s, c, 0) + +def translate(tx, ty): + return Transform(1, 0, tx, 0, 1, ty) + +def scale(x, y): + return Transform(x, 0, 0, 0, y, 0) + +# To perform a transformation 'around' some point, +# we first translate that point to the origin, perform a transformation, +# then translate back that same amount +def around(v, t): + return translate(v.x, v.y) * t * translate(-v.x, -v.y) diff --git a/incomplete/rasterizer/rasterizer/image.py b/incomplete/rasterizer/rasterizer/image.py index 9c6de0b11..1c62a1cc2 100644 --- a/incomplete/rasterizer/rasterizer/image.py +++ b/incomplete/rasterizer/rasterizer/image.py @@ -1,13 +1,13 @@ -from color import Color -from geometry import AABox, Vector +from .color import Color +from .geometry import AABox, Vector class PPMImage: def __init__(self, resolution, bg=Color()): self.resolution = resolution self.pixels = [] - for i in xrange(self.resolution): + for i in range(self.resolution): lst = [] - for j in xrange(self.resolution): + for j in range(self.resolution): lst.append(Color(rgb=bg.rgb, a=bg.a)) self.pixels.append(lst) def bounds(self): @@ -19,6 +19,6 @@ def __setitem__(self, a, color): def write_ppm(self, out): n = self.resolution out.write("P6\n%s\n%s\n255\n" % (n,n)) - for y in xrange(n-1, -1, -1): - for x in xrange(n): + for y in range(n-1, -1, -1): + for x in range(n): out.write(self.pixels[y][x].as_ppm()) diff --git a/incomplete/rasterizer/rasterizer/image.py.bak b/incomplete/rasterizer/rasterizer/image.py.bak new file mode 100644 index 000000000..9c6de0b11 --- /dev/null +++ b/incomplete/rasterizer/rasterizer/image.py.bak @@ -0,0 +1,24 @@ +from color import Color +from geometry import AABox, Vector + +class PPMImage: + def __init__(self, resolution, bg=Color()): + self.resolution = resolution + self.pixels = [] + for i in xrange(self.resolution): + lst = [] + for j in xrange(self.resolution): + lst.append(Color(rgb=bg.rgb, a=bg.a)) + self.pixels.append(lst) + def bounds(self): + return AABox(Vector(0,0), Vector(1,1)) + def __getitem__(self, a): + return self.pixels[a.y][a.x] + def __setitem__(self, a, color): + self.pixels[a.y][a.x] = color + def write_ppm(self, out): + n = self.resolution + out.write("P6\n%s\n%s\n255\n" % (n,n)) + for y in xrange(n-1, -1, -1): + for x in xrange(n): + out.write(self.pixels[y][x].as_ppm()) diff --git a/incomplete/rasterizer/rasterizer/poly.py b/incomplete/rasterizer/rasterizer/poly.py index d759195dd..981ef2255 100644 --- a/incomplete/rasterizer/rasterizer/poly.py +++ b/incomplete/rasterizer/rasterizer/poly.py @@ -1,5 +1,5 @@ -from shape import Shape -from geometry import HalfPlane, Vector, AABox +from .shape import Shape +from .geometry import HalfPlane, Vector, AABox class ConvexPoly(Shape): # a *convex* poly, in ccw order, with no repeating vertices def __init__(self, ps, color=None): @@ -7,7 +7,7 @@ def __init__(self, ps, color=None): self.vs = ps self.bound = AABox.from_vectors(*self.vs) self.half_planes = [] - for i in xrange(len(self.vs)): + for i in range(len(self.vs)): h = HalfPlane(self.vs[i], self.vs[(i+1) % len(self.vs)]) self.half_planes.append(h) def signed_distance_bound(self, p): @@ -20,7 +20,7 @@ def signed_distance_bound(self, p): max_outside = d if d >= 0 and d < min_inside: min_inside = d - return max_outside if max_outside <> -1e30 else min_inside + return max_outside if max_outside != -1e30 else min_inside def contains(self, p): for plane in self.half_planes: if plane.signed_distance(p) < 0: diff --git a/incomplete/rasterizer/rasterizer/poly.py.bak b/incomplete/rasterizer/rasterizer/poly.py.bak new file mode 100644 index 000000000..d759195dd --- /dev/null +++ b/incomplete/rasterizer/rasterizer/poly.py.bak @@ -0,0 +1,52 @@ +from shape import Shape +from geometry import HalfPlane, Vector, AABox + +class ConvexPoly(Shape): # a *convex* poly, in ccw order, with no repeating vertices + def __init__(self, ps, color=None): + Shape.__init__(self, color) + self.vs = ps + self.bound = AABox.from_vectors(*self.vs) + self.half_planes = [] + for i in xrange(len(self.vs)): + h = HalfPlane(self.vs[i], self.vs[(i+1) % len(self.vs)]) + self.half_planes.append(h) + def signed_distance_bound(self, p): + plane = self.half_planes[0] + min_inside = 1e30 + max_outside = -1e30 + for plane in self.half_planes: + d = plane.signed_distance(p) + if d <= 0 and d > max_outside: + max_outside = d + if d >= 0 and d < min_inside: + min_inside = d + return max_outside if max_outside <> -1e30 else min_inside + def contains(self, p): + for plane in self.half_planes: + if plane.signed_distance(p) < 0: + return False + return True + def transform(self, xform): + return ConvexPoly(list(xform * v for v in self.vs), color=self.color) + +Triangle, Quad = ConvexPoly, ConvexPoly + +def Rectangle(v1, v2, color=None): + return Quad([Vector(min(v1.x, v2.x), min(v1.y, v2.y)), + Vector(max(v1.x, v2.x), min(v1.y, v2.y)), + Vector(max(v1.x, v2.x), max(v1.y, v2.y)), + Vector(min(v1.x, v2.x), max(v1.y, v2.y))], + color=color) + +def LineSegment(v1, v2, thickness, color=None): + d = v2 - v1 + d.x, d.y = -d.y, d.x + d *= thickness / d.length() / 2 + return Quad([v1 + d, v1 - d, v2 - d, v2 + d], color=color) + +# A simple polygon +# It will work by triangulating a polygon (via simple, slow ear-clipping) +# and then rendering it by a CSG union of the triangles +class Poly(Shape): + def __init__(self, *args, **kwargs): + raise Exception("Unimplemented") diff --git a/incomplete/rasterizer/rasterizer/scene.py b/incomplete/rasterizer/rasterizer/scene.py index ff93c42ca..456a9a754 100644 --- a/incomplete/rasterizer/rasterizer/scene.py +++ b/incomplete/rasterizer/rasterizer/scene.py @@ -1,5 +1,5 @@ -from geometry import identity -from shape import Shape, SceneObject +from .geometry import identity +from .shape import Shape, SceneObject class Scene(SceneObject): def __init__(self, nodes=None, transform=None): diff --git a/incomplete/rasterizer/rasterizer/scene.py.bak b/incomplete/rasterizer/rasterizer/scene.py.bak new file mode 100644 index 000000000..ff93c42ca --- /dev/null +++ b/incomplete/rasterizer/rasterizer/scene.py.bak @@ -0,0 +1,24 @@ +from geometry import identity +from shape import Shape, SceneObject + +class Scene(SceneObject): + def __init__(self, nodes=None, transform=None): + if transform is None: + transform = identity() + if nodes is None: + nodes = [] + self.transform = transform + self.nodes = nodes + def add(self, node): + self.nodes.append(node) + def draw(self, image): + for scene_object in self.traverse(identity()): + scene_object.draw(image) + def traverse(self, xform): + this_xform = xform * self.transform + for node in self.nodes: + if isinstance(node, Scene): + for n in node.traverse(this_xform): + yield n + elif isinstance(node, Shape): + yield node.transform(this_xform) diff --git a/incomplete/rasterizer/rasterizer/shape.py b/incomplete/rasterizer/rasterizer/shape.py index faf735301..eb41d112a 100644 --- a/incomplete/rasterizer/rasterizer/shape.py +++ b/incomplete/rasterizer/rasterizer/shape.py @@ -1,6 +1,6 @@ -from color import Color +from .color import Color from itertools import product -from geometry import Vector +from .geometry import Vector import random class SceneObject: @@ -22,13 +22,13 @@ def draw(self, image, super_sampling = 6): r = float(image.resolution) jitter = [Vector((x + random.random()) / super_sampling / r, (y + random.random()) / super_sampling / r) - for (x, y) in product(xrange(super_sampling), repeat=2)] + for (x, y) in product(list(range(super_sampling)), repeat=2)] lj = len(jitter) l_x = max(int(self.bound.low.x * r), 0) l_y = max(int(self.bound.low.y * r), 0) h_x = min(int(self.bound.high.x * r), r-1) h_y = min(int(self.bound.high.y * r), r-1) - for y in xrange(l_y, int(h_y+1)): + for y in range(l_y, int(h_y+1)): x = l_x while x <= h_x: corner = Vector(x / r, y / r) @@ -36,7 +36,7 @@ def draw(self, image, super_sampling = 6): pixel_diameter = (2 ** 0.5) / r if b > pixel_diameter: steps = int(r * (b - (pixel_diameter - 1.0/r))) - for x_ in xrange(x, min(x + steps, int(h_x+1))): + for x_ in range(x, min(x + steps, int(h_x+1))): image.pixels[y][x_].draw(color) x += steps elif b < -pixel_diameter: diff --git a/incomplete/rasterizer/rasterizer/shape.py.bak b/incomplete/rasterizer/rasterizer/shape.py.bak new file mode 100644 index 000000000..5aa48bbd9 --- /dev/null +++ b/incomplete/rasterizer/rasterizer/shape.py.bak @@ -0,0 +1,51 @@ +from .color import Color +from itertools import product +from .geometry import Vector +import random + +class SceneObject: + def draw(self, image): + raise NotImplementedError("Undefined method") + +class Shape(SceneObject): + def __init__(self, color=None): + self.color = color if color is not None else Color() + self.bound = None + def contains(self, p): + raise NotImplementedError("Undefined method") + def signed_distance_bound(self, p): + raise NotImplementedError("Undefined method") + def draw(self, image, super_sampling = 6): + if not self.bound.overlaps(image.bounds()): + return + color = self.color + r = float(image.resolution) + jitter = [Vector((x + random.random()) / super_sampling / r, + (y + random.random()) / super_sampling / r) + for (x, y) in product(range(super_sampling), repeat=2)] + lj = len(jitter) + l_x = max(int(self.bound.low.x * r), 0) + l_y = max(int(self.bound.low.y * r), 0) + h_x = min(int(self.bound.high.x * r), r-1) + h_y = min(int(self.bound.high.y * r), r-1) + for y in range(l_y, int(h_y+1)): + x = l_x + while x <= h_x: + corner = Vector(x / r, y / r) + b = self.signed_distance_bound(corner) + pixel_diameter = (2 ** 0.5) / r + if b > pixel_diameter: + steps = int(r * (b - (pixel_diameter - 1.0/r))) + for x_ in range(x, min(x + steps, int(h_x+1))): + image.pixels[y][x_].draw(color) + x += steps + elif b < -pixel_diameter: + steps = int(r * (-b - (pixel_diameter - 1.0/r))) + x += steps + else: + coverage = 0 + for j in jitter: + if self.contains(corner + j): + coverage += 1.0 + image.pixels[y][x].draw(color.fainter(coverage / lj)) + x += 1 diff --git a/incomplete/rasterizer/test_rasterizer.py b/incomplete/rasterizer/test_rasterizer.py index 88e56be19..023c05aa4 100644 --- a/incomplete/rasterizer/test_rasterizer.py +++ b/incomplete/rasterizer/test_rasterizer.py @@ -53,7 +53,7 @@ def do_it(): f.close() def test_quadratic(): - for i in xrange(1000): + for i in range(1000): a = random.random() b = random.random() c = random.random() @@ -63,7 +63,7 @@ def test_quadratic(): raise Exception("fail") if v2 * v2 * a + v2 * b + c > 1e-5: raise Exception("fail") - for i in xrange(1000): + for i in range(1000): a = 0 b = random.random() c = random.random() @@ -75,15 +75,15 @@ def test_quadratic(): raise Exception("fail") def test_inverse(): - for i in xrange(10000): + for i in range(10000): f = Transform(random.random(), random.random(), random.random(), random.random(), random.random(), random.random()) v = Vector(random.random(), random.random()) m = f * f.inverse() if (m * v - v).length() > 1e-4: - print >>sys.stderr, "inverse failed!" - print f.m - print m.m + print("inverse failed!", file=sys.stderr) + print(f.m) + print(m.m) raise Exception("foo") if __name__ == '__main__': diff --git a/incomplete/rasterizer/test_rasterizer.py.bak b/incomplete/rasterizer/test_rasterizer.py.bak new file mode 100644 index 000000000..88e56be19 --- /dev/null +++ b/incomplete/rasterizer/test_rasterizer.py.bak @@ -0,0 +1,92 @@ +from rasterizer import * +import sys +import cProfile +import random +import math + +def do_it(): + f = open(sys.argv[1], 'w') + i = PPMImage(512, Color(1,1,1,1)) + s = Scene() + s3 = Scene() + s3.add(Union(Circle(Vector(0.3, 0.1), 0.1), + Circle(Vector(0.35, 0.1), 0.1), + Color(0,0,0,0.5))) + s3.add(Intersection(Circle(Vector(0.3, 0.3), 0.1), + Circle(Vector(0.35, 0.3), 0.1), + Color(0,0.5,0,1))) + s3.add(Subtraction(Circle(Vector(0.3, 0.5), 0.1), + Circle(Vector(0.35, 0.5), 0.1), + Color(0,0,0.5,1))) + s3.add(Subtraction(Circle(Vector(0.35, 0.7), 0.1), + Circle(Vector(0.3, 0.7), 0.1), + Color(0,0.5,0.5,1))) + s2 = Scene([ + LineSegment(Vector(0.0,0), Vector(0.0,1), 0.01, Color(1,0,0,1)), + LineSegment(Vector(0.1,0), Vector(0.1,1), 0.01, Color(1,0,0,1)), + LineSegment(Vector(0.2,0), Vector(0.2,1), 0.01, Color(1,0,0,1)), + LineSegment(Vector(0.3,0), Vector(0.3,1), 0.01, Color(1,0,0,1)), + LineSegment(Vector(0.4,0), Vector(0.4,1), 0.01, Color(1,0,0,1)), + LineSegment(Vector(0.5,0), Vector(0.5,1), 0.01, Color(1,0,0,1)), + LineSegment(Vector(0.6,0), Vector(0.6,1), 0.01, Color(1,0,0,1)), + LineSegment(Vector(0.2,0.1), Vector(0.7, 0.4), 0.01, Color(0,0.5,0,1))]) + for shape in [ + Circle(Vector(0.0, 0.0), 0.8, Color(1,0.5,0,1)).transform(scale(0.05,1)), + Triangle([Vector(0.2,0.1), Vector(0.9,0.3), Vector(0.1,0.4)], + Color(1,0,0,0.5)), + Triangle([Vector(0.5,0.2), Vector(1,0.4), Vector(0.2,0.7)], + Color(0,1,0,0.5)), + Rectangle(Vector(0.1,0.7), Vector(0.6,0.8), + Color(0,0.5,0.8,0.5)), + Triangle([Vector(-1, 0.6), Vector(0.2, 0.8), Vector(-2, 0.7)], + Color(1,0,1,0.9)), + Circle(Vector(0.5,0.9), 0.2, Color(0,0,0,1)), + Circle(Vector(0.9, 0.5), 0.2, + Color(0,1,1,0.5)).transform(around(Vector(0.9, 0.5), scale(1.0, 0.5))), + s2, + Scene([s2], around(Vector(0.5, 0.5), rotate(math.radians(90)))), + Scene([s3], translate(0.5, 0)) + ]: + s.add(shape) + s.draw(i) + i.write_ppm(f) + f.close() + +def test_quadratic(): + for i in xrange(1000): + a = random.random() + b = random.random() + c = random.random() + if b * b - 4 * a * c >= 0: + v1, v2 = quadratic(a, b, c) + if v1 * v1 * a + v1 * b + c > 1e-5: + raise Exception("fail") + if v2 * v2 * a + v2 * b + c > 1e-5: + raise Exception("fail") + for i in xrange(1000): + a = 0 + b = random.random() + c = random.random() + if b * b - 4 * a * c >= 0: + v1, v2 = quadratic(a, b, c) + if v1 * v1 * a + v1 * b + c > 1e-5: + raise Exception("fail") + if v2 * v2 * a + v2 * b + c > 1e-5: + raise Exception("fail") + +def test_inverse(): + for i in xrange(10000): + f = Transform(random.random(), random.random(), random.random(), + random.random(), random.random(), random.random()) + v = Vector(random.random(), random.random()) + m = f * f.inverse() + if (m * v - v).length() > 1e-4: + print >>sys.stderr, "inverse failed!" + print f.m + print m.m + raise Exception("foo") + +if __name__ == '__main__': + test_inverse() + test_quadratic() + do_it() diff --git a/incomplete/search-engine/index.py b/incomplete/search-engine/index.py index 1a651d052..4ab16a06b 100755 --- a/incomplete/search-engine/index.py +++ b/incomplete/search-engine/index.py @@ -9,14 +9,14 @@ import sys import shutil # to remove directory trees import traceback -import urllib # for quote and unquote +import urllib.request, urllib.parse, urllib.error # for quote and unquote def write_tuples(outfile, tuples): for item in tuples: - line = ' '.join(urllib.quote(str(field)) for field in item) + line = ' '.join(urllib.parse.quote(str(field)) for field in item) outfile.write(line + "\n") def read_tuples(infile): for line in infile: - yield tuple(urllib.unquote(field) for field in line.split()) + yield tuple(urllib.parse.unquote(field) for field in line.split()) # XXX maybe these functions don't need to exist? def read_metadata(index_path): with path['documents'].open() as metadata_file: @@ -84,7 +84,7 @@ def search_ui(index_path, terms): # Use the crudest possible ranking: newest (largest mtime) first. for path in sorted(paths(index_path, terms), key=get_metadata, reverse=True): - print(path.name) + print((path.name)) # At the moment, our doc_ids are just pathnames; this converts them to Path objects. def paths(index_path, terms): parent = index_path.parent() diff --git a/incomplete/search-engine/index.py.bak b/incomplete/search-engine/index.py.bak new file mode 100755 index 000000000..1a651d052 --- /dev/null +++ b/incomplete/search-engine/index.py.bak @@ -0,0 +1,263 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import gzip +import heapq # for heapq.merge +import itertools +import os +import re # used to extract words from input +import sys +import shutil # to remove directory trees +import traceback +import urllib # for quote and unquote +def write_tuples(outfile, tuples): + for item in tuples: + line = ' '.join(urllib.quote(str(field)) for field in item) + outfile.write(line + "\n") +def read_tuples(infile): + for line in infile: + yield tuple(urllib.unquote(field) for field in line.split()) +# XXX maybe these functions don't need to exist? +def read_metadata(index_path): + with path['documents'].open() as metadata_file: + return dict((pathname, (int(mtime), int(size))) + for pathname, size, mtime in read_tuples(metadata_file)) + +# XXX we probably want files_unchanged +def file_unchanged(metadatas, path): + return any(metadata.get(path.name) == get_metadata(path) + for metadata in metadatas) +def doc_ids(index_path, terms): + "Actually evaluate a query." + doc_id_sets = (set(term_doc_ids(index_path, term)) for term in terms) + return set.intersection(*doc_id_sets) +def term_doc_ids(index_path, term): + doc_id_sets = (segment_term_doc_ids(segment, term) + for segment in index_segments(index_path)) + return itertools.chain(*doc_id_sets) + +# XXX make this a method of the Index object, perhaps returning Segment objects +def index_segments(index_path): + return [path for path in index_path if path.basename().startswith('seg_')] +class Path: # like java.lang.File + def __init__(self, name): + self.name = name + __getitem__ = lambda self, child: Path(os.path.join(self.name, str(child))) + __contains__ = lambda self, child: os.path.exists(self[child].name) + __iter__ = lambda self: (self[child] for child in os.listdir(self.name)) + open = lambda self, *args: open(self.name, *args) + open_gzipped = lambda self, *args: gzip.GzipFile(self.name, *args) + basename = lambda self: os.path.basename(self.name) + parent = lambda self: Path(os.path.dirname(self.name)) + abspath = lambda self: os.path.abspath(self.name) +def segment_term_doc_ids(segment, needle_term): + for chunk_name in segment_term_chunks(segment, needle_term): + with segment[chunk_name].open_gzipped() as chunk_file: + for haystack_term, doc_id in read_tuples(chunk_file): + if haystack_term == needle_term: + yield doc_id + # Once we reach an alphabetically later term, we're done: + if haystack_term > needle_term: + break +# XXX maybe return Path objects? +def segment_term_chunks(segment, term): + previous_chunk = None + for headword, chunk in skip_file_entries(segment): + if headword >= term: + if previous_chunk is not None: + yield previous_chunk + if headword > term: + break + + previous_chunk = chunk + else: # executed if we don't break + if previous_chunk is not None: + yield previous_chunk +def skip_file_entries(segment_path): + with segment_path['skip'].open() as skip_file: + return list(read_tuples(skip_file)) +def get_metadata(path): + s = os.stat(path.name) + return int(s.st_mtime), int(s.st_size) + +def search_ui(index_path, terms): + # Use the crudest possible ranking: newest (largest mtime) first. + for path in sorted(paths(index_path, terms), + key=get_metadata, reverse=True): + print(path.name) +# At the moment, our doc_ids are just pathnames; this converts them to Path objects. +def paths(index_path, terms): + parent = index_path.parent() + for doc_id in doc_ids(index_path, terms): + yield Path(os.path.relpath(parent[doc_id].abspath(), start='.')) +# 2**20 is chosen as the maximum segment size because that uses +# typically about a quarter gig, which is a reasonable size these +# days. +segment_size = 2**20 + +def build_index(index_path, corpus_path, postings_filters): + os.mkdir(index_path.name) + + # XXX hmm, these should match the doc_ids in the index + corpus_paths = list(find_documents(corpus_path)) + with index_path['documents'].open('w') as outfile: + write_tuples(outfile, ((path.name,) + get_metadata(path) + for path in corpus_paths)) + + postings = tokenize_documents(corpus_paths) + for filter_function in postings_filters: + postings = filter_function(postings) + + # XXX at this point we should just pass the fucking doc_id into + # the analyzer function :( + parent = index_path.parent() + rel_paths = dict((path.name, os.path.relpath(path.name, start=parent.name)) + for path in corpus_paths) + rel_postings = ((term, rel_paths[doc_id]) for term, doc_id in postings) + for ii, chunk in enumerate(blocked(rel_postings, segment_size)): + write_new_segment(index_path['seg_%s' % ii], sorted(chunk)) + + merge_segments(index_path, index_segments(index_path)) +# From nikipore on Stack Overflow +def blocked(seq, block_size): + seq = iter(seq) + while True: + # XXX for some reason using list(), and then later sorting in + # place, makes the whole program run twice as slow and doesn't + # reduce its memory usage. No idea why. + block = tuple(itertools.islice(seq, block_size)) + if block: + yield block + else: + raise StopIteration +def find_documents(path): + for dir_name, _, filenames in os.walk(path.name): + dir_path = Path(dir_name) + for filename in filenames: + yield dir_path[filename] + +def tokenize_documents(paths): + for path in paths: + for posting in remove_duplicates(tokenize_file(path)): + yield posting + +# Demonstrate a crude set of smart tokenizer frontends. +def tokenize_file(file_path): + if file_path.name.endswith('.html'): + return tokenize_html(file_path) + else: + return tokenize_text(file_path) + +def tokenize_text(file_path): + word_re = re.compile(r'\w+') + with file_path.open() as fo: + for line in fo: + for word in word_re.findall(line): + yield word, file_path.name + +def remove_duplicates(seq): + seen_items = set() + for item in seq: + if item not in seen_items: + yield item + seen_items.add(item) +chunk_size = 4096 +def write_new_segment(path, postings): + os.mkdir(path.name) + chunks = blocked(postings, chunk_size) + skip_file_contents = (write_chunk(path, '%s.gz' % ii, chunk) + for ii, chunk in enumerate(chunks)) + with path['skip'].open('w') as skip_file: + write_tuples(skip_file, itertools.chain(*skip_file_contents)) + +# Yields one skip file entry, or, in the edge case of an empty chunk, none. +def write_chunk(path, filename, chunk): + with path[filename].open_gzipped('w') as chunk_file: + write_tuples(chunk_file, chunk) + if chunk: + yield chunk[0][0], filename +# Crude approximation of HTML tokenization. Note that for proper +# excerpt generation (as in the "grep" command) the postings generated +# need to contain position information, because we need to run this +# tokenizer during excerpt generation too. +def tokenize_html(file_path): + tag_re = re.compile('<.*?>') + tag_start_re = re.compile('<.*') + tag_end_re = re.compile('.*?>') + word_re = re.compile(r'\w+') + + with file_path.open() as fo: + in_tag = False + for line in fo: + + if in_tag and tag_end_re.search(line): + line = tag_end_re.sub('', line) + in_tag = False + + elif not in_tag: + line = tag_re.subn('', line)[0] + if tag_start_re.search(line): + in_tag = True + line = tag_start_re.sub('', line) + for term in word_re.findall(line): + yield term, file_path.name +def merge_segments(path, segments): + if len(segments) == 1: + return + + postings = heapq.merge(*(read_segment(segment) + for segment in segments)) + write_new_segment(path['seg_merged'], postings) + + for segment in segments: + shutil.rmtree(segment.name) +def read_segment(path): + for _, chunk in skip_file_entries(path): + # XXX refactor chunk reading? We open_gzipped in three places now. + with path[chunk].open_gzipped() as chunk_file: + for item in read_tuples(chunk_file): + yield item +def make_stopwords_filter(stopwords): + stopwords = set(stopwords) + stopwords |= ( set(word.upper() for word in stopwords) + | set(word.capitalize() for word in stopwords)) + return lambda postings: ((term, doc_id) for term, doc_id in postings + if term not in stopwords) +word_len = 20 # to eliminate nonsense words +def discard_long_nonsense_words_filter(postings): + """Drop postings for nonsense words.""" + return ((term, doc_id) for term, doc_id in postings if len(term) < word_len) +def case_insensitive_filter(postings): + for term, doc_id in postings: + yield term, doc_id + if term.lower() != term: + yield term.lower(), doc_id +def grep(index_path, terms): + for path in paths(index_path, terms): + try: + with path.open() as text: + for ii, line in enumerate(text, start=1): + if any(term in line for term in terms): + sys.stdout.write("%s:%s:%s" % (path.name, ii, line)) + except Exception: # The file might e.g. no longer exist. + traceback.print_exc() +def main(argv): + # Eliminate the most common English words from queries and indices. + stopwords = 'the of and to a in it is was that i for on you he be'.split() + + if argv[1] == 'index': + build_index(index_path=Path(argv[2]), corpus_path=Path(argv[3]), + postings_filters=[discard_long_nonsense_words_filter, + make_stopwords_filter(stopwords), + case_insensitive_filter]) + elif argv[1] == 'query': + search_ui(Path(argv[2]), [term for term in argv[3:] + if term not in stopwords]) + elif argv[1] == 'grep': + grep(Path(argv[2]), [term for term in argv[3:] + if term not in stopwords]) + else: + raise Exception("%s (index|query|grep) index_dir ..." % (argv[0])) + +if __name__ == '__main__': + main(sys.argv) diff --git a/incomplete/search-engine/sizes.py b/incomplete/search-engine/sizes.py index 6cadb21bc..214f11b4f 100755 --- a/incomplete/search-engine/sizes.py +++ b/incomplete/search-engine/sizes.py @@ -7,12 +7,12 @@ def xcumsum(xs): total += xx def describe(sizes): - print ' ' * 4 + ' '.join("%4d" % size for size in sizes) - print ' ' * 4 + ' '.join("%4d" % tots for tots in xcumsum(sizes)) + print((' ' * 4 + ' '.join("%4d" % size for size in sizes))) + print((' ' * 4 + ' '.join("%4d" % tots for tots in xcumsum(sizes)))) sizes = [] while True: - line = raw_input("+ ") + line = eval(input("+ ")) if line == '': sizes = [] diff --git a/incomplete/search-engine/sizes.py.bak b/incomplete/search-engine/sizes.py.bak new file mode 100755 index 000000000..42e4e620d --- /dev/null +++ b/incomplete/search-engine/sizes.py.bak @@ -0,0 +1,30 @@ +#!/usr/bin/python + +def xcumsum(xs): + total = 0 + for xx in xs: + yield total + total += xx + +def describe(sizes): + print(' ' * 4 + ' '.join("%4d" % size for size in sizes)) + print(' ' * 4 + ' '.join("%4d" % tots for tots in xcumsum(sizes))) + +sizes = [] +while True: + line = input("+ ") + + if line == '': + sizes = [] + else: + sizes.append(int(line)) + sizes.sort() + + describe(sizes) + + merge = [cumsum >= size for cumsum, size in zip(xcumsum(sizes), sizes)] + if any(merge): + max_merge = max(ii for ii in range(len(merge)) if merge[ii]) + 1 + sizes[:max_merge] = [sum(sizes[:max_merge])] + sizes.sort() + describe(sizes) diff --git a/incomplete/search-engine/skipdiagram.py b/incomplete/search-engine/skipdiagram.py index 4bce05fca..98ef6c623 100755 --- a/incomplete/search-engine/skipdiagram.py +++ b/incomplete/search-engine/skipdiagram.py @@ -19,7 +19,7 @@ def main(): elif sys.argv[1] == 'skiplist': emit(sys.stdout, "Skip list", Skiplist()) else: - emit(sys.stdout, u"Skip files", Btree()) + emit(sys.stdout, "Skip files", Btree()) def emit(output, title_string, structure): "Write an SVG file to output representing structure." @@ -130,19 +130,24 @@ def title(s): builder.data(s) builder.end("text") -def arrow((x0, y0), (x1, y1)): +def arrow(xxx_todo_changeme, xxx_todo_changeme1): "Draw a drafting-style arrow from (x0, y0) to (x1, y1)." + (x0, y0) = xxx_todo_changeme + (x1, y1) = xxx_todo_changeme1 direction = normalize(displacement((x0, y0), (x1, y1))) dx2, dy2 = rotate(direction, (-3.5, -2)) dx3, dy3 = rotate(direction, (-3.5, 2)) path(['M', x0, y0, 'L', x1, y1, 'm', dx2, dy2, 'l', -dx2, -dy2, 'l', dx3, dy3]) -def displacement((x0, y0), (x1, y1)): +def displacement(xxx_todo_changeme2, xxx_todo_changeme3): "Compute displacement 2-vector from first to second argument." + (x0, y0) = xxx_todo_changeme2 + (x1, y1) = xxx_todo_changeme3 return x1 - x0, y1 - y0 -def normalize((x, y)): +def normalize(xxx_todo_changeme4): "Normalize a 2-vector to L₂-norm of unity, yielding direction cosines." + (x, y) = xxx_todo_changeme4 mag = (x**2 + y**2)**0.5 return x/mag, y/mag @@ -150,14 +155,18 @@ def normalize((x, y)): assert normalize((1, 0)) == (1, 0) assert normalize((-1, 0)) == (-1, 0) -def rotate((cos, sin), (x, y)): +def rotate(xxx_todo_changeme5, xxx_todo_changeme6): "Rotate a 2-vector to the angle specified by given direction cosines." + (cos, sin) = xxx_todo_changeme5 + (x, y) = xxx_todo_changeme6 return (cos * x - sin * y), (sin * x + cos * y) assert rotate((1, 0), (.2, .3)) == (.2, .3) -def box((x0, y0), (x1, y1), attrs={}): +def box(xxx_todo_changeme7, xxx_todo_changeme8, attrs={}): "Draw a paraxial box." + (x0, y0) = xxx_todo_changeme7 + (x1, y1) = xxx_todo_changeme8 path(['M', x0, y0, 'L', x1, y0, 'L', x1, y1, 'L', x0, y1, 'Z'], attrs) def path(d, attrs=dict(fill="none", stroke="#000")): diff --git a/incomplete/search-engine/skipdiagram.py.bak b/incomplete/search-engine/skipdiagram.py.bak new file mode 100755 index 000000000..4bce05fca --- /dev/null +++ b/incomplete/search-engine/skipdiagram.py.bak @@ -0,0 +1,169 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Draw SVG diagrams of skip lists and skip files to contrast. +""" +import sys + +from xml.etree.ElementTree import ElementTree, TreeBuilder + +font = "font-family: Palatino, \"URW Palladio R\", serif;" +width = 288 +height = 144 + +def main(): + "Command-line UI for emit()." + if len(sys.argv) != 2 or sys.argv[1] not in ['skiplist', 'skipfiles']: + sys.stderr.write("Usage: %(me)s skiplist or %(me)s skipfiles\n" % + dict(me=sys.argv[0])) + return + elif sys.argv[1] == 'skiplist': + emit(sys.stdout, "Skip list", Skiplist()) + else: + emit(sys.stdout, u"Skip files", Btree()) + +def emit(output, title_string, structure): + "Write an SVG file to output representing structure." + global builder + + builder = TreeBuilder() + + builder.start("svg", dict(xmlns="http://www.w3.org/2000/svg", + width=str(width), + height=str(height))) + + title(title_string) + structure.depth(3) + for key, depth in [(3, 3), + (1, 5), + (2, 6), + (1, 10), + (1, 11), + (2, 15), + (3, 16), + (1, 22), + (2, 25), + (1, 25), + ]: + structure.key(key, depth) + + builder.end("svg") + + ElementTree(builder.close()).write(output, + encoding='utf-8', + xml_declaration=True) + output.write("\n") + +class Btree: + "Draw a skip-files-style B+-tree." + y_top = 50 + def depth(self, depth): + "Set the depth of the B+-tree." + self._depth = depth + self.x = [20 + (depth-i) * 39 for i in range(depth)] + + def key(self, depth, key): + "Put a key into the B+-tree at a given number of levels." + width = height = 20 + padding = 3 + margin = 16 + prev = None + yoff = 4 + for i in range(self._depth - depth, self._depth): + y = self.y_top + i * (height + margin) + box((self.x[i], y), + (self.x[i] + width, y + height), + {'stroke': 'white', 'fill': '#ccc'}) + cx, cy = self.x[i] + width/2, y + height/2 + if prev is not None: + arrow(prev, (cx, y)) + + label(cx, y + height - padding - yoff, key) + + prev = cx, y + height - yoff + self.x[i] += width + +class Skiplist: + "Draw a skip list." + def __init__(self): + self.pending = [] + self.x = 20 + + def depth(self, depth): + "No-op." + pass + + def key(self, node_height, key): + "Create a skiplist node with the given height." + height = width = 20 + y_base = 130 + padding = 3 + for i in range(node_height + 1): + box((self.x, y_base - i * height), + (self.x + width, y_base - (i+1) * height), + {'stroke': 'white', 'fill': '#ccc'}) + cx = self.x + width/2 + cy = y_base - (i+0.5) * height + + if i > 0 and i < len(self.pending): + arrow(self.pending[i], (self.x, self.pending[i][1])) + elif i == len(self.pending): + self.pending.append(None) + self.pending[i] = cx, cy + + label(cx, y_base - padding, key) + + margin = 5 + self.x += width + margin + +def label(x, y, text): + "Add a cell label centered at a given baseline point." + builder.start('text', + dict(x=str(x), + y=str(y), + style="font-size: 12px; %s text-anchor: middle" % font)) + builder.data(str(text)) + builder.end('text') + +def title(s): + "Add a title to the drawing." + builder.start("text", dict(style="font-size: 40px; text-anchor: middle; %s" % font, x=str(width/2), y="40")) + builder.data(s) + builder.end("text") + +def arrow((x0, y0), (x1, y1)): + "Draw a drafting-style arrow from (x0, y0) to (x1, y1)." + direction = normalize(displacement((x0, y0), (x1, y1))) + dx2, dy2 = rotate(direction, (-3.5, -2)) + dx3, dy3 = rotate(direction, (-3.5, 2)) + path(['M', x0, y0, 'L', x1, y1, 'm', dx2, dy2, 'l', -dx2, -dy2, 'l', dx3, dy3]) + +def displacement((x0, y0), (x1, y1)): + "Compute displacement 2-vector from first to second argument." + return x1 - x0, y1 - y0 + +def normalize((x, y)): + "Normalize a 2-vector to L₂-norm of unity, yielding direction cosines." + mag = (x**2 + y**2)**0.5 + return x/mag, y/mag + +assert normalize((0, 1)) == (0, 1) +assert normalize((1, 0)) == (1, 0) +assert normalize((-1, 0)) == (-1, 0) + +def rotate((cos, sin), (x, y)): + "Rotate a 2-vector to the angle specified by given direction cosines." + return (cos * x - sin * y), (sin * x + cos * y) + +assert rotate((1, 0), (.2, .3)) == (.2, .3) + +def box((x0, y0), (x1, y1), attrs={}): + "Draw a paraxial box." + path(['M', x0, y0, 'L', x1, y0, 'L', x1, y1, 'L', x0, y1, 'Z'], attrs) + +def path(d, attrs=dict(fill="none", stroke="#000")): + attrs = attrs.copy() + attrs['d'] = ' '.join(map(str, d)) + builder.start("path", attrs) + builder.end("path") + +if __name__ == '__main__': main() diff --git a/incomplete/tiled-renderer/index.py b/incomplete/tiled-renderer/index.py index c1c29918d..5dd303eb3 100644 --- a/incomplete/tiled-renderer/index.py +++ b/incomplete/tiled-renderer/index.py @@ -1,5 +1,5 @@ import os -import urllib +import urllib.request, urllib.parse, urllib.error import jinja2 import webapp2 diff --git a/incomplete/tiled-renderer/index.py.bak b/incomplete/tiled-renderer/index.py.bak new file mode 100644 index 000000000..c1c29918d --- /dev/null +++ b/incomplete/tiled-renderer/index.py.bak @@ -0,0 +1,17 @@ +import os +import urllib + +import jinja2 +import webapp2 + +JINJA_ENVIRONMENT = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), + extensions=['jinja2.ext.autoescape'], + autoescape=True) + +class MainPage(webapp2.RequestHandler): + def get(self): + template = JINJA_ENVIRONMENT.get_template("index.html") + self.response.write(template.render()) + +application = webapp2.WSGIApplication([('/', MainPage)], debug=True) diff --git a/interpreter/code/byterun/__main__.py b/interpreter/code/byterun/__main__.py index a5f406e70..66d127dc6 100644 --- a/interpreter/code/byterun/__main__.py +++ b/interpreter/code/byterun/__main__.py @@ -1,7 +1,7 @@ """A main program for Byterun.""" import sys, imp -from pyvm2 import VirtualMachine +from .pyvm2 import VirtualMachine def run_python_file(filename): """Run a python file as if it were the main program on the command line. diff --git a/interpreter/code/byterun/__main__.py.bak b/interpreter/code/byterun/__main__.py.bak new file mode 100644 index 000000000..a5f406e70 --- /dev/null +++ b/interpreter/code/byterun/__main__.py.bak @@ -0,0 +1,26 @@ +"""A main program for Byterun.""" + +import sys, imp +from pyvm2 import VirtualMachine + +def run_python_file(filename): + """Run a python file as if it were the main program on the command line. + `filename` is the path to the file to execute. + """ + old_main_mod = sys.modules['__main__'] + main_mod = imp.new_module('__main__') # Create a module to serve as __main__ + sys.modules['__main__'] = main_mod + main_mod.__builtins__ = sys.modules['builtins'] + + with open(filename, 'rU') as f: + source = f.read() + + if not source or source[-1] != '\n': + source += '\n' # `compile` needs the last line to be clean + code = compile(source, filename, "exec") + + vm = VirtualMachine() + vm.run_code(code, global_names=main_mod.__dict__) + +if __name__ == '__main__': + run_python_file(sys.argv[1]) diff --git a/interpreter/code/byterun/pyvm2.py b/interpreter/code/byterun/pyvm2.py index 954d82c3c..5f725989b 100644 --- a/interpreter/code/byterun/pyvm2.py +++ b/interpreter/code/byterun/pyvm2.py @@ -78,18 +78,18 @@ class Function(object): def __init__(self, name, code, globs, defaults, closure, vm): self._vm = vm - self.func_code = code - self.func_name = self.__name__ = name or code.co_name - self.func_defaults = tuple(defaults) - self.func_globals = globs + self.__code__ = code + self.__name__ = self.__name__ = name or code.co_name + self.__defaults__ = tuple(defaults) + self.__globals__ = globs self.func_locals = self._vm.frame.local_names self.__dict__ = {} - self.func_closure = closure + self.__closure__ = closure self.__doc__ = code.co_consts[0] if code.co_consts else None # Sometimes, we need a real Python function. This is for that. kw = { - 'argdefs': self.func_defaults, + 'argdefs': self.__defaults__, } if closure: kw['closure'] = tuple(make_cell(0) for _ in closure) @@ -98,7 +98,7 @@ def __init__(self, name, code, globs, defaults, closure, vm): def __call__(self, *args, **kwargs): callargs = inspect.getcallargs(self._func, *args, **kwargs) frame = self._vm.make_frame( - self.func_code, callargs, self.func_globals, {} + self.__code__, callargs, self.__globals__, {} ) return self._vm.run_frame(frame) diff --git a/interpreter/code/byterun/pyvm2.py.bak b/interpreter/code/byterun/pyvm2.py.bak new file mode 100644 index 000000000..954d82c3c --- /dev/null +++ b/interpreter/code/byterun/pyvm2.py.bak @@ -0,0 +1,589 @@ +"""A pure-Python Python bytecode interpreter.""" +# Adapted from: +# 1. pyvm2 by Paul Swartz (z3p), from http://www.twistedmatrix.com/users/z3p/ +# 2. byterun by Ned Batchelder, github.com/nedbat/byterun + +import dis, operator, sys, collections, inspect, types + +class Frame(object): + def __init__(self, code_obj, global_names, local_names, prev_frame): + self.code_obj = code_obj + self.global_names = global_names + self.local_names = local_names + self.prev_frame = prev_frame + self.stack = [] + if prev_frame: + self.builtin_names = prev_frame.builtin_names + else: + self.builtin_names = local_names['__builtins__'] + if hasattr(self.builtin_names, '__dict__'): + self.builtin_names = self.builtin_names.__dict__ + + self.last_instruction = 0 + self.block_stack = [] + + # Data stack manipulation + def top(self): + return self.stack[-1] + + def pop(self): + return self.stack.pop() + + def push(self, *vals): + self.stack.extend(vals) + + def popn(self, n): + """Pop a number of values from the value stack. + A list of `n` values is returned, the deepest value first. + """ + if n: + ret = self.stack[-n:] + self.stack[-n:] = [] + return ret + else: + return [] + + # Block stack manipulation + def push_block(self, b_type, handler=None): + stack_height = len(self.stack) + self.block_stack.append(Block(b_type, handler, stack_height)) + + def pop_block(self): + return self.block_stack.pop() + + def unwind_block(self, block): + """Unwind the values on the data stack when a given block is finished.""" + if block.type == 'except-handler': + offset = 3 + else: + offset = 0 + + while len(self.stack) > block.stack_height + offset: + self.pop() + + if block.type == 'except-handler': + traceback, value, exctype = self.popn(3) + return exctype, value, traceback + +Block = collections.namedtuple("Block", "type, handler, stack_height") + +class Function(object): + __slots__ = [ + 'func_code', 'func_name', 'func_defaults', 'func_globals', + 'func_locals', 'func_dict', 'func_closure', + '__name__', '__dict__', '__doc__', + '_vm', '_func', + ] + + + def __init__(self, name, code, globs, defaults, closure, vm): + self._vm = vm + self.func_code = code + self.func_name = self.__name__ = name or code.co_name + self.func_defaults = tuple(defaults) + self.func_globals = globs + self.func_locals = self._vm.frame.local_names + self.__dict__ = {} + self.func_closure = closure + self.__doc__ = code.co_consts[0] if code.co_consts else None + + # Sometimes, we need a real Python function. This is for that. + kw = { + 'argdefs': self.func_defaults, + } + if closure: + kw['closure'] = tuple(make_cell(0) for _ in closure) + self._func = types.FunctionType(code, globs, **kw) + + def __call__(self, *args, **kwargs): + callargs = inspect.getcallargs(self._func, *args, **kwargs) + frame = self._vm.make_frame( + self.func_code, callargs, self.func_globals, {} + ) + return self._vm.run_frame(frame) + +def make_cell(value): + # Thanks to Alex Gaynor for help with this bit of twistiness. + fn = (lambda x: lambda: x)(value) + return fn.__closure__[0] + + + +class VirtualMachineError(Exception): + pass + +class VirtualMachine(object): + def __init__(self): + self.frames = [] # The call stack of frames. + self.frame = None # The current frame. + self.return_value = None + self.last_exception = None + + # Frame manipulation + def make_frame(self, code, callargs={}, global_names=None, local_names=None): + if global_names is not None and local_names is not None: + local_names = global_names + elif self.frames: + global_names = self.frame.global_names + local_names = {} + else: + global_names = local_names = { + '__builtins__': __builtins__, + '__name__': '__main__', + '__doc__': None, + '__package__': None, + } + local_names.update(callargs) + frame = Frame(code, global_names, local_names, self.frame) + return frame + + def push_frame(self, frame): + self.frames.append(frame) + self.frame = frame + + def pop_frame(self): + self.frames.pop() + if self.frames: + self.frame = self.frames[-1] + else: + self.frame = None + + # Jumping through bytecode + def jump(self, jump): + """Move the bytecode pointer to `jump`, so it will execute next.""" + self.frame.last_instruction = jump + + def run_code(self, code, global_names=None, local_names=None): + """ An entry point to execute code using the virtual machine.""" + frame = self.make_frame(code, global_names=global_names, local_names=local_names) + + self.run_frame(frame) + # Check some invariants + # if self.frames: + # raise VirtualMachineError("Frames left over!") + # if self.frame and self.frame.stack: + # raise VirtualMachineError("Data left on stack! %r" % self.frame.stack) + + # for testing, was val = self.run_frame(frame) + # return val # for testing + + def parse_byte_and_args(self): + f = self.frame + opoffset = f.last_instruction + byteCode = f.code_obj.co_code[opoffset] + f.last_instruction += 1 + byte_name = dis.opname[byteCode] + if byteCode >= dis.HAVE_ARGUMENT: + arg = f.code_obj.co_code[f.last_instruction:f.last_instruction+2] # index into the bytecode + f.last_instruction += 2 # advance the instruction pointer + arg_val = arg[0] + (arg[1] << 8) + if byteCode in dis.hasconst: # Look up a constant + arg = f.code_obj.co_consts[arg_val] + elif byteCode in dis.hasname: # Look up a name + arg = f.code_obj.co_names[arg_val] + elif byteCode in dis.haslocal: # Look up a local name + arg = f.code_obj.co_varnames[arg_val] + elif byteCode in dis.hasjrel: # Calculate a relative jump + arg = f.last_instruction + arg_val + else: + arg = arg_val + argument = [arg] + else: + argument = [] + + return byte_name, argument + + def dispatch(self, byte_name, argument): + """ Dispatch by bytename to the corresponding methods. + Exceptions are caught and set on the virtual machine.""" + + # When later unwinding the block stack, + # we need to keep track of why we are doing it. + why = None + try: + bytecode_fn = getattr(self, 'byte_%s' % byte_name, None) + if bytecode_fn is None: + if byte_name.startswith('UNARY_'): + self.unaryOperator(byte_name[6:]) + elif byte_name.startswith('BINARY_'): + self.binaryOperator(byte_name[7:]) + else: + raise VirtualMachineError( + "unsupported bytecode type: %s" % byte_name + ) + else: + why = bytecode_fn(*argument) + except: + # deal with exceptions encountered while executing the op. + self.last_exception = sys.exc_info()[:2] + (None,) + why = 'exception' + + return why + + def manage_block_stack(self, why): + block = self.frame.block_stack[-1] + + if block.type == 'loop' and why == 'continue': + self.jump(self.return_value) + why = None + return why + + self.frame.pop_block() + current_exc = self.frame.unwind_block(block) + if current_exc is not None: + self.last_exception = current_exc + + if block.type == 'loop' and why == 'break': + self.jump(block.handler) + why = None + + elif (block.type in ['setup-except', 'finally'] and why == 'exception'): + self.frame.push_block('except-handler') + exctype, value, tb = self.last_exception + self.frame.push(tb, value, exctype) + self.frame.push(tb, value, exctype) # yes, twice + self.jump(block.handler) + why = None + + elif block.type == 'finally': + if why in ('return', 'continue'): + self.frame.push(self.return_value) + self.frame.push(why) + self.jump(block.handler) + why = None + + return why + + + def run_frame(self, frame): + """Run a frame until it returns (somehow). + Exceptions are raised, the return value is returned. + """ + self.push_frame(frame) + while True: + byte_name, argument = self.parse_byte_and_args() + + why = self.dispatch(byte_name, argument) + + # Deal with any block management we need to do + while why and frame.block_stack: + why = self.manage_block_stack(why) + + if why: + break + + self.pop_frame() + + if why == 'exception': + exc, val, tb = self.last_exception + e = exc(val) + e.__traceback__ = tb + raise e + + return self.return_value + + ## Stack manipulation + + def byte_LOAD_CONST(self, const): + self.frame.push(const) + + def byte_POP_TOP(self): + self.frame.pop() + + def byte_DUP_TOP(self): + self.frame.push(self.frame.top()) + + ## Names + def byte_LOAD_NAME(self, name): + frame = self.frame + if name in frame.local_names: + val = frame.local_names[name] + elif name in frame.global_names: + val = frame.global_names[name] + elif name in frame.builtin_names: + val = frame.builtin_names[name] + else: + raise NameError("name '%s' is not defined" % name) + self.frame.push(val) + + def byte_STORE_NAME(self, name): + self.frame.local_names[name] = self.frame.pop() + + def byte_DELETE_NAME(self, name): + del self.frame.local_names[name] + + def byte_LOAD_FAST(self, name): + if name in self.frame.local_names: + val = self.frame.local_names[name] + else: + raise UnboundLocalError( + "local variable '%s' referenced before assignment" % name + ) + self.frame.push(val) + + def byte_STORE_FAST(self, name): + self.frame.local_names[name] = self.frame.pop() + + def byte_LOAD_GLOBAL(self, name): + f = self.frame + if name in f.global_names: + val = f.global_names[name] + elif name in f.builtin_names: + val = f.builtin_names[name] + else: + raise NameError("global name '%s' is not defined" % name) + f.push(val) + + ## Operators + + UNARY_OPERATORS = { + 'POSITIVE': operator.pos, + 'NEGATIVE': operator.neg, + 'NOT': operator.not_, + 'INVERT': operator.invert, + } + + def unaryOperator(self, op): + x = self.frame.pop() + self.frame.push(self.UNARY_OPERATORS[op](x)) + + BINARY_OPERATORS = { + 'POWER': pow, + 'MULTIPLY': operator.mul, + 'FLOOR_DIVIDE': operator.floordiv, + 'TRUE_DIVIDE': operator.truediv, + 'MODULO': operator.mod, + 'ADD': operator.add, + 'SUBTRACT': operator.sub, + 'SUBSCR': operator.getitem, + 'LSHIFT': operator.lshift, + 'RSHIFT': operator.rshift, + 'AND': operator.and_, + 'XOR': operator.xor, + 'OR': operator.or_, + } + + def binaryOperator(self, op): + x, y = self.frame.popn(2) + self.frame.push(self.BINARY_OPERATORS[op](x, y)) + + COMPARE_OPERATORS = [ + operator.lt, + operator.le, + operator.eq, + operator.ne, + operator.gt, + operator.ge, + lambda x, y: x in y, + lambda x, y: x not in y, + lambda x, y: x is y, + lambda x, y: x is not y, + lambda x, y: issubclass(x, Exception) and issubclass(x, y), + ] + + def byte_COMPARE_OP(self, opnum): + x, y = self.frame.popn(2) + self.frame.push(self.COMPARE_OPERATORS[opnum](x, y)) + + ## Attributes and indexing + + def byte_LOAD_ATTR(self, attr): + obj = self.frame.pop() + val = getattr(obj, attr) + self.frame.push(val) + + def byte_STORE_ATTR(self, name): + val, obj = self.frame.popn(2) + setattr(obj, name, val) + + def byte_STORE_SUBSCR(self): + val, obj, subscr = self.frame.popn(3) + obj[subscr] = val + + ## Building + + def byte_BUILD_TUPLE(self, count): + elts = self.frame.popn(count) + self.frame.push(tuple(elts)) + + def byte_BUILD_LIST(self, count): + elts = self.frame.popn(count) + self.frame.push(elts) + + def byte_BUILD_MAP(self, size): + self.frame.push({}) + + def byte_STORE_MAP(self): + the_map, val, key = self.frame.popn(3) + the_map[key] = val + self.frame.push(the_map) + + def byte_UNPACK_SEQUENCE(self, count): + seq = self.frame.pop() + for x in reversed(seq): + self.frame.push(x) + + def byte_BUILD_SLICE(self, count): + if count == 2: + x, y = self.frame.popn(2) + self.frame.push(slice(x, y)) + elif count == 3: + x, y, z = self.frame.popn(3) + self.frame.push(slice(x, y, z)) + else: # pragma: no cover + raise VirtualMachineError("Strange BUILD_SLICE count: %r" % count) + + def byte_LIST_APPEND(self, count): + val = self.frame.pop() + the_list = self.frame.stack[-count] # peek + the_list.append(val) + + + ## Jumps + + def byte_JUMP_FORWARD(self, jump): + self.jump(jump) + + def byte_JUMP_ABSOLUTE(self, jump): + self.jump(jump) + + def byte_POP_JUMP_IF_TRUE(self, jump): + val = self.frame.pop() + if val: + self.jump(jump) + + def byte_POP_JUMP_IF_FALSE(self, jump): + val = self.frame.pop() + if not val: + self.jump(jump) + + def byte_JUMP_IF_TRUE_OR_POP(self, jump): + val = self.frame.top() + if val: + self.jump(jump) + else: + self.frame.pop() + + def byte_JUMP_IF_FALSE_OR_POP(self, jump): + val = self.frame.top() + if not val: + self.jump(jump) + else: + self.frame.pop() + + ## Blocks + + def byte_SETUP_LOOP(self, dest): + self.frame.push_block('loop', dest) + + def byte_GET_ITER(self): + self.frame.push(iter(self.frame.pop())) + + def byte_FOR_ITER(self, jump): + iterobj = self.frame.top() + try: + v = next(iterobj) + self.frame.push(v) + except StopIteration: + self.frame.pop() + self.jump(jump) + + def byte_BREAK_LOOP(self): + return 'break' + + def byte_CONTINUE_LOOP(self, dest): + # This is a trick with the return value. + # While unrolling blocks, continue and return both have to preserve + # state as the finally blocks are executed. For continue, it's + # where to jump to, for return, it's the value to return. It gets + # pushed on the stack for both, so continue puts the jump destination + # into return_value. + self.return_value = dest + return 'continue' + + def byte_SETUP_EXCEPT(self, dest): + self.frame.push_block('setup-except', dest) + + def byte_SETUP_FINALLY(self, dest): + self.frame.push_block('finally', dest) + + def byte_POP_BLOCK(self): + self.frame.pop_block() + + def byte_RAISE_VARARGS(self, argc): + cause = exc = None + if argc == 2: + cause = self.frame.pop() + exc = self.frame.pop() + elif argc == 1: + exc = self.frame.pop() + return self.do_raise(exc, cause) + + def do_raise(self, exc, cause): + if exc is None: # reraise + exc_type, val, tb = self.last_exception + + elif type(exc) == type: # As in `raise ValueError` + exc_type = exc + val = exc() # Make an instance. + elif isinstance(exc, BaseException): + # As in `raise ValueError('foo')` + exc_type = type(exc) + val = exc + else: + return 'exception' # failure + + self.last_exception = exc_type, val, val.__traceback__ + return 'exception' + + def byte_POP_EXCEPT(self): + block = self.frame.pop_block() + if block.type != 'except-handler': + raise Exception("popped block is not an except handler") + current_exc = self.frame.unwind_block(block) + if current_exc is not None: + self.last_exception = current_exc + + ## Functions + + def byte_MAKE_FUNCTION(self, argc): + name = self.frame.pop() + code = self.frame.pop() + defaults = self.frame.popn(argc) + globs = self.frame.global_names + #TODO: if we're not supporting kwargs, do we need the defaults? + fn = Function(name, code, globs, defaults, None, self) + self.frame.push(fn) + + def byte_CALL_FUNCTION(self, arg): + lenKw, lenPos = divmod(arg, 256) # KWargs not supported in byterun + posargs = self.frame.popn(lenPos) + + func = self.frame.pop() + frame = self.frame + retval = func(*posargs) + self.frame.push(retval) + + def byte_RETURN_VALUE(self): + self.return_value = self.frame.pop() + return "return" + + ## Importing + + def byte_IMPORT_NAME(self, name): + level, fromlist = self.frame.popn(2) + frame = self.frame + self.frame.push(__import__(name, frame.global_names, frame.local_names, fromlist, level)) + + def byte_IMPORT_FROM(self, name): + mod = self.frame.top() + self.frame.push(getattr(mod, name)) + + ## And the rest... + def byte_LOAD_BUILD_CLASS(self): + self.frame.push(__build_class__) + + def byte_STORE_LOCALS(self): + self.frame.local_names = self.frame.pop() + + diff --git a/interpreter/code/check_completeness.py b/interpreter/code/check_completeness.py index 949e60316..4ffbb33cb 100644 --- a/interpreter/code/check_completeness.py +++ b/interpreter/code/check_completeness.py @@ -12,12 +12,12 @@ for line in source: if line and line not in chapter_set: if zero_missing: - print "Missing lines of source!" + print("Missing lines of source!") zero_missing = False - print line, + print(line, end=' ') for line in chapter: if line not in source_set and line not in ["```", "~~~~"]: - print line, + print(line, end=' ') diff --git a/interpreter/code/check_completeness.py.bak b/interpreter/code/check_completeness.py.bak new file mode 100644 index 000000000..949e60316 --- /dev/null +++ b/interpreter/code/check_completeness.py.bak @@ -0,0 +1,23 @@ + + +if __name__ == '__main__': + with open('chapter.txt') as f: + chapter = f.readlines() + chapter_set = set(chapter) + with open('byterun/pyvm2.py') as g: + source = g.readlines() + source_set = set(source) + + zero_missing = True + for line in source: + if line and line not in chapter_set: + if zero_missing: + print "Missing lines of source!" + zero_missing = False + print line, + + for line in chapter: + if line not in source_set and line not in ["```", "~~~~"]: + print line, + + diff --git a/interpreter/code/conditionals_interpreter.py b/interpreter/code/conditionals_interpreter.py index 18f21df85..c43518115 100644 --- a/interpreter/code/conditionals_interpreter.py +++ b/interpreter/code/conditionals_interpreter.py @@ -28,7 +28,7 @@ def BINARY_LESS_THAN(self): def STORE_NAME(self, name): val = self.stack.pop() - print("storing name %s: %s" % (name, val)) + print(("storing name %s: %s" % (name, val))) self.environment[name] = val def LOAD_NAME(self, name): @@ -67,10 +67,10 @@ def execute(self, what_to_execute): while True: if self.next_i >= len(instructions) or self.should_stop: break - print(self.next_i) - print(self.environment) + print((self.next_i)) + print((self.environment)) instruction, argument = instructions[self.next_i] - print(instructions[self.next_i]) + print((instructions[self.next_i])) argument = self.parse_argument(instruction, argument, what_to_execute) if instruction == "LOAD_VALUE": diff --git a/interpreter/code/conditionals_interpreter.py.bak b/interpreter/code/conditionals_interpreter.py.bak new file mode 100644 index 000000000..18f21df85 --- /dev/null +++ b/interpreter/code/conditionals_interpreter.py.bak @@ -0,0 +1,164 @@ +# Abandoned this because calculating jump targets by hand is *so tedious* + +class SimpleInterpreter(object): + def __init__(self): + self.stack = [] + self.environment = {} + self.should_stop = False + # instruction_map = {1: self.LOAD_VALUE, + # 2: self.ADD_TWO_VALUES, + # 3: self.POP_FROM_STACK } + + def LOAD_VALUE(self, number): + self.stack.append(number) + + def PRINT_ANSWER(self): + answer = self.stack.pop() + print(answer) + + def ADD_TWO_VALUES(self): + first_num = self.stack.pop() + second_num = self.stack.pop() + total = first_num + second_num + self.stack.append(total) + + def BINARY_LESS_THAN(self): + a, b = self.stack.pop(), self.stack.pop() + self.stack.append((a < b)) + + def STORE_NAME(self, name): + val = self.stack.pop() + print("storing name %s: %s" % (name, val)) + self.environment[name] = val + + def LOAD_NAME(self, name): + val = self.environment[name] + self.stack.append(val) + + def JUMP(self, jump_target): + self.next_i = jump_target + + def JUMP_IF_FALSE(self, jump_target): + cond = self.stack.pop() + if not cond: + self.next_i = jump_target + + def RETURN(self): + self.should_stop = True + + def parse_argument(self, instruction, argument, what_to_execute): + argument_meaning = {"values": ["LOAD_VALUE"], + "names": ["LOAD_NAME", "STORE_NAME"], + "jumps": ["JUMP_IF_FALSE", "JUMP"]} + + if instruction in argument_meaning["values"]: + argument = what_to_execute["values"][argument] + elif instruction in argument_meaning["names"]: + argument = what_to_execute["names"][argument] + elif instruction in argument_meaning["jumps"]: + pass + + return argument + + + def execute(self, what_to_execute): + instructions = what_to_execute["instructions"] + self.next_i = 0 + while True: + if self.next_i >= len(instructions) or self.should_stop: + break + print(self.next_i) + print(self.environment) + instruction, argument = instructions[self.next_i] + print(instructions[self.next_i]) + argument = self.parse_argument(instruction, argument, what_to_execute) + + if instruction == "LOAD_VALUE": + self.LOAD_VALUE(argument) + elif instruction == "ADD_TWO_VALUES": + self.ADD_TWO_VALUES() + elif instruction == "PRINT_ANSWER": + self.PRINT_ANSWER() + elif instruction == "STORE_NAME": + self.STORE_NAME(argument) + elif instruction == "LOAD_NAME": + self.LOAD_NAME(argument) + elif instruction == "JUMP_IF_FALSE": + self.JUMP_IF_FALSE(argument) + elif instruction == "JUMP": + self.JUMP(argument) + elif instruction == "RETURN": + self.RETURN() + + self.next_i += 1 + +def test_simple_interpreter(): + def test_conditional_true(): + what_to_execute = { + "instructions": [("LOAD_VALUE", 0), + ("STORE_NAME", 0), + ("LOAD_NAME", 0), + ("JUMP_IF_FALSE", 6), + ("LOAD_VALUE", 1), + ("PRINT_ANSWER", None), + ("RETURN", None), + ("LOAD_VALUE", 2), + ("PRINT_ANSWER", None), + ("RETURN", None)], + "values": [True, 'yes', 'no'], + "names": ["a"] } + + interpreter = SimpleInterpreter() + interpreter.execute(what_to_execute) + print("== 'yes'") + test_conditional_true() + + def test_conditional_false(): + what_to_execute = { + "instructions": [("LOAD_VALUE", 0), + ("STORE_NAME", 0), + ("LOAD_NAME", 0), + ("JUMP_IF_FALSE", 6), + ("LOAD_VALUE", 1), + ("PRINT_ANSWER", None), + ("RETURN", None), + ("LOAD_VALUE", 2), + ("PRINT_ANSWER", None), + ("RETURN", None)], + "values": [False, 'yes', 'no'], + "names": ["a"] } + interpreter = SimpleInterpreter() + interpreter.execute(what_to_execute) + print("== 'no'") + test_conditional_false() + + # def test_loop(): + # >>> def loop(): + # x = 1 + # ... while x < 5: + # ... print("going") + # what_to_execute = { + # "instructions": [("LOAD_VALUE", 0), + # ("STORE_NAME", 0), + # ("LOAD_NAME", 0), + # ("LOAD_VALUE", 1), + # ("BINARY_LESS_THAN", None), + # ("JUMP_IF_FALSE", 8), # <- decide whether or not to jump FIXME + # ("LOAD_VALUE", 2), + # ("PRINT_ANSWER", None), + # ("LOAD_NAME", 0), + # ("LOAD_VALUE", 1), + # ("ADD_TWO_VALUES", None), + # ("STORE_NAME", 0), + # ("JUMP", 1), # <- Jump to top of loop + # ("RETURN", None)], # <- Jump to here when done looping + # "values": [1, 5, 'going'], + # "names": ["x"] } + # interpreter = SimpleInterpreter() + # interpreter.execute(what_to_execute) + # print("==" + "going\n"*5) + # test_loop() + +if __name__ == '__main__': + test_simple_interpreter() + diff --git a/interpreter/code/simplest_interpreter.py b/interpreter/code/simplest_interpreter.py index 4b5832bfe..c8765971f 100644 --- a/interpreter/code/simplest_interpreter.py +++ b/interpreter/code/simplest_interpreter.py @@ -21,7 +21,7 @@ def ADD_TWO_VALUES(self): def STORE_NAME(self, name): val = self.stack.pop() - print("storing name %s: %s" % (name, val)) + print(("storing name %s: %s" % (name, val))) self.environment[name] = val def LOAD_NAME(self, name): diff --git a/interpreter/code/simplest_interpreter.py.bak b/interpreter/code/simplest_interpreter.py.bak new file mode 100644 index 000000000..4b5832bfe --- /dev/null +++ b/interpreter/code/simplest_interpreter.py.bak @@ -0,0 +1,139 @@ +class SimpleInterpreter(object): + def __init__(self): + self.stack = [] + self.environment = {} + # instruction_map = {1: self.LOAD_VALUE, + # 2: self.ADD_TWO_VALUES, + # 3: self.POP_FROM_STACK } + + def LOAD_VALUE(self, number): + self.stack.append(number) + + def PRINT_ANSWER(self): + answer = self.stack.pop() + print(answer) + + def ADD_TWO_VALUES(self): + first_num = self.stack.pop() + second_num = self.stack.pop() + total = first_num + second_num + self.stack.append(total) + + def STORE_NAME(self, name): + val = self.stack.pop() + print("storing name %s: %s" % (name, val)) + self.environment[name] = val + + def LOAD_NAME(self, name): + val = self.environment[name] + self.stack.append(val) + + def JUMP_IF_FALSE(self, jump_target): + cond = self.stack.pop() + if not cond: + self.next_i = jump_target + + def STOP(self): + self.should_stop = True + + def parse_argument(self, instruction, argument, what_to_execute): + argument_meaning = {"numbers": ["LOAD_VALUE"], + "names": ["LOAD_NAME", "STORE_NAME"], + "jumps": ["JUMP_IF_FALSE"]} + + if instruction in argument_meaning["numbers"]: + argument = what_to_execute["numbers"][argument] + elif instruction in argument_meaning["names"]: + argument = what_to_execute["names"][argument] + elif instruction in argument_meaning["jumps"]: + pass + + return argument + + + # def execute(self, what_to_execute): + # instructions = what_to_execute["instructions"] + # self.next_i = 0 + # while True: + # if self.next_i >= len(instructions) or self.should_stop: + # break + # instruction, argument = instructions[self.next_i] + # argument = self.parse_argument(instruction, argument, what_to_execute) + + # if instruction == "LOAD_VALUE": + # self.LOAD_VALUE(argument) + # elif instruction == "ADD_TWO_VALUES": + # self.ADD_TWO_VALUES() + # elif instruction == "PRINT_ANSWER": + # self.PRINT_ANSWER() + # elif instruction == "STORE_NAME": + # self.STORE_NAME(argument) + # elif instruction == "LOAD_NAME": + # self.LOAD_NAME(argument) + # elif instruction == "JUMP_IF_FALSE": + # self.JUMP_IF_FALSE(argument) + # elif instruction == "STOP": + # self.STOP() + + # self.next_i += 1 + + def execute(self, what_to_execute): + instructions = what_to_execute["instructions"] + for each_step in instructions: + instruction, argument = each_step + argument = self.parse_argument(instruction, argument, what_to_execute) + bytecode_method = getattr(self, instruction) + if argument is None: + bytecode_method() + else: + bytecode_method(argument) + +def test_simple_interpreter(): + simple = SimpleInterpreter() + what_to_execute = { + "instructions": [("LOAD_VALUE", 0), # the first number + ("LOAD_VALUE", 1), # the second number + ("ADD_TWO_VALUES", None), + ("PRINT_ANSWER", None)], + "numbers": [7,5] } + simple.execute(what_to_execute) + print(" == 12") + + what_to_execute = { + "instructions": [("LOAD_VALUE", 0), # the first number + ("LOAD_VALUE", 1), # the second number + ("ADD_TWO_VALUES", None), + ("LOAD_VALUE", 2), # the second number + ("ADD_TWO_VALUES", None), + ("PRINT_ANSWER", None)], + "numbers": [7,5, 8] } + simple.execute(what_to_execute) + print(" == 20") + + what_to_execute = { + "instructions": [("LOAD_VALUE", 0), + ("STORE_NAME", 0), + ("LOAD_VALUE", 1), + ("STORE_NAME", 1), + ("LOAD_NAME", 0), + ("LOAD_NAME", 1), + ("ADD_TWO_VALUES", None), + ("PRINT_ANSWER", None)], + "numbers": [1, 2], + "names": ["a", "b"] } + simple.execute(what_to_execute) + print(" == 3") + + what_to_execute = { + "instructions": [("LOAD_VALUE", 0), + ("JUMP_IF_FALSE", None), + ("LOAD_VALUE", 1), # the second number + ("PRINT_ANSWER", None), + ("LOAD_VALUE", 2), # the second number + ("PRINT_ANSWER", None)], + "values": [True, 'yes', 'no'] } + + +if __name__ == '__main__': + test_simple_interpreter() + diff --git a/interpreter/code/tests/test_basic.py b/interpreter/code/tests/test_basic.py index c5187c6ca..50d944a38 100644 --- a/interpreter/code/tests/test_basic.py +++ b/interpreter/code/tests/test_basic.py @@ -1,6 +1,6 @@ """Basic tests for Byterun.""" -from __future__ import print_function + from . import vmtest class TestIt(vmtest.VmTestCase): diff --git a/interpreter/code/tests/test_basic.py.bak b/interpreter/code/tests/test_basic.py.bak new file mode 100644 index 000000000..c5187c6ca --- /dev/null +++ b/interpreter/code/tests/test_basic.py.bak @@ -0,0 +1,489 @@ +"""Basic tests for Byterun.""" + +from __future__ import print_function +from . import vmtest + +class TestIt(vmtest.VmTestCase): + def test_constant(self): + self.assert_ok("17") + + def test_for_loop(self): + self.assert_ok("""\ + out = "" + for i in range(5): + out = out + str(i) + print(out) + """) + + # def test_inplace_operators(self): + # self.assert_ok("""\ + # x, y = 2, 3 + # x **= y + # assert x == 8 and y == 3 + # x *= y + # assert x == 24 and y == 3 + # x //= y + # assert x == 8 and y == 3 + # x %= y + # assert x == 2 and y == 3 + # x += y + # assert x == 5 and y == 3 + # x -= y + # assert x == 2 and y == 3 + # x <<= y + # assert x == 16 and y == 3 + # x >>= y + # assert x == 2 and y == 3 + + # x = 0x8F + # x &= 0xA5 + # assert x == 0x85 + # x |= 0x10 + # assert x == 0x95 + # x ^= 0x33 + # assert x == 0xA6 + # """) + + # def test_inplace_division(self): + # self.assert_ok("""\ + # x, y = 24, 3 + # x /= y + # assert x == 8.0 and y == 3 + # assert isinstance(x, float) + # x /= y + # assert x == (8.0/3.0) and y == 3 + # assert isinstance(x, float) + # """) + + def test_slice(self): + self.assert_ok("""\ + print("hello, world"[3:8]) + """) + self.assert_ok("""\ + print("hello, world"[:8]) + """) + self.assert_ok("""\ + print("hello, world"[3:]) + """) + self.assert_ok("""\ + print("hello, world"[:]) + """) + self.assert_ok("""\ + print("hello, world"[::-1]) + """) + self.assert_ok("""\ + print("hello, world"[3:8:2]) + """) + + def test_slice_assignment(self): + self.assert_ok("""\ + l = list(range(10)) + l[3:8] = ["x"] + print(l) + """) + self.assert_ok("""\ + l = list(range(10)) + l[:8] = ["x"] + print(l) + """) + self.assert_ok("""\ + l = list(range(10)) + l[3:] = ["x"] + print(l) + """) + self.assert_ok("""\ + l = list(range(10)) + l[:] = ["x"] + print(l) + """) + + # def test_slice_deletion(self): + # self.assert_ok("""\ + # l = list(range(10)) + # del l[3:8] + # print(l) + # """) + # self.assert_ok("""\ + # l = list(range(10)) + # del l[:8] + # print(l) + # """) + # self.assert_ok("""\ + # l = list(range(10)) + # del l[3:] + # print(l) + # """) + # self.assert_ok("""\ + # l = list(range(10)) + # del l[:] + # print(l) + # """) + # self.assert_ok("""\ + # l = list(range(10)) + # del l[::2] + # print(l) + # """) + + def test_building_stuff(self): + self.assert_ok("""\ + print((1+1, 2+2, 3+3)) + """) + self.assert_ok("""\ + print([1+1, 2+2, 3+3]) + """) + self.assert_ok("""\ + print({1:1+1, 2:2+2, 3:3+3}) + """) + + def test_subscripting(self): + self.assert_ok("""\ + l = list(range(10)) + print("%s %s %s" % (l[0], l[3], l[9])) + """) + self.assert_ok("""\ + l = list(range(10)) + l[5] = 17 + print(l) + """) + # self.assert_ok("""\ + # l = list(range(10)) + # del l[5] + # print(l) + # """) + + # def test_generator_expression(self): + # self.assert_ok("""\ + # x = "-".join(str(z) for z in range(5)) + # assert x == "0-1-2-3-4" + # """) + # # From test_regr.py + # # This failed a different way than the previous join when genexps were + # # broken: + # self.assert_ok("""\ + # from textwrap import fill + # x = set(['test_str']) + # width = 70 + # indent = 4 + # blanks = ' ' * indent + # res = fill(' '.join(str(elt) for elt in sorted(x)), width, + # initial_indent=blanks, subsequent_indent=blanks) + # print(res) + # """) + def test_list_comprehension(self): + self.assert_ok("""\ + x = [z*z for z in range(5)] + assert x == [0, 1, 4, 9, 16] + """) + + # def test_dict_comprehension(self): + # """Requires MAP_ADD, not supported""" + # self.assert_ok("""\ + # x = {z:z*z for z in range(5)} + # assert x == {0:0, 1:1, 2:4, 3:9, 4:16} + # """) + + # def test_set_comprehension(self): + # """Requires BUILD_SET, not supported""" + # self.assert_ok("""\ + # x = {z*z for z in range(5)} + # assert x == {0, 1, 4, 9, 16} + # """) + + # def test_strange_sequence_ops(self): # relies on DUP_TOP_TWO + # # from stdlib: test/test_augassign.py + # self.assert_ok("""\ + # x = [1,2] + # x += [3,4] + # x *= 2 + + # assert x == [1, 2, 3, 4, 1, 2, 3, 4] + + # x = [1, 2, 3] + # y = x + # x[1:2] *= 2 + # y[1:2] += [1] + + # assert x == [1, 2, 1, 2, 3] + # assert x is y + # """) + + def test_unary_operators(self): + self.assert_ok("""\ + x = 8 + print(-x, ~x, not x) + """) + + def test_attributes(self): + self.assert_ok("""\ + l = lambda: 1 # Just to have an object... + l.foo = 17 + print(hasattr(l, "foo"), l.foo) + # del l.foo + # print(hasattr(l, "foo")) + """) + + # def test_attribute_inplace_ops(self): + # self.assert_ok("""\ + # l = lambda: 1 # Just to have an object... + # l.foo = 17 + # l.foo -= 3 + # print(l.foo) + # """) + + # def test_deleting_names(self): + # self.assert_ok("""\ + # g = 17 + # assert g == 17 + # del g + # g + # """, raises=NameError) + + # def test_deleting_local_names(self): + # self.assert_ok("""\ + # def f(): + # l = 23 + # assert l == 23 + # del l + # l + # f() + # """, raises=NameError) + + def test_import(self): + self.assert_ok("""\ + import math + print(math.pi, math.e) + from math import sqrt + print(sqrt(2)) + # from math import * # not supported + # print(sin(2)) + """) + + # def test_classes(self): + # self.assert_ok("""\ + # class Thing(object): + # def __init__(self, x): + # self.x = x + # def meth(self, y): + # return self.x * y + # thing1 = Thing(2) + # thing2 = Thing(3) + # print(thing1.x, thing2.x) + # print(thing1.meth(4), thing2.meth(5)) + # """) + + # def test_calling_methods_wrong(self): + # self.assert_ok("""\ + # class Thing(object): + # def __init__(self, x): + # self.x = x + # def meth(self, y): + # return self.x * y + # thing1 = Thing(2) + # print(Thing.meth(14)) + # """, raises=TypeError) + + # def test_calling_subclass_methods(self): + # self.assert_ok("""\ + # class Thing(object): + # def foo(self): + # return 17 + + # class SubThing(Thing): + # pass + + # st = SubThing() + # print(st.foo()) + # """) + + # def test_attribute_access(self): + # self.assert_ok("""\ + # class Thing(object): + # z = 17 + # def __init__(self): + # self.x = 23 + # t = Thing() + # print(Thing.z) + # print(t.z) + # print(t.x) + # """) + + # self.assert_ok("""\ + # class Thing(object): + # z = 17 + # def __init__(self): + # self.x = 23 + # t = Thing() + # print(t.xyzzy) + # """, raises=AttributeError) + + def test_staticmethods(self): + self.assert_ok("""\ + class Thing(object): + @staticmethod + def smeth(x): + print(x) + @classmethod + def cmeth(cls, x): + print(x) + + Thing.smeth(1492) + Thing.cmeth(1776) + """) + + def test_unbound_methods(self): + self.assert_ok("""\ + class Thing(object): + def meth(self, x): + print(x) + m = Thing.meth + m(Thing(), 1815) + """) + + # def test_callback(self): + # # Relies on KW arguments, not supported + # self.assert_ok("""\ + # def lcase(s): + # return s.lower() + # l = ["xyz", "ABC"] + # l.sort(key=lcase) + # print(l) + # assert l == ["ABC", "xyz"] + # """) + + def test_unpacking(self): + self.assert_ok("""\ + a, b, c = (1, 2, 3) + assert a == 1 + assert b == 2 + assert c == 3 + """) + + def test_exec_function(self): + self.assert_ok("""\ + g = {} + exec("a = 11", g, g) + assert g['a'] == 11 + """) + + def test_jump_if_true_or_pop(self): + self.assert_ok("""\ + def f(a, b): + return a or b + assert f(17, 0) == 17 + assert f(0, 23) == 23 + assert f(0, "") == "" + """) + + def test_jump_if_false_or_pop(self): + self.assert_ok("""\ + def f(a, b): + return not(a and b) + assert f(17, 0) is True + assert f(0, 23) is True + assert f(0, "") is True + assert f(17, 23) is False + """) + + def test_pop_jump_if_true(self): + self.assert_ok("""\ + def f(a): + if not a: + return 'foo' + else: + return 'bar' + assert f(0) == 'foo' + assert f(1) == 'bar' + """) + + # def test_decorator(self): + # self.assert_ok("""\ + # def verbose(func): + # def _wrapper(*args, **kwargs): + # return func(*args, **kwargs) + # return _wrapper + + # @verbose + # def add(x, y): + # return x+y + + # add(7, 3) + # """) + + +class TestLoops(vmtest.VmTestCase): + def test_for(self): + self.assert_ok("""\ + for i in range(10): + print(i) + print("done") + """) + + def test_break(self): + self.assert_ok("""\ + for i in range(10): + print(i) + if i == 7: + break + print("done") + """) + + def test_continue(self): + # fun fact: this doesn't use CONTINUE_LOOP + self.assert_ok("""\ + for i in range(10): + if i % 3 == 0: + continue + print(i) + print("done") + """) + + def test_continue_in_try_except(self): + self.assert_ok("""\ + for i in range(10): + try: + if i % 3 == 0: + continue + print(i) + except ValueError: + pass + print("done") + """) + + # def test_continue_in_try_finally(self): + # """Requires END_FINALLY, not supported""" + # self.assert_ok("""\ + # for i in range(10): + # try: + # if i % 3 == 0: + # continue + # print(i) + # finally: + # print(".") + # print("done") + # """) + + +class TestComparisons(vmtest.VmTestCase): + def test_in(self): + self.assert_ok("""\ + assert "x" in "xyz" + assert "x" not in "abc" + assert "x" in ("x", "y", "z") + assert "x" not in ("a", "b", "c") + """) + + def test_less(self): + self.assert_ok("""\ + assert 1 < 3 + assert 1 <= 2 and 1 <= 1 + assert "a" < "b" + assert "a" <= "b" and "a" <= "a" + """) + + def test_greater(self): + self.assert_ok("""\ + assert 3 > 1 + assert 3 >= 1 and 3 >= 3 + assert "z" > "a" + assert "z" >= "a" and "z" >= "z" + """) diff --git a/interpreter/code/tests/test_exceptions.py b/interpreter/code/tests/test_exceptions.py index cb44df10e..43ca420d8 100644 --- a/interpreter/code/tests/test_exceptions.py +++ b/interpreter/code/tests/test_exceptions.py @@ -1,6 +1,6 @@ """Test exceptions for Byterun.""" -from __future__ import print_function + from . import vmtest diff --git a/interpreter/code/tests/test_exceptions.py.bak b/interpreter/code/tests/test_exceptions.py.bak new file mode 100644 index 000000000..cb44df10e --- /dev/null +++ b/interpreter/code/tests/test_exceptions.py.bak @@ -0,0 +1,149 @@ +"""Test exceptions for Byterun.""" + +from __future__ import print_function +from . import vmtest + + +class TestExceptions(vmtest.VmTestCase): + def test_catching_exceptions(self): + # Catch the exception precisely + self.assert_ok("""\ + try: + [][1] + print("Shouldn't be here...") + except IndexError: + print("caught it!") + """) + # Catch the exception by a parent class + self.assert_ok("""\ + try: + [][1] + print("Shouldn't be here...") + except Exception: + print("caught it!") + """) + # Catch all exceptions + self.assert_ok("""\ + try: + [][1] + print("Shouldn't be here...") + except: + print("caught it!") + """) + + def test_raise_exception(self): + self.assert_ok("raise Exception('oops')", raises=Exception) + + def test_raise_exception_class(self): + self.assert_ok("raise ValueError", raises=ValueError) + + + def test_raise_exception_from(self): + self.assert_ok("""raise ValueError from NameError""", raises=ValueError) + + # def test_raise_and_catch_exception(self): + # """ Requires END_FINALLY, not supported """ + # self.assert_ok("""\ + # try: + # raise ValueError("oops") + # except ValueError as e: + # print("Caught: %s" % e) + # print("All done") + # """) + + def test_raise_exception_from(self): + self.assert_ok( + "raise ValueError from NameError", + raises=ValueError + ) + + # def test_raise_and_catch_exception_in_function(self): + # """ Requires END_FINALLY, not supported """ + # self.assert_ok("""\ + # def fn(): + # raise ValueError("oops") + + # try: + # fn() + # except ValueError as e: + # print("Caught: %s" % e) + # print("done") + # """) + + def test_global_name_error(self): + self.assert_ok("fooey", raises=NameError) + self.assert_ok("""\ + try: + fooey + print("Yes fooey?") + except NameError: + print("No fooey") + """) + + def test_local_name_error(self): + self.assert_ok("""\ + def fn(): + fooey + fn() + """, raises=NameError) + + def test_catch_local_name_error(self): + self.assert_ok("""\ + def fn(): + try: + fooey + print("Yes fooey?") + except NameError: + print("No fooey") + fn() + """) + + def test_reraise(self): + self.assert_ok("""\ + def fn(): + try: + fooey + print("Yes fooey?") + except NameError: + print("No fooey") + raise + fn() + """, raises=NameError) + + def test_reraise_explicit_exception(self): + self.assert_ok("""\ + def fn(): + try: + raise ValueError("ouch") + except ValueError as e: + print("Caught %s" % e) + raise + fn() + """, raises=ValueError) + + def test_finally_while_throwing(self): + self.assert_ok("""\ + def fn(): + try: + print("About to..") + raise ValueError("ouch") + finally: + print("Finally") + fn() + print("Done") + """, raises=ValueError) + + # def test_coverage_issue_92(self): + # """ Relies on END_FINALLY, not supported""" + # self.assert_ok("""\ + # l = [] + # for i in range(3): + # try: + # l.append(i) + # finally: + # l.append('f') + # l.append('e') + # l.append('r') + # print(l) + # assert l == [0, 'f', 'e', 1, 'f', 'e', 2, 'f', 'e', 'r'] + # """) diff --git a/interpreter/code/tests/test_functions.py b/interpreter/code/tests/test_functions.py index 03c84bbcf..9f951ab63 100644 --- a/interpreter/code/tests/test_functions.py +++ b/interpreter/code/tests/test_functions.py @@ -1,6 +1,6 @@ """Test functions etc, for Byterun.""" -from __future__ import print_function + from . import vmtest diff --git a/interpreter/code/tests/test_functions.py.bak b/interpreter/code/tests/test_functions.py.bak new file mode 100644 index 000000000..03c84bbcf --- /dev/null +++ b/interpreter/code/tests/test_functions.py.bak @@ -0,0 +1,229 @@ +"""Test functions etc, for Byterun.""" + +from __future__ import print_function +from . import vmtest + + +class TestFunctions(vmtest.VmTestCase): + def test_functions(self): + self.assert_ok("""\ + def fn(a, b=17, c="Hello", d=[]): + d.append(99) + print(a, b, c, d) + fn(1) + fn(2, 3) + # fn(3, c="Bye") # not supporting KWargs + # fn(4, d=["What?"]) + fn(5, "b", "c") + """) + + def test_recursion(self): + self.assert_ok("""\ + def fact(n): + if n <= 1: + return 1 + else: + return n * fact(n-1) + f6 = fact(6) + print(f6) + assert f6 == 720 + """) + + # def test_calling_functions_with_args_kwargs(self): + # """ KW args unsupported""" + # self.assert_ok("""\ + # def fn(a, b=17, c="Hello", d=[]): + # d.append(99) + # print(a, b, c, d) + # fn(6, *[77, 88]) + # fn(**{'c': 23, 'a': 7}) + # fn(6, *[77], **{'c': 23, 'd': [123]}) + # """) + + def test_defining_functions_with_args_kwargs(self): + self.assert_ok("""\ + def fn(*args): + print("args is %r" % (args,)) + fn(1, 2) + """) + # self.assert_ok("""\ + # def fn(**kwargs): + # print("kwargs is %r" % (kwargs,)) + # fn(red=True, blue=False) + # """) + # self.assert_ok("""\ + # def fn(*args, **kwargs): + # print("args is %r" % (args,)) + # print("kwargs is %r" % (kwargs,)) + # fn(1, 2, red=True, blue=False) + # """) + # self.assert_ok("""\ + # def fn(x, y, *args, **kwargs): + # print("x is %r, y is %r" % (x, y)) + # print("args is %r" % (args,)) + # print("kwargs is %r" % (kwargs,)) + # fn('a', 'b', 1, 2, red=True, blue=False) + # """) + + def test_defining_functions_with_empty_args_kwargs(self): + self.assert_ok("""\ + def fn(*args): + print("args is %r" % (args,)) + fn() + """) + self.assert_ok("""\ + def fn(**kwargs): + print("kwargs is %r" % (kwargs,)) + fn() + """) + self.assert_ok("""\ + def fn(*args, **kwargs): + print("args is %r, kwargs is %r" % (args, kwargs)) + fn() + """) + + def test_partial(self): + self.assert_ok("""\ + from _functools import partial + + def f(a,b): + return a-b + + f7 = partial(f, 7) + four = f7(3) + assert four == 4 + """) + + # def test_weird_splatting(self): + # self.assert_ok("""\ + # def foo(arg): + # pass + # li = [[]] + # foo(*li) + # """) + + # def test_partial_with_kwargs(self): + # """ KW args not suppoted""" + # self.assert_ok("""\ + # from _functools import partial + + # def f(a,b,c=0,d=0): + # return (a,b,c,d) + + # f7 = partial(f, b=7, c=1) + # them = f7(10) + # assert them == (10,7,1,0) + # """) + + # def test_wraps(self): + # self.assert_ok("""\ + # from functools import wraps + # def my_decorator(f): + # dec = wraps(f) + # def wrapper(*args, **kwds): + # print('Calling decorated function') + # return f(*args, **kwds) + # wrapper = dec(wrapper) + # return wrapper + + # @my_decorator + # def example(): + # '''Docstring''' + # return 17 + + # assert example() == 17 + # """) + + +# class TestClosures(vmtest.VmTestCase): +# def test_closures(self): +# self.assert_ok("""\ +# def make_adder(x): +# def add(y): +# return x+y +# return add +# a = make_adder(10) +# print(a(7)) +# assert a(7) == 17 +# """) + +# def test_closures_store_deref(self): +# self.assert_ok("""\ +# def make_adder(x): +# z = x+1 +# def add(y): +# return x+y+z +# return add +# a = make_adder(10) +# print(a(7)) +# assert a(7) == 28 +# """) + +# def test_closures_in_loop(self): +# self.assert_ok("""\ +# def make_fns(x): +# fns = [] +# for i in range(x): +# fns.append(lambda i=i: i) +# return fns +# fns = make_fns(3) +# for f in fns: +# print(f()) +# assert (fns[0](), fns[1](), fns[2]()) == (0, 1, 2) +# """) + +# def test_closures_with_defaults(self): +# self.assert_ok("""\ +# def make_adder(x, y=13, z=43): +# def add(q, r=11): +# return x+y+z+q+r +# return add +# a = make_adder(10, 17) +# print(a(7)) +# assert a(7) == 88 +# """) + +# def test_deep_closures(self): +# self.assert_ok("""\ +# def f1(a): +# b = 2*a +# def f2(c): +# d = 2*c +# def f3(e): +# f = 2*e +# def f4(g): +# h = 2*g +# return a+b+c+d+e+f+g+h +# return f4 +# return f3 +# return f2 +# answer = f1(3)(4)(5)(6) +# print(answer) +# assert answer == 54 +# """) + + +# class TestGenerators(vmtest.VmTestCase): +# def test_first(self): +# self.assert_ok("""\ +# def two(): +# yield 1 +# yield 2 +# for i in two(): +# print(i) +# """) + +# def test_partial_generator(self): +# self.assert_ok("""\ +# from _functools import partial + +# def f(a,b): +# num = a+b +# while num: +# yield num +# num -= 1 + +# f2 = partial(f, 2) +# three = f2(1) +# assert list(three) == [3,2,1] +# """) diff --git a/interpreter/code/tests/vmtest.py b/interpreter/code/tests/vmtest.py index 24abf697a..2732cec34 100644 --- a/interpreter/code/tests/vmtest.py +++ b/interpreter/code/tests/vmtest.py @@ -1,6 +1,6 @@ """Testing tools for byterun.""" -from __future__ import print_function + import dis import io diff --git a/interpreter/code/tests/vmtest.py.bak b/interpreter/code/tests/vmtest.py.bak new file mode 100644 index 000000000..24abf697a --- /dev/null +++ b/interpreter/code/tests/vmtest.py.bak @@ -0,0 +1,96 @@ +"""Testing tools for byterun.""" + +from __future__ import print_function + +import dis +import io +import sys +import textwrap +import types +import unittest + +from byterun.pyvm2 import VirtualMachine, VirtualMachineError + +# Make this false if you need to run the debugger inside a test. +CAPTURE_STDOUT = ('-s' not in sys.argv) +# Make this false to see the traceback from a failure inside pyvm2. +CAPTURE_EXCEPTION = 1 + + +def dis_code(code): + """Disassemble `code` and all the code it refers to.""" + for const in code.co_consts: + if isinstance(const, types.CodeType): + dis_code(const) + + print("") + print(code) + dis.dis(code) + + +class VmTestCase(unittest.TestCase): + + def assert_ok(self, code, raises=None): + """Run `code` in our VM and in real Python: they behave the same.""" + + code = textwrap.dedent(code) + code = compile(code, "<%s>" % self.id(), "exec", 0, 1) + + # Print the disassembly so we'll see it if the test fails. + dis_code(code) + + real_stdout = sys.stdout + + # Run the code through our VM. + + vm_stdout = io.StringIO() + if CAPTURE_STDOUT: # pragma: no branch + sys.stdout = vm_stdout + vm = VirtualMachine() + + vm_value = vm_exc = None + try: + vm_value = vm.run_code(code) + except VirtualMachineError: # pragma: no cover + # If the VM code raises an error, show it. + raise + except AssertionError: # pragma: no cover + # If test code fails an assert, show it. + raise + except Exception as e: + # Otherwise, keep the exception for comparison later. + if not CAPTURE_EXCEPTION: # pragma: no cover + raise + vm_exc = e + finally: + real_stdout.write("-- stdout ----------\n") + real_stdout.write(vm_stdout.getvalue()) + + # Run the code through the real Python interpreter, for comparison. + + py_stdout = io.StringIO() + sys.stdout = py_stdout + + py_value = py_exc = None + globs = {} + try: + py_value = eval(code, globs, globs) + except AssertionError: # pragma: no cover + raise + except Exception as e: + py_exc = e + + sys.stdout = real_stdout + + self.assert_same_exception(vm_exc, py_exc) + self.assertEqual(vm_stdout.getvalue(), py_stdout.getvalue()) + self.assertEqual(vm_value, py_value) + if raises: + self.assertIsInstance(vm_exc, raises) + else: + self.assertIsNone(vm_exc) + + def assert_same_exception(self, e1, e2): + """Exceptions don't implement __eq__, check it ourselves.""" + self.assertEqual(str(e1), str(e2)) + self.assertIs(type(e1), type(e2)) diff --git a/modeller/code/primitive.py b/modeller/code/primitive.py index d286c0d7b..5b8f21e47 100644 --- a/modeller/code/primitive.py +++ b/modeller/code/primitive.py @@ -11,7 +11,7 @@ def make_plane(): glNewList(G_OBJ_PLANE, GL_COMPILE) glBegin(GL_LINES) glColor3f(0, 0, 0) - for i in xrange(41): + for i in range(41): glVertex3f(-10.0 + 0.5 * i, 0, -10) glVertex3f(-10.0 + 0.5 * i, 0, 10) glVertex3f(-10.0, 0, -10 + 0.5 * i) @@ -87,9 +87,9 @@ def make_cube(): normals = [(-1.0, 0.0, 0.0), (0.0, 0.0, -1.0), (1.0, 0.0, 0.0), (0.0, 0.0, 1.0), (0.0, -1.0, 0.0), (0.0, 1.0, 0.0)] glBegin(GL_QUADS) - for i in xrange(6): + for i in range(6): glNormal3f(normals[i][0], normals[i][1], normals[i][2]) - for j in xrange(4): + for j in range(4): glVertex3f(vertices[i][j][0], vertices[i][j][1], vertices[i][j][2]) glEnd() glEndList() diff --git a/modeller/code/primitive.py.bak b/modeller/code/primitive.py.bak new file mode 100644 index 000000000..d286c0d7b --- /dev/null +++ b/modeller/code/primitive.py.bak @@ -0,0 +1,101 @@ +from OpenGL.GL import glBegin, glColor3f, glEnd, glEndList, glLineWidth, glNewList, glNormal3f, glVertex3f, \ + GL_COMPILE, GL_LINES, GL_QUADS +from OpenGL.GLU import gluDeleteQuadric, gluNewQuadric, gluSphere + +G_OBJ_PLANE = 1 +G_OBJ_SPHERE = 2 +G_OBJ_CUBE = 3 + + +def make_plane(): + glNewList(G_OBJ_PLANE, GL_COMPILE) + glBegin(GL_LINES) + glColor3f(0, 0, 0) + for i in xrange(41): + glVertex3f(-10.0 + 0.5 * i, 0, -10) + glVertex3f(-10.0 + 0.5 * i, 0, 10) + glVertex3f(-10.0, 0, -10 + 0.5 * i) + glVertex3f(10.0, 0, -10 + 0.5 * i) + + # Axes + glEnd() + glLineWidth(5) + + glBegin(GL_LINES) + glColor3f(0.5, 0.7, 0.5) + glVertex3f(0.0, 0.0, 0.0) + glVertex3f(5, 0.0, 0.0) + glEnd() + + glBegin(GL_LINES) + glColor3f(0.5, 0.7, 0.5) + glVertex3f(0.0, 0.0, 0.0) + glVertex3f(0.0, 5, 0.0) + glEnd() + + glBegin(GL_LINES) + glColor3f(0.5, 0.7, 0.5) + glVertex3f(0.0, 0.0, 0.0) + glVertex3f(0.0, 0.0, 5) + glEnd() + + # Draw the Y. + glBegin(GL_LINES) + glColor3f(0.0, 0.0, 0.0) + glVertex3f(0.0, 5.0, 0.0) + glVertex3f(0.0, 5.5, 0.0) + glVertex3f(0.0, 5.5, 0.0) + glVertex3f(-0.5, 6.0, 0.0) + glVertex3f(0.0, 5.5, 0.0) + glVertex3f(0.5, 6.0, 0.0) + + # Draw the Z. + glVertex3f(-0.5, 0.0, 5.0) + glVertex3f(0.5, 0.0, 5.0) + glVertex3f(0.5, 0.0, 5.0) + glVertex3f(-0.5, 0.0, 6.0) + glVertex3f(-0.5, 0.0, 6.0) + glVertex3f(0.5, 0.0, 6.0) + + # Draw the X. + glVertex3f(5.0, 0.0, 0.5) + glVertex3f(6.0, 0.0, -0.5) + glVertex3f(5.0, 0.0, -0.5) + glVertex3f(6.0, 0.0, 0.5) + + glEnd() + glLineWidth(1) + glEndList() + + +def make_sphere(): + glNewList(G_OBJ_SPHERE, GL_COMPILE) + quad = gluNewQuadric() + gluSphere(quad, 0.5, 30, 30) + gluDeleteQuadric(quad) + glEndList() + + +def make_cube(): + glNewList(G_OBJ_CUBE, GL_COMPILE) + vertices = [((-0.5, -0.5, -0.5), (-0.5, -0.5, 0.5), (-0.5, 0.5, 0.5), (-0.5, 0.5, -0.5)), + ((-0.5, -0.5, -0.5), (-0.5, 0.5, -0.5), (0.5, 0.5, -0.5), (0.5, -0.5, -0.5)), + ((0.5, -0.5, -0.5), (0.5, 0.5, -0.5), (0.5, 0.5, 0.5), (0.5, -0.5, 0.5)), + ((-0.5, -0.5, 0.5), (0.5, -0.5, 0.5), (0.5, 0.5, 0.5), (-0.5, 0.5, 0.5)), + ((-0.5, -0.5, 0.5), (-0.5, -0.5, -0.5), (0.5, -0.5, -0.5), (0.5, -0.5, 0.5)), + ((-0.5, 0.5, -0.5), (-0.5, 0.5, 0.5), (0.5, 0.5, 0.5), (0.5, 0.5, -0.5))] + normals = [(-1.0, 0.0, 0.0), (0.0, 0.0, -1.0), (1.0, 0.0, 0.0), (0.0, 0.0, 1.0), (0.0, -1.0, 0.0), (0.0, 1.0, 0.0)] + + glBegin(GL_QUADS) + for i in xrange(6): + glNormal3f(normals[i][0], normals[i][1], normals[i][2]) + for j in xrange(4): + glVertex3f(vertices[i][j][0], vertices[i][j][1], vertices[i][j][2]) + glEnd() + glEndList() + + +def init_primitives(): + make_plane() + make_sphere() + make_cube() diff --git a/modeller/code/scene.py b/modeller/code/scene.py index 77e6f85ff..b89539a5a 100644 --- a/modeller/code/scene.py +++ b/modeller/code/scene.py @@ -33,7 +33,7 @@ def pick(self, start, direction, mat): self.selected_node = None # Keep track of the closest hit. - mindist = sys.maxint + mindist = sys.maxsize closest_node = None for node in self.node_list: hit, distance = node.pick(start, direction, mat) diff --git a/modeller/code/scene.py.bak b/modeller/code/scene.py.bak new file mode 100644 index 000000000..77e6f85ff --- /dev/null +++ b/modeller/code/scene.py.bak @@ -0,0 +1,102 @@ +import sys +import numpy +from node import Sphere, Cube, SnowFigure + + +class Scene(object): + + # the default depth from the camera to place an object at + PLACE_DEPTH = 15.0 + + def __init__(self): + # The scene keeps a list of nodes that are displayed + self.node_list = list() + # Keep track of the currently selected node. + # Actions may depend on whether or not something is selected + self.selected_node = None + + def add_node(self, node): + """ Add a new node to the scene """ + self.node_list.append(node) + + def render(self): + """ Render the scene. This function simply calls the render function for each node. """ + for node in self.node_list: + node.render() + + def pick(self, start, direction, mat): + """ Execute selection. + Consume: start, direction describing a Ray + mat is the inverse of the current modelview matrix for the scene """ + if self.selected_node is not None: + self.selected_node.select(False) + self.selected_node = None + + # Keep track of the closest hit. + mindist = sys.maxint + closest_node = None + for node in self.node_list: + hit, distance = node.pick(start, direction, mat) + if hit and distance < mindist: + mindist, closest_node = distance, node + + # If we hit something, keep track of it. + if closest_node is not None: + closest_node.select() + closest_node.depth = mindist + closest_node.selected_loc = start + direction * mindist + self.selected_node = closest_node + + def move_selected(self, start, direction, inv_modelview): + """ Move the selected node, if there is one. + Consume: start, direction describes the Ray to move to + inv_modelview is the inverse modelview matrix for the scene """ + if self.selected_node is None: return + + # Find the current depth and location of the selected node + node = self.selected_node + depth = node.depth + oldloc = node.selected_loc + + # The new location of the node is the same depth along the new ray + newloc = (start + direction * depth) + + # transform the translation with the modelview matrix + translation = newloc - oldloc + pre_tran = numpy.array([translation[0], translation[1], translation[2], 0]) + translation = inv_modelview.dot(pre_tran) + + # translate the node and track its location + node.translate(translation[0], translation[1], translation[2]) + node.selected_loc = newloc + + def place(self, shape, start, direction, inv_modelview): + """ Place a new node. + Consume: shape the shape to add + start, direction describes the Ray to move to + inv_modelview is the inverse modelview matrix for the scene """ + new_node = None + if shape == 'sphere': new_node = Sphere() + elif shape == 'cube': new_node = Cube() + elif shape == 'figure': new_node = SnowFigure() + + self.add_node(new_node) + + # place the node at the cursor in camera-space + translation = (start + direction * self.PLACE_DEPTH) + + # convert the translation to world-space + pre_tran = numpy.array([translation[0], translation[1], translation[2], 1]) + translation = inv_modelview.dot(pre_tran) + + new_node.translate(translation[0], translation[1], translation[2]) + + def rotate_selected_color(self, forwards): + """ Rotate the color of the currently selected node """ + if self.selected_node is None: return + self.selected_node.rotate_color(forwards) + + def scale_selected(self, up): + """ Scale the current selection """ + if self.selected_node is None: return + self.selected_node.scale(up) diff --git a/objmodel/code/02-attr-based/objmodel.py b/objmodel/code/02-attr-based/objmodel.py index 704aefc6c..835abb443 100644 --- a/objmodel/code/02-attr-based/objmodel.py +++ b/objmodel/code/02-attr-based/objmodel.py @@ -1,3 +1,4 @@ +import collections MISSING = object() class Base(object): @@ -42,7 +43,7 @@ def _write_dict(self, fieldname, value): self._fields[fieldname] = value def _is_bindable(meth): - return callable(meth) + return isinstance(meth, collections.Callable) def _make_boundmethod(meth, self): def bound(*args): diff --git a/objmodel/code/02-attr-based/objmodel.py.bak b/objmodel/code/02-attr-based/objmodel.py.bak new file mode 100644 index 000000000..704aefc6c --- /dev/null +++ b/objmodel/code/02-attr-based/objmodel.py.bak @@ -0,0 +1,94 @@ +MISSING = object() + +class Base(object): + """ The base class that all of the object model classes inherit from. """ + + def __init__(self, cls, fields): + """ Every object has a class. """ + self.cls = cls + self._fields = fields + + def read_attr(self, fieldname): + """ read field 'fieldname' out of the object """ + result = self._read_dict(fieldname) + if result is not MISSING: + return result + result = self.cls._read_from_class(fieldname) + if _is_bindable(result): + return _make_boundmethod(result, self) + if result is not MISSING: + return result + raise AttributeError(fieldname) + + def write_attr(self, fieldname, value): + """ write field 'fieldname' into the object """ + self._write_dict(fieldname, value) + + def isinstance(self, cls): + """ return True if the object is an instance of class cls """ + return self.cls.issubclass(cls) + + def callmethod(self, methname, *args): + """ call method 'methname' with arguments 'args' on object """ + meth = self.read_attr(methname) + return meth(*args) + + def _read_dict(self, fieldname): + """ read an field 'fieldname' out of the object's dict """ + return self._fields.get(fieldname, MISSING) + + def _write_dict(self, fieldname, value): + """ write a field 'fieldname' into the object's dict """ + self._fields[fieldname] = value + +def _is_bindable(meth): + return callable(meth) + +def _make_boundmethod(meth, self): + def bound(*args): + return meth(self, *args) + return bound + + +class Instance(Base): + """Instance of a user-defined class. """ + + def __init__(self, cls): + assert isinstance(cls, Class) + Base.__init__(self, cls, {}) + + +class Class(Base): + """ A User-defined class. """ + + def __init__(self, name, base_class, fields, metaclass): + Base.__init__(self, metaclass, fields) + self.name = name + self.base_class = base_class + + def method_resolution_order(self): + """ compute the method resolution order of the class """ + if self.base_class is None: + return [self] + else: + return [self] + self.base_class.method_resolution_order() + + def issubclass(self, cls): + """ is self a subclass of cls? """ + return cls in self.method_resolution_order() + + def _read_from_class(self, methname): + for cls in self.method_resolution_order(): + if methname in cls._fields: + return cls._fields[methname] + return MISSING + +# set up the base hierarchy like in Python (the ObjVLisp model) +# the ultimate base class is OBJECT +OBJECT = Class(name="object", base_class=None, fields={}, metaclass=None) +# TYPE is a subclass of OBJECT +TYPE = Class(name="type", base_class=OBJECT, fields={}, metaclass=None) +# TYPE is an instance of itself +TYPE.cls = TYPE +# OBJECT is an instance of TYPE +OBJECT.cls = TYPE diff --git a/objmodel/code/countlines.py b/objmodel/code/countlines.py index 63644a6bb..33a9cbe43 100644 --- a/objmodel/code/countlines.py +++ b/objmodel/code/countlines.py @@ -8,7 +8,7 @@ total = 0 -print dirs[0] +print((dirs[0])) prev = {} for fn in sorted(os.listdir(dirs[0])): fulln = os.path.join(dirs[0], fn) @@ -16,23 +16,23 @@ continue with file(fulln) as f: lines = f.readlines() - print len(lines), fn + print((len(lines), fn)) total += len(lines) prev[fn] = lines -print +print() for d in dirs[1:]: - print d + print(d) for fn, prevlines in sorted(prev.items()): fulln = os.path.join(d, fn) with file(fulln) as f: lines = f.readlines() diffsize = len(list(difflib.unified_diff(prevlines, lines, n=1))) - print diffsize, fn + print((diffsize, fn)) #print "".join(difflib.unified_diff(prevlines, lines)) prev[fn] = lines total += diffsize - print + print() -print "------------" -print total, "total" +print("------------") +print((total, "total")) diff --git a/objmodel/code/countlines.py.bak b/objmodel/code/countlines.py.bak new file mode 100644 index 000000000..449486987 --- /dev/null +++ b/objmodel/code/countlines.py.bak @@ -0,0 +1,38 @@ +import os +import difflib + +currdir = os.path.dirname(os.path.abspath(__file__)) + +dirs = [p for p in os.listdir(currdir) if os.path.isdir(p) and "0" in p] +dirs.sort() + +total = 0 + +print(dirs[0]) +prev = {} +for fn in sorted(os.listdir(dirs[0])): + fulln = os.path.join(dirs[0], fn) + if not fn.endswith(".py") or not os.path.isfile(fulln): + continue + with file(fulln) as f: + lines = f.readlines() + print(len(lines), fn) + total += len(lines) + prev[fn] = lines + +print() +for d in dirs[1:]: + print(d) + for fn, prevlines in sorted(prev.items()): + fulln = os.path.join(d, fn) + with file(fulln) as f: + lines = f.readlines() + diffsize = len(list(difflib.unified_diff(prevlines, lines, n=1))) + print(diffsize, fn) + #print "".join(difflib.unified_diff(prevlines, lines)) + prev[fn] = lines + total += diffsize + print() + +print("------------") +print(total, "total") diff --git a/ocr/code/neural_network_design.py b/ocr/code/neural_network_design.py index bb41d7bdf..63175a3b4 100644 --- a/ocr/code/neural_network_design.py +++ b/ocr/code/neural_network_design.py @@ -14,7 +14,7 @@ def test(data_matrix, data_labels, test_indices, nn): avg_sum = 0 - for j in xrange(100): + for j in range(100): correct_guess_count = 0 for i in test_indices: test = data_matrix[i] @@ -33,11 +33,11 @@ def test(data_matrix, data_labels, test_indices, nn): # Create training and testing sets. train_indices, test_indices = train_test_split(list(range(5000))) -print "PERFORMANCE" -print "-----------" +print("PERFORMANCE") +print("-----------") # Try various number of hidden nodes and see what performs best -for i in xrange(5, 50, 5): +for i in range(5, 50, 5): nn = OCRNeuralNetwork(i, data_matrix, data_labels, train_indices, False) performance = str(test(data_matrix, data_labels, test_indices, nn)) - print "{i} Hidden Nodes: {val}".format(i=i, val=performance) \ No newline at end of file + print(("{i} Hidden Nodes: {val}".format(i=i, val=performance))) \ No newline at end of file diff --git a/ocr/code/neural_network_design.py.bak b/ocr/code/neural_network_design.py.bak new file mode 100644 index 000000000..60fe4963f --- /dev/null +++ b/ocr/code/neural_network_design.py.bak @@ -0,0 +1,43 @@ +""" +In order to decide how many hidden nodes the hidden layer should have, +split up the data set into training and testing data and create networks +with various hidden node counts (5, 10, 15, ... 45), testing the performance +for each. + +The best-performing node count is used in the actual system. If multiple counts +perform similarly, choose the smallest count for a smaller network with fewer computations. +""" + +import numpy as np +from ocr import OCRNeuralNetwork +from sklearn.cross_validation import train_test_split + +def test(data_matrix, data_labels, test_indices, nn): + avg_sum = 0 + for j in range(100): + correct_guess_count = 0 + for i in test_indices: + test = data_matrix[i] + prediction = nn.predict(test) + if data_labels[i] == prediction: + correct_guess_count += 1 + + avg_sum += (correct_guess_count / float(len(test_indices))) + return avg_sum / 100 + + +# Load data samples and labels into matrix +data_matrix = np.loadtxt(open('data.csv', 'rb'), delimiter = ',').tolist() +data_labels = np.loadtxt(open('dataLabels.csv', 'rb')).tolist() + +# Create training and testing sets. +train_indices, test_indices = train_test_split(list(range(5000))) + +print("PERFORMANCE") +print("-----------") + +# Try various number of hidden nodes and see what performs best +for i in range(5, 50, 5): + nn = OCRNeuralNetwork(i, data_matrix, data_labels, train_indices, False) + performance = str(test(data_matrix, data_labels, test_indices, nn)) + print("{i} Hidden Nodes: {val}".format(i=i, val=performance)) \ No newline at end of file diff --git a/ocr/code/ocr.py b/ocr/code/ocr.py index fb304fe6e..7494e2057 100644 --- a/ocr/code/ocr.py +++ b/ocr/code/ocr.py @@ -56,8 +56,8 @@ def _sigmoid_prime_scalar(self, z): return self.sigmoid(z) * (1 - self.sigmoid(z)) def _draw(self, sample): - pixelArray = [sample[j:j+self.WIDTH_IN_PIXELS] for j in xrange(0, len(sample), self.WIDTH_IN_PIXELS)] - plt.imshow(zip(*pixelArray), cmap = cm.Greys_r, interpolation="nearest") + pixelArray = [sample[j:j+self.WIDTH_IN_PIXELS] for j in range(0, len(sample), self.WIDTH_IN_PIXELS)] + plt.imshow(list(zip(*pixelArray)), cmap = cm.Greys_r, interpolation="nearest") plt.show() def train(self, training_data_array): diff --git a/ocr/code/ocr.py.bak b/ocr/code/ocr.py.bak new file mode 100644 index 000000000..fb304fe6e --- /dev/null +++ b/ocr/code/ocr.py.bak @@ -0,0 +1,120 @@ +import csv +import matplotlib.pyplot as plt +import matplotlib.cm as cm +import numpy as np +from numpy import matrix +from math import pow +from collections import namedtuple +import math +import random +import os +import json + +""" +This class does some initial training of a neural network for predicting drawn +digits based on a data set in data_matrix and data_labels. It can then be used to +train the network further by calling train() with any array of data or to predict +what a drawn digit is by calling predict(). + +The weights that define the neural network can be saved to a file, NN_FILE_PATH, +to be reloaded upon initilization. +""" +class OCRNeuralNetwork: + LEARNING_RATE = 0.1 + WIDTH_IN_PIXELS = 20 + NN_FILE_PATH = 'nn.json' + + def __init__(self, num_hidden_nodes, data_matrix, data_labels, training_indices, use_file=True): + self.sigmoid = np.vectorize(self._sigmoid_scalar) + self.sigmoid_prime = np.vectorize(self._sigmoid_prime_scalar) + self._use_file = use_file + self.data_matrix = data_matrix + self.data_labels = data_labels + + if (not os.path.isfile(OCRNeuralNetwork.NN_FILE_PATH) or not use_file): + # Step 1: Initialize weights to small numbers + self.theta1 = self._rand_initialize_weights(400, num_hidden_nodes) + self.theta2 = self._rand_initialize_weights(num_hidden_nodes, 10) + self.input_layer_bias = self._rand_initialize_weights(1, num_hidden_nodes) + self.hidden_layer_bias = self._rand_initialize_weights(1, 10) + + # Train using sample data + TrainData = namedtuple('TrainData', ['y0', 'label']) + self.train([TrainData(self.data_matrix[i], int(self.data_labels[i])) for i in training_indices]) + self.save() + else: + self._load() + + def _rand_initialize_weights(self, size_in, size_out): + return [((x * 0.12) - 0.06) for x in np.random.rand(size_out, size_in)] + + # The sigmoid activation function. Operates on scalars. + def _sigmoid_scalar(self, z): + return 1 / (1 + math.e ** -z) + + def _sigmoid_prime_scalar(self, z): + return self.sigmoid(z) * (1 - self.sigmoid(z)) + + def _draw(self, sample): + pixelArray = [sample[j:j+self.WIDTH_IN_PIXELS] for j in xrange(0, len(sample), self.WIDTH_IN_PIXELS)] + plt.imshow(zip(*pixelArray), cmap = cm.Greys_r, interpolation="nearest") + plt.show() + + def train(self, training_data_array): + for data in training_data_array: + # Step 2: Forward propagation + y1 = np.dot(np.mat(self.theta1), np.mat(data['y0']).T) + sum1 = y1 + np.mat(self.input_layer_bias) # Add the bias + y1 = self.sigmoid(sum1) + + y2 = np.dot(np.array(self.theta2), y1) + y2 = np.add(y2, self.hidden_layer_bias) # Add the bias + y2 = self.sigmoid(y2) + + # Step 3: Back propagation + actual_vals = [0] * 10 # actual_vals is a python list for easy initialization and is later turned into an np matrix (2 lines down). + actual_vals[data['label']] = 1 + output_errors = np.mat(actual_vals).T - np.mat(y2) + hidden_errors = np.multiply(np.dot(np.mat(self.theta2).T, output_errors), self.sigmoid_prime(sum1)) + + # Step 4: Update weights + self.theta1 += self.LEARNING_RATE * np.dot(np.mat(hidden_errors), np.mat(data['y0'])) + self.theta2 += self.LEARNING_RATE * np.dot(np.mat(output_errors), np.mat(y1).T) + self.hidden_layer_bias += self.LEARNING_RATE * output_errors + self.input_layer_bias += self.LEARNING_RATE * hidden_errors + + def predict(self, test): + y1 = np.dot(np.mat(self.theta1), np.mat(test).T) + y1 = y1 + np.mat(self.input_layer_bias) # Add the bias + y1 = self.sigmoid(y1) + + y2 = np.dot(np.array(self.theta2), y1) + y2 = np.add(y2, self.hidden_layer_bias) # Add the bias + y2 = self.sigmoid(y2) + + results = y2.T.tolist()[0] + return results.index(max(results)) + + def save(self): + if not self._use_file: + return + + json_neural_network = { + "theta1":[np_mat.tolist()[0] for np_mat in self.theta1], + "theta2":[np_mat.tolist()[0] for np_mat in self.theta2], + "b1":self.input_layer_bias[0].tolist()[0], + "b2":self.hidden_layer_bias[0].tolist()[0] + }; + with open(OCRNeuralNetwork.NN_FILE_PATH,'w') as nnFile: + json.dump(json_neural_network, nnFile) + + def _load(self): + if not self._use_file: + return + + with open(OCRNeuralNetwork.NN_FILE_PATH) as nnFile: + nn = json.load(nnFile) + self.theta1 = [np.array(li) for li in nn['theta1']] + self.theta2 = [np.array(li) for li in nn['theta2']] + self.input_layer_bias = [np.array(nn['b1'][0])] + self.hidden_layer_bias = [np.array(nn['b2'][0])] diff --git a/ocr/code/server.py b/ocr/code/server.py index b8a3e77f6..da757665c 100644 --- a/ocr/code/server.py +++ b/ocr/code/server.py @@ -1,4 +1,4 @@ -import BaseHTTPServer +import http.server import json from ocr import OCRNeuralNetwork import numpy as np @@ -20,7 +20,7 @@ # for hidden nodes nn = OCRNeuralNetwork(HIDDEN_NODE_COUNT, data_matrix, data_labels, list(range(5000))); -class JSONHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class JSONHandler(http.server.BaseHTTPRequestHandler): def do_POST(s): response_code = 200 response = "" @@ -48,7 +48,7 @@ def do_POST(s): return if __name__ == '__main__': - server_class = BaseHTTPServer.HTTPServer; + server_class = http.server.HTTPServer; httpd = server_class((HOST_NAME, PORT_NUMBER), JSONHandler) try: @@ -56,6 +56,6 @@ def do_POST(s): except KeyboardInterrupt: pass else: - print "Unexpected server exception occurred." + print("Unexpected server exception occurred.") finally: httpd.server_close() diff --git a/ocr/code/server.py.bak b/ocr/code/server.py.bak new file mode 100644 index 000000000..b8a3e77f6 --- /dev/null +++ b/ocr/code/server.py.bak @@ -0,0 +1,61 @@ +import BaseHTTPServer +import json +from ocr import OCRNeuralNetwork +import numpy as np + +HOST_NAME = 'localhost' +PORT_NUMBER = 8000 +HIDDEN_NODE_COUNT = 15 + +# Load data samples and labels into matrix +data_matrix = np.loadtxt(open('data.csv', 'rb'), delimiter = ',') +data_labels = np.loadtxt(open('dataLabels.csv', 'rb')) + +# Convert from numpy ndarrays to python lists +data_matrix = data_matrix.tolist() +data_labels = data_labels.tolist() + +# If a neural network file does not exist, train it using all 5000 existing data samples. +# Based on data collected from neural_network_design.py, 15 is the optimal number +# for hidden nodes +nn = OCRNeuralNetwork(HIDDEN_NODE_COUNT, data_matrix, data_labels, list(range(5000))); + +class JSONHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def do_POST(s): + response_code = 200 + response = "" + var_len = int(s.headers.get('Content-Length')) + content = s.rfile.read(var_len); + payload = json.loads(content); + + if payload.get('train'): + nn.train(payload['trainArray']) + nn.save() + elif payload.get('predict'): + try: + response = {"type":"test", "result":nn.predict(str(payload['image']))} + except: + response_code = 500 + else: + response_code = 400 + + s.send_response(response_code) + s.send_header("Content-type", "application/json") + s.send_header("Access-Control-Allow-Origin", "*") + s.end_headers() + if response: + s.wfile.write(json.dumps(response)) + return + +if __name__ == '__main__': + server_class = BaseHTTPServer.HTTPServer; + httpd = server_class((HOST_NAME, PORT_NUMBER), JSONHandler) + + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + else: + print "Unexpected server exception occurred." + finally: + httpd.server_close() diff --git a/sampler/code/rpg.py b/sampler/code/rpg.py index ff163e4ad..1c7da8e6b 100644 --- a/sampler/code/rpg.py +++ b/sampler/code/rpg.py @@ -45,7 +45,7 @@ def sample(self): """ stats = self._sample_stats() - item_stats = dict(zip(self.stats_names, stats)) + item_stats = dict(list(zip(self.stats_names, stats))) return item_stats def log_pmf(self, item): @@ -228,7 +228,7 @@ def sample(self): """ # First, we need to randomly generate items (the number of # which was passed into the constructor). - items = [self.item_dist.sample() for i in xrange(self.num_items)] + items = [self.item_dist.sample() for i in range(self.num_items)] # Based on the item stats (in particular, strength), compute # the number of dice we get to roll. diff --git a/sampler/code/rpg.py.bak b/sampler/code/rpg.py.bak new file mode 100644 index 000000000..ff163e4ad --- /dev/null +++ b/sampler/code/rpg.py.bak @@ -0,0 +1,240 @@ +import numpy as np +from multinomial import MultinomialDistribution + + +class MagicItemDistribution(object): + + # these are the names (and order) of the stats that all magical + # items will have + stats_names = ("dexterity", "constitution", "strength", + "intelligence", "wisdom", "charisma") + + def __init__(self, bonus_probs, stats_probs, rso=np.random): + """Initialize a magic item distribution parameterized by `bonus_probs` + and `stats_probs`. + + Parameters + ---------- + bonus_probs: numpy array of length m + The probabilities of the overall bonuses. Each index in + the array corresponds to the bonus of that amount (e.g. + index 0 is +0, index 1 is +1, etc.) + + stats_probs: numpy array of length 6 + The probabilities of how the overall bonus is distributed + among the different stats. `stats_probs[i]` corresponds to + the probability of giving a bonus point to the ith stat, + i.e. the value at `MagicItemDistribution.stats_names[i]`. + + rso: numpy RandomState object (default: np.random) + The random number generator + + """ + # Create the multinomial distributions we'll be using + self.bonus_dist = MultinomialDistribution(bonus_probs, rso=rso) + self.stats_dist = MultinomialDistribution(stats_probs, rso=rso) + + def sample(self): + """Sample a random magical item. + + Returns + ------- + dictionary + The keys are the names of the stats, and the values are + the bonus conferred to the corresponding stat. + + """ + stats = self._sample_stats() + item_stats = dict(zip(self.stats_names, stats)) + return item_stats + + def log_pmf(self, item): + """Compute the log probability the given magical item. + + Parameters + ---------- + item: dictionary + The keys are the names of the stats, and the values are + the bonus conferred to the corresponding stat. + + Returns + ------- + float + The value corresponding to log(p(item)) + + """ + # First pull out the bonus points for each stat, in the + # correct order, then pass that to _stats_log_pmf. + stats = np.array([item[stat] for stat in self.stats_names]) + log_pmf = self._stats_log_pmf(stats) + return log_pmf + + def pmf(self, item): + """Compute the probability the given magical item. + + Parameters + ---------- + item: dictionary + The keys are the names of the stats, and the values are + the bonus conferred to the corresponding stat. + + Returns + ------- + float + The value corresponding to p(item) + + """ + return np.exp(self.log_pmf(item)) + + def _sample_bonus(self): + """Sample a value of the overall bonus. + + Returns + ------- + integer + The overall bonus + + """ + # The bonus is essentially just a sample from a multinomial + # distribution with n=1; i.e., only one event occurs. + sample = self.bonus_dist.sample(1) + + # `sample` is an array of zeros and a single one at the + # location corresponding to the bonus. We want to convert this + # one into the actual value of the bonus. + bonus = np.argmax(sample) + return bonus + + def _sample_stats(self): + """Sample the overall bonus and how it is distributed across the + different stats. + + Returns + ------- + numpy array of length 6 + The number of bonus points for each stat + + """ + # First we need to sample the overall bonus + bonus = self._sample_bonus() + + # Then, we use a different multinomial distribution to sample + # how that bonus is distributed. The bonus corresponds to the + # number of events. + stats = self.stats_dist.sample(bonus) + return stats + + def _bonus_log_pmf(self, bonus): + """Evaluate the log-PMF for the given bonus. + + Parameters + ---------- + bonus: integer + The total bonus. + + Returns + ------- + float + The value corresponding to log(p(bonus)) + + """ + # Make sure the value that is passed in is within the + # appropriate bounds + if bonus < 0 or bonus >= len(self.bonus_dist.p): + return -np.inf + + # Convert the scalar bonus value into a vector of event + # occurrences + x = np.zeros(len(self.bonus_dist.p)) + x[bonus] = 1 + + return self.bonus_dist.log_pmf(x) + + def _stats_log_pmf(self, stats): + """Evaluate the log-PMF for the given distribution of bonus points + across the different stats. + + Parameters + ---------- + stats: numpy array of length 6 + The distribution of bonus points across the stats + + Returns + ------- + float + The value corresponding to log(p(stats)) + + """ + # There are never any leftover bonus points, so the sum of the + # stats gives us the total bonus. + total_bonus = np.sum(stats) + + # First calculate the probability of the total bonus + logp_bonus = self._bonus_log_pmf(total_bonus) + + # Then calculate the probability of the stats + logp_stats = self.stats_dist.log_pmf(stats) + + # Then multiply them together (using addition, because we are + # working with logs) + log_pmf = logp_bonus + logp_stats + return log_pmf + + +class DamageDistribution(object): + + def __init__(self, num_items, item_dist, + num_dice_sides=12, num_hits=1, rso=np.random): + """Initialize a distribution over attack damage. This object can + sample possible values for the attack damage dealt over + `num_hits` hits when the player has `num_items` items, and + where attack damage is computed by rolling dice with + `num_dice_sides` sides. + + Parameters + ---------- + num_items: int + The number of items the player has. + item_dist: MagicItemDistribution object + The distribution over magic items. + num_dice_sides: int (default: 12) + The number of sides on each die. + num_hits: int (default: 1) + The number of hits across which we want to calculate damage. + rso: numpy RandomState object (default: np.random) + The random number generator + + """ + # This is an array of integers corresponding to the sides of a + # single die. + self.dice_sides = np.arange(1, num_dice_sides + 1) + # Create a multinomial distribution corresponding to one of + # these dice. Each side has equal probabilities. + self.dice_dist = MultinomialDistribution( + np.ones(num_dice_sides) / float(num_dice_sides), rso=rso) + + self.num_hits = num_hits + self.num_items = num_items + self.item_dist = item_dist + + def sample(self): + """Sample the attack damage. + + Returns + ------- + int + The sampled damage + + """ + # First, we need to randomly generate items (the number of + # which was passed into the constructor). + items = [self.item_dist.sample() for i in xrange(self.num_items)] + + # Based on the item stats (in particular, strength), compute + # the number of dice we get to roll. + num_dice = 1 + np.sum([item['strength'] for item in items]) + + # Roll the dice and compute the resulting damage. + dice_rolls = self.dice_dist.sample(self.num_hits * num_dice) + damage = np.sum(self.dice_sides * dice_rolls) + return damage diff --git a/sampler/code/test_multinomial.py b/sampler/code/test_multinomial.py index afcf43d15..29988f38b 100644 --- a/sampler/code/test_multinomial.py +++ b/sampler/code/test_multinomial.py @@ -80,10 +80,10 @@ def test_sample_1(): p = np.array([1.0]) rso = np.random.RandomState(29348) dist = MultinomialDistribution(p, rso=rso) - samples = np.array([dist.sample(1) for i in xrange(100)]) + samples = np.array([dist.sample(1) for i in range(100)]) assert samples.shape == (100, 1) assert (samples == 1).all() - samples = np.array([dist.sample(3) for i in xrange(100)]) + samples = np.array([dist.sample(3) for i in range(100)]) assert samples.shape == (100, 1) assert (samples == 3).all() @@ -93,20 +93,20 @@ def test_sample_2(): p = np.array([1.0, 0.0]) rso = np.random.RandomState(29348) dist = MultinomialDistribution(p, rso=rso) - samples = np.array([dist.sample(1) for i in xrange(100)]) + samples = np.array([dist.sample(1) for i in range(100)]) assert samples.shape == (100, 2) assert (samples == np.array([1, 0])).all() - samples = np.array([dist.sample(3) for i in xrange(100)]) + samples = np.array([dist.sample(3) for i in range(100)]) assert samples.shape == (100, 2) assert (samples == np.array([3, 0])).all() p = np.array([0.0, 1.0]) rso = np.random.RandomState(29348) dist = MultinomialDistribution(p, rso=rso) - samples = np.array([dist.sample(1) for i in xrange(100)]) + samples = np.array([dist.sample(1) for i in range(100)]) assert samples.shape == (100, 2) assert (samples == np.array([0, 1])).all() - samples = np.array([dist.sample(3) for i in xrange(100)]) + samples = np.array([dist.sample(3) for i in range(100)]) assert samples.shape == (100, 2) assert (samples == np.array([0, 3])).all() @@ -116,11 +116,11 @@ def test_sample_3(): p = np.array([0.5, 0.5]) rso = np.random.RandomState(29348) dist = MultinomialDistribution(p, rso=rso) - samples = np.array([dist.sample(1) for i in xrange(100)]) + samples = np.array([dist.sample(1) for i in range(100)]) assert samples.shape == (100, 2) assert ((samples == np.array([1, 0])) | (samples == np.array([0, 1]))).all() - samples = np.array([dist.sample(3) for i in xrange(100)]) + samples = np.array([dist.sample(3) for i in range(100)]) assert samples.shape == (100, 2) assert ((samples == np.array([3, 0])) | (samples == np.array([2, 1])) | diff --git a/sampler/code/test_multinomial.py.bak b/sampler/code/test_multinomial.py.bak new file mode 100644 index 000000000..afcf43d15 --- /dev/null +++ b/sampler/code/test_multinomial.py.bak @@ -0,0 +1,128 @@ +import numpy as np +import pytest +from multinomial import MultinomialDistribution + + +def test_init_without_rso(): + """Initialize without rso""" + p = np.array([0.1, 0.5, 0.3, 0.1]) + dist = MultinomialDistribution(p) + assert (dist.p == p).all() + assert (dist.logp == np.log(p)).all() + assert dist.rso is np.random + + +def test_init_with_rso(): + """Initialize with rso""" + p = np.array([0.1, 0.5, 0.3, 0.1]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + assert (dist.p == p).all() + assert (dist.logp == np.log(p)).all() + assert dist.rso == rso + + +def test_init_bad_probabilities(): + """Initialize with probabilities that don't sum to 1""" + p = np.array([0.1, 0.5, 0.3, 0.0]) + rso = np.random.RandomState(29348) + with pytest.raises(ValueError): + MultinomialDistribution(p, rso=rso) + + +def test_pmf_1(): + """Test PMF with only one possible event""" + p = np.array([1.0]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + assert dist.pmf(np.array([1])) == 1.0 + assert dist.pmf(np.array([2])) == 1.0 + assert dist.pmf(np.array([10])) == 1.0 + + +def test_pmf_2(): + """Test PMF with two possible events, one with zero probability""" + p = np.array([1.0, 0.0]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + assert dist.pmf(np.array([1, 0])) == 1.0 + assert dist.pmf(np.array([0, 1])) == 0.0 + assert dist.pmf(np.array([2, 0])) == 1.0 + assert dist.pmf(np.array([2, 2])) == 0.0 + assert dist.pmf(np.array([10, 0])) == 1.0 + assert dist.pmf(np.array([10, 3])) == 0.0 + + p = np.array([0.0, 1.0]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + assert dist.pmf(np.array([0, 1])) == 1.0 + assert dist.pmf(np.array([1, 0])) == 0.0 + assert dist.pmf(np.array([0, 2])) == 1.0 + assert dist.pmf(np.array([2, 2])) == 0.0 + assert dist.pmf(np.array([0, 10])) == 1.0 + assert dist.pmf(np.array([3, 10])) == 0.0 + + +def test_pmf_3(): + """Test PMF with two possible events, both with nonzero probability""" + p = np.array([0.5, 0.5]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + assert dist.pmf(np.array([1, 0])) == 0.5 + assert dist.pmf(np.array([0, 1])) == 0.5 + assert dist.pmf(np.array([2, 0])) == 0.25 + assert dist.pmf(np.array([0, 2])) == 0.25 + assert dist.pmf(np.array([1, 1])) == 0.5 + + +def test_sample_1(): + """Test sampling with only one possible event""" + p = np.array([1.0]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + samples = np.array([dist.sample(1) for i in xrange(100)]) + assert samples.shape == (100, 1) + assert (samples == 1).all() + samples = np.array([dist.sample(3) for i in xrange(100)]) + assert samples.shape == (100, 1) + assert (samples == 3).all() + + +def test_sample_2(): + """Test sampling with two possible events, one with zero probability""" + p = np.array([1.0, 0.0]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + samples = np.array([dist.sample(1) for i in xrange(100)]) + assert samples.shape == (100, 2) + assert (samples == np.array([1, 0])).all() + samples = np.array([dist.sample(3) for i in xrange(100)]) + assert samples.shape == (100, 2) + assert (samples == np.array([3, 0])).all() + + p = np.array([0.0, 1.0]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + samples = np.array([dist.sample(1) for i in xrange(100)]) + assert samples.shape == (100, 2) + assert (samples == np.array([0, 1])).all() + samples = np.array([dist.sample(3) for i in xrange(100)]) + assert samples.shape == (100, 2) + assert (samples == np.array([0, 3])).all() + + +def test_sample_3(): + """Test sampling with two possible events, both with nonzero probability""" + p = np.array([0.5, 0.5]) + rso = np.random.RandomState(29348) + dist = MultinomialDistribution(p, rso=rso) + samples = np.array([dist.sample(1) for i in xrange(100)]) + assert samples.shape == (100, 2) + assert ((samples == np.array([1, 0])) | + (samples == np.array([0, 1]))).all() + samples = np.array([dist.sample(3) for i in xrange(100)]) + assert samples.shape == (100, 2) + assert ((samples == np.array([3, 0])) | + (samples == np.array([2, 1])) | + (samples == np.array([1, 2])) | + (samples == np.array([0, 3]))).all() diff --git a/template-engine/code/templite.py b/template-engine/code/templite.py index 53824e08f..3e6ab139b 100644 --- a/template-engine/code/templite.py +++ b/template-engine/code/templite.py @@ -3,6 +3,7 @@ # Coincidentally named the same as http://code.activestate.com/recipes/496702/ import re +import collections class TempliteSyntaxError(ValueError): @@ -247,6 +248,6 @@ def _do_dots(self, value, *dots): value = getattr(value, dot) except AttributeError: value = value[dot] - if callable(value): + if isinstance(value, collections.Callable): value = value() return value diff --git a/template-engine/code/templite.py.bak b/template-engine/code/templite.py.bak new file mode 100644 index 000000000..53824e08f --- /dev/null +++ b/template-engine/code/templite.py.bak @@ -0,0 +1,252 @@ +"""A simple Python template renderer, for a nano-subset of Django syntax.""" + +# Coincidentally named the same as http://code.activestate.com/recipes/496702/ + +import re + + +class TempliteSyntaxError(ValueError): + """Raised when a template has a syntax error.""" + pass + + +class CodeBuilder(object): + """Build source code conveniently.""" + + def __init__(self, indent=0): + self.code = [] + self.indent_level = indent + + def __str__(self): + return "".join(str(c) for c in self.code) + + def add_line(self, line): + """Add a line of source to the code. + + Indentation and newline will be added for you, don't provide them. + + """ + self.code.extend([" " * self.indent_level, line, "\n"]) + + def add_section(self): + """Add a section, a sub-CodeBuilder.""" + section = CodeBuilder(self.indent_level) + self.code.append(section) + return section + + INDENT_STEP = 4 # PEP8 says so! + + def indent(self): + """Increase the current indent for following lines.""" + self.indent_level += self.INDENT_STEP + + def dedent(self): + """Decrease the current indent for following lines.""" + self.indent_level -= self.INDENT_STEP + + def get_globals(self): + """Execute the code, and return a dict of globals it defines.""" + # A check that the caller really finished all the blocks they started. + assert self.indent_level == 0 + # Get the Python source as a single string. + python_source = str(self) + # Execute the source, defining globals, and return them. + global_namespace = {} + exec(python_source, global_namespace) + return global_namespace + + +class Templite(object): + """A simple template renderer, for a nano-subset of Django syntax. + + Supported constructs are extended variable access:: + + {{var.modifer.modifier|filter|filter}} + + loops:: + + {% for var in list %}...{% endfor %} + + and ifs:: + + {% if var %}...{% endif %} + + Comments are within curly-hash markers:: + + {# This will be ignored #} + + Construct a Templite with the template text, then use `render` against a + dictionary context to create a finished string:: + + templite = Templite(''' +

Hello {{name|upper}}!

+ {% for topic in topics %} +

You are interested in {{topic}}.

+ {% endif %} + ''', + {'upper': str.upper}, + ) + text = templite.render({ + 'name': "Ned", + 'topics': ['Python', 'Geometry', 'Juggling'], + }) + + """ + def __init__(self, text, *contexts): + """Construct a Templite with the given `text`. + + `contexts` are dictionaries of values to use for future renderings. + These are good for filters and global values. + + """ + self.context = {} + for context in contexts: + self.context.update(context) + + self.all_vars = set() + self.loop_vars = set() + + # We construct a function in source form, then compile it and hold onto + # it, and execute it to render the template. + code = CodeBuilder() + + code.add_line("def render_function(context, do_dots):") + code.indent() + vars_code = code.add_section() + code.add_line("result = []") + code.add_line("append_result = result.append") + code.add_line("extend_result = result.extend") + code.add_line("to_str = str") + + buffered = [] + def flush_output(): + """Force `buffered` to the code builder.""" + if len(buffered) == 1: + code.add_line("append_result(%s)" % buffered[0]) + elif len(buffered) > 1: + code.add_line("extend_result([%s])" % ", ".join(buffered)) + del buffered[:] + + ops_stack = [] + + # Split the text to form a list of tokens. + tokens = re.split(r"(?s)({{.*?}}|{%.*?%}|{#.*?#})", text) + + for token in tokens: + if token.startswith('{#'): + # Comment: ignore it and move on. + continue + elif token.startswith('{{'): + # An expression to evaluate. + expr = self._expr_code(token[2:-2].strip()) + buffered.append("to_str(%s)" % expr) + elif token.startswith('{%'): + # Action tag: split into words and parse further. + flush_output() + words = token[2:-2].strip().split() + if words[0] == 'if': + # An if statement: evaluate the expression to determine if. + if len(words) != 2: + self._syntax_error("Don't understand if", token) + ops_stack.append('if') + code.add_line("if %s:" % self._expr_code(words[1])) + code.indent() + elif words[0] == 'for': + # A loop: iterate over expression result. + if len(words) != 4 or words[2] != 'in': + self._syntax_error("Don't understand for", token) + ops_stack.append('for') + self._variable(words[1], self.loop_vars) + code.add_line( + "for c_%s in %s:" % ( + words[1], + self._expr_code(words[3]) + ) + ) + code.indent() + elif words[0].startswith('end'): + # Endsomething. Pop the ops stack. + if len(words) != 1: + self._syntax_error("Don't understand end", token) + end_what = words[0][3:] + if not ops_stack: + self._syntax_error("Too many ends", token) + start_what = ops_stack.pop() + if start_what != end_what: + self._syntax_error("Mismatched end tag", end_what) + code.dedent() + else: + self._syntax_error("Don't understand tag", words[0]) + else: + # Literal content. If it isn't empty, output it. + if token: + buffered.append(repr(token)) + + if ops_stack: + self._syntax_error("Unmatched action tag", ops_stack[-1]) + + flush_output() + + for var_name in self.all_vars - self.loop_vars: + vars_code.add_line("c_%s = context[%r]" % (var_name, var_name)) + + code.add_line("return ''.join(result)") + code.dedent() + self._render_function = code.get_globals()['render_function'] + + def _expr_code(self, expr): + """Generate a Python expression for `expr`.""" + if "|" in expr: + pipes = expr.split("|") + code = self._expr_code(pipes[0]) + for func in pipes[1:]: + self._variable(func, self.all_vars) + code = "c_%s(%s)" % (func, code) + elif "." in expr: + dots = expr.split(".") + code = self._expr_code(dots[0]) + args = ", ".join(repr(d) for d in dots[1:]) + code = "do_dots(%s, %s)" % (code, args) + else: + self._variable(expr, self.all_vars) + code = "c_%s" % expr + return code + + def _syntax_error(self, msg, thing): + """Raise a syntax error using `msg`, and showing `thing`.""" + raise TempliteSyntaxError("%s: %r" % (msg, thing)) + + def _variable(self, name, vars_set): + """Track that `name` is used as a variable. + + Adds the name to `vars_set`, a set of variable names. + + Raises an syntax error if `name` is not a valid name. + + """ + if not re.match(r"[_a-zA-Z][_a-zA-Z0-9]*$", name): + self._syntax_error("Not a valid name", name) + vars_set.add(name) + + def render(self, context=None): + """Render this template by applying it to `context`. + + `context` is a dictionary of values to use in this rendering. + + """ + # Make the complete context we'll use. + render_context = dict(self.context) + if context: + render_context.update(context) + return self._render_function(render_context, self._do_dots) + + def _do_dots(self, value, *dots): + """Evaluate dotted expressions at runtime.""" + for dot in dots: + try: + value = getattr(value, dot) + except AttributeError: + value = value[dot] + if callable(value): + value = value() + return value diff --git a/template-engine/code/test_templite.py b/template-engine/code/test_templite.py index d821677c6..21b55a83d 100644 --- a/template-engine/code/test_templite.py +++ b/template-engine/code/test_templite.py @@ -15,7 +15,7 @@ class AnyOldObject(object): """ def __init__(self, **attrs): - for n, v in attrs.items(): + for n, v in list(attrs.items()): setattr(self, n, v) diff --git a/template-engine/code/test_templite.py.bak b/template-engine/code/test_templite.py.bak new file mode 100644 index 000000000..d821677c6 --- /dev/null +++ b/template-engine/code/test_templite.py.bak @@ -0,0 +1,275 @@ +"""Tests for templite.""" + +import re +from templite import Templite, TempliteSyntaxError +from unittest import TestCase + +# pylint: disable=W0612,E1101 +# Disable W0612 (Unused variable) and +# E1101 (Instance of 'foo' has no 'bar' member) + +class AnyOldObject(object): + """Simple testing object. + + Use keyword arguments in the constructor to set attributes on the object. + + """ + def __init__(self, **attrs): + for n, v in attrs.items(): + setattr(self, n, v) + + +class TempliteTest(TestCase): + """Tests for Templite.""" + + def try_render(self, text, ctx=None, result=None): + """Render `text` through `ctx`, and it had better be `result`. + + Result defaults to None so we can shorten the calls where we expect + an exception and never get to the result comparison. + """ + actual = Templite(text).render(ctx or {}) + if result: + self.assertEqual(actual, result) + + def assertSynErr(self, msg): + pat = "^" + re.escape(msg) + "$" + return self.assertRaisesRegexp(TempliteSyntaxError, pat) + + def test_passthrough(self): + # Strings without variables are passed through unchanged. + self.assertEqual(Templite("Hello").render(), "Hello") + self.assertEqual( + Templite("Hello, 20% fun time!").render(), + "Hello, 20% fun time!" + ) + + def test_variables(self): + # Variables use {{var}} syntax. + self.try_render("Hello, {{name}}!", {'name':'Ned'}, "Hello, Ned!") + + def test_undefined_variables(self): + # Using undefined names is an error. + with self.assertRaises(Exception): + self.try_render("Hi, {{name}}!") + + def test_pipes(self): + # Variables can be filtered with pipes. + data = { + 'name': 'Ned', + 'upper': lambda x: x.upper(), + 'second': lambda x: x[1], + } + self.try_render("Hello, {{name|upper}}!", data, "Hello, NED!") + + # Pipes can be concatenated. + self.try_render("Hello, {{name|upper|second}}!", data, "Hello, E!") + + def test_reusability(self): + # A single Templite can be used more than once with different data. + globs = { + 'upper': lambda x: x.upper(), + 'punct': '!', + } + + template = Templite("This is {{name|upper}}{{punct}}", globs) + self.assertEqual(template.render({'name':'Ned'}), "This is NED!") + self.assertEqual(template.render({'name':'Ben'}), "This is BEN!") + + def test_attribute(self): + # Variables' attributes can be accessed with dots. + obj = AnyOldObject(a="Ay") + self.try_render("{{obj.a}}", locals(), "Ay") + + obj2 = AnyOldObject(obj=obj, b="Bee") + self.try_render("{{obj2.obj.a}} {{obj2.b}}", locals(), "Ay Bee") + + def test_member_function(self): + # Variables' member functions can be used, as long as they are nullary. + class WithMemberFns(AnyOldObject): + """A class to try out member function access.""" + def ditto(self): + """Return twice the .txt attribute.""" + return self.txt + self.txt + obj = WithMemberFns(txt="Once") + self.try_render("{{obj.ditto}}", locals(), "OnceOnce") + + def test_item_access(self): + # Variables' items can be used. + d = {'a':17, 'b':23} + self.try_render("{{d.a}} < {{d.b}}", locals(), "17 < 23") + + def test_loops(self): + # Loops work like in Django. + nums = [1,2,3,4] + self.try_render( + "Look: {% for n in nums %}{{n}}, {% endfor %}done.", + locals(), + "Look: 1, 2, 3, 4, done." + ) + # Loop iterables can be filtered. + def rev(l): + """Return the reverse of `l`.""" + l = l[:] + l.reverse() + return l + + self.try_render( + "Look: {% for n in nums|rev %}{{n}}, {% endfor %}done.", + locals(), + "Look: 4, 3, 2, 1, done." + ) + + def test_empty_loops(self): + self.try_render( + "Empty: {% for n in nums %}{{n}}, {% endfor %}done.", + {'nums':[]}, + "Empty: done." + ) + + def test_multiline_loops(self): + self.try_render( + "Look: \n{% for n in nums %}\n{{n}}, \n{% endfor %}done.", + {'nums':[1,2,3]}, + "Look: \n\n1, \n\n2, \n\n3, \ndone." + ) + + def test_multiple_loops(self): + self.try_render( + "{% for n in nums %}{{n}}{% endfor %} and " + "{% for n in nums %}{{n}}{% endfor %}", + {'nums': [1,2,3]}, + "123 and 123" + ) + + def test_comments(self): + # Single-line comments work: + self.try_render( + "Hello, {# Name goes here: #}{{name}}!", + {'name':'Ned'}, "Hello, Ned!" + ) + # and so do multi-line comments: + self.try_render( + "Hello, {# Name\ngoes\nhere: #}{{name}}!", + {'name':'Ned'}, "Hello, Ned!" + ) + + def test_if(self): + self.try_render( + "Hi, {% if ned %}NED{% endif %}{% if ben %}BEN{% endif %}!", + {'ned': 1, 'ben': 0}, + "Hi, NED!" + ) + self.try_render( + "Hi, {% if ned %}NED{% endif %}{% if ben %}BEN{% endif %}!", + {'ned': 0, 'ben': 1}, + "Hi, BEN!" + ) + self.try_render( + "Hi, {% if ned %}NED{% if ben %}BEN{% endif %}{% endif %}!", + {'ned': 0, 'ben': 0}, + "Hi, !" + ) + self.try_render( + "Hi, {% if ned %}NED{% if ben %}BEN{% endif %}{% endif %}!", + {'ned': 1, 'ben': 0}, + "Hi, NED!" + ) + self.try_render( + "Hi, {% if ned %}NED{% if ben %}BEN{% endif %}{% endif %}!", + {'ned': 1, 'ben': 1}, + "Hi, NEDBEN!" + ) + + def test_complex_if(self): + class Complex(AnyOldObject): + """A class to try out complex data access.""" + def getit(self): + """Return it.""" + return self.it + obj = Complex(it={'x':"Hello", 'y': 0}) + self.try_render( + "@" + "{% if obj.getit.x %}X{% endif %}" + "{% if obj.getit.y %}Y{% endif %}" + "{% if obj.getit.y|str %}S{% endif %}" + "!", + { 'obj': obj, 'str': str }, + "@XS!" + ) + + def test_loop_if(self): + self.try_render( + "@{% for n in nums %}{% if n %}Z{% endif %}{{n}}{% endfor %}!", + {'nums': [0,1,2]}, + "@0Z1Z2!" + ) + self.try_render( + "X{%if nums%}@{% for n in nums %}{{n}}{% endfor %}{%endif%}!", + {'nums': [0,1,2]}, + "X@012!" + ) + self.try_render( + "X{%if nums%}@{% for n in nums %}{{n}}{% endfor %}{%endif%}!", + {'nums': []}, + "X!" + ) + + def test_nested_loops(self): + self.try_render( + "@" + "{% for n in nums %}" + "{% for a in abc %}{{a}}{{n}}{% endfor %}" + "{% endfor %}" + "!", + {'nums': [0,1,2], 'abc': ['a', 'b', 'c']}, + "@a0b0c0a1b1c1a2b2c2!" + ) + + def test_exception_during_evaluation(self): + # TypeError: Couldn't evaluate {{ foo.bar.baz }}: + # 'NoneType' object is unsubscriptable + with self.assertRaises(TypeError): + self.try_render( + "Hey {{foo.bar.baz}} there", {'foo': None}, "Hey ??? there" + ) + + def test_bad_names(self): + with self.assertSynErr("Not a valid name: 'var%&!@'"): + self.try_render("Wat: {{ var%&!@ }}") + with self.assertSynErr("Not a valid name: 'filter%&!@'"): + self.try_render("Wat: {{ foo|filter%&!@ }}") + with self.assertSynErr("Not a valid name: '@'"): + self.try_render("Wat: {% for @ in x %}{% endfor %}") + + def test_bogus_tag_syntax(self): + with self.assertSynErr("Don't understand tag: 'bogus'"): + self.try_render("Huh: {% bogus %}!!{% endbogus %}??") + + def test_malformed_if(self): + with self.assertSynErr("Don't understand if: '{% if %}'"): + self.try_render("Buh? {% if %}hi!{% endif %}") + with self.assertSynErr("Don't understand if: '{% if this or that %}'"): + self.try_render("Buh? {% if this or that %}hi!{% endif %}") + + def test_malformed_for(self): + with self.assertSynErr("Don't understand for: '{% for %}'"): + self.try_render("Weird: {% for %}loop{% endfor %}") + with self.assertSynErr("Don't understand for: '{% for x from y %}'"): + self.try_render("Weird: {% for x from y %}loop{% endfor %}") + with self.assertSynErr("Don't understand for: '{% for x, y in z %}'"): + self.try_render("Weird: {% for x, y in z %}loop{% endfor %}") + + def test_bad_nesting(self): + with self.assertSynErr("Unmatched action tag: 'if'"): + self.try_render("{% if x %}X") + with self.assertSynErr("Mismatched end tag: 'for'"): + self.try_render("{% if x %}X{% endfor %}") + with self.assertSynErr("Too many ends: '{% endif %}'"): + self.try_render("{% if x %}{% endif %}{% endif %}") + + def test_malformed_end(self): + with self.assertSynErr("Don't understand end: '{% end if %}'"): + self.try_render("{% if x %}X{% end if %}") + with self.assertSynErr("Don't understand end: '{% endif now %}'"): + self.try_render("{% if x %}X{% endif now %}") diff --git a/web-server/code/00-hello-web/server.py b/web-server/code/00-hello-web/server.py index 2e95f8bee..d7daf39ca 100644 --- a/web-server/code/00-hello-web/server.py +++ b/web-server/code/00-hello-web/server.py @@ -1,8 +1,8 @@ -import BaseHTTPServer +import http.server #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): '''Handle HTTP requests by returning a fixed 'page'.''' # Page to send back. @@ -26,5 +26,5 @@ def do_GET(self): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/00-hello-web/server.py.bak b/web-server/code/00-hello-web/server.py.bak new file mode 100644 index 000000000..2e95f8bee --- /dev/null +++ b/web-server/code/00-hello-web/server.py.bak @@ -0,0 +1,30 @@ +import BaseHTTPServer + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + '''Handle HTTP requests by returning a fixed 'page'.''' + + # Page to send back. + Page = '''\ + + +

Hello, web!

+ + +''' + + # Handle a GET request. + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(self.Page))) + self.end_headers() + self.wfile.write(self.Page) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/01-echo-request-info/server.py b/web-server/code/01-echo-request-info/server.py index f18f7bc48..782eeea40 100644 --- a/web-server/code/01-echo-request-info/server.py +++ b/web-server/code/01-echo-request-info/server.py @@ -1,8 +1,8 @@ -import BaseHTTPServer +import http.server #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): '''Respond to HTTP requests with info about the request.''' # Template for page to send back. @@ -51,5 +51,5 @@ def send_page(self, page): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/01-echo-request-info/server.py.bak b/web-server/code/01-echo-request-info/server.py.bak new file mode 100644 index 000000000..f18f7bc48 --- /dev/null +++ b/web-server/code/01-echo-request-info/server.py.bak @@ -0,0 +1,55 @@ +import BaseHTTPServer + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + '''Respond to HTTP requests with info about the request.''' + + # Template for page to send back. + Page = '''\ + + + + + + + + + +
Header Value
Date and time {date_time}
Client host {client_host}
Client port {client_port}s
Command {command}
Path {path}
+ + +''' + + # Handle a request by constructing an HTML page that echoes the + # request back to the caller. + def do_GET(self): + page = self.create_page() + self.send_page(page) + + # Create an information page to send. + def create_page(self): + values = { + 'date_time' : self.date_time_string(), + 'client_host' : self.client_address[0], + 'client_port' : self.client_address[1], + 'command' : self.command, + 'path' : self.path + } + page = self.Page.format(**values) + return page + + # Send the created page. + def send_page(self, page): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(page))) + self.end_headers() + self.wfile.write(page) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/02-serve-static/server-status-code.py b/web-server/code/02-serve-static/server-status-code.py index f34d899f9..7a6590f2f 100644 --- a/web-server/code/02-serve-static/server-status-code.py +++ b/web-server/code/02-serve-static/server-status-code.py @@ -1,4 +1,4 @@ -import sys, os, BaseHTTPServer +import sys, os, http.server #------------------------------------------------------------------------------- @@ -8,7 +8,7 @@ class ServerException(Exception): #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): ''' If the requested path maps to a file, that file is served. If anything goes wrong, an error page is constructed. @@ -73,5 +73,5 @@ def send_content(self, content, status=200): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/02-serve-static/server-status-code.py.bak b/web-server/code/02-serve-static/server-status-code.py.bak new file mode 100644 index 000000000..f34d899f9 --- /dev/null +++ b/web-server/code/02-serve-static/server-status-code.py.bak @@ -0,0 +1,77 @@ +import sys, os, BaseHTTPServer + +#------------------------------------------------------------------------------- + +class ServerException(Exception): + '''For internal error reporting.''' + pass + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + ''' + If the requested path maps to a file, that file is served. + If anything goes wrong, an error page is constructed. + ''' + + # How to display an error. + Error_Page = """\ + + +

Error accessing {path}

+

{msg}

+ + + """ + + # Classify and handle request. + def do_GET(self): + try: + + # Figure out what exactly is being requested. + full_path = os.getcwd() + self.path + + # It doesn't exist... + if not os.path.exists(full_path): + raise ServerException("'{0}' not found".format(self.path)) + + # ...it's a file... + elif os.path.isfile(full_path): + self.handle_file(full_path) + + # ...it's something we don't handle. + else: + raise ServerException("Unknown object '{0}'".format(self.path)) + + # Handle errors. + except Exception as msg: + self.handle_error(msg) + + def handle_file(self, full_path): + try: + with open(full_path, 'rb') as reader: + content = reader.read() + self.send_content(content) + except IOError as msg: + msg = "'{0}' cannot be read: {1}".format(self.path, msg) + self.handle_error(msg) + + # Handle unknown objects. + def handle_error(self, msg): + content = self.Error_Page.format(path=self.path, msg=msg) + self.send_content(content, 404) + + # Send actual content. + def send_content(self, content, status=200): + self.send_response(status) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/02-serve-static/server.py b/web-server/code/02-serve-static/server.py index 65c3df8ac..039c40a7a 100644 --- a/web-server/code/02-serve-static/server.py +++ b/web-server/code/02-serve-static/server.py @@ -1,4 +1,4 @@ -import sys, os, BaseHTTPServer +import sys, os, http.server #------------------------------------------------------------------------------- @@ -8,7 +8,7 @@ class ServerException(Exception): #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): ''' If the requested path maps to a file, that file is served. If anything goes wrong, an error page is constructed. @@ -73,5 +73,5 @@ def send_content(self, content): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/02-serve-static/server.py.bak b/web-server/code/02-serve-static/server.py.bak new file mode 100644 index 000000000..65c3df8ac --- /dev/null +++ b/web-server/code/02-serve-static/server.py.bak @@ -0,0 +1,77 @@ +import sys, os, BaseHTTPServer + +#------------------------------------------------------------------------------- + +class ServerException(Exception): + '''For internal error reporting.''' + pass + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + ''' + If the requested path maps to a file, that file is served. + If anything goes wrong, an error page is constructed. + ''' + + # How to display an error. + Error_Page = """\ + + +

Error accessing {path}

+

{msg}

+ + + """ + + # Classify and handle request. + def do_GET(self): + try: + + # Figure out what exactly is being requested. + full_path = os.getcwd() + self.path + + # It doesn't exist... + if not os.path.exists(full_path): + raise ServerException("'{0}' not found".format(self.path)) + + # ...it's a file... + elif os.path.isfile(full_path): + self.handle_file(full_path) + + # ...it's something we don't handle. + else: + raise ServerException("Unknown object '{0}'".format(self.path)) + + # Handle errors. + except Exception as msg: + self.handle_error(msg) + + def handle_file(self, full_path): + try: + with open(full_path, 'rb') as reader: + content = reader.read() + self.send_content(content) + except IOError as msg: + msg = "'{0}' cannot be read: {1}".format(self.path, msg) + self.handle_error(msg) + + # Handle unknown objects. + def handle_error(self, msg): + content = self.Error_Page.format(path=self.path, msg=msg) + self.send_content(content) + + # Send actual content. + def send_content(self, content): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/03-handlers/server-index-page.py b/web-server/code/03-handlers/server-index-page.py index f74055af3..41c3c4215 100644 --- a/web-server/code/03-handlers/server-index-page.py +++ b/web-server/code/03-handlers/server-index-page.py @@ -1,4 +1,4 @@ -import sys, os, BaseHTTPServer +import sys, os, http.server #------------------------------------------------------------------------------- @@ -56,7 +56,7 @@ def act(self, handler): #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): ''' If the requested path maps to a file, that file is served. If anything goes wrong, an error page is constructed. @@ -120,5 +120,5 @@ def send_content(self, content, status=200): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/03-handlers/server-index-page.py.bak b/web-server/code/03-handlers/server-index-page.py.bak new file mode 100644 index 000000000..f74055af3 --- /dev/null +++ b/web-server/code/03-handlers/server-index-page.py.bak @@ -0,0 +1,124 @@ +import sys, os, BaseHTTPServer + +#------------------------------------------------------------------------------- + +class ServerException(Exception): + '''For internal error reporting.''' + pass + +#------------------------------------------------------------------------------- + +class case_no_file(object): + '''File or directory does not exist.''' + + def test(self, handler): + return not os.path.exists(handler.full_path) + + def act(self, handler): + raise ServerException("'{0}' not found".format(handler.path)) + +#------------------------------------------------------------------------------- + +class case_existing_file(object): + '''File exists.''' + + def test(self, handler): + return os.path.isfile(handler.full_path) + + def act(self, handler): + handler.handle_file(handler.full_path) + +#------------------------------------------------------------------------------- + +class case_directory_index_file(object): + '''Serve index.html page for a directory.''' + + def index_path(self, handler): + return os.path.join(handler.full_path, 'index.html') + + def test(self, handler): + return os.path.isdir(handler.full_path) and \ + os.path.isfile(self.index_path(handler)) + + def act(self, handler): + handler.handle_file(self.index_path(handler)) + +#------------------------------------------------------------------------------- + +class case_always_fail(object): + '''Base case if nothing else worked.''' + + def test(self, handler): + return True + + def act(self, handler): + raise ServerException("Unknown object '{0}'".format(handler.path)) + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + ''' + If the requested path maps to a file, that file is served. + If anything goes wrong, an error page is constructed. + ''' + + Cases = [case_no_file(), + case_existing_file(), + case_directory_index_file(), + case_always_fail()] + + # How to display an error. + Error_Page = """\ + + +

Error accessing {path}

+

{msg}

+ + + """ + + # Classify and handle request. + def do_GET(self): + try: + + # Figure out what exactly is being requested. + self.full_path = os.getcwd() + self.path + + # Figure out how to handle it. + for case in self.Cases: + if case.test(self): + case.act(self) + break + + # Handle errors. + except Exception as msg: + self.handle_error(msg) + + def handle_file(self, full_path): + try: + with open(full_path, 'rb') as reader: + content = reader.read() + self.send_content(content) + except IOError as msg: + msg = "'{0}' cannot be read: {1}".format(self.path, msg) + self.handle_error(msg) + + # Handle unknown objects. + def handle_error(self, msg): + content = self.Error_Page.format(path=self.path, msg=msg) + self.send_content(content, 404) + + # Send actual content. + def send_content(self, content, status=200): + self.send_response(status) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/03-handlers/server-no-index-page.py b/web-server/code/03-handlers/server-no-index-page.py index 4851b08de..78ce518ad 100644 --- a/web-server/code/03-handlers/server-no-index-page.py +++ b/web-server/code/03-handlers/server-no-index-page.py @@ -1,4 +1,4 @@ -import sys, os, BaseHTTPServer +import sys, os, http.server #------------------------------------------------------------------------------- @@ -71,7 +71,7 @@ def act(self, handler): #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): ''' If the requested path maps to a file, that file is served. If anything goes wrong, an error page is constructed. @@ -157,5 +157,5 @@ def send_content(self, content, status=200): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/03-handlers/server-no-index-page.py.bak b/web-server/code/03-handlers/server-no-index-page.py.bak new file mode 100644 index 000000000..4851b08de --- /dev/null +++ b/web-server/code/03-handlers/server-no-index-page.py.bak @@ -0,0 +1,161 @@ +import sys, os, BaseHTTPServer + +#------------------------------------------------------------------------------- + +class ServerException(Exception): + '''For internal error reporting.''' + pass + +#------------------------------------------------------------------------------- + +class case_no_file(object): + '''File or directory does not exist.''' + + def test(self, handler): + return not os.path.exists(handler.full_path) + + def act(self, handler): + raise ServerException("'{0}' not found".format(handler.path)) + +#------------------------------------------------------------------------------- + +class case_existing_file(object): + '''File exists.''' + + def test(self, handler): + return os.path.isfile(handler.full_path) + + def act(self, handler): + handler.handle_file(handler.full_path) + +#------------------------------------------------------------------------------- + +class case_directory_index_file(object): + '''Serve index.html page for a directory.''' + + def index_path(self, handler): + return os.path.join(handler.full_path, 'index.html') + + def test(self, handler): + return os.path.isdir(handler.full_path) and \ + os.path.isfile(self.index_path(handler)) + + def act(self, handler): + handler.handle_file(self.index_path(handler)) + +#------------------------------------------------------------------------------- + +class case_directory_no_index_file(object): + '''Serve listing for a directory without an index.html page.''' + + def index_path(self, handler): + return os.path.join(handler.full_path, 'index.html') + + def test(self, handler): + return os.path.isdir(handler.full_path) and \ + not os.path.isfile(self.index_path(handler)) + + def act(self, handler): + handler.list_dir(handler.full_path) + +#------------------------------------------------------------------------------- + +class case_always_fail(object): + '''Base case if nothing else worked.''' + + def test(self, handler): + return True + + def act(self, handler): + raise ServerException("Unknown object '{0}'".format(handler.path)) + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + ''' + If the requested path maps to a file, that file is served. + If anything goes wrong, an error page is constructed. + ''' + + Cases = [case_no_file(), + case_existing_file(), + case_directory_index_file(), + case_directory_no_index_file(), + case_always_fail()] + + # How to display an error. + Error_Page = """\ + + +

Error accessing {path}

+

{msg}

+ + + """ + + # How to display a directory listing. + Listing_Page = '''\ + + +
    + {0} +
+ + + ''' + + # Classify and handle request. + def do_GET(self): + try: + + # Figure out what exactly is being requested. + self.full_path = os.getcwd() + self.path + + # Figure out how to handle it. + for case in self.Cases: + if case.test(self): + case.act(self) + break + + # Handle errors. + except Exception as msg: + self.handle_error(msg) + + def handle_file(self, full_path): + try: + with open(full_path, 'rb') as reader: + content = reader.read() + self.send_content(content) + except IOError as msg: + msg = "'{0}' cannot be read: {1}".format(self.path, msg) + self.handle_error(msg) + + def list_dir(self, full_path): + try: + entries = os.listdir(full_path) + bullets = ['
  • {0}
    • '.format(e) for e in entries if not e.startswith('.')] + page = self.Listing_Page.format('\n'.join(bullets)) + self.send_content(page) + except OSError as msg: + msg = "'{0}' cannot be listed: {1}".format(self.path, msg) + self.handle_error(msg) + + # Handle unknown objects. + def handle_error(self, msg): + content = self.Error_Page.format(path=self.path, msg=msg) + self.send_content(content, 404) + + # Send actual content. + def send_content(self, content, status=200): + self.send_response(status) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/03-handlers/server.py b/web-server/code/03-handlers/server.py index c41402f01..0cc99584d 100644 --- a/web-server/code/03-handlers/server.py +++ b/web-server/code/03-handlers/server.py @@ -1,4 +1,4 @@ -import sys, os, BaseHTTPServer +import sys, os, http.server #------------------------------------------------------------------------------- @@ -41,7 +41,7 @@ def act(self, handler): #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): ''' If the requested path maps to a file, that file is served. If anything goes wrong, an error page is constructed. @@ -104,5 +104,5 @@ def send_content(self, content, status=200): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/03-handlers/server.py.bak b/web-server/code/03-handlers/server.py.bak new file mode 100644 index 000000000..c41402f01 --- /dev/null +++ b/web-server/code/03-handlers/server.py.bak @@ -0,0 +1,108 @@ +import sys, os, BaseHTTPServer + +#------------------------------------------------------------------------------- + +class ServerException(Exception): + '''For internal error reporting.''' + pass + +#------------------------------------------------------------------------------- + +class case_no_file(object): + '''File or directory does not exist.''' + + def test(self, handler): + return not os.path.exists(handler.full_path) + + def act(self, handler): + raise ServerException("'{0}' not found".format(handler.path)) + +#------------------------------------------------------------------------------- + +class case_existing_file(object): + '''File exists.''' + + def test(self, handler): + return os.path.isfile(handler.full_path) + + def act(self, handler): + handler.handle_file(handler.full_path) + +#------------------------------------------------------------------------------- + +class case_always_fail(object): + '''Base case if nothing else worked.''' + + def test(self, handler): + return True + + def act(self, handler): + raise ServerException("Unknown object '{0}'".format(handler.path)) + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + ''' + If the requested path maps to a file, that file is served. + If anything goes wrong, an error page is constructed. + ''' + + Cases = [case_no_file(), + case_existing_file(), + case_always_fail()] + + # How to display an error. + Error_Page = """\ + + +

    Error accessing {path}

    +

    {msg}

    + + + """ + + # Classify and handle request. + def do_GET(self): + try: + + # Figure out what exactly is being requested. + self.full_path = os.getcwd() + self.path + + # Figure out how to handle it. + for case in self.Cases: + if case.test(self): + case.act(self) + break + + # Handle errors. + except Exception as msg: + self.handle_error(msg) + + def handle_file(self, full_path): + try: + with open(full_path, 'rb') as reader: + content = reader.read() + self.send_content(content) + except IOError as msg: + msg = "'{0}' cannot be read: {1}".format(self.path, msg) + self.handle_error(msg) + + # Handle unknown objects. + def handle_error(self, msg): + content = self.Error_Page.format(path=self.path, msg=msg) + self.send_content(content, 404) + + # Send actual content. + def send_content(self, content, status=200): + self.send_response(status) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/04-cgi/server.py b/web-server/code/04-cgi/server.py index 8dda3f7c8..41afe156c 100644 --- a/web-server/code/04-cgi/server.py +++ b/web-server/code/04-cgi/server.py @@ -1,4 +1,4 @@ -import sys, os, BaseHTTPServer +import sys, os, http.server #------------------------------------------------------------------------------- @@ -83,7 +83,7 @@ def act(self, handler): #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): ''' If the requested path maps to a file, that file is served. If anything goes wrong, an error page is constructed. @@ -178,5 +178,5 @@ def send_content(self, content, status=200): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/04-cgi/server.py.bak b/web-server/code/04-cgi/server.py.bak new file mode 100644 index 000000000..8dda3f7c8 --- /dev/null +++ b/web-server/code/04-cgi/server.py.bak @@ -0,0 +1,182 @@ +import sys, os, BaseHTTPServer + +#------------------------------------------------------------------------------- + +class ServerException(Exception): + '''For internal error reporting.''' + pass + +#------------------------------------------------------------------------------- + +class case_no_file(object): + '''File or directory does not exist.''' + + def test(self, handler): + return not os.path.exists(handler.full_path) + + def act(self, handler): + raise ServerException("'{0}' not found".format(handler.path)) + +#------------------------------------------------------------------------------- + +class case_cgi_file(object): + '''Something runnable.''' + + def test(self, handler): + return os.path.isfile(handler.full_path) and \ + handler.full_path.endswith('.py') + + def act(self, handler): + handler.run_cgi(handler.full_path) + +#------------------------------------------------------------------------------- + +class case_existing_file(object): + '''File exists.''' + + def test(self, handler): + return os.path.isfile(handler.full_path) + + def act(self, handler): + handler.handle_file(handler.full_path) + +#------------------------------------------------------------------------------- + +class case_directory_index_file(object): + '''Serve index.html page for a directory.''' + + def index_path(self, handler): + return os.path.join(handler.full_path, 'index.html') + + def test(self, handler): + return os.path.isdir(handler.full_path) and \ + os.path.isfile(self.index_path(handler)) + + def act(self, handler): + handler.handle_file(self.index_path(handler)) + +#------------------------------------------------------------------------------- + +class case_directory_no_index_file(object): + '''Serve listing for a directory without an index.html page.''' + + def index_path(self, handler): + return os.path.join(handler.full_path, 'index.html') + + def test(self, handler): + return os.path.isdir(handler.full_path) and \ + not os.path.isfile(self.index_path(handler)) + + def act(self, handler): + handler.list_dir(handler.full_path) + +#------------------------------------------------------------------------------- + +class case_always_fail(object): + '''Base case if nothing else worked.''' + + def test(self, handler): + return True + + def act(self, handler): + raise ServerException("Unknown object '{0}'".format(handler.path)) + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + ''' + If the requested path maps to a file, that file is served. + If anything goes wrong, an error page is constructed. + ''' + + Cases = [case_no_file(), + case_cgi_file(), + case_existing_file(), + case_directory_index_file(), + case_directory_no_index_file(), + case_always_fail()] + + # How to display an error. + Error_Page = """\ + + +

    Error accessing {path}

    +

    {msg}

    + + + """ + + # How to display a directory listing. + Listing_Page = '''\ + + +
      + {0} +
    + + + ''' + + # Classify and handle request. + def do_GET(self): + try: + + # Figure out what exactly is being requested. + self.full_path = os.getcwd() + self.path + + # Figure out how to handle it. + for case in self.Cases: + if case.test(self): + case.act(self) + break + + # Handle errors. + except Exception as msg: + self.handle_error(msg) + + def handle_file(self, full_path): + try: + with open(full_path, 'rb') as reader: + content = reader.read() + self.send_content(content) + except IOError as msg: + msg = "'{0}' cannot be read: {1}".format(self.path, msg) + self.handle_error(msg) + + def list_dir(self, full_path): + try: + entries = os.listdir(full_path) + bullets = ['
  • {0}
    • '.format(e) for e in entries if not e.startswith('.')] + page = self.Listing_Page.format('\n'.join(bullets)) + self.send_content(page) + except OSError as msg: + msg = "'{0}' cannot be listed: {1}".format(self.path, msg) + self.handle_error(msg) + + def run_cgi(self, full_path): + cmd = "python " + full_path + child_stdin, child_stdout = os.popen2(cmd) + child_stdin.close() + data = child_stdout.read() + child_stdout.close() + self.send_content(data) + + # Handle unknown objects. + def handle_error(self, msg): + content = self.Error_Page.format(path=self.path, msg=msg) + self.send_content(content, 404) + + # Send actual content. + def send_content(self, content, status=200): + self.send_response(status) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/04-cgi/simple.py b/web-server/code/04-cgi/simple.py index be002a2c5..4492d9e31 100644 --- a/web-server/code/04-cgi/simple.py +++ b/web-server/code/04-cgi/simple.py @@ -1,7 +1,7 @@ from datetime import datetime -print '''\ +print(('''\

    Generated {0}

    -'''.format(datetime.now()) +'''.format(datetime.now()))) diff --git a/web-server/code/04-cgi/simple.py.bak b/web-server/code/04-cgi/simple.py.bak new file mode 100644 index 000000000..22c0b3cc7 --- /dev/null +++ b/web-server/code/04-cgi/simple.py.bak @@ -0,0 +1,7 @@ +from datetime import datetime +print('''\ + + +

    Generated {0}

    + +'''.format(datetime.now())) diff --git a/web-server/code/05-refactored/server.py b/web-server/code/05-refactored/server.py index 4c3974e16..42a9c2f62 100644 --- a/web-server/code/05-refactored/server.py +++ b/web-server/code/05-refactored/server.py @@ -1,4 +1,4 @@ -import sys, os, BaseHTTPServer +import sys, os, http.server #------------------------------------------------------------------------------- @@ -129,7 +129,7 @@ def act(self, handler): #------------------------------------------------------------------------------- -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): +class RequestHandler(http.server.BaseHTTPRequestHandler): ''' If the requested path maps to a file, that file is served. If anything goes wrong, an error page is constructed. @@ -186,5 +186,5 @@ def send_content(self, content, status=200): if __name__ == '__main__': serverAddress = ('', 8080) - server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server = http.server.HTTPServer(serverAddress, RequestHandler) server.serve_forever() diff --git a/web-server/code/05-refactored/server.py.bak b/web-server/code/05-refactored/server.py.bak new file mode 100644 index 000000000..4c3974e16 --- /dev/null +++ b/web-server/code/05-refactored/server.py.bak @@ -0,0 +1,190 @@ +import sys, os, BaseHTTPServer + +#------------------------------------------------------------------------------- + +class ServerException(Exception): + '''For internal error reporting.''' + pass + +#------------------------------------------------------------------------------- + +class base_case(object): + '''Parent for case handlers.''' + + def handle_file(self, handler, full_path): + try: + with open(full_path, 'rb') as reader: + content = reader.read() + handler.send_content(content) + except IOError as msg: + msg = "'{0}' cannot be read: {1}".format(full_path, msg) + handler.handle_error(msg) + + def index_path(self, handler): + return os.path.join(handler.full_path, 'index.html') + + def test(self, handler): + assert False, 'Not implemented.' + + def act(self, handler): + assert False, 'Not implemented.' + +#------------------------------------------------------------------------------- + +class case_no_file(base_case): + '''File or directory does not exist.''' + + def test(self, handler): + return not os.path.exists(handler.full_path) + + def act(self, handler): + raise ServerException("'{0}' not found".format(handler.path)) + +#------------------------------------------------------------------------------- + +class case_cgi_file(base_case): + '''Something runnable.''' + + def run_cgi(self, handler): + cmd = "python " + handler.full_path + child_stdin, child_stdout = os.popen2(cmd) + child_stdin.close() + data = child_stdout.read() + child_stdout.close() + handler.send_content(data) + + def test(self, handler): + return os.path.isfile(handler.full_path) and \ + handler.full_path.endswith('.py') + + def act(self, handler): + self.run_cgi(handler) + +#------------------------------------------------------------------------------- + +class case_existing_file(base_case): + '''File exists.''' + + def test(self, handler): + return os.path.isfile(handler.full_path) + + def act(self, handler): + self.handle_file(handler, handler.full_path) + +#------------------------------------------------------------------------------- + +class case_directory_index_file(base_case): + '''Serve index.html page for a directory.''' + + def test(self, handler): + return os.path.isdir(handler.full_path) and \ + os.path.isfile(self.index_path(handler)) + + def act(self, handler): + self.handle_file(handler, self.index_path(handler)) + +#------------------------------------------------------------------------------- + +class case_directory_no_index_file(base_case): + '''Serve listing for a directory without an index.html page.''' + + # How to display a directory listing. + Listing_Page = '''\ + + +
      + {0} +
    + + + ''' + + def list_dir(self, handler, full_path): + try: + entries = os.listdir(full_path) + bullets = ['
  • {0}
    • '.format(e) for e in entries if not e.startswith('.')] + page = self.Listing_Page.format('\n'.join(bullets)) + handler.send_content(page) + except OSError as msg: + msg = "'{0}' cannot be listed: {1}".format(self.path, msg) + handler.handle_error(msg) + + def test(self, handler): + return os.path.isdir(handler.full_path) and \ + not os.path.isfile(self.index_path(handler)) + + def act(self, handler): + self.list_dir(handler, handler.full_path) + +#------------------------------------------------------------------------------- + +class case_always_fail(base_case): + '''Base case if nothing else worked.''' + + def test(self, handler): + return True + + def act(self, handler): + raise ServerException("Unknown object '{0}'".format(handler.path)) + +#------------------------------------------------------------------------------- + +class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + ''' + If the requested path maps to a file, that file is served. + If anything goes wrong, an error page is constructed. + ''' + + Cases = [case_no_file(), + case_cgi_file(), + case_existing_file(), + case_directory_index_file(), + case_directory_no_index_file(), + case_always_fail()] + + # How to display an error. + Error_Page = """\ + + +

    Error accessing {path}

    +

    {msg}

    + + + """ + + # Classify and handle request. + def do_GET(self): + try: + + # Figure out what exactly is being requested. + self.full_path = os.getcwd() + self.path + + # Figure out how to handle it. + for case in self.Cases: + if case.test(self): + case.act(self) + break + + # Handle errors. + except Exception as msg: + self.handle_error(msg) + + # Handle unknown objects. + def handle_error(self, msg): + content = self.Error_Page.format(path=self.path, msg=msg) + self.send_content(content, 404) + + # Send actual content. + def send_content(self, content, status=200): + self.send_response(status) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + +#------------------------------------------------------------------------------- + +if __name__ == '__main__': + serverAddress = ('', 8080) + server = BaseHTTPServer.HTTPServer(serverAddress, RequestHandler) + server.serve_forever() diff --git a/web-server/code/05-refactored/simple.py b/web-server/code/05-refactored/simple.py index be002a2c5..4492d9e31 100644 --- a/web-server/code/05-refactored/simple.py +++ b/web-server/code/05-refactored/simple.py @@ -1,7 +1,7 @@ from datetime import datetime -print '''\ +print(('''\

    Generated {0}

    -'''.format(datetime.now()) +'''.format(datetime.now()))) diff --git a/web-server/code/05-refactored/simple.py.bak b/web-server/code/05-refactored/simple.py.bak new file mode 100644 index 000000000..22c0b3cc7 --- /dev/null +++ b/web-server/code/05-refactored/simple.py.bak @@ -0,0 +1,7 @@ +from datetime import datetime +print('''\ + + +

    Generated {0}

    + +'''.format(datetime.now())) diff --git a/web-server/code/requests-01.py b/web-server/code/requests-01.py index 580938378..e8c2ebeb3 100644 --- a/web-server/code/requests-01.py +++ b/web-server/code/requests-01.py @@ -1,5 +1,5 @@ import requests response = requests.get("http://aosabook.org/en/posa/introduction.html") -print 'status code:', response.status_code -print 'content length:', response.headers['content-length'] -print response.text +print(('status code:', response.status_code)) +print(('content length:', response.headers['content-length'])) +print((response.text)) diff --git a/web-server/code/requests-01.py.bak b/web-server/code/requests-01.py.bak new file mode 100644 index 000000000..4ce93250e --- /dev/null +++ b/web-server/code/requests-01.py.bak @@ -0,0 +1,5 @@ +import requests +response = requests.get("http://aosabook.org/en/posa/introduction.html") +print('status code:', response.status_code) +print('content length:', response.headers['content-length']) +print(response.text) diff --git a/web-server/code/requests-02.py b/web-server/code/requests-02.py index 17bdbf043..744faa5a6 100644 --- a/web-server/code/requests-02.py +++ b/web-server/code/requests-02.py @@ -1,4 +1,4 @@ import requests parameters = {'q' : 'Python', 'client' : 'Firefox'} response = requests.get('http://www.google.com/search', params=parameters) -print 'actual URL:', response.url +print(('actual URL:', response.url)) diff --git a/web-server/code/requests-02.py.bak b/web-server/code/requests-02.py.bak new file mode 100644 index 000000000..f9243dedf --- /dev/null +++ b/web-server/code/requests-02.py.bak @@ -0,0 +1,4 @@ +import requests +parameters = {'q' : 'Python', 'client' : 'Firefox'} +response = requests.get('http://www.google.com/search', params=parameters) +print('actual URL:', response.url)