# Copyright 2015 The Swarming Authors. All rights reserved.
# Use of this source code is governed by the Apache v2.0 license that can be
# found in the LICENSE file.

"""Android specific utility functions.

This file serves as an API to bot_config.py. bot_config.py can be replaced on
the server to allow additional server-specific functionality.
"""

import collections
import logging
import os

from adb import adb_commands_safe
from adb import adb_protocol
from adb import common
from adb import high

# Master switch that can easily be temporarily increased to INFO or even DEBUG
# when needed by simply pushing a new tainted swarming server version. This
# helps quickly debugging issues. On the other hand, even INFO level is quite
# verbose so keep it at WARNING by default.
LEVEL = logging.WARNING
adb_commands_safe._LOG.setLevel(LEVEL)
adb_protocol._LOG.setLevel(LEVEL)
common._LOG.setLevel(LEVEL)
high._LOG.setLevel(LEVEL)

def initialize(pub_key, priv_key):
  return high.Initialize(pub_key, priv_key)

def get_devices(bot):
  return high.GetDevices(
      'swarming', 10000, 10000, on_error=bot.post_error if bot else None,
      as_root=True)

def close_devices(devices):
  return high.CloseDevices(devices)

def kill_adb():
  return adb_commands_safe.KillADB()

Response:
# Copyright 2015 The Swarming Authors. All rights reserved.
# Use of this source code is governed by the Apache v2.0 license that can be
# found in the LICENSE file.

"""Android specific utility functions.

This file serves as an API to bot_config.py. bot_config.py can be replaced on
the server to allow additional server-specific functionality.
"""

import collections
import logging
import os

from adb import adb_commands_safe
from adb import adb_protocol
from adb import common
from adb import high

# Master switch that can easily be temporarily increased to INFO or even DEBUG
# when needed by simply pushing a new tainted swarming server version. This
# helps quickly debugging issues. On the other hand, even INFO level is quite
# verbose so keep it at WARNING by default.
LEVEL = logging.WARNING
adb_commands_safe._LOG.setLevel(LEVEL)
adb_protocol._LOG.setLevel(LEVEL)
common._LOG.setLevel(LEVEL)
high._LOG.setLevel(LEVEL)

def initialize(pub_key, priv_key):
  return high.Initialize(pub_key, priv_key)

def get_devices(bot):
  return high.GetDevices(
      'swarming', 10000, 10000, on_error=bot.post_error if bot else None,
      as_root=True)

def close_devices(devices):
  return high.CloseDevices(devices)

def kill_adb():
  return adb_commands_safe.KillADB()

Prompt:
Add kill_adb() and increase python-adb logging to WARNING.

It gives a master switch that can easily be temporarily increased to INFO or
even DEBUG when needed by simply pushing a new tainted swarming server version.
This helps quickly debugging issues. On the other hand, even INFO level is quite
verbose so keep it at WARNING by default.

Adding kill_adb() is useful, since it's better to forcibly kill adb upfront in
on_after_task() than try to kill it after trying USB enumeration then killing
adb then enumerating again.

R=stip@chromium.org
BUG=chromium:525873

Review URL: https://codereview.chromium.org/1430073003

# Copyright 2015 The Swarming Authors. All rights reserved.
# Use of this source code is governed by the Apache v2.0 license that can be
# found in the LICENSE file.

"""Android specific utility functions.

This file serves as an API to bot_config.py. bot_config.py can be replaced on
the server to allow additional server-specific functionality.
"""

import collections
import logging
import os

from adb import adb_commands_safe
from adb import high

def initialize(pub_key, priv_key):
  return high.Initialize(pub_key, priv_key)

def get_devices(bot):
  return high.GetDevices(
      'swarming', 10000, 10000, on_error=bot.post_error if bot else None,
      as_root=True)

def close_devices(devices):
  return high.CloseDevices(devices)


#!/usr/bin/python2

"""
packet_loss.py: Simulates packet loss by dropping all packets with a
probability of 25%.
"""

import random

from pox.core import core
import pox.openflow.libopenflow_01 as of

def packet_in(event):
    if random.random() >= 0.25:
        msg = of.ofp_packet_out(data = event.ofp)
        msg.actions.append(of.ofp_action_output(port = of.OFPP_FLOOD))
        event.connection.send(msg)

def launch():
    core.openflow.addListenerByName("PacketIn", packet_in)

Fix bug in packet loss

Fix bug in packet loss

Response:
#!/usr/bin/python2

"""
packet_loss.py: Simulates packet loss by dropping all packets with a
probability of 25%.
"""

import random

from pox.core import core
import pox.openflow.libopenflow_01 as of

def packet_in(event):
    if random.random() >= 0.25:
        msg = of.ofp_packet_out(data = event.ofp)
        msg.actions.append(of.ofp_action_output(port = of.OFPP_FLOOD))
        event.connection.send(msg)

def launch():
    core.openflow.addListenerByName("PacketIn", packet_in)

Prompt:
Fix bug in packet loss

#!/usr/bin/python2

"""
packet_loss.py: Simulates packet loss by dropping all packets with a
probability of 25%.
"""

import random

from pox.core import core
import pox.openflow.libopenflow_01 as of

def packet_in(event):.
    if random.random() >= 0.25:
        msg = of.ofp_packet_out(data = event.ofp)
        msg.actions.append(of.ofp_action_output(port = of.OFPP_FLOOD))
        conn.send(event.connection)

def launch():
    core.openflow.addListenerByName("PacketIn", packet_in)


#!/usr/bin/python

import argparse
import os
import struct
import sys

CP_STRUCT_SIZES = { 0xa:3 }

###############
### CLASSES ###
###############
class MyParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

###################
### SUBROUTINES ###
###################
def lenCpStruct(tag):
  if tag in CP_STRUCT_SIZES:
    return CP_STRUCT_SIZES[tag]
  else:
    return -1

############
### MAIN ###
############
parser = MyParser('Run bytecode in jjvm')
parser.add_argument('path', help='path to class')
args = parser.parse_args()

with open(args.path, "rb") as c:
  c.seek(8)
  cpCount = struct.unpack(">H", c.read(2))[0] - 1

  print "Constant pool count: %d" % cpCount;

  while cpCount >= 0:
    cpTag = ord(c.read(1))

    print "Got tag: %d" % cpTag
    cpStructSize = lenCpStruct(cpTag)

    if cpStructSize < 0:
      print "ERROR: cpStructSize %d for tag %d" % (cpStructSize, cpTag)
      sys.exit(1)

    print "Size: %d" % cpStructSize

    cpCount -= 1
    c.seek(cpStructSize, os.SEEK_CUR)

Response:
#!/usr/bin/python

import argparse
import os
import struct
import sys

CP_STRUCT_SIZES = { 0xa:3 }

###############
### CLASSES ###
###############
class MyParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

###################
### SUBROUTINES ###
###################
def lenCpStruct(tag):
  if tag in CP_STRUCT_SIZES:
    return CP_STRUCT_SIZES[tag]
  else:
    return -1

############
### MAIN ###
############
parser = MyParser('Run bytecode in jjvm')
parser.add_argument('path', help='path to class')
args = parser.parse_args()

with open(args.path, "rb") as c:
  c.seek(8)
  cpCount = struct.unpack(">H", c.read(2))[0] - 1

  print "Constant pool count: %d" % cpCount;

  while cpCount >= 0:
    cpTag = ord(c.read(1))

    print "Got tag: %d" % cpTag
    cpStructSize = lenCpStruct(cpTag)

    if cpStructSize < 0:
      print "ERROR: cpStructSize %d for tag %d" % (cpStructSize, cpTag)
      sys.exit(1)

    print "Size: %d" % cpStructSize

    cpCount -= 1
    c.seek(cpStructSize, os.SEEK_CUR)

Prompt:
Use constants dictionary in lenCpStruct()

#!/usr/bin/python

import argparse
import os
import struct
import sys

###############
### CLASSES ###
###############
class MyParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(2)

###################
### SUBROUTINES ###
###################
def lenCpStruct(tag):
  if tag == 0xa:
    return 3
  else:
    return -1

############
### MAIN ###
############
parser = MyParser('Run bytecode in jjvm')
parser.add_argument('path', help='path to class')
args = parser.parse_args()

with open(args.path, "rb") as c:
  c.seek(8)
  cpCount = struct.unpack(">H", c.read(2))[0] - 1

  print "Constant pool count: %d" % cpCount;

  while cpCount >= 0:
    cpTag = ord(c.read(1))

    print "Got tag: %d" % cpTag
    cpStructSize = lenCpStruct(cpTag)

    if cpStructSize < 0:
      print "ERROR: cpStructSize %d for tag %d" % (cpStructSize, cpTag)
      sys.exit(1)

    print "Size: %d" % cpStructSize

    cpCount -= 1
    c.seek(cpStructSize, os.SEEK_CUR)


# -*- coding: utf-8 -*-
# Generated by Django 1.10.7 on 2017-09-08 10:23
from __future__ import unicode_literals

from django.db import migrations

from corehq.blobs import get_blob_db
from corehq.sql_db.operations import HqRunPython

FIXTURE_BUCKET = 'domain-fixtures'

def rm_blobdb_domain_fixtures(apps, schema_editor):
    get_blob_db().delete(bucket=FIXTURE_BUCKET)

def noop_reverse_migration(apps, schema_editor):
    pass

class Migration(migrations.Migration):

    dependencies = [
        ('fixtures', '0001_initial'),
    ]

    operations = [
        HqRunPython(rm_blobdb_domain_fixtures, noop_reverse_migration),
    ]

Add migration to delete existing cache values

Response:
# -*- coding: utf-8 -*-
# Generated by Django 1.10.7 on 2017-09-08 10:23
from __future__ import unicode_literals

from django.db import migrations

from corehq.blobs import get_blob_db
from corehq.sql_db.operations import HqRunPython

FIXTURE_BUCKET = 'domain-fixtures'

def rm_blobdb_domain_fixtures(apps, schema_editor):
    get_blob_db().delete(bucket=FIXTURE_BUCKET)

def noop_reverse_migration(apps, schema_editor):
    pass

class Migration(migrations.Migration):

    dependencies = [
        ('fixtures', '0001_initial'),
    ]

    operations = [
        HqRunPython(rm_blobdb_domain_fixtures, noop_reverse_migration),
    ]

Prompt: Add migration to delete existing cache values


from django.core.management.base import BaseCommand
from django.core import serializers

from data_log import models
import json

class Command(BaseCommand):
    help = 'Create Data Log Report fixtures'

    def handle(self, *args, **kwargs):
        self.stdout.write(self.style.HTTP_INFO('Creating fixtures for Data Log Reports...'))
        JSONSerializer = serializers.get_serializer("json")
        j = JSONSerializer()
        data = []
        models_to_serialize = [
            models.LevelReport,
            models.SummonReport,
            models.MagicShopRefreshReport,
            models.MagicBoxCraftingReport,
            models.WishReport,
            models.RuneCraftingReport
        ]

        pks = []
        for model in models_to_serialize:
            self.stdout.write(self.style.WARNING(model.__name__))
            serialized_data = json.loads(j.serialize(model.objects.order_by('-generated_on')[:100]))
            pks += [d['pk'] for d in serialized_data]
            data += serialized_data

        self.stdout.write(self.style.WARNING(models.Report.__name__))
        reports = json.loads(j.serialize(models.Report.objects.order_by('-generated_on')[:100]))

        self.stdout.write(self.style.HTTP_INFO("Finishing special reports..."))
        reports += json.loads(j.serialize(models.Report.objects.filter(pk__in=pks)))

        data = reports + data

        self.stdout.write(self.style.HTTP_INFO('Saving fixtures to file'))
        with open("fixture_reports.json", "w+") as f:
            json.dump(data, f)

        self.stdout.write(self.style.SUCCESS('Done!'))

Fix data log fixture foreign keys

Fix data log fixture foreign keys

Response:
from django.core.management.base import BaseCommand
from django.core import serializers

from data_log import models
import json

class Command(BaseCommand):
    help = 'Create Data Log Report fixtures'

    def handle(self, *args, **kwargs):
        self.stdout.write(self.style.HTTP_INFO('Creating fixtures for Data Log Reports...'))
        JSONSerializer = serializers.get_serializer("json")
        j = JSONSerializer()
        data = []
        models_to_serialize = [
            models.LevelReport,
            models.SummonReport,
            models.MagicShopRefreshReport,
            models.MagicBoxCraftingReport,
            models.WishReport,
            models.RuneCraftingReport
        ]

        pks = []
        for model in models_to_serialize:
            self.stdout.write(self.style.WARNING(model.__name__))
            serialized_data = json.loads(j.serialize(model.objects.order_by('-generated_on')[:100]))
            pks += [d['pk'] for d in serialized_data]
            data += serialized_data

        self.stdout.write(self.style.WARNING(models.Report.__name__))
        reports = json.loads(j.serialize(models.Report.objects.order_by('-generated_on')[:100]))

        self.stdout.write(self.style.HTTP_INFO("Finishing special reports..."))
        reports += json.loads(j.serialize(models.Report.objects.filter(pk__in=pks)))

        data = reports + data

        self.stdout.write(self.style.HTTP_INFO('Saving fixtures to file'))
        with open("fixture_reports.json", "w+") as f:
            json.dump(data, f)

        self.stdout.write(self.style.SUCCESS('Done!'))

Prompt:
Fix data log fixture foreign keys

from django.core.management.base import BaseCommand
from django.core import serializers

from data_log import models
import json

class Command(BaseCommand):
    help = 'Create Data Log Report fixtures'

    def handle(self, *args, **kwargs):
        self.stdout.write(self.style.HTTP_INFO('Creating fixtures for Data Log Reports...'))
        JSONSerializer = serializers.get_serializer("json")
        j = JSONSerializer()
        data = []
        models_to_serialize = [
            models.LevelReport,
            models.SummonReport,
            models.MagicShopRefreshReport,
            models.MagicBoxCraftingReport,
            models.WishReport,
            models.RuneCraftingReport
        ]

        for model in models_to_serialize:
            self.stdout.write(self.style.WARNING(model.__name__))
            data += json.loads(j.serialize(model.objects.order_by('-generated_on')[:100]))

        self.stdout.write(self.style.WARNING(models.Report.__name__))
        data += json.loads(j.serialize(models.Report.objects.order_by('-generated_on')[:1000]))

        with open("fixture_reports.json", "w+") as f:
            json.dump(data, f)

        self.stdout.write(self.style.SUCCESS('Done!'))


#! /usr/bin/env python
#
# Copyright (c) 2011 Red Hat, Inc.
#
# This software is licensed to you under the GNU Lesser General Public
# License as published by the Free Software Foundation; either version
# 2 of the License (LGPLv2) or (at your option) any later version.
# There is NO WARRANTY for this software, express or implied,
# including the implied warranties of MERCHANTABILITY,
# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should
# have received a copy of LGPLv2 along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/lgpl-2.0.txt.
#
# Jeff Ortel <jortel@redhat.com>
#

"""
Provides a debugging agent that loads configuration and plugins
from standard locations.  But, does not need to run as root
since it does use /var directories.
IMPORTANT: be sure the installed daemon is stopped.
"""

import os
import time

from gofer.agent import logutil

class TestAgent:

    ROOT = '/tmp/gofer'

    def start(self):
        self.mkdir()
        logutil.LOGDIR = self.ROOT

        from gofer.agent.main import PluginLoader, Agent, AgentLock, eager
        from gofer.rmi.store import PendingQueue

        AgentLock.PATH = os.path.join(self.ROOT, 'gofer.pid')
        PendingQueue.ROOT = os.path.join(self.ROOT, 'messaging/pending')
        self.mkdir(PendingQueue.ROOT)

        pl = PluginLoader()
        plugins = pl.load(eager())
        agent = Agent(plugins)
        agent.start(False)
        print 'Agent: started'
        while True:
            time.sleep(10)
            print 'Agent: sleeping...'

    def mkdir(self, path=ROOT):
        if not os.path.exists(path):
            os.makedirs(path)
        return path

if __name__ == '__main__':
    agent = TestAgent()
    agent.start()
    print 'Done'

Add for debugging w/o running as root.

Add for debugging w/o running as root.

Response:
#! /usr/bin/env python
#
# Copyright (c) 2011 Red Hat, Inc.
#
# This software is licensed to you under the GNU Lesser General Public
# License as published by the Free Software Foundation; either version
# 2 of the License (LGPLv2) or (at your option) any later version.
# There is NO WARRANTY for this software, express or implied,
# including the implied warranties of MERCHANTABILITY,
# NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should
# have received a copy of LGPLv2 along with this software; if not, see
# http://www.gnu.org/licenses/old-licenses/lgpl-2.0.txt.
#
# Jeff Ortel <jortel@redhat.com>
#

"""
Provides a debugging agent that loads configuration and plugins
from standard locations.  But, does not need to run as root
since it does use /var directories.
IMPORTANT: be sure the installed daemon is stopped.
"""

import os
import time

from gofer.agent import logutil

class TestAgent:

    ROOT = '/tmp/gofer'

    def start(self):
        self.mkdir()
        logutil.LOGDIR = self.ROOT

        from gofer.agent.main import PluginLoader, Agent, AgentLock, eager
        from gofer.rmi.store import PendingQueue

        AgentLock.PATH = os.path.join(self.ROOT, 'gofer.pid')
        PendingQueue.ROOT = os.path.join(self.ROOT, 'messaging/pending')
        self.mkdir(PendingQueue.ROOT)

        pl = PluginLoader()
        plugins = pl.load(eager())
        agent = Agent(plugins)
        agent.start(False)
        print 'Agent: started'
        while True:
            time.sleep(10)
            print 'Agent: sleeping...'

    def mkdir(self, path=ROOT):
        if not os.path.exists(path):
            os.makedirs(path)
        return path

if __name__ == '__main__':
    agent = TestAgent()
    agent.start()
    print 'Done'

Prompt: Add for debugging w/o running as root.


import unittest
import numpy as np
import numpy.testing as test
from word2gauss.words import Vocabulary, iter_pairs

DTYPE = np.float32

class TestIterPairs(unittest.TestCase):
    def test_iter_pairs(self):
        np.random.seed(1234)
        vocab = Vocabulary({'zero': 0, 'one': 1, 'two': 2})

        actual = list(iter_pairs(['zero one two', 'one two zero'],
            vocab, batch_size=2, nsamples=1))
        expected = np.array([[0, 1, 0, 2, 0],
                            [0, 1, 1, 1, 1],
                            [0, 2, 0, 0, 0],
                            [0, 2, 0, 2, 1],
                            [1, 2, 1, 0, 0],
                            [1, 2, 1, 2, 1],
                            [1, 2, 1, 1, 0],
                            [1, 2, 1, 2, 1],
                            [1, 0, 1, 2, 0],
                            [1, 0, 2, 0, 1],
                            [2, 0, 2, 2, 0],
                            [2, 0, 0, 0, 1]], dtype=DTYPE)
        self.assertEqual(len(actual), 1)
        self.assertTrue(np.all(actual[0] == expected))

if __name__ == '__main__':
    unittest.main()

Add a test for iter_pairs

Add a test for iter_pairs

Response:
import unittest
import numpy as np
import numpy.testing as test
from word2gauss.words import Vocabulary, iter_pairs

DTYPE = np.float32

class TestIterPairs(unittest.TestCase):
    def test_iter_pairs(self):
        np.random.seed(1234)
        vocab = Vocabulary({'zero': 0, 'one': 1, 'two': 2})

        actual = list(iter_pairs(['zero one two', 'one two zero'],
            vocab, batch_size=2, nsamples=1))
        expected = np.array([[0, 1, 0, 2, 0],
                            [0, 1, 1, 1, 1],
                            [0, 2, 0, 0, 0],
                            [0, 2, 0, 2, 1],
                            [1, 2, 1, 0, 0],
                            [1, 2, 1, 2, 1],
                            [1, 2, 1, 1, 0],
                            [1, 2, 1, 2, 1],
                            [1, 0, 1, 2, 0],
                            [1, 0, 2, 0, 1],
                            [2, 0, 2, 2, 0],
                            [2, 0, 0, 0, 1]], dtype=DTYPE)
        self.assertEqual(len(actual), 1)
        self.assertTrue(np.all(actual[0] == expected))

if __name__ == '__main__':
    unittest.main()

Prompt: Add a test for iter_pairs


# -*- coding: utf-8 -*-

import sys
from traceback import format_exception

from pyramid.decorator import reify
from pyramid.httpexceptions import HTTPInternalServerError
from pyramid.interfaces import IRequestFactory

from ines.convert import force_string
from ines.middlewares import Middleware
from ines.utils import format_error_to_json

class LoggingMiddleware(Middleware):
    name = 'logging'

    @property
    def request_factory(self):
        return self.config.registry.queryUtility(IRequestFactory)

    @reify
    def api_name(self):
        return self.settings.get('logging_api_name') or 'logging'

    def __call__(self, environ, start_response):
        try:
            for chunk in self.application(environ, start_response):
                yield chunk
        except (BaseException, Exception) as error:
            type_, value, tb = sys.exc_info()
            message = ''.join(format_exception(type_, value, tb))

            # Save / log error
            request = self.request_factory(environ)
            request.registry = self.config.registry

            try:
                small_message = '%s: %s' % (error.__class__.__name__, force_string(error))
            except (BaseException, Exception):
                small_message = error

            try:
                getattr(request.api, self.api_name).log_critical(
                    'internal_server_error',
                    str(small_message))
            except (BaseException, Exception):
                print message

            internal_server_error = HTTPInternalServerError()
            headers = [('Content-type', 'application/json')]
            start_response(internal_server_error.status, headers)
            yield format_error_to_json(internal_server_error, request=request)

Set log with small message

Set log with small message

Response:
# -*- coding: utf-8 -*-

import sys
from traceback import format_exception

from pyramid.decorator import reify
from pyramid.httpexceptions import HTTPInternalServerError
from pyramid.interfaces import IRequestFactory

from ines.convert import force_string
from ines.middlewares import Middleware
from ines.utils import format_error_to_json

class LoggingMiddleware(Middleware):
    name = 'logging'

    @property
    def request_factory(self):
        return self.config.registry.queryUtility(IRequestFactory)

    @reify
    def api_name(self):
        return self.settings.get('logging_api_name') or 'logging'

    def __call__(self, environ, start_response):
        try:
            for chunk in self.application(environ, start_response):
                yield chunk
        except (BaseException, Exception) as error:
            type_, value, tb = sys.exc_info()
            message = ''.join(format_exception(type_, value, tb))

            # Save / log error
            request = self.request_factory(environ)
            request.registry = self.config.registry

            try:
                small_message = '%s: %s' % (error.__class__.__name__, force_string(error))
            except (BaseException, Exception):
                small_message = error

            try:
                getattr(request.api, self.api_name).log_critical(
                    'internal_server_error',
                    str(small_message))
            except (BaseException, Exception):
                print message

            internal_server_error = HTTPInternalServerError()
            headers = [('Content-type', 'application/json')]
            start_response(internal_server_error.status, headers)
            yield format_error_to_json(internal_server_error, request=request)

Prompt:
Set log with small message

# -*- coding: utf-8 -*-

import sys
from traceback import format_exception

from pyramid.decorator import reify
from pyramid.httpexceptions import HTTPInternalServerError
from pyramid.interfaces import IRequestFactory

from ines.middlewares import Middleware
from ines.utils import format_error_to_json

class LoggingMiddleware(Middleware):
    name = 'logging'

    @property
    def request_factory(self):
        return self.config.registry.queryUtility(IRequestFactory)

    @reify
    def api_name(self):
        return self.settings.get('logging_api_name') or 'logging'

    def __call__(self, environ, start_response):
        try:
            for chunk in self.application(environ, start_response):
                yield chunk
        except (BaseException, Exception):
            type_, value, tb = sys.exc_info()
            error = ''.join(format_exception(type_, value, tb))

            # Save / log error
            request = self.request_factory(environ)
            request.registry = self.config.registry

            try:
                message = error.split()[-1]
                getattr(request.api, self.api_name).log_critical('internal_server_error', message)
            except (BaseException, Exception):
                print error

            internal_server_error = HTTPInternalServerError()
            headers = [('Content-type', 'application/json')]
            start_response(internal_server_error.status, headers)
            yield format_error_to_json(internal_server_error, request=request)


from django import forms

from .models import Account

class AccountForm(forms.ModelForm):
    class Meta:
        model = Account
        fields = ('name', 'desc', 'address_one',
                  'address_two', 'city', 'state', 'phone',
        )
        widgets = {
            'name': forms.TextInput(
                attrs={
                    'placeholder':'Company',
                    'class':'col-md-12 form-control'
                }
            ),
            'desc': forms.Textarea(
                attrs={
                    'placeholder':'Enter a description',
                    'class':'form-control'
                }
            ),
            'address_one': forms.TextInput(
                attrs={
                    'placeholder':'Street Address',
                    'class':'gi-form-addr form-control'
                }
            ),
            'address_two': forms.TextInput(
                attrs={
                    'placeholder':'Suite, PO, etc',
                    'class':'gi-form-addr form-control'
                }
            ),
            'city': forms.TextInput(
                attrs={
                    'placeholder':'City',
                    'class':'gi-form-addr form-control'
                }
            ),
            'state': forms.TextInput(
                attrs={
                    'placeholder':'State',
                    'class':'gi-form-addr form-control'
                }
            ),
            'phone': forms.TextInput(
                attrs={
                    'placeholder':'Phone',
                    'class':'gi-form-addr form-control'
                }
            ),
        }

Create the Account Detail Page - Part II > New Account - Create Form

Response:
from django import forms

from .models import Account

class AccountForm(forms.ModelForm):
    class Meta:
        model = Account
        fields = ('name', 'desc', 'address_one',
                  'address_two', 'city', 'state', 'phone',
        )
        widgets = {
            'name': forms.TextInput(
                attrs={
                    'placeholder':'Company',
                    'class':'col-md-12 form-control'
                }
            ),
            'desc': forms.Textarea(
                attrs={
                    'placeholder':'Enter a description',
                    'class':'form-control'
                }
            ),
            'address_one': forms.TextInput(
                attrs={
                    'placeholder':'Street Address',
                    'class':'gi-form-addr form-control'
                }
            ),
            'address_two': forms.TextInput(
                attrs={
                    'placeholder':'Suite, PO, etc',
                    'class':'gi-form-addr form-control'
                }
            ),
            'city': forms.TextInput(
                attrs={
                    'placeholder':'City',
                    'class':'gi-form-addr form-control'
                }
            ),
            'state': forms.TextInput(
                attrs={
                    'placeholder':'State',
                    'class':'gi-form-addr form-control'
                }
            ),
            'phone': forms.TextInput(
                attrs={
                    'placeholder':'Phone',
                    'class':'gi-form-addr form-control'
                }
            ),
        }

Prompt: Create the Account Detail Page - Part II > New Account - Create Form


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from django.views.generic.detail import DetailView
from django.views.generic.list import ListView

from opps.article.models import Post

class OppsList(ListView):

    context_object_name = "context"

    @property
    def template_name(self):
        long_slug = self.kwargs.get('channel__long_slug', 'home')
        return 'channel/{0}.html'.format(long_slug)

    @property
    def queryset(self):
        if not self.kwargs.get('channel__long_slug'):
            return Post.objects.filter(channel__homepage=True).all()
        return Post.objects.filter(
                channel__long_slug=self.kwargs['channel__long_slug']).all()

class OppsDetail(DetailView):

    context_object_name = "context"

    @property
    def template_name(self):
        long_slug = self.kwargs.get('channel__long_slug', 'home')
        return 'article/{0}/{1}.html'.format(long_slug, self.kwargs['slug'])

    @property
    def queryset(self):
        long_slug = self.kwargs.get('channel__long_slug', 'home')
        return Post.objects.filter(channel__long_slug=long_slug,
                slug=self.kwargs['slug']).all()

Response:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from django.views.generic.detail import DetailView
from django.views.generic.list import ListView

from opps.article.models import Post

class OppsList(ListView):

    context_object_name = "context"

    @property
    def template_name(self):
        long_slug = self.kwargs.get('channel__long_slug', 'home')
        return 'channel/{0}.html'.format(long_slug)

    @property
    def queryset(self):
        if not self.kwargs.get('channel__long_slug'):
            return Post.objects.filter(channel__homepage=True).all()
        return Post.objects.filter(
                channel__long_slug=self.kwargs['channel__long_slug']).all()

class OppsDetail(DetailView):

    context_object_name = "context"

    @property
    def template_name(self):
        long_slug = self.kwargs.get('channel__long_slug', 'home')
        return 'article/{0}/{1}.html'.format(long_slug, self.kwargs['slug'])

    @property
    def queryset(self):
        long_slug = self.kwargs.get('channel__long_slug', 'home')
        return Post.objects.filter(channel__long_slug=long_slug,
                slug=self.kwargs['slug']).all()

Prompt:
Fix template name on entry home page (/) on detail page

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from django.views.generic.detail import DetailView
from django.views.generic.list import ListView

from opps.article.models import Post

class OppsList(ListView):

    context_object_name = "context"

    @property
    def template_name(self):
        long_slug = self.kwargs.get('channel__long_slug', 'home')
        return 'channel/{0}.html'.format(long_slug)

    @property
    def queryset(self):
        if not self.kwargs.get('channel__long_slug'):
            return Post.objects.filter(channel__homepage=True).all()
        return Post.objects.filter(
                channel__long_slug=self.kwargs['channel__long_slug']).all()

class OppsDetail(DetailView):

    context_object_name = "context"

    @property
    def template_name(self):
        return 'article/{0}/{1}.html'.format(
                self.kwargs['channel__long_slug'], self.kwargs['slug'])

    @property
    def queryset(self):
        long_slug = self.kwargs.get('channel__long_slug', 'home')
        return Post.objects.filter(channel__long_slug=long_slug,
                slug=self.kwargs['slug']).all()


# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2019-11-03 20:56
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
import journal.models

class Migration(migrations.Migration):

    dependencies = [
        ('journal', '0034_migrate_issue_types'),
        ('core', '0037_journal_xsl_files'),

    ]

    operations = [
        migrations.AddField(
            model_name='journal',
            name='xsl',
            field=models.ForeignKey(default=journal.models.default_xsl, on_delete=django.db.models.deletion.SET_DEFAULT, to='core.XSLFile'),
            preserve_default=False,
        ),
    ]

Fix migration tree for journal 0035

Fix migration tree for journal 0035

Response:
# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2019-11-03 20:56
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
import journal.models

class Migration(migrations.Migration):

    dependencies = [
        ('journal', '0034_migrate_issue_types'),
        ('core', '0037_journal_xsl_files'),

    ]

    operations = [
        migrations.AddField(
            model_name='journal',
            name='xsl',
            field=models.ForeignKey(default=journal.models.default_xsl, on_delete=django.db.models.deletion.SET_DEFAULT, to='core.XSLFile'),
            preserve_default=False,
        ),
    ]

Prompt:
Fix migration tree for journal 0035

# -*- coding: utf-8 -*-
# Generated by Django 1.11.23 on 2019-11-03 20:56
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion
import journal.models

class Migration(migrations.Migration):

    dependencies = [
        ('journal', '0034_migrate_issue_types'),
    ]

    operations = [
        migrations.AddField(
            model_name='journal',
            name='xsl',
            field=models.ForeignKey(default=journal.models.default_xsl, on_delete=django.db.models.deletion.SET_DEFAULT, to='core.XSLFile'),
            preserve_default=False,
        ),
    ]


from django.conf import settings
from purl import Template
import requests

from .models import SimilarResponse

API_URL = Template("http://developer.echonest.com/api/v4/artist/similar"
                   "?api_key=%s&results=100&name={name}"
                   % settings.ECHONEST_API_KEY)

def get_similar_from_api(name):
    url = API_URL.expand({'name': name})
    r = requests.get(str(url))
    r.raise_for_status()
    return SimilarResponse.objects.create(name=name, response=r.json())

def get_similar_from_db(name):
    return SimilarResponse.objects.get(normalized_name=name.upper())

def get_similar(name):
    try:
        response = get_similar_from_db(name)
    except SimilarResponse.DoesNotExist:
        response = get_similar_from_api(name)
    return response.artist_names

Fix bug in The Echo Nest API call

Fix bug in The Echo Nest API call

Response:
from django.conf import settings
from purl import Template
import requests

from .models import SimilarResponse

API_URL = Template("http://developer.echonest.com/api/v4/artist/similar"
                   "?api_key=%s&results=100&name={name}"
                   % settings.ECHONEST_API_KEY)

def get_similar_from_api(name):
    url = API_URL.expand({'name': name})
    r = requests.get(str(url))
    r.raise_for_status()
    return SimilarResponse.objects.create(name=name, response=r.json())

def get_similar_from_db(name):
    return SimilarResponse.objects.get(normalized_name=name.upper())

def get_similar(name):
    try:
        response = get_similar_from_db(name)
    except SimilarResponse.DoesNotExist:
        response = get_similar_from_api(name)
    return response.artist_names

Prompt:
Fix bug in The Echo Nest API call

from django.conf import settings
from purl import Template
import requests

from .models import SimilarResponse

API_URL = Template("http://developer.echonest.com/api/v4/artist/similar"
                   "?api_key=%s&results=100&name={name}"
                   % settings.ECHONEST_API_KEY)

def get_similar_from_api(name):
    url = API_URL.expand({'name': name})
    r = requests.get(url)
    r.raise_for_status()
    return SimilarResponse.objects.create(name=name, response=r.json())

def get_similar_from_db(name):
    return SimilarResponse.objects.get(normalized_name=name.upper())

def get_similar(name):
    try:
        response = get_similar_from_db(name)
    except SimilarResponse.DoesNotExist:
        response = get_similar_from_api(name)
    return response.artist_names


import os
import imp

_formula_cache = {}

_default_search_path = [os.path.join(os.path.dirname(__file__), "..", "Formula")]

class FormulaManager:
    @staticmethod
    def _find(name, search_path=[]):
        file_path = ""
        for path in search_path + default_search_path:
            if os.path.exists(os.path.join(path, name + ".py")):
                file_path = os.path.join(path, name + ".py")
        return file_path

    @staticmethod
    def get(name, search_path=[]):
        file_path = ""

        #name can be a path
        if os.path.isfile(name):
            file_path = name

        name = os.path.splitext(os.path.basename(file_path))[0]

        if _formula_cache.has_key(name):
            return _formula_cache[name]
        else:
            if not file_path:
                file_path = FormulaManager._find(name, search_path)

            formula = imp.load_source(name, file_path)
            _formula_cache[name] = formula

            return formula

Implement class for managing formulas

Implement class for managing formulas

Response:
import os
import imp

_formula_cache = {}

_default_search_path = [os.path.join(os.path.dirname(__file__), "..", "Formula")]

class FormulaManager:
    @staticmethod
    def _find(name, search_path=[]):
        file_path = ""
        for path in search_path + default_search_path:
            if os.path.exists(os.path.join(path, name + ".py")):
                file_path = os.path.join(path, name + ".py")
        return file_path

    @staticmethod
    def get(name, search_path=[]):
        file_path = ""

        #name can be a path
        if os.path.isfile(name):
            file_path = name

        name = os.path.splitext(os.path.basename(file_path))[0]

        if _formula_cache.has_key(name):
            return _formula_cache[name]
        else:
            if not file_path:
                file_path = FormulaManager._find(name, search_path)

            formula = imp.load_source(name, file_path)
            _formula_cache[name] = formula

            return formula

Prompt: Implement class for managing formulas


from rest_framework import serializers as ser

from api.base.serializers import (
    JSONAPISerializer, LinksField, IDField, TypeField
)
from api.base.utils import absolute_reverse

class LicenseSerializer(JSONAPISerializer):
    filterable_fields = frozenset([
        'name',
        'id',
    ])
    non_anonymized_fields = ['type']
    id = IDField(source='_id', read_only=True)
    type = TypeField()
    name = ser.CharField(required=True, help_text='License name')
    text = ser.CharField(required=True, help_text='Full text of the license')
    url = ser.URLField(required=False, help_text='URL for the license')
    required_fields = ser.ListField(source='properties', read_only=True,
                                    help_text='Fields required for this license (provided to help front-end validators)')
    links = LinksField({'self': 'get_absolute_url'})

    class Meta:
        type_ = 'licenses'

    def get_absolute_url(self, obj):
        return absolute_reverse('licenses:license-detail', kwargs={
            'license_id': obj._id,
            'version': self.context['request'].parser_context['kwargs']['version']
        })

Add url to the license api serializer

Add url to the license api serializer

Response:
from rest_framework import serializers as ser

from api.base.serializers import (
    JSONAPISerializer, LinksField, IDField, TypeField
)
from api.base.utils import absolute_reverse

class LicenseSerializer(JSONAPISerializer):
    filterable_fields = frozenset([
        'name',
        'id',
    ])
    non_anonymized_fields = ['type']
    id = IDField(source='_id', read_only=True)
    type = TypeField()
    name = ser.CharField(required=True, help_text='License name')
    text = ser.CharField(required=True, help_text='Full text of the license')
    url = ser.URLField(required=False, help_text='URL for the license')
    required_fields = ser.ListField(source='properties', read_only=True,
                                    help_text='Fields required for this license (provided to help front-end validators)')
    links = LinksField({'self': 'get_absolute_url'})

    class Meta:
        type_ = 'licenses'

    def get_absolute_url(self, obj):
        return absolute_reverse('licenses:license-detail', kwargs={
            'license_id': obj._id,
            'version': self.context['request'].parser_context['kwargs']['version']
        })

Prompt:
Add url to the license api serializer

from rest_framework import serializers as ser

from api.base.serializers import (
    JSONAPISerializer, LinksField, IDField, TypeField
)
from api.base.utils import absolute_reverse

class LicenseSerializer(JSONAPISerializer):
    filterable_fields = frozenset([
        'name',
        'id',
    ])
    non_anonymized_fields = ['type']
    id = IDField(source='_id', read_only=True)
    type = TypeField()
    name = ser.CharField(required=True, help_text='License name')
    text = ser.CharField(required=True, help_text='Full text of the license')
    required_fields = ser.ListField(source='properties', read_only=True,
                                    help_text='Fields required for this license (provided to help front-end validators)')
    links = LinksField({'self': 'get_absolute_url'})

    class Meta:
        type_ = 'licenses'

    def get_absolute_url(self, obj):
        return absolute_reverse('licenses:license-detail', kwargs={
            'license_id': obj._id,
            'version': self.context['request'].parser_context['kwargs']['version']
        })


from setuptools import setup, find_packages

__version__ = '0.2.1'
__pkg_name__ = 'textsift'

setup(
    name = __pkg_name__,
    version = __version__,
    description = 'Text modelling framework',
    author='Andrew Chisholm',
    packages = find_packages(),
    license = 'MIT',
    url = 'https://github.com/wikilinks/sift',
    scripts = [
        'scripts/download-wikipedia'
    ],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 2.7',
        'Topic :: Text Processing :: Linguistic'
    ],
    install_requires = [
        "ujson",
        "numpy",
        "pattern",
        "gensim",
        "msgpack-python"
    ],
    test_suite = __pkg_name__ + '.test'
)

Response:
from setuptools import setup, find_packages

__version__ = '0.2.1'
__pkg_name__ = 'textsift'

setup(
    name = __pkg_name__,
    version = __version__,
    description = 'Text modelling framework',
    author='Andrew Chisholm',
    packages = find_packages(),
    license = 'MIT',
    url = 'https://github.com/wikilinks/sift',
    scripts = [
        'scripts/download-wikipedia'
    ],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 2.7',
        'Topic :: Text Processing :: Linguistic'
    ],
    install_requires = [
        "ujson",
        "numpy",
        "pattern",
        "gensim",
        "msgpack-python"
    ],
    test_suite = __pkg_name__ + '.test'
)

Prompt:
Remove all sift cli script command from package

from setuptools import setup, find_packages

__version__ = '0.2.0'
__pkg_name__ = 'textsift'

setup(
    name = __pkg_name__,
    version = __version__,
    description = 'Text modelling framework',
    author='Andrew Chisholm',
    packages = find_packages(),
    license = 'MIT',
    url = 'https://github.com/wikilinks/sift',
    scripts = [
        'scripts/sift',
        'scripts/download-wikipedia'
    ],
    classifiers=[
        'Development Status :: 4 - Beta',
        'Environment :: Console',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 2.7',
        'Topic :: Text Processing :: Linguistic'
    ],
    install_requires = [
        "ujson",
        "numpy",
        "pattern",
        "gensim",
        "msgpack-python"
    ],
    test_suite = __pkg_name__ + '.test'
)


__version__ = '1.6.0'
__author__ = 'Cameron Higby-Naquin'

Response:
__version__ = '1.6.0'
__author__ = 'Cameron Higby-Naquin'

Prompt:
Increment minor version for new feature release.

__version__ = '1.5.2'
__author__ = 'Cameron Higby-Naquin'


from twisted.words.protocols import irc

class HeufyBotConnection(irc.IRC):
    def __init__(self, protocol):
        self.protocol = protocol
        self.nick = "PyHeufyBot"  # TODO This will be set by a configuration at some point
        self.ident = "PyHeufyBot"  # TODO This will be set by a configuration at some point
        self.gecos = "PyHeufyBot IRC Bot"  # TODO This will be set by a configuration at some point
        self.channels = {}
        self.usermodes = {}

    def connectionMade(self):
        self.cmdNICK(self.nick)
        self.cmdUSER(self.ident, self.gecos)

    def connectionLost(self, reason=""):
        print reason

    def dataReceived(self, data):
        print data

    def sendMessage(self, command, *parameter_list, **prefix):
        print command, " ".join(parameter_list)
        irc.IRC.sendMessage(self, command, *parameter_list, **prefix)

    def cmdNICK(self, nick):
        self.sendMessage("NICK", nick)

    def cmdUSER(self, ident, gecos):
        # RFC2812 allows usermodes to be set, but this isn't implemented much in IRCds at all.
        # Pass 0 for usermodes instead.
        self.sendMessage("USER", ident, "0", "*", ":{}".format(gecos))

Response:
from twisted.words.protocols import irc

class HeufyBotConnection(irc.IRC):
    def __init__(self, protocol):
        self.protocol = protocol
        self.nick = "PyHeufyBot"  # TODO This will be set by a configuration at some point
        self.ident = "PyHeufyBot"  # TODO This will be set by a configuration at some point
        self.gecos = "PyHeufyBot IRC Bot"  # TODO This will be set by a configuration at some point
        self.channels = {}
        self.usermodes = {}

    def connectionMade(self):
        self.cmdNICK(self.nick)
        self.cmdUSER(self.ident, self.gecos)

    def connectionLost(self, reason=""):
        print reason

    def dataReceived(self, data):
        print data

    def sendMessage(self, command, *parameter_list, **prefix):
        print command, " ".join(parameter_list)
        irc.IRC.sendMessage(self, command, *parameter_list, **prefix)

    def cmdNICK(self, nick):
        self.sendMessage("NICK", nick)

    def cmdUSER(self, ident, gecos):
        # RFC2812 allows usermodes to be set, but this isn't implemented much in IRCds at all.
        # Pass 0 for usermodes instead.
        self.sendMessage("USER", ident, "0", "*", ":{}".format(gecos))

Prompt:
Add dictionaries for channels and usermodes

from twisted.words.protocols import irc

class HeufyBotConnection(irc.IRC):
    def __init__(self, protocol):
        self.protocol = protocol
        self.nick = "PyHeufyBot"  # TODO This will be set by a configuration at some point
        self.ident = "PyHeufyBot"  # TODO This will be set by a configuration at some point
        self.gecos = "PyHeufyBot IRC Bot"  # TODO This will be set by a configuration at some point

    def connectionMade(self):
        self.cmdNICK(self.nick)
        self.cmdUSER(self.ident, self.gecos)

    def connectionLost(self, reason=""):
        print reason

    def dataReceived(self, data):
        print data

    def sendMessage(self, command, *parameter_list, **prefix):
        print command, " ".join(parameter_list)
        irc.IRC.sendMessage(self, command, *parameter_list, **prefix)

    def cmdNICK(self, nick):
        self.sendMessage("NICK", nick)

    def cmdUSER(self, ident, gecos):
        # RFC2812 allows usermodes to be set, but this isn't implemented much in IRCds at all.
        # Pass 0 for usermodes instead.
        self.sendMessage("USER", ident, "0", "*", ":{}".format(gecos))


# -*- encoding: utf-8 *-*
#!/usr/bin/env python
import os
import sys
from glob import glob
from setuptools import setup, Extension
from setuptools.command.sdist import sdist
hashindex_sources = ['darc/hashindex.pyx', 'darc/_hashindex.c']

try:
    from Cython.Distutils import build_ext
    import Cython.Compiler.Main as cython_compiler
    class Sdist(sdist):
        def __init__(self, *args, **kwargs):
            for src in glob('darc/*.pyx'):
                print 'src', src
                cython_compiler.compile(glob('darc/*.pyx'),
                                        cython_compiler.default_options)
            sdist.__init__(self, *args, **kwargs)

        def run(self):
            sdist.run(self)
            self.filelist.append('darc/hashindex.c', 'darc/hashindex.h')

except ImportError:
    hashindex_sources[0] = hashindex_sources[0].replace('.pyx', '.c')
    from setuptools.command.build_ext import build_ext
    Sdist = sdist
    if not os.path.exists('darc/hashindex.c'):
        raise ImportError('The GIT version of darc needs Cython. Install Cython or use a released version')

dependencies = ['pycrypto', 'msgpack-python', 'pbkdf2.py', 'xattr', 'paramiko']
if sys.version_info < (2, 7):
    dependencies.append('argparse')

setup(name='darc',
      version='0.1',
      author='Jonas Borgström',
      author_email='jonas@borgstrom.se',
      url='http://github.com/jborg/darc/',
      packages=['darc'],
      cmdclass = {'build_ext': build_ext, 'sdist': Sdist},
      ext_modules=[
      Extension('darc._speedups', ['darc/_speedups.c']),
      Extension('darc.hashindex', hashindex_sources)],
      install_requires=dependencies,
      entry_points = {
        'console_scripts': [
            'darc = darc.archiver:main',
        ]
    })

Include Cython output in sdist

Include Cython output in sdist

Response:
# -*- encoding: utf-8 *-*
#!/usr/bin/env python
import os
import sys
from glob import glob
from setuptools import setup, Extension
from setuptools.command.sdist import sdist
hashindex_sources = ['darc/hashindex.pyx', 'darc/_hashindex.c']

try:
    from Cython.Distutils import build_ext
    import Cython.Compiler.Main as cython_compiler
    class Sdist(sdist):
        def __init__(self, *args, **kwargs):
            for src in glob('darc/*.pyx'):
                print 'src', src
                cython_compiler.compile(glob('darc/*.pyx'),
                                        cython_compiler.default_options)
            sdist.__init__(self, *args, **kwargs)

        def run(self):
            sdist.run(self)
            self.filelist.append('darc/hashindex.c', 'darc/hashindex.h')

except ImportError:
    hashindex_sources[0] = hashindex_sources[0].replace('.pyx', '.c')
    from setuptools.command.build_ext import build_ext
    Sdist = sdist
    if not os.path.exists('darc/hashindex.c'):
        raise ImportError('The GIT version of darc needs Cython. Install Cython or use a released version')

dependencies = ['pycrypto', 'msgpack-python', 'pbkdf2.py', 'xattr', 'paramiko']
if sys.version_info < (2, 7):
    dependencies.append('argparse')

setup(name='darc',
      version='0.1',
      author='Jonas Borgström',
      author_email='jonas@borgstrom.se',
      url='http://github.com/jborg/darc/',
      packages=['darc'],
      cmdclass = {'build_ext': build_ext, 'sdist': Sdist},
      ext_modules=[
      Extension('darc._speedups', ['darc/_speedups.c']),
      Extension('darc.hashindex', hashindex_sources)],
      install_requires=dependencies,
      entry_points = {
        'console_scripts': [
            'darc = darc.archiver:main',
        ]
    })

Prompt:
Include Cython output in sdist

# -*- encoding: utf-8 *-*
#!/usr/bin/env python
import sys
from setuptools import setup, Extension
from Cython.Distutils import build_ext

dependencies = ['pycrypto', 'msgpack-python', 'pbkdf2.py', 'xattr', 'paramiko', 'Pyrex', 'Cython']
if sys.version_info < (2, 7):
    dependencies.append('argparse')

setup(name='darc',
      version='0.1',
      author='Jonas Borgström',
      author_email='jonas@borgstrom.se',
      packages=['darc'],
      cmdclass = {'build_ext': build_ext},
      ext_modules=[
      Extension('darc._speedups', ['darc/_speedups.c']),
                   Extension('darc.hashindex', ['darc/hashindex.pyx', 'darc/_hashindex.c'])],
      install_requires=dependencies,
      entry_points = {
        'console_scripts': [
            'darc = darc.archiver:main',
        ]
    })


# Licensed under an MIT open source license - see LICENSE
from __future__ import print_function, absolute_import, division

class TurbuStatTestingWarning(Warning):
    '''
    Turbustat.statistics warning for untested methods.
    '''

class TurbuStatMetricWarning(Warning):
    '''
    Turbustat.statistics warning for misusing a distance metric.
    '''

Response:
# Licensed under an MIT open source license - see LICENSE
from __future__ import print_function, absolute_import, division

class TurbuStatTestingWarning(Warning):
    '''
    Turbustat.statistics warning for untested methods.
    '''

class TurbuStatMetricWarning(Warning):
    '''
    Turbustat.statistics warning for misusing a distance metric.
    '''

Prompt:
Add warning for where a distance metric is being misused

# Licensed under an MIT open source license - see LICENSE
from __future__ import print_function, absolute_import, division

class TurbuStatTestingWarning(Warning):
    '''
    Turbustat.statistics warning for untested methods.
    '''


import packages.project as project
import packages.primitive as primitive
import packages.utils as utils
import packages.processing
import packages.io
import argparse
from matplotlib import pyplot as plt
import matplotlib.mlab as mlab
import numpy as np
from scipy.stats import norm,kstest,skewtest,kurtosistest,normaltest

compareToGt = False

itmin=0
itmax=3

################################################################################
## Command line parsing
parser = argparse.ArgumentParser(description='Collect full run statistics.')
parser.add_argument('projectdir')

args = parser.parse_args()

projectdir = args.projectdir
if projectdir[-1] == '/':
    projectdir = projectdir[:-1]

projectname  = projectdir.split('/')[-1]
cloudfile    = projectdir+'/cloud.ply'
cloud        = packages.io.readPointCloudFromPly(cloudfile)

print 'Processing project ', projectname

sigma_ref = 1.

if compareToGt:
    projectfile  = projectdir+'/gt/'+projectname+'.prj'
    gtlinesfile  = projectdir+'/gt/primitives.csv'
    gtassignfile = projectdir+'/gt/points_primitives.csv'

    project  = project.PyProject(projectfile)
    gtlines  = primitive.readPrimitivesFromFile(gtlinesfile)
    gtassign = packages.io.readPointAssignementFromFiles(gtassignfile)

    gtlines  = packages.processing.removeUnassignedPrimitives(gtlines, gtassign)
    gtassign = packages.processing.removeUnassignedPoint(gtlines, gtassign)

    ############################################################################
    ## Process noise
    sigma_ref = project.kernels[0].stdev()
    gtDistrib   = utils.distanceToPrimitives(cloud, gtassign,   gtlines)
    print "Ground truth sigma ", np.sqrt(np.var(gtDistrib)) / sigma_ref

for it in range(itmin, itmax):
    print "Processing iteration",it

    linesfile_it  = projectdir+'/primitives_merged_it'+str(it)+'.csv'
    assignfile_it = projectdir+'/points_primitives_it'+str(it)+'.csv'

    ################################################################################
    ## Reading input files
    lines_it  = primitive.readPrimitivesFromFile(linesfile_it)
    assign_it = packages.io.readPointAssignementFromFiles(assignfile_it)

    lines_it  = packages.processing.removeUnassignedPrimitives(lines_it, assign_it)
    assign_it = packages.processing.removeUnassignedPoint(lines_it, assign_it)

    ################################################################################
    ## Process noise

    # Compute the distance between each point and its assigned primitive
    distrib_it = utils.distanceToPrimitives(cloud, assign_it, lines_it)
    print "Estimated sigma ", np.sqrt(np.var(distrib_it)) / sigma_ref

Add a new script processing different iterations of a run and computing statistics

Response:
import packages.project as project
import packages.primitive as primitive
import packages.utils as utils
import packages.processing
import packages.io
import argparse
from matplotlib import pyplot as plt
import matplotlib.mlab as mlab
import numpy as np
from scipy.stats import norm,kstest,skewtest,kurtosistest,normaltest

compareToGt = False

itmin=0
itmax=3

################################################################################
## Command line parsing
parser = argparse.ArgumentParser(description='Collect full run statistics.')
parser.add_argument('projectdir')

args = parser.parse_args()

projectdir = args.projectdir
if projectdir[-1] == '/':
    projectdir = projectdir[:-1]

projectname  = projectdir.split('/')[-1]
cloudfile    = projectdir+'/cloud.ply'
cloud        = packages.io.readPointCloudFromPly(cloudfile)

print 'Processing project ', projectname

sigma_ref = 1.

if compareToGt:
    projectfile  = projectdir+'/gt/'+projectname+'.prj'
    gtlinesfile  = projectdir+'/gt/primitives.csv'
    gtassignfile = projectdir+'/gt/points_primitives.csv'

    project  = project.PyProject(projectfile)
    gtlines  = primitive.readPrimitivesFromFile(gtlinesfile)
    gtassign = packages.io.readPointAssignementFromFiles(gtassignfile)

    gtlines  = packages.processing.removeUnassignedPrimitives(gtlines, gtassign)
    gtassign = packages.processing.removeUnassignedPoint(gtlines, gtassign)

    ############################################################################
    ## Process noise
    sigma_ref = project.kernels[0].stdev()
    gtDistrib   = utils.distanceToPrimitives(cloud, gtassign,   gtlines)
    print "Ground truth sigma ", np.sqrt(np.var(gtDistrib)) / sigma_ref

for it in range(itmin, itmax):
    print "Processing iteration",it

    linesfile_it  = projectdir+'/primitives_merged_it'+str(it)+'.csv'
    assignfile_it = projectdir+'/points_primitives_it'+str(it)+'.csv'

    ################################################################################
    ## Reading input files
    lines_it  = primitive.readPrimitivesFromFile(linesfile_it)
    assign_it = packages.io.readPointAssignementFromFiles(assignfile_it)

    lines_it  = packages.processing.removeUnassignedPrimitives(lines_it, assign_it)
    assign_it = packages.processing.removeUnassignedPoint(lines_it, assign_it)

    ################################################################################
    ## Process noise

    # Compute the distance between each point and its assigned primitive
    distrib_it = utils.distanceToPrimitives(cloud, assign_it, lines_it)
    print "Estimated sigma ", np.sqrt(np.var(distrib_it)) / sigma_ref

Prompt: Add a new script processing different iterations of a run and computing statistics


# -*- coding: utf-8 -*-

from django.db import models
from django.contrib.auth.models import User
from django.contrib import admin
import os.path

class UploadedFile(models.Model):
    user = models.ForeignKey(User, verbose_name=u'Käyttäjä')
    description = models.TextField(u'Kuvaus', help_text=u'Lyhyt kuvaus siitä, mihin/missä tiedostoa käytetään.', blank=True)
    file = models.FileField(u'Tiedosto', upload_to='admin_upload/')
    date = models.DateTimeField(u'Aika')

    def __unicode__(self):
        return self.file.name + ' by' + self.user.username + ')'

    class Meta:
        verbose_name=u"tiedosto"
        verbose_name_plural=u"tiedostot"

    def name(self):
        return os.path.basename(self.file.name)

try:
    admin.site.register(UploadedFile)
except:
    pass

Response:
# -*- coding: utf-8 -*-

from django.db import models
from django.contrib.auth.models import User
from django.contrib import admin
import os.path

class UploadedFile(models.Model):
    user = models.ForeignKey(User, verbose_name=u'Käyttäjä')
    description = models.TextField(u'Kuvaus', help_text=u'Lyhyt kuvaus siitä, mihin/missä tiedostoa käytetään.', blank=True)
    file = models.FileField(u'Tiedosto', upload_to='admin_upload/')
    date = models.DateTimeField(u'Aika')

    def __unicode__(self):
        return self.file.name + ' by' + self.user.username + ')'

    class Meta:
        verbose_name=u"tiedosto"
        verbose_name_plural=u"tiedostot"

    def name(self):
        return os.path.basename(self.file.name)

try:
    admin.site.register(UploadedFile)
except:
    pass

Prompt:
admin_upload: Add helper function for getting name from UploadedFile, add model to admin.

# -*- coding: utf-8 -*-

from django.db import models
from django.contrib.auth.models import User
from django.contrib import admin

class UploadedFile(models.Model):
    user = models.ForeignKey(User, verbose_name=u'Käyttäjä')
    description = models.TextField(u'Kuvaus', help_text=u'Lyhyt kuvaus siitä, mihin/missä tiedostoa käytetään.', blank=True)
    file = models.FileField(u'Tiedosto', upload_to='admin_upload/')
    date = models.DateTimeField(u'Aika')
    def __unicode__(self):
        return self.file.name + ' by' + self.user.username + ')'
    class Meta:
        verbose_name=u"tiedosto"
        verbose_name_plural=u"tiedostot"


from flask import Flask
import relay_api.api.backend as backend

server = Flask(__name__)

backend.init_relays()

@server.route("/relay-api/relays/", methods=["GET"])
def get_all_relays():
    js = backend.get_all_relays()
    return js, 200

@server.route("/relay-api/relays/<relay_name>", methods=["GET"])
def get_relay(relay_name):
    js = backend.get_relay(relay_name)
    if not js:
        return "", 404
    return js, 200

Fix issue with backend import

Fix issue with backend import

Response:
from flask import Flask
import relay_api.api.backend as backend

server = Flask(__name__)

backend.init_relays()

@server.route("/relay-api/relays/", methods=["GET"])
def get_all_relays():
    js = backend.get_all_relays()
    return js, 200

@server.route("/relay-api/relays/<relay_name>", methods=["GET"])
def get_relay(relay_name):
    js = backend.get_relay(relay_name)
    if not js:
        return "", 404
    return js, 200

Prompt:
Fix issue with backend import

from flask import Flask
import relay_api.api.server as backend

server = Flask(__name__)

backend.init_relays()

@server.route("/relay-api/relays/", methods=["GET"])
def get_all_relays():
    js = backend.get_all_relays()
    return js, 200

@server.route("/relay-api/relays/<relay_name>", methods=["GET"])
def get_relay(relay_name):
    js = backend.get_relay(relay_name)
    if not js:
        return "", 404
    return js, 200


import re

try:
  import javascript
  import html
except:
  from . import javascript
  from . import html

def expand(string, start, end, extension=None):

  if(re.compile("html|htm|xml").search(extension)):
    return html.expand(string, start, end)

  return javascript.expand(string, start, end)

Response:
import re

try:
  import javascript
  import html
except:
  from . import javascript
  from . import html

def expand(string, start, end, extension=None):

  if(re.compile("html|htm|xml").search(extension)):
    return html.expand(string, start, end)

  return javascript.expand(string, start, end)

Prompt:
Use html strategy for any file that has xml/html in file extension. This will will match shtml, xhtml and so on.

try:
  import javascript
  import html
except:
  from . import javascript
  from . import html

def expand(string, start, end, extension=None):

  if(extension in ["html", "htm", "xml"]):
    return html.expand(string, start, end)

  return javascript.expand(string, start, end)


# osm_id: 43950409 name: San Pablo Bay
assert_has_feature(
    14, 2623, 6318, 'water',
    { 'kind': 'bay', 'label_placement': 'yes' })

# osm_id: 360566115 name: Byron strait
assert_has_feature(
    14, 15043, 8311, 'water',
    { 'kind': 'strait', 'label_placement': 'yes' })

# osm_id: -1451065 name: Horsens Fjord
assert_has_feature(
    14, 8645, 5114, 'water',
    { 'kind': 'fjord', 'label_placement': 'yes' })

Add tests for strait and fjord

Add tests for strait and fjord

Response:
# osm_id: 43950409 name: San Pablo Bay
assert_has_feature(
    14, 2623, 6318, 'water',
    { 'kind': 'bay', 'label_placement': 'yes' })

# osm_id: 360566115 name: Byron strait
assert_has_feature(
    14, 15043, 8311, 'water',
    { 'kind': 'strait', 'label_placement': 'yes' })

# osm_id: -1451065 name: Horsens Fjord
assert_has_feature(
    14, 8645, 5114, 'water',
    { 'kind': 'fjord', 'label_placement': 'yes' })

Prompt:
Add tests for strait and fjord

assert_has_feature(
    14, 2623, 6318, 'water',
    { 'kind': 'bay', 'label_placement': 'yes' })


from django import VERSION as DJANGO_VERSION
from django import template
from django.conf import settings
if DJANGO_VERSION[0] == 1 and DJANGO_VERSION[1] <= 9:
    from django.core.urlresolvers import reverse
else:
    from django.urls import reverse

register = template.Library()

@register.simple_tag(takes_context=True)
def active_link(context, viewname, css_class=None, strict=None):
    """
    Renders the given CSS class if the request path matches the path of the view.
    :param context: The context where the tag was called. Used to access the request object.
    :param viewname: The name of the view (include namespaces if any).
    :param css_class: The CSS class to render.
    :param strict: If True, the tag will perform an exact match with the request path.
    :return:
    """
    if css_class is None:
        css_class = getattr(settings, 'ACTIVE_LINK_CSS_CLASS', 'active')

    if strict is None:
        strict = getattr(settings, 'ACTIVE_LINK_STRICT', False)

    request = context.get('request')
    if request is None:
        # Can't work without the request object.
        return ''
    path = reverse(viewname)
    if strict:
        active = request.path == path
    else:
        active = request.path.find(path) == 0
    if active:
        return css_class
    return ''

Improve how the active link is checked

Improve how the active link is checked

Response:
from django import VERSION as DJANGO_VERSION
from django import template
from django.conf import settings
if DJANGO_VERSION[0] == 1 and DJANGO_VERSION[1] <= 9:
    from django.core.urlresolvers import reverse
else:
    from django.urls import reverse

register = template.Library()

@register.simple_tag(takes_context=True)
def active_link(context, viewname, css_class=None, strict=None):
    """
    Renders the given CSS class if the request path matches the path of the view.
    :param context: The context where the tag was called. Used to access the request object.
    :param viewname: The name of the view (include namespaces if any).
    :param css_class: The CSS class to render.
    :param strict: If True, the tag will perform an exact match with the request path.
    :return:
    """
    if css_class is None:
        css_class = getattr(settings, 'ACTIVE_LINK_CSS_CLASS', 'active')

    if strict is None:
        strict = getattr(settings, 'ACTIVE_LINK_STRICT', False)

    request = context.get('request')
    if request is None:
        # Can't work without the request object.
        return ''
    path = reverse(viewname)
    if strict:
        active = request.path == path
    else:
        active = request.path.find(path) == 0
    if active:
        return css_class
    return ''

Prompt:
Improve how the active link is checked

from django import VERSION as DJANGO_VERSION
from django import template
from django.conf import settings
if DJANGO_VERSION[0] == 1 and DJANGO_VERSION[1] <= 9:
    from django.core.urlresolvers import reverse
else:
    from django.urls import reverse

register = template.Library()

@register.simple_tag(takes_context=True)
def active_link(context, viewname, css_class=None, strict=None):
    """
    Renders the given CSS class if the request path matches the path of the view.
    :param context: The context where the tag was called. Used to access the request object.
    :param viewname: The name of the view (include namespaces if any).
    :param css_class: The CSS class to render.
    :param strict: If True, the tag will perform an exact match with the request path.
    :return:
    """
    if css_class is None:
        css_class = getattr(settings, 'ACTIVE_LINK_CSS_CLASS', 'active')

    if strict is None:
        strict = getattr(settings, 'ACTIVE_LINK_STRICT', False)

    request = context.get('request')
    if request is None:
        # Can't work without the request object.
        return ''
    path = reverse(viewname)
    if strict:
        active = request.path == path
    else:
        active = path in request.path
    if active:
        return css_class
    return ''


from Helpers.string import *

def binop_aexp(commands, env, op, left, right):
    left.compile_vm(commands, env)
    right.compile_vm(commands, env)
    if op == '+':
        value = assemble(Add)
    elif op == '-':
        value = assemble(Sub)
    elif op == '*':
        value = assemble(Mul)
    elif op == '/':
        value = assemble(Div)
    elif op == '%':
        value = assemble(Mod)
    else:
        raise RuntimeError('unknown operator: ' + op)
    commands.append(value)

def int_aexp(commands, env, i):
    commands.append(assemble(Push, i))

def var_aexp(commands, env, name):
    var_type = Environment.get_var_type(env, name)
    var_value = Environment.get_var(env, name)
    if var_type == 'String':
        String.compile_get(commands, env, var_value)
    else:
        commands.append(assemble(Load, var_value))

Response:
from Helpers.string import *

def binop_aexp(commands, env, op, left, right):
    left.compile_vm(commands, env)
    right.compile_vm(commands, env)
    if op == '+':
        value = assemble(Add)
    elif op == '-':
        value = assemble(Sub)
    elif op == '*':
        value = assemble(Mul)
    elif op == '/':
        value = assemble(Div)
    elif op == '%':
        value = assemble(Mod)
    else:
        raise RuntimeError('unknown operator: ' + op)
    commands.append(value)

def int_aexp(commands, env, i):
    commands.append(assemble(Push, i))

def var_aexp(commands, env, name):
    var_type = Environment.get_var_type(env, name)
    var_value = Environment.get_var(env, name)
    if var_type == 'String':
        String.compile_get(commands, env, var_value)
    else:
        commands.append(assemble(Load, var_value))

Prompt:
Fix compiling problem for runtime variables

from Helpers.string import *

def binop_aexp(commands, env, op, left, right):
    left.compile_vm(commands, env)
    right.compile_vm(commands, env)
    if op == '+':
        value = assemble(Add)
    elif op == '-':
        value = assemble(Sub)
    elif op == '*':
        value = assemble(Mul)
    elif op == '/':
        value = assemble(Div)
    elif op == '%':
        value = assemble(Mod)
    else:
        raise RuntimeError('unknown operator: ' + op)
    commands.append(value)

def int_aexp(commands, env, i):
    commands.append(assemble(Push, i))

def var_aexp(commands, env, name):
    var_type = Environment.get_var_type(env, name)
    var_value = Environment.get_var(env, name)
    if var_type == 'IntAexp':
        commands.append(assemble(Load, var_value))
    elif var_type == 'Char':
        commands.append(assemble(Load, var_value))
    elif var_type == 'String':
        String.compile_get(commands, env, var_value)


from datetime import date, timedelta

class DateWithCalendar(object):
    def __init__(self, calendar_class, date):
        self.calendar = calendar_class
        self.date = date

    def convert_to(self, calendar):
        return calendar.from_date(self.date)

    def __eq__(self, other):
        return self.calendar == other.calendar and self.date == other.date

class ProlepticGregorianCalendar(object):
    def date(self, year, month, day):
        d = date(year, month, day)
        return DateWithCalendar(ProlepticGregorianCalendar, d)

    def from_date(self, date):
        return DateWithCalendar(ProlepticGregorianCalendar, date)

class JulianCalendar(object):
    def date(self, year, month, day):
        d = date(year, month, day)
        d = d + timedelta(days=10)
        return DateWithCalendar(JulianCalendar, d)

Response:
from datetime import date, timedelta

class DateWithCalendar(object):
    def __init__(self, calendar_class, date):
        self.calendar = calendar_class
        self.date = date

    def convert_to(self, calendar):
        return calendar.from_date(self.date)

    def __eq__(self, other):
        return self.calendar == other.calendar and self.date == other.date

class ProlepticGregorianCalendar(object):
    def date(self, year, month, day):
        d = date(year, month, day)
        return DateWithCalendar(ProlepticGregorianCalendar, d)

    def from_date(self, date):
        return DateWithCalendar(ProlepticGregorianCalendar, date)

class JulianCalendar(object):
    def date(self, year, month, day):
        d = date(year, month, day)
        d = d + timedelta(days=10)
        return DateWithCalendar(JulianCalendar, d)

Prompt:
Allow conversion from Julian to ProlepticGregorian, also comparison of dates.

from datetime import date

class DateWithCalendar(object):
    def __init__(self, calendar_class, date):
        self.calendar = calendar_class
        self.date = date

class ProlepticGregorianCalendar(object):
    def date(self, year, month, day):
        d = date(year, month, day)
        return DateWithCalendar(ProlepticGregorianCalendar, d)

class JulianCalendar(object):
    def date(self, year, month, day):
        d = date(year, month, day)
        return DateWithCalendar(JulianCalendar, d)


# -*- coding: utf-8 -*-
from django import forms
from django.db import models
from django.utils.translation import ugettext_lazy as _

from .widgets import AppHookConfigWidget

class AppHookConfigFormField(forms.ModelChoiceField):

    def __init__(self, queryset, empty_label="---------", required=True,
            widget=AppHookConfigWidget, *args, **kwargs):
        super(AppHookConfigFormField, self).__init__(queryset, empty_label,
            required, widget, *args, **kwargs)

class AppHookConfigField(models.ForeignKey):

    def __init__(self, *args, **kwargs):
        kwargs.update({'help_text': _(u'When selecting a value, the form is reloaded to get the updated default')})
        super(AppHookConfigField, self).__init__(*args, **kwargs)

    def formfield(self, **kwargs):
        kwargs.update({'form_class': AppHookConfigFormField})
        return super(AppHookConfigField, self).formfield(**kwargs)

Response:
# -*- coding: utf-8 -*-
from django import forms
from django.db import models
from django.utils.translation import ugettext_lazy as _

from .widgets import AppHookConfigWidget

class AppHookConfigFormField(forms.ModelChoiceField):

    def __init__(self, queryset, empty_label="---------", required=True,
            widget=AppHookConfigWidget, *args, **kwargs):
        super(AppHookConfigFormField, self).__init__(queryset, empty_label,
            required, widget, *args, **kwargs)

class AppHookConfigField(models.ForeignKey):

    def __init__(self, *args, **kwargs):
        kwargs.update({'help_text': _(u'When selecting a value, the form is reloaded to get the updated default')})
        super(AppHookConfigField, self).__init__(*args, **kwargs)

    def formfield(self, **kwargs):
        kwargs.update({'form_class': AppHookConfigFormField})
        return super(AppHookConfigField, self).formfield(**kwargs)

Prompt:
Improve the ability for developers to extend or modify

# -*- coding: utf-8 -*-
from django import forms
from django.db import models
from django.utils.translation import ugettext_lazy as _

from .widgets import AppHookConfigWidget

class AppHookConfigField(models.ForeignKey):

    def __init__(self, *args, **kwargs):
        kwargs.update({'help_text': _(u'When selecting a value, the form is reloaded to get the updated default')})
        super(AppHookConfigField, self).__init__(*args, **kwargs)

    def formfield(self, **kwargs):
        class AppHookConfigFormField(forms.ModelChoiceField):
            widget = AppHookConfigWidget
        kwargs.update({'form_class': AppHookConfigFormField})
        return super(AppHookConfigField, self).formfield(**kwargs)


#
# Copyright 2014, NICTA
#
# This software may be distributed and modified according to the terms of
# the BSD 2-Clause license. Note that NO WARRANTY is provided.
# See "LICENSE_BSD2.txt" for details.
#
# @TAG(NICTA_BSD)
#

'''Various helpers for doing advanced things with dictionaries.'''

import re

def get_fields(s):
    '''Return a set of field names referenced as formatting keys in the given
    string. I thought there would be an easier way to get this, but I can't
    find one. E.g. get_fields('%(hello)s %(world)s') returns
    set('hello', 'world').'''
    return set(re.findall(r'%\(([^)]+)\)', s))

class Guard(object):
    '''Representation of a condition required for some action. See usage in
    Template.py.'''
    def __init__(self, guard_fn):
        self.guard_fn = guard_fn

    def __call__(self, arg):
        return self.guard_fn(arg)

Response:
#
# Copyright 2014, NICTA
#
# This software may be distributed and modified according to the terms of
# the BSD 2-Clause license. Note that NO WARRANTY is provided.
# See "LICENSE_BSD2.txt" for details.
#
# @TAG(NICTA_BSD)
#

'''Various helpers for doing advanced things with dictionaries.'''

import re

def get_fields(s):
    '''Return a set of field names referenced as formatting keys in the given
    string. I thought there would be an easier way to get this, but I can't
    find one. E.g. get_fields('%(hello)s %(world)s') returns
    set('hello', 'world').'''
    return set(re.findall(r'%\(([^)]+)\)', s))

class Guard(object):
    '''Representation of a condition required for some action. See usage in
    Template.py.'''
    def __init__(self, guard_fn):
        self.guard_fn = guard_fn

    def __call__(self, arg):
        return self.guard_fn(arg)

Prompt:
Support non-string formats in `get_fields`.

This commit rewrites the `get_fields` function to be simpler and more general.
The previous implementation relied on ad hoc dict mocking and only supported
format strings involving string-valued formats ('%(name)s'). From this commit
other formats (e.g. '%(name)04d') are supported. The motivation for this is
upcoming changes to how the DMA pool frames are named.

#
# Copyright 2014, NICTA
#
# This software may be distributed and modified according to the terms of
# the BSD 2-Clause license. Note that NO WARRANTY is provided.
# See "LICENSE_BSD2.txt" for details.
#
# @TAG(NICTA_BSD)
#

'''Various helpers for doing advanced things with dictionaries.'''

def get_fields(s):
    '''Return a set of field names referenced as formatting keys in the given
    string. I thought there would be an easier way to get this, but I can't
    find one. E.g. get_fields('%(hello)s %(world)s') returns
    set('hello', 'world').'''
    class FakeDict(dict):
        def __init__(self):
            super(FakeDict, self).__init__()
            self.referenced = set()
        def __getitem__(self, key):
            self.referenced.add(key)
            return ''
    f = FakeDict()
    s % f # Value deliberately discarded
    return f.referenced

class Guard(object):
    '''Representation of a condition required for some action. See usage in
    Template.py.'''
    def __init__(self, guard_fn):
        self.guard_fn = guard_fn

    def __call__(self, arg):
        return self.guard_fn(arg)


VERSION = (0, 9, 0, 'beta', 1)

def get_version():
    try:
        from django.utils.version import get_version as django_get_version
        return django_get_version(VERSION) # pragma: no cover
    except ImportError:
        return ".".join(str(n) for n in VERSION)

Response:
VERSION = (0, 9, 0, 'beta', 1)

def get_version():
    try:
        from django.utils.version import get_version as django_get_version
        return django_get_version(VERSION) # pragma: no cover
    except ImportError:
        return ".".join(str(n) for n in VERSION)

Prompt:
Fix dependency order issue in get_version

VERSION = (0, 9, 0, 'beta', 1)

def get_version():
    from django.utils.version import get_version as django_get_version
    return django_get_version(VERSION) # pragma: no cover


"""
Convert Google Analytics buckets from Europe/London to UTC
"""
import logging
import copy
from itertools import imap, ifilter
from datetime import timedelta

from backdrop.core.records import Record
from backdrop.core.timeutils import utc

log = logging.getLogger(__name__)

GA_BUCKETS_TO_MIGRATE = [
    "carers_allowance_journey",
    "deposit_foreign_marriage_journey",
    "pay_foreign_marriage_certificates_journey",
    "pay_legalisation_drop_off_journey",
    "pay_legalisation_post_journey",
    "pay_register_birth_abroad_journey",
    "pay_register_death_abroad_journey",
    "lpa_journey",
    "licensing_journey",
]

def fix_timestamp(document):
    """Return a new dict with the _timestamp field fixed
    """
    document = copy.deepcopy(document)
    document['_timestamp'] = document['_timestamp'].replace(tzinfo=None) + \
        timedelta(hours=1)
    return document

def strip_internal_fields(document):
    """Return a new dict with all internal fields removed

    Leaves _timestamp and _id in place
    """
    def allowed_field(key):
        return not key.startswith('_') or key in ['_timestamp', '_id']

    return dict(
        (key, value) for key, value in document.items() if allowed_field(key))

def is_bst(document):
    """Return true if a document looks like it's BST"""
    return document['_timestamp'].hour == 23

def create_record(document):
    """Return a dict with internal fields applied"""
    return Record(document).to_mongo()

def fix_id(document):
    """Return a new dict with the _id field recalculated"""
    def _format(timestamp):
        return to_utc(timestamp).strftime("%Y%m%d%H%M%S")

    def data_id(data_type, timestamp, period):
        """_id generation function copied from backdrop-ga-collector"""
        return base64.urlsafe_b64encode("_".join(
            [data_type, _format(timestamp), period]))

    document = copy.deepcopy(document)
    document['_id'] = data_id(
        document['dataType'], document['_timestamp'],
        document['timeSpan'])

    return document

def up(db):
    for name in GA_BUCKETS_TO_MIGRATE:
        collection = db.get_repository(name)

        documents = collection.find({})

        documents = ifilter(is_bst, documents)
        documents = imap(strip_internal_fields, documents)
        documents = imap(fix_timestamp, documents)
        documents = map(create_record, documents)

        if len(documents) > 0:
            log.info("Convert GA timezone: {0}".format(name))

        map(collection.save, documents)

Add migration to convert GA timezone to UTC

We are ignoring the timezone from GA requests so previous data needs to
be converted to account for this.

See https://www.pivotaltracker.com/story/show/62779118 for details.

Response:
"""
Convert Google Analytics buckets from Europe/London to UTC
"""
import logging
import copy
from itertools import imap, ifilter
from datetime import timedelta

from backdrop.core.records import Record
from backdrop.core.timeutils import utc

log = logging.getLogger(__name__)

GA_BUCKETS_TO_MIGRATE = [
    "carers_allowance_journey",
    "deposit_foreign_marriage_journey",
    "pay_foreign_marriage_certificates_journey",
    "pay_legalisation_drop_off_journey",
    "pay_legalisation_post_journey",
    "pay_register_birth_abroad_journey",
    "pay_register_death_abroad_journey",
    "lpa_journey",
    "licensing_journey",
]

def fix_timestamp(document):
    """Return a new dict with the _timestamp field fixed
    """
    document = copy.deepcopy(document)
    document['_timestamp'] = document['_timestamp'].replace(tzinfo=None) + \
        timedelta(hours=1)
    return document

def strip_internal_fields(document):
    """Return a new dict with all internal fields removed

    Leaves _timestamp and _id in place
    """
    def allowed_field(key):
        return not key.startswith('_') or key in ['_timestamp', '_id']

    return dict(
        (key, value) for key, value in document.items() if allowed_field(key))

def is_bst(document):
    """Return true if a document looks like it's BST"""
    return document['_timestamp'].hour == 23

def create_record(document):
    """Return a dict with internal fields applied"""
    return Record(document).to_mongo()

def fix_id(document):
    """Return a new dict with the _id field recalculated"""
    def _format(timestamp):
        return to_utc(timestamp).strftime("%Y%m%d%H%M%S")

    def data_id(data_type, timestamp, period):
        """_id generation function copied from backdrop-ga-collector"""
        return base64.urlsafe_b64encode("_".join(
            [data_type, _format(timestamp), period]))

    document = copy.deepcopy(document)
    document['_id'] = data_id(
        document['dataType'], document['_timestamp'],
        document['timeSpan'])

    return document

def up(db):
    for name in GA_BUCKETS_TO_MIGRATE:
        collection = db.get_repository(name)

        documents = collection.find({})

        documents = ifilter(is_bst, documents)
        documents = imap(strip_internal_fields, documents)
        documents = imap(fix_timestamp, documents)
        documents = map(create_record, documents)

        if len(documents) > 0:
            log.info("Convert GA timezone: {0}".format(name))

        map(collection.save, documents)

Prompt:
Add migration to convert GA timezone to UTC

We are ignoring the timezone from GA requests so previous data needs to
be converted to account for this.

See https://www.pivotaltracker.com/story/show/62779118 for details.


"""
dialogKit: easy bake dialogs
"""

# determine the environment
haveFL = False
haveVanilla = False
try:
    import FL
    haveFL = True
except ImportError:
    pass
if not haveFL:
    try:
        import vanilla
        haveVanilla = True
    except ImportError:
        pass
# perform the environment specific import
if haveFL:
    from _dkFL import *
elif haveVanilla:
    from _dkVanilla import *
else:
    raise ImportError, 'dialogKit is not available in this environment'

numberVersion = (0, 0, "beta", 1)
version = "0.0.1b"

Response:
"""
dialogKit: easy bake dialogs
"""

# determine the environment
haveFL = False
haveVanilla = False
try:
    import FL
    haveFL = True
except ImportError:
    pass
if not haveFL:
    try:
        import vanilla
        haveVanilla = True
    except ImportError:
        pass
# perform the environment specific import
if haveFL:
    from _dkFL import *
elif haveVanilla:
    from _dkVanilla import *
else:
    raise ImportError, 'dialogKit is not available in this environment'

numberVersion = (0, 0, "beta", 1)
version = "0.0.1b"

Prompt:
Stop trying imports after something has been successfully loaded.

"""
dialogKit: easy bake dialogs
"""

# determine the environment
try:
    import FL
    haveFL = True
except ImportError:
    haveFL = False
try:
    import vanilla
    haveVanilla = True
except ImportError:
    haveVanilla = False
# perform the environment specific import
if haveFL:
    from _dkFL import *
if haveVanilla:
    from _dkVanilla import *
else:
    raise ImportError, 'dialogKit is not available in this environment'

numberVersion = (0, 0, "beta", 1)
version = "0.0.1b"


__author__ = 'katharine'

import sys
from setuptools import setup, find_packages

requires = [
  'backports.ssl-match-hostname==3.4.0.2',
  'gevent>=1.1b5',
  'gevent-websocket==0.9.3',
  'greenlet==0.4.9',
  'peewee==2.4.7',
  'pygeoip==0.3.2',
  'pypng==0.0.17',
  'python-dateutil==2.4.1',
  'requests==2.5.0',
  'sh==1.09',
  'six==1.9.0',
  'websocket-client==0.31.0',
  'wsgiref==0.1.2',
  'libpebble2==0.0.12',
  'netaddr==0.7.18'
]

setup(name='pypkjs',
      version='3.6',
      description='PebbleKit JS in Python!',
      url='https://github.com/pebble/pypkjs',
      author='Pebble Technology Corporation',
      author_email='katharine@pebble.com',
      license='MIT',
      packages=find_packages(),
      install_requires=requires,
      entry_points={
          'console_scripts': [
            'pypkjs=runner.websocket:run_tool'
          ],
      })

Response:
__author__ = 'katharine'

import sys
from setuptools import setup, find_packages

requires = [
  'backports.ssl-match-hostname==3.4.0.2',
  'gevent>=1.1b5',
  'gevent-websocket==0.9.3',
  'greenlet==0.4.9',
  'peewee==2.4.7',
  'pygeoip==0.3.2',
  'pypng==0.0.17',
  'python-dateutil==2.4.1',
  'requests==2.5.0',
  'sh==1.09',
  'six==1.9.0',
  'websocket-client==0.31.0',
  'wsgiref==0.1.2',
  'libpebble2==0.0.12',
  'netaddr==0.7.18'
]

setup(name='pypkjs',
      version='3.6',
      description='PebbleKit JS in Python!',
      url='https://github.com/pebble/pypkjs',
      author='Pebble Technology Corporation',
      author_email='katharine@pebble.com',
      license='MIT',
      packages=find_packages(),
      install_requires=requires,
      entry_points={
          'console_scripts': [
            'pypkjs=runner.websocket:run_tool'
          ],
      })

Prompt:
Address review comments, remove zip_safe=True because GeoIP

__author__ = 'katharine'

import sys
from setuptools import setup, find_packages

requires = [
  'backports.ssl-match-hostname==3.4.0.2',
  'gevent>=1.1b5',
  'gevent-websocket==0.9.3',
  'greenlet==0.4.9',
  'peewee==2.4.7',
  'pygeoip==0.3.2',
  'pypng==0.0.17',
  'python-dateutil==2.4.1',
  'requests==2.5.0',
  'sh==1.09',
  'six==1.9.0',
  'websocket-client==0.31.0',
  'wsgiref==0.1.2',
  'libpebble2==0.0.12',
  'netaddr==0.7.18'
]

packages = find_packages()
print packages

setup(name='pypkjs',
      version='3.6',
      description='PebbleKit JS in Python!',
      url='https://github.com/pebble/pypkjs',
      author='Pebble Technology Corporation',
      author_email='katharine@pebble.com',
      license='MIT',
      packages=packages,
      install_requires=requires,
      entry_points={
          'console_scripts': [
            'pypkjs=runner.websocket:run_tool'
          ],
      },
      zip_safe=True)


import os
import re

def hdfs_make_qualified(path):
    return path if (re.match(r'[.]*://[.]*', path) or 'CF_HADOOP_DEFAULT_FS' not in os.environ) \
        else os.environ['CF_HADOOP_DEFAULT_FS'] + path

Response:
import os
import re

def hdfs_make_qualified(path):
    return path if (re.match(r'[.]*://[.]*', path) or 'CF_HADOOP_DEFAULT_FS' not in os.environ) \
        else os.environ['CF_HADOOP_DEFAULT_FS'] + path

Prompt:
Update python script util to detect if paths are already fully qualified

import os

def hdfs_make_qualified(path):
    return path if 'CF_HADOOP_DEFAULT_FS' not in os.environ else os.environ['CF_HADOOP_DEFAULT_FS'] + path


# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""python-keystoneclient's pep8 extensions.

In order to make the review process faster and easier for core devs we are
adding some python-keystoneclient specific pep8 checks. This will catch common
errors so that core devs don't have to.

"""

import re

def check_oslo_namespace_imports(logical_line, blank_before, filename):
    oslo_namespace_imports = re.compile(
        r"(((from)|(import))\s+oslo\.)|(from\s+oslo\s+import\s+)")

    if re.match(oslo_namespace_imports, logical_line):
        msg = ("K333: '%s' must be used instead of '%s'.") % (
            logical_line.replace('oslo.', 'oslo_'),
            logical_line)
        yield(0, msg)

def factory(register):
    register(check_oslo_namespace_imports)

Response:
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""python-keystoneclient's pep8 extensions.

In order to make the review process faster and easier for core devs we are
adding some python-keystoneclient specific pep8 checks. This will catch common
errors so that core devs don't have to.

"""

import re

def check_oslo_namespace_imports(logical_line, blank_before, filename):
    oslo_namespace_imports = re.compile(
        r"(((from)|(import))\s+oslo\.)|(from\s+oslo\s+import\s+)")

    if re.match(oslo_namespace_imports, logical_line):
        msg = ("K333: '%s' must be used instead of '%s'.") % (
            logical_line.replace('oslo.', 'oslo_'),
            logical_line)
        yield(0, msg)

def factory(register):
    register(check_oslo_namespace_imports)

Prompt:
Change hacking check to verify all oslo imports

The hacking check was verifying that specific oslo imports
weren't using the oslo-namespaced package. Since all the oslo
libraries used by keystoneclient are now changed to use the
new package name the hacking check can be simplified.

bp drop-namespace-packages

Change-Id: I6466e857c6eda0add6918e9fb14dc9296ed98600

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""python-keystoneclient's pep8 extensions.

In order to make the review process faster and easier for core devs we are
adding some python-keystoneclient specific pep8 checks. This will catch common
errors so that core devs don't have to.

"""

import re

def check_oslo_namespace_imports(logical_line, blank_before, filename):
    oslo_namespace_imports = re.compile(
        r"(((from)|(import))\s+oslo\."
        "((config)|(serialization)|(utils)|(i18n)))|"
        "(from\s+oslo\s+import\s+((config)|(serialization)|(utils)|(i18n)))")

    if re.match(oslo_namespace_imports, logical_line):
        msg = ("K333: '%s' must be used instead of '%s'.") % (
            logical_line.replace('oslo.', 'oslo_'),
            logical_line)
        yield(0, msg)

def factory(register):
    register(check_oslo_namespace_imports)