view env/bin/mturk @ 0:4f3585e2f14b draft default tip

"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"

#!/Users/cmdms/OneDrive-UOB/Development/Projects/2021/sam-consensus-v3/env/bin/python3
# Copyright 2012, 2014 Kodi Arfer
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish, dis-
# tribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the fol-
# lowing conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS

import argparse # Hence, Python 2.7 is required.
import sys
import os.path
import string
import inspect
import datetime, calendar
import boto.mturk.connection, boto.mturk.price, boto.mturk.question, boto.mturk.qualification
from boto.compat import json

# --------------------------------------------------
# Globals
# -------------------------------------------------

interactive = False
con = None
mturk_website = None

default_nicknames_path = os.path.expanduser('~/.boto_mturkcli_hit_nicknames')
nicknames = {}
nickname_pool = set(string.ascii_lowercase)

get_assignments_page_size = 100

time_units = dict(
    s = 1,
    min = 60,
    h = 60 * 60,
    d = 24 * 60 * 60)

qual_requirements = dict(
    Adult = '00000000000000000060',
    Locale = '00000000000000000071',
    NumberHITsApproved = '00000000000000000040',
    PercentAssignmentsSubmitted = '00000000000000000000',
    PercentAssignmentsAbandoned = '00000000000000000070',
    PercentAssignmentsReturned = '000000000000000000E0',
    PercentAssignmentsApproved = '000000000000000000L0',
    PercentAssignmentsRejected = '000000000000000000S0')

qual_comparators = {v : k for k, v in dict(
    LessThan = '<', LessThanOrEqualTo = '<=',
    GreaterThan = '>', GreaterThanOrEqualTo = '>=',
    EqualTo = '==', NotEqualTo = '!=',
    Exists = 'exists').items()}

example_config_file = '''Example configuration file:

  {
    "title": "Pick your favorite color",
    "description": "In this task, you are asked to pick your favorite color.",
    "reward": 0.50,
    "assignments": 10,
    "duration": "20 min",
    "keywords": ["color", "favorites", "survey"],
    "lifetime": "7 d",
    "approval_delay": "14 d",
    "qualifications": [
        "PercentAssignmentsApproved > 90",
        "Locale == US",
        "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6 exists"
    ],
    "question_url": "http://example.com/myhit",
    "question_frame_height": 450
  }'''

# --------------------------------------------------
# Subroutines
# --------------------------------------------------

def unjson(path):
    with open(path) as o:
        return json.load(o)

def add_argparse_arguments(parser):
    parser.add_argument('-P', '--production',
        dest = 'sandbox', action = 'store_false', default = True,
        help = 'use the production site (default: use the sandbox)')
    parser.add_argument('--nicknames',
        dest = 'nicknames_path', metavar = 'PATH',
        default = default_nicknames_path,
        help = 'where to store HIT nicknames (default: {})'.format(
            default_nicknames_path))

def init_by_args(args):
    init(args.sandbox, args.nicknames_path)

def init(sandbox = False, nicknames_path = default_nicknames_path):
    global con, mturk_website, nicknames, original_nicknames

    mturk_website = 'workersandbox.mturk.com' if sandbox else 'www.mturk.com'
    con = boto.mturk.connection.MTurkConnection(
        host = 'mechanicalturk.sandbox.amazonaws.com' if sandbox else 'mechanicalturk.amazonaws.com')

    try:
        nicknames = unjson(nicknames_path)
    except IOError:
        nicknames = {}
    original_nicknames = nicknames.copy()

def save_nicknames(nicknames_path = default_nicknames_path):
    if nicknames != original_nicknames:
        with open(nicknames_path, 'w') as o:
            json.dump(nicknames, o, sort_keys = True, indent = 4)
            print >>o

def parse_duration(s):
    '''Parses durations like "2 d", "48 h", "2880 min",
"172800 s", or "172800".'''
    x = s.split()
    return int(x[0]) * time_units['s' if len(x) == 1 else x[1]]
def display_duration(n):
    for unit, m in sorted(time_units.items(), key = lambda x: -x[1]):
        if n % m == 0:
            return '{} {}'.format(n / m, unit)

def parse_qualification(inp):
    '''Parses qualifications like "PercentAssignmentsApproved > 90",
"Locale == US", and "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6 exists".'''
    inp = inp.split()
    name, comparator, value = inp.pop(0), inp.pop(0), (inp[0] if len(inp) else None)
    qtid = qual_requirements.get(name)
    if qtid is None:
      # Treat "name" as a Qualification Type ID.
        qtid = name
    if qtid == qual_requirements['Locale']:
        return boto.mturk.qualification.LocaleRequirement(
            qual_comparators[comparator],
            value,
            required_to_preview = False)
    return boto.mturk.qualification.Requirement(
        qtid,
        qual_comparators[comparator],
        value,
        required_to_preview = qtid == qual_requirements['Adult'])
          # Thus required_to_preview is true only for the
          # Worker_Adult requirement.

def preview_url(hit):
    return 'https://{}/mturk/preview?groupId={}'.format(
        mturk_website, hit.HITTypeId)

def parse_timestamp(s):
    '''Takes a timestamp like "2012-11-24T16:34:41Z".

Returns a datetime object in the local time zone.'''
    return datetime.datetime.fromtimestamp(
        calendar.timegm(
        datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ').timetuple()))

def get_hitid(nickname_or_hitid):
    return nicknames.get(nickname_or_hitid) or nickname_or_hitid

def get_nickname(hitid):
    for k, v in nicknames.items():
        if v == hitid:
            return k
    return None

def display_datetime(dt):
    return dt.strftime('%e %b %Y, %l:%M %P')

def display_hit(hit, verbose = False):
    et = parse_timestamp(hit.Expiration)
    return '\n'.join([
        '{} - {} ({}, {}, {})'.format(
            get_nickname(hit.HITId),
            hit.Title,
            hit.FormattedPrice,
            display_duration(int(hit.AssignmentDurationInSeconds)),
            hit.HITStatus),
        'HIT ID: ' + hit.HITId,
        'Type ID: ' + hit.HITTypeId,
        'Group ID: ' + hit.HITGroupId,
        'Preview: ' + preview_url(hit),
        'Created {}   {}'.format(
            display_datetime(parse_timestamp(hit.CreationTime)),
            'Expired' if et <= datetime.datetime.now() else
                'Expires ' + display_datetime(et)),
        'Assignments: {} -- {} avail, {} pending, {} reviewable, {} reviewed'.format(
            hit.MaxAssignments,
            hit.NumberOfAssignmentsAvailable,
            hit.NumberOfAssignmentsPending,
            int(hit.MaxAssignments) - (int(hit.NumberOfAssignmentsAvailable) + int(hit.NumberOfAssignmentsPending) + int(hit.NumberOfAssignmentsCompleted)),
            hit.NumberOfAssignmentsCompleted)
            if hasattr(hit, 'NumberOfAssignmentsAvailable')
            else 'Assignments: {} total'.format(hit.MaxAssignments),
            # For some reason, SearchHITs includes the
            # NumberOfAssignmentsFoobar fields but GetHIT doesn't.
        ] + ([] if not verbose else [
            '\nDescription: ' + hit.Description,
            '\nKeywords: ' + hit.Keywords
        ])) + '\n'

def digest_assignment(a):
    return dict(
        answers = {str(x.qid): str(x.fields[0]) for x in a.answers[0]},
        **{k: str(getattr(a, k)) for k in (
            'AcceptTime', 'SubmitTime',
            'HITId', 'AssignmentId', 'WorkerId',
            'AssignmentStatus')})

# --------------------------------------------------
# Commands
# --------------------------------------------------

def get_balance():
    return con.get_account_balance()

def show_hit(hit):
    return display_hit(con.get_hit(hit)[0], verbose = True)

def list_hits():
    'Lists your 10 most recently created HITs, with the most recent last.'
    return '\n'.join(reversed(map(display_hit, con.search_hits(
        sort_by = 'CreationTime',
        sort_direction = 'Descending',
        page_size = 10))))

def make_hit(title, description, keywords, reward, question_url, question_frame_height, duration, assignments, approval_delay, lifetime, qualifications = []):
    r = con.create_hit(
        title = title,
        description = description,
        keywords = con.get_keywords_as_string(keywords),
        reward = con.get_price_as_price(reward),
        question = boto.mturk.question.ExternalQuestion(
            question_url,
            question_frame_height),
        duration = parse_duration(duration),
        qualifications = boto.mturk.qualification.Qualifications(
            map(parse_qualification, qualifications)),
        max_assignments = assignments,
        approval_delay = parse_duration(approval_delay),
        lifetime = parse_duration(lifetime))
    nick = None
    available_nicks = nickname_pool - set(nicknames.keys())
    if available_nicks:
        nick = min(available_nicks)
        nicknames[nick] = r[0].HITId
    if interactive:
        print 'Nickname:', nick
        print 'HIT ID:', r[0].HITId
        print 'Preview:', preview_url(r[0])
    else:
        return r[0]

def extend_hit(hit, assignments_increment = None, expiration_increment = None):
    con.extend_hit(hit, assignments_increment, expiration_increment)

def expire_hit(hit):
    con.expire_hit(hit)

def delete_hit(hit):
    '''Deletes a HIT using DisableHIT.

Unreviewed assignments get automatically approved. Unsubmitted
assignments get automatically approved upon submission.

The API docs say DisableHIT doesn't work with Reviewable HITs,
but apparently, it does.'''
    con.disable_hit(hit)
    global nicknames
    nicknames = {k: v for k, v in nicknames.items() if v != hit}

def list_assignments(hit, only_reviewable = False):
    # Accumulate all relevant assignments, one page of results at
    # a time.
    assignments = []
    page = 1
    while True:
        rs = con.get_assignments(
            hit_id = hit,
            page_size = get_assignments_page_size,
            page_number = page,
            status = 'Submitted' if only_reviewable else None)
        assignments += map(digest_assignment, rs)
        if len(assignments) >= int(rs.TotalNumResults):
            break
        page += 1
    if interactive:
        print json.dumps(assignments, sort_keys = True, indent = 4)
        print ' '.join([a['AssignmentId'] for a in assignments])
        print ' '.join([a['WorkerId'] + ',' + a['AssignmentId'] for a in assignments])
    else:
        return assignments

def grant_bonus(message, amount, pairs):
    for worker, assignment in pairs:
        con.grant_bonus(worker, assignment, con.get_price_as_price(amount), message)
        if interactive: print 'Bonused', worker

def approve_assignments(message, assignments):
    for a in assignments:
        con.approve_assignment(a, message)
        if interactive: print 'Approved', a

def reject_assignments(message, assignments):
    for a in assignments:
        con.reject_assignment(a, message)
        if interactive: print 'Rejected', a

def unreject_assignments(message, assignments):
    for a in assignments:
        con.approve_rejected_assignment(a, message)
        if interactive: print 'Unrejected', a

def notify_workers(subject, text, workers):
    con.notify_workers(workers, subject, text)

def give_qualification(qualification, workers, value = 1, notify = True):
    for w in workers:
        con.assign_qualification(qualification, w, value, notify)
        if interactive: print 'Gave to', w

def revoke_qualification(qualification, workers, message = None):
    for w in workers:
        con.revoke_qualification(w, qualification, message)
        if interactive: print 'Revoked from', w

# --------------------------------------------------
# Mainline code
# --------------------------------------------------

if __name__ == '__main__':
    interactive = True

    parser = argparse.ArgumentParser()
    add_argparse_arguments(parser)
    subs = parser.add_subparsers()

    sub = subs.add_parser('bal',
        help = 'display your prepaid balance')
    sub.set_defaults(f = get_balance, a = lambda: [])

    sub = subs.add_parser('hit',
        help = 'get information about a HIT')
    sub.add_argument('HIT',
        help = 'nickname or ID of the HIT to show')
    sub.set_defaults(f = show_hit, a = lambda:
        [get_hitid(args.HIT)])

    sub = subs.add_parser('hits',
        help = 'list all your HITs')
    sub.set_defaults(f = list_hits, a = lambda: [])

    sub = subs.add_parser('new',
        help = 'create a new HIT (external questions only)',
        epilog = example_config_file,
        formatter_class = argparse.RawDescriptionHelpFormatter)
    sub.add_argument('JSON_PATH',
        help = 'path to JSON configuration file for the HIT')
    sub.add_argument('-u', '--question-url', dest = 'question_url',
        metavar = 'URL',
        help = 'URL for the external question')
    sub.add_argument('-a', '--assignments', dest = 'assignments',
        type = int, metavar = 'N',
        help = 'number of assignments')
    sub.add_argument('-r', '--reward', dest = 'reward',
        type = float, metavar = 'PRICE',
        help = 'reward amount, in USD')
    sub.set_defaults(f = make_hit, a = lambda: dict(
        unjson(args.JSON_PATH).items() + [(k, getattr(args, k))
            for k in ('question_url', 'assignments', 'reward')
            if getattr(args, k) is not None]))

    sub = subs.add_parser('extend',
        help = 'add assignments or time to a HIT')
    sub.add_argument('HIT',
        help = 'nickname or ID of the HIT to extend')
    sub.add_argument('-a', '--assignments', dest = 'assignments',
        metavar = 'N', type = int,
        help = 'number of assignments to add')
    sub.add_argument('-t', '--time', dest = 'time',
        metavar = 'T',
        help = 'amount of time to add to the expiration date')
    sub.set_defaults(f = extend_hit, a = lambda:
        [get_hitid(args.HIT), args.assignments,
            args.time and parse_duration(args.time)])

    sub = subs.add_parser('expire',
        help = 'force a HIT to expire without deleting it')
    sub.add_argument('HIT',
        help = 'nickname or ID of the HIT to expire')
    sub.set_defaults(f = expire_hit, a = lambda:
        [get_hitid(args.HIT)])

    sub = subs.add_parser('rm',
        help = 'delete a HIT')
    sub.add_argument('HIT',
        help = 'nickname or ID of the HIT to delete')
    sub.set_defaults(f = delete_hit, a = lambda:
        [get_hitid(args.HIT)])

    sub = subs.add_parser('as',
        help = "list a HIT's submitted assignments")
    sub.add_argument('HIT',
        help = 'nickname or ID of the HIT to get assignments for')
    sub.add_argument('-r', '--reviewable', dest = 'only_reviewable',
        action = 'store_true',
        help = 'show only unreviewed assignments')
    sub.set_defaults(f = list_assignments, a = lambda:
        [get_hitid(args.HIT), args.only_reviewable])

    for command, fun, helpmsg in [
            ('approve', approve_assignments, 'approve assignments'),
            ('reject', reject_assignments, 'reject assignments'),
            ('unreject', unreject_assignments, 'approve previously rejected assignments')]:
        sub = subs.add_parser(command, help = helpmsg)
        sub.add_argument('ASSIGNMENT', nargs = '+',
            help = 'ID of an assignment')
        sub.add_argument('-m', '--message', dest = 'message',
            metavar = 'TEXT',
            help = 'feedback message shown to workers')
        sub.set_defaults(f = fun, a = lambda:
            [args.message, args.ASSIGNMENT])

    sub = subs.add_parser('bonus',
        help = 'give some workers a bonus')
    sub.add_argument('AMOUNT', type = float,
        help = 'bonus amount, in USD')
    sub.add_argument('MESSAGE',
        help = 'the reason for the bonus (shown to workers in an email sent by MTurk)')
    sub.add_argument('WIDAID', nargs = '+',
        help = 'a WORKER_ID,ASSIGNMENT_ID pair')
    sub.set_defaults(f = grant_bonus, a = lambda:
        [args.MESSAGE, args.AMOUNT,
            [p.split(',') for p in args.WIDAID]])

    sub = subs.add_parser('notify',
        help = 'send a message to some workers')
    sub.add_argument('SUBJECT',
        help = 'subject of the message')
    sub.add_argument('MESSAGE',
        help = 'text of the message')
    sub.add_argument('WORKER', nargs = '+',
        help = 'ID of a worker')
    sub.set_defaults(f = notify_workers, a = lambda:
        [args.SUBJECT, args.MESSAGE, args.WORKER])

    sub = subs.add_parser('give-qual',
        help = 'give a qualification to some workers')
    sub.add_argument('QUAL',
        help = 'ID of the qualification')
    sub.add_argument('WORKER', nargs = '+',
        help = 'ID of a worker')
    sub.add_argument('-v', '--value', dest = 'value',
        metavar = 'N', type = int, default = 1,
        help = 'value of the qualification')
    sub.add_argument('--dontnotify', dest = 'notify',
        action = 'store_false', default = True,
        help = "don't notify workers")
    sub.set_defaults(f = give_qualification, a = lambda:
        [args.QUAL, args.WORKER, args.value, args.notify])

    sub = subs.add_parser('revoke-qual',
        help = 'revoke a qualification from some workers')
    sub.add_argument('QUAL',
        help = 'ID of the qualification')
    sub.add_argument('WORKER', nargs = '+',
        help = 'ID of a worker')
    sub.add_argument('-m', '--message', dest = 'message',
        metavar = 'TEXT',
        help = 'the reason the qualification was revoked (shown to workers in an email sent by MTurk)')
    sub.set_defaults(f = revoke_qualification, a = lambda:
        [args.QUAL, args.WORKER, args.message])

    args = parser.parse_args()

    init_by_args(args)

    f = args.f
    a = args.a()
    if isinstance(a, dict):
        # We do some introspective gymnastics so we can produce a
        # less incomprehensible error message if some arguments
        # are missing.
        spec = inspect.getargspec(f)
        missing = set(spec.args[: len(spec.args) - len(spec.defaults)]) - set(a.keys())
        if missing:
            raise ValueError('Missing arguments: ' + ', '.join(missing))
        doit = lambda: f(**a)
    else:
        doit = lambda: f(*a)

    try:
        x = doit()
    except boto.mturk.connection.MTurkRequestError as e:
        print 'MTurk error:', e.error_message
        sys.exit(1)

    if x is not None:
        print x

    save_nicknames()