initial commit

3 years ago · 4b51098f86
8 changed files with 691 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,5 @@
 env/
 data/
 .idea/
 **__pycache__**
 outbox.json
--- a/LICENSE.md
+++ b/LICENSE.md
@ -0,0 +1,483 @@
 THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS
 COOPERATIVE NON-VIOLENT PUBLIC LICENSE (\"LICENSE\"). THE WORK IS
 PROTECTED BY COPYRIGHT AND ALL OTHER APPLICABLE LAWS. ANY USE OF THE
 WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS
 PROHIBITED. BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED IN THIS
 LICENSE, YOU AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE.TO THE
 EXTENT THIS LICENSE MAY BE CONSIDERED TO BE A CONTRACT, THE LICENSOR
 GRANTS YOU THE RIGHTS CONTAINED HERE IN AS CONSIDERATION FOR ACCEPTING
 THE TERMS AND CONDITIONS OF THIS LICENSE AND FOR AGREEING TO BE BOUND BY
 THE TERMS AND CONDITIONS OF THIS LICENSE.
 # Definitions
 An Act of War is any action of one country against any group either with
 an intention to provoke a conflict or an action that occurs during a
 declared war or during armed conflict between military forces of any
 origin. This includes but is not limited to enforcing sanctions or
 sieges, supplying armed forces, or profiting from the manufacture of
 tools or weaponry used in military conflict.
 An Adaptation is a work based upon the Work, or upon the Work and other
 pre-existing works, such as a translation, adaptation, derivative work,
 arrangement of music or other alterations of a literary or artistic
 work, or phonogram or performance and includes cinematographic
 adaptations or any other form in which the Work may be recast,
 transformed, or adapted including in any form recognizably derived from
 the original, except that a work that constitutes a Collection will not
 be considered an Adaptation for the purpose of this License. For the
 avoidance of doubt, where the Work is a musical work, performance or
 phonogram, the synchronization of the Work in timed-relation with a
 moving image (\"synching\") will be considered an Adaptation for the
 purpose of this License. In addition, where the Work is designed to
 output a neural network the output of the neural network will be
 considered an Adaptation for the purpose of this license.
 Bodily Harm is any physical hurt or injury to a person that interferes
 with the health or comfort of the person and that is more than merely
 transient or trifling in nature.
 Distribute is to make available to the public the original and copies of
 the Work or Adaptation, as appropriate, through sale, gift or any other
 transfer of possession or ownership.
 Incarceration is Confinement in a jail, prison, or any other place where
 individuals of any kind are held against either their will or (if their
 will cannot be determined) the will of their legal guardian or
 guardians. In the case of a conflict between the will of the individual
 and the will of their legal guardian or guardians, the will of the
 individual will take precedence.
 Licensor is The individual, individuals, entity, or entities that
 offer(s) the Work under the terms of this License
 Original Author is in the case of a literary or artistic work, the
 individual, individuals, entity or entities who created the Work or if
 no individual or entity can be identified, the publisher; and in
 addition
 -   in the case of a performance the actors, singers, musicians,
    dancers, and other persons who act, sing, deliver, declaim, play in,
    interpret or otherwise perform literary or artistic works or
    expressions of folklore;
 -   in the case of a phonogram the producer being the person or legal
    entity who first fixes the sounds of a performance or other sounds;
    and,
 -   in the case of broadcasts, the organization that transmits the
    broadcast.
 Work is the literary and/or artistic work offered under the terms of
 this License including without limitation any production in the
 literary, scientific and artistic domain, whatever may be the mode or
 form of its expression including digital form, such as a book, pamphlet
 and other writing; a lecture, address, sermon or other work of the same
 nature; a dramatic or dramatico-musical work; a choreographic work or
 entertainment in dumb show; a musical composition with or without words;
 a cinematographic work to which are assimilated works expressed by a
 process analogous to cinematography; a work of drawing, painting,
 architecture, sculpture, engraving or lithography; a photographic work
 to which are assimilated works expressed by a process analogous to
 photography; a work of applied art; an illustration, map, plan, sketch
 or three-dimensional work relative to geography, topography,
 architecture or science; a performance; a broadcast; a phonogram; a
 compilation of data to the extent it is protected as a copyrightable
 work; or a work performed by a variety or circus performer to the extent
 it is not otherwise considered a literary or artistic work.
 You means an individual or entity exercising rights under this License
 who has not previously violated the terms of this License with respect
 to the Work, or who has received express permission from the Licensor to
 exercise rights under this License despite a previous violation.
 Publicly Perform means to perform public recitations of the Work and to
 communicate to the public those public recitations, by any means or
 process, including by wire or wireless means or public digital
 performances; to make available to the public Works in such a way that
 members of the public may access these Works from a place and at a place
 individually chosen by them; to perform the Work to the public by any
 means or process and the communication to the public of the performances
 of the Work, including by public digital performance; to broadcast and
 rebroadcast the Work by any means including signs, sounds or images.
 Reproduce is to make copies of the Work by any means including without
 limitation by sound or visual recordings and the right of fixation and
 reproducing fixations of the Work, including storage of a protected
 performance or phonogram in digital form or other electronic medium.
 Software is any digital Work which, through use of a third-party piece
 of Software or through the direct usage of itself on a computer system,
 the memory of the computer is modified dynamically or semi-dynamically.
 \"Software\", secondly, processes or interprets information.
 Source Code is Any digital Work which, through use of a third-party
 piece of Software or through the direct usage of itself on a computer
 system, the memory of the computer is modified dynamically or
 semi-dynamically. \"Software\", secondly, processes or interprets
 information.
 Surveilling is the use of the Work to either overtly or covertly observe
 and record persons and or their activities.
 A Network Service is the use of a piece of Software to interpret or
 modify information that is subsequently and directly served to users
 over the Internet.
 To Discriminate is the use of a piece of Software to interpret or modify
 information that is subsequently and directly served to users over the
 Internet.
 Hate Speech is Communication or any form of expression which is solely
 for the purpose of expressing hatred for some group or advocating a form
 of Discrimination between humans.
 Coercion is leveraging of the threat of force or use of force to
 intimidate a person in order to gain compliance, or to offer large
 incentives which aim to entice a person to act against their will.
 # Fair Dealing Rights
 Nothing in this License is intended to reduce, limit, or restrict any
 uses free from copyright or rights arising from limitations or
 exceptions that are provided for in connection with the copyright
 protection under copyright law or other applicable laws.
 # License Grant
 Subject to the terms and conditions of this License, Licensor hereby
 grants You a worldwide, royalty-free, non-exclusive, perpetual (for the
 duration of the applicable copyright) license to exercise the rights in
 the Work as stated below:
 To Reproduce the Work, to incorporate the Work into one or more
 Collections, and to Reproduce the Work as incorporated in the
 Collections
 To create and Reproduce Adaptations provided that any such Adaptation,
 including any translation in any medium, takes reasonable steps to
 clearly label, demarcate or otherwise identify that changes were made to
 the original Work. For example, a translation could be marked \"The
 original work was translated from English to Spanish,\" or a
 modification could indicate \"The original work has been modified.\"
 To Distribute and Publicly Perform the Work including as incorporated in
 Collections.
 To Distribute and Publicly Perform Adaptations. The above rights may be
 exercised in all media and formats whether now known or hereafter
 devised. The above rights include the right to make such modifications
 as are technically necessary to exercise the rights in other media and
 formats. This License constitutes the entire agreement between the
 parties with respect to the Work licensed here. There are no
 understandings, agreements or representations with respect to the Work
 not specified here. Licensor shall not be bound by any additional
 provisions that may appear in any communication from You. This License
 may not be modified without the mutual written agreement of the Licensor
 and You. All rights not expressly granted by Licensor are hereby
 reserved, including but not limited to the rights set forth in
 Non-waivable Compulsory License Schemes, Waivable Compulsory License
 Schemes, and Voluntary License Schemes in the restrictions.
 # Restrictions
 The license granted in the license grant above is expressly made subject
 to and limited by the following restrictions:
 You may Distribute or Publicly Perform the Work only under the terms of
 this License. You must include a copy of, or the Uniform Resource
 Identifier (URI) for, this License with every copy of the Work You
 Distribute or Publicly Perform. You may not offer or impose any terms on
 the Work that restrict the terms of this License or the ability of the
 recipient of the Work to exercise the rights granted to that recipient
 under the terms of the License. You may not sublicense the Work. You
 must keep intact all notices that refer to this License and to the
 disclaimer of warranties with every copy of the Work You Distribute or
 Publicly Perform. When You Distribute or Publicly Perform the Work, You
 may not impose any effective technological measures on the Work that
 restrict the ability of a recipient of the Work from You to exercise the
 rights granted to that recipient under the terms of the License. This
 Section applies to the Work as incorporated in a Collection, but this
 does not require the Collection apart from the Work itself to be made
 subject to the terms of this License. If You create a Collection, upon
 notice from any Licensor You must, to the extent practicable, remove
 from the Collection any credit as requested. If You create an
 Adaptation, upon notice from any Licensor You must, to the extent
 practicable, remove from the Adaptation any credit as requested.
 ## Commercial Restrictions
 You may not exercise any of the rights granted to You in the above
 section in any manner that is primarily intended for or directed toward
 commercial advantage or private monetary compensation unless you meet
 the following requirements.
 i.  You are a worker-owned business or worker-owned collective.
 ii. after tax, all financial gain, surplus, profits and benefits
    produced by the business or collective are distributed among the
    worker-owners unless a set amount is to be allocated towards
    community projects as decided by a previously-established consensus
    agreement between the worker-owners where all worker-owners agreed.
 iii. You are not using such rights on behalf of a business other than
     those specified in (i) or (ii) above, nor are using such rights as
     a proxy on behalf of a business with the intent to circumvent the
     aforementioned restrictions on such a business.
 The exchange of the Work for other copyrighted works by means of digital
 file-sharing or otherwise shall not be considered to be intended for or
 directed toward commercial advantage or private monetary compensation,
 provided there is no payment of any monetary compensation in connection
 with the exchange of copyrighted works.
 If the Work meets the definition of Software, You may exercise the
 rights granted in the license grant only if You provide a copy of the
 corresponding Source Code from which the Work was derived in digital
 form, or You provide a URI for the corresponding Source Code of the
 Work, to any recipients upon request.
 If the Work is used as or for a Network Service, You may exercise the
 rights granted in the license grant only if You provide a copy of the
 corresponding Source Code from which the Work was derived in digital
 form, or You provide a URI for the corresponding Source Code to the
 Work, to any recipients of the data served or modified by the Web
 Service.
 Any use by a business that is privately owned and managed, and that
 seeks to generate profit from the labor of employees paid by salary or
 other wages, is not permitted under this license.
 ##
 You may exercise the rights granted in the license grant for any
 purposes only if:
 i.  You do not use the Work for the purpose of inflicting Bodily Harm on
    human beings (subject to criminal prosecution or otherwise) outside
    of providing medical aid or undergoing a voluntary procedure under
    no form of Coercion.
 ii. You do not use the Work for the purpose of Surveilling or tracking
    individuals for financial gain.
 iii. You do not use the Work in an Act of War.
 iv. You do not use the Work for the purpose of supporting or profiting
    from an Act of War.
 v.  You do not use the Work for the purpose of Incarceration.
 vi. You do not use the Work for the purpose of extracting, processing,
    or refining, oil, gas, or coal. Or to in any other way to
    deliberately pollute the environment as a byproduct of manufacturing
    or irresponsible disposal of hazardous materials.
 vii. You do not use the Work for the purpose of expediting,
     coordinating, or facilitating paid work undertaken by individuals
     under the age of 12 years.
 viii. You do not use the Work to either Discriminate or spread Hate
      Speech on the basis of sex, sexual orientation, gender identity,
      race, age, disability, color, national origin, religion, caste, or
      lower economic status.
 ##
 If You Distribute, or Publicly Perform the Work or any Adaptations or
 Collections, You must, unless a request has been made by any Licensor to
 remove credit from a Collection or Adaptation, keep intact all copyright
 notices for the Work and provide, reasonable to the medium or means You
 are utilizing:
 i.  the name of the Original Author (or pseudonym, if applicable) if
    supplied, and/or if the Original Author and/or Licensor designate
    another party or parties (e.g., a sponsor institute, publishing
    entity, journal) for attribution (\"Attribution Parties\") in
    Licensor\'s copyright notice, terms of service or by other
    reasonable means, the name of such party or parties;
 ii. the title of the Work if supplied;
 iii. to the extent reasonably practicable, the URI, if any, that
     Licensor to be associated with the Work, unless such URI does not
     refer to the copyright notice or licensing information for the
     Work; and,
 iv. in the case of an Adaptation, a credit identifying the use of the
    Work in the Adaptation (e.g., \"French translation of the Work by
    Original Author,\" or \"Screenplay based on original Work by
    Original Author\").
 If any Licensor has sent notice to request removing credit, You must, to
 the extent practicable, remove any credit as requested. The credit
 required by this Section may be implemented in any reasonable manner;
 provided, however, that in the case of an Adaptation or Collection, at a
 minimum such credit will appear, if a credit for all contributing
 authors of the Adaptation or Collection appears, then as part of these
 credits and in a manner at least as prominent as the credits for the
 other contributing authors. For the avoidance of doubt, You may only use
 the credit required by this Section for the purpose of attribution in
 the manner set out above and, by exercising Your rights under this
 License, You may not implicitly or explicitly assert or imply any
 connection with, sponsorship or endorsement by the Original Author,
 Licensor and/or Attribution Parties, as appropriate, of You or Your use
 of the Work, without the separate, express prior written permission of
 the Original Author, Licensor and/or Attribution Parties.
 Non-waivable Compulsory License Schemes. In those jurisdictions in which
 the right to collect royalties through any statutory or compulsory
 licensing scheme cannot be waived, the Licensor reserves the exclusive
 right to collect such royalties for any exercise by You of the rights
 granted under this License
 Waivable Compulsory License Schemes. In those jurisdictions in which the
 right to collect royalties through any statutory or compulsory licensing
 scheme can be waived, the Licensor reserves the exclusive right to
 collect such royalties for any exercise by You of the rights granted
 under this License if Your exercise of such rights is for a purpose or
 use which is otherwise than noncommercial as permitted under Commercial
 Restrictions and otherwise waives the right to collect royalties through
 any statutory or compulsory licensing scheme.
 Voluntary License Schemes. The Licensor reserves the right to collect
 royalties, whether individually or, in the event that the Licensor is a
 member of a collecting society that administers voluntary licensing
 schemes, via that society, from any exercise by You of the rights
 granted under this License that is for a purpose or use which is
 otherwise than noncommercial as permitted under the license grant.
 Except as otherwise agreed in writing by the Licensor or as may be
 otherwise permitted by applicable law, if You Reproduce, Distribute or
 Publicly Perform the Work either by itself or as part of any Adaptations
 or Collections, You must not distort, mutilate, modify or take other
 derogatory action in relation to the Work which would be prejudicial to
 the Original Author\'s honor or reputation. Licensor agrees that in
 those jurisdictions (e.g. Japan), in which any exercise of the right
 granted in the license grant of this License (the right to make
 Adaptations) would be deemed to be a distortion, mutilation,
 modification or other derogatory action prejudicial to the Original
 Author\'s honor and reputation, the Licensor will waive or not assert,
 as appropriate, this Section, to the fullest extent permitted by the
 applicable national law, to enable You to reasonably exercise Your right
 under the license grant of this License (right to make Adaptations) but
 not otherwise.
 Do not make any legal claim against anyone accusing the Work, with or
 without changes, alone or with other works, of infringing any patent
 claim.
 # Representations Warranties and Disclaimer
 UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR
 OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY
 KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE,
 INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY,
 FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF
 LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS,
 WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE
 EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
 # Limitation on Liability
 EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL
 LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL,
 INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF
 THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED
 OF THE POSSIBILITY OF SUCH DAMAGES.
 # Termination
 This License and the rights granted hereunder will terminate
 automatically upon any breach by You of the terms of this License.
 Individuals or entities who have received Adaptations or Collections
 from You under this License, however, will not have their licenses
 terminated provided such individuals or entities remain in full
 compliance with those licenses. The Sections on definitions, fair
 dealing rights, representations, warranties, and disclaimer, limitation
 on liability, termination, and revised license versions will survive any
 termination of this License.
 Subject to the above terms and conditions, the license granted here is
 perpetual (for the duration of the applicable copyright in the Work).
 Notwithstanding the above, Licensor reserves the right to release the
 Work under different license terms or to stop distributing the Work at
 any time; provided, however that any such election will not serve to
 withdraw this License (or any other license that has been, or is
 required to be, granted under the terms of this License), and this
 License will continue in full force and effect unless terminated as
 stated above.
 # Revised License Versions
 This License may receive future revisions in the original spirit of the
 license intended to strengthen This License. Each version of This
 License has an incrementing version number.
 Unless otherwise specified like in the below subsection The Licensor has
 only granted this current version of This License for The Work. In this
 case future revisions do not apply.
 The Licensor may specify that the latest available revision of This
 License be used for The Work by either explicitly writing so or by
 suffixing the License URI with a \"+\" symbol.
 The Licensor may specify that The Work is also available under the terms
 of This License\'s current revision as well as specific future
 revisions. The Licensor may do this by writing it explicitly or
 suffixing the License URI with any additional version numbers each
 separated by a comma.
 # Miscellaneous
 Each time You Distribute or Publicly Perform the Work or a Collection,
 the Licensor offers to the recipient a license to the Work on the same
 terms and conditions as the license granted to You under this License.
 Each time You Distribute or Publicly Perform an Adaptation, Licensor
 offers to the recipient a license to the original Work on the same terms
 and conditions as the license granted to You under this License.
 If the Work is classified as Software, each time You Distribute or
 Publicly Perform an Adaptation, Licensor offers to the recipient a copy
 and/or URI of the corresponding Source Code on the same terms and
 conditions as the license granted to You under this License.
 If the Work is used as a Network Service, each time You Distribute or
 Publicly Perform an Adaptation, or serve data derived from the Software,
 the Licensor offers to any recipients of the data a copy and/or URI of
 the corresponding Source Code on the same terms and conditions as the
 license granted to You under this License.
 If any provision of this License is invalid or unenforceable under
 applicable law, it shall not affect the validity or enforceability of
 the remainder of the terms of this License, and without further action
 by the parties to this agreement, such provision shall be reformed to
 the minimum extent necessary to make such provision valid and
 enforceable.
 No term or provision of this License shall be deemed waived and no
 breach consented to unless such waiver or consent shall be in writing
 and signed by the party to be charged with such waiver or consent.
 This License constitutes the entire agreement between the parties with
 respect to the Work licensed here. There are no understandings,
 agreements or representations with respect to the Work not specified
 here. Licensor shall not be bound by any additional provisions that may
 appear in any communication from You. This License may not be modified
 without the mutual written agreement of the Licensor and You.
 The rights granted under, and the subject matter referenced, in this
 License were drafted utilizing the terminology of the Berne Convention
 for the Protection of Literary and Artistic Works (as amended on
 September 28, 1979), the Rome Convention of 1961, the WIPO Copyright
 Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 and
 the Universal Copyright Convention (as revised on July 24, 1971). These
 rights and subject matter take effect in the relevant jurisdiction in
 which the License terms are sought to be enforced according to the
 corresponding provisions of the implementation of those treaty
 provisions in the applicable national law. If the standard suite of
 rights granted under applicable copyright law includes additional rights
 not granted under this License, such additional rights are deemed to be
 included in the License; this License is not intended to restrict the
 license of any rights under applicable law.
--- a/config.py
+++ b/config.py
@ -0,0 +1,38 @@
 """
 config file for gpt2-fedibooks
 this file is a simple python file that will be imported by gpt2-fedibooks. it looks for the following keys:
 datadir: PathLike  # the working directory of fedibooks. it will recursively create all directories needed, and exclusively create files there.
 parsing_arbitrary_exclude_fn: callable  # this function takes a string (the post) and returns True if that post should be excluded
 parsing_exclude_mentions: bool  # strip out any word starting with @
 parsed_posts_file: str  # filename that the outbox parser will save into
 tokenizer_output_prefix: str  # file prefix for the merges and vocab files generated by the tokenizer
 model_size: enum[str]  # CURRENTLY NOT IMPLEMENTED! s/m/l/xl to pick the gpt2 model size (124M, 355M, 774M, and 1558M)
 model_folder: str  # name of the folder (relative to datadir) for trained model storage
 use_gpu: bool  # NOT YET IMPLEMENTED!
 prompt_before_training: bool
 training_block_size: int
 training_num_workers: int  # seems to have absolutely no effect?
 training_batch_size: int
 training_num_steps: int
 training_sample_frequency: int  # print out sample generations every n training steps
 training_save_frequency: int  # save model snapshots every n steps
 generation_zwsp_mentions: bool  # add a zero width space after every @ in generated texts
 generation_prompt: str | None  # prompt for gpt2 generation
 generation_include_prompt: bool  # whether to include the prompt in the output
 generation_max_length: int
 generation_temperature: fload  # 0.0 to 1.0, how "crazy" the output is, higher is more
 configuration is done by defining python variables / functions, like so:
 ```py
 model_size = 'l'
 training_block_size = 64
 generation_temperature = 0.8
 prompt_before_training = False
 def parsing_arbitrary_exclude_fn(post):
    return random.randint(0, 1)
 ```
 any defined variables that aren't in the above list will be ignored. any scripts are possible.
 """
--- a/requirements.txt
+++ b/requirements.txt
@ -0,0 +1,5 @@
 typer[all]
 aitextgen
 tensorflow
 bs4
 xenua @ git+https://git.julia.network/julia/xenua_tools.git
--- a/src/init.py
+++ b/src/init.py
--- a/src/config.py
+++ b/src/config.py
@ -0,0 +1,45 @@
 import os
 from pathlib import Path
 from xenua import LayeredObject, import_module
 gpt2_name_mapper = {
    's': '124M',
    'm': '355M',
    'l': '774M',
    'xl': '1558M',
 }
 defaults = {
    'datadir': Path(__file__).parent.parent / 'data',
    'parsing_arbitrary_exclude_fn': lambda content: False,
    'parsing_exclude_mentions': True,
    'parsed_posts_file': 'posts.txt',
    'tokenizer_output_prefix': 'fedibooks',
    'model_size': 's',  # s/m/l/xl, meaning 124M, 355M, 774M, and 1558M, respectively
    'model_folder': 'trained_model',
    'use_gpu': False,  # todo: implement
    'prompt_before_training': True,
    'training_block_size': 64,
    'training_num_workers': 4,  # seems to have no effect
    'training_batch_size': 8,  # no clue what this does, docs kinda suck
    'training_num_steps': 50000,
    'training_sample_frequency': 5000,
    'training_save_frequency': 5000,
    'generation_zwsp_mentions': True,
    'generation_prompt': None,
    'generation_include_prompt': False,
    'generation_max_length': 400,
    'generation_temperature': 0.7,
 }
 try:
    user_config = import_module(Path(__file__).parent.parent / 'config.py')
 except ModuleNotFoundError:
    user_config = None
 c = LayeredObject(os.environ, user_config, defaults)
 c.datadir = Path(c.datadir)
 c.datadir.mkdir(parents=True, exist_ok=True)
--- a/src/fedibooks.py
+++ b/src/fedibooks.py
@ -0,0 +1,84 @@
 import json
 import os
 from pathlib import Path
 import typer
 from aitextgen import aitextgen
 from aitextgen.TokenDataset import TokenDataset
 from aitextgen.tokenizers import train_tokenizer
 from aitextgen.utils import GPT2ConfigCPU
 from outbox_parser import parse_outbox
 from config import c, gpt2_name_mapper
 app = typer.Typer()
@app.command()
 def generate(prompt: str = c.generation_prompt, include_prompt: bool = c.generation_include_prompt,
             max_length: int = c.generation_max_length, temperature: float = c.generation_temperature):
    orig_path = os.getcwd()
    os.chdir(c.datadir)
    ai = aitextgen(model_folder=c.model_folder, vocab_file=f'{c.tokenizer_output_prefix}-vocab.json',
                   merges_file=f'{c.tokenizer_output_prefix}-merges.txt')
    generated_strings = []
    while not generated_strings:
        generated_strings = ai.generate(prompt=prompt or '', include_prompt=include_prompt,
                                       max_length=max_length, temperature=temperature, return_as_list=True)
    generated_string = generated_strings[0].strip('<|endoftext|>').strip()
    if c.generation_zwsp_mentions:
        generated_string.replace('@', '@')  # <- zwsp after the second @
    typer.echo(generated_string)
    os.chdir(orig_path)
@app.command()
 def init(outbox_json_file: Path):
    with open(outbox_json_file) as f:
        outbox = json.load(f)
    orig_path = os.getcwd()
    os.chdir(c.datadir)
    out_text = parse_outbox(outbox, c.parsing_exclude_mentions, c.parsing_arbitrary_exclude_fn)
    with open(c.parsed_posts_file, 'w') as f:
        f.write(out_text)
    train_tokenizer(c.parsed_posts_file, serialize=False, save_path=str(c.datadir), prefix=c.tokenizer_output_prefix)
    merges_file = f'{c.tokenizer_output_prefix}-merges.txt'
    vocab_file = f'{c.tokenizer_output_prefix}-vocab.json'
    config = GPT2ConfigCPU()  # todo: maybe add params, probably not needed idfk
    ai = aitextgen(
        # tf_gpt2=gpt2_name_mapper[c.model_size],  # for some reason this makes it Really slow (4 days kind of slow)
                   merges_file=merges_file, vocab_file=vocab_file, config=config)
    data = TokenDataset(c.parsed_posts_file,
                        vocab_file=vocab_file,
                        merges_file=merges_file,
                        block_size=c.training_block_size)
    typer.secho('About to start training. This will take a long time!', fg=typer.colors.BRIGHT_RED)
    if c.prompt_before_training:
        typer.confirm('Start training?', default=True, abort=True)
    ai.train(data, batch_size=c.training_batch_size, num_steps=c.training_num_steps,
             generate_every=c.training_sample_frequency, save_every=c.training_save_frequency,
             num_workers=c.training_num_workers)
    typer.secho('Training finished.', fg=typer.colors.GREEN, bold=True)
    os.chdir(orig_path)
 if __name__ == '__main__':
    app()
--- a/src/outbox_parser.py
+++ b/src/outbox_parser.py
@ -0,0 +1,31 @@
 from bs4 import BeautifulSoup
 def parse_outbox(
        outbox: dict,
        exclude_mentions: bool,
        arbitrary_exclude_fn: callable,
 ):
    posts = outbox['orderedItems']
    post_list = []
    for post in posts:
        obj = post['object']
        # filter out boosts (where obj is a str/url) and posts without content
        if isinstance(obj, dict) and 'content' in obj:
            cont = obj['content']
            if cont and not arbitrary_exclude_fn(cont):
                post_list.append(cont)
    for i, post in enumerate(post_list):
        soup = BeautifulSoup(post)
        stripped = soup.get_text()
        post_list[i] = stripped
    if exclude_mentions:
        for i, post in enumerate(post_list):
            words = post.split(' ')
            post_list[i] = ' '.join([w for w in words if not w.startswith('@')])
    return '\n\n'.join(post_list)