From a2b2fe7da78378dc6a64f58c14e907496c9adea9 Mon Sep 17 00:00:00 2001 From: Hristo Venev Date: Thu, 26 Aug 2021 17:30:35 +0300 Subject: Initial commit --- .gitignore | 1 + check_reverts.py | 58 ++++ export_series.py | 154 +++++++++ import_series.py | 86 +++++ linux.sh | 23 ++ migrate_series.py | 47 +++ patchstate.py | 383 ++++++++++++++++++++++ patchtheory.py | 960 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ prune_aliases.py | 21 ++ ungone_series.py | 70 ++++ 10 files changed, 1803 insertions(+) create mode 100644 .gitignore create mode 100644 check_reverts.py create mode 100644 export_series.py create mode 100644 import_series.py create mode 100755 linux.sh create mode 100644 migrate_series.py create mode 100644 patchstate.py create mode 100644 patchtheory.py create mode 100644 prune_aliases.py create mode 100644 ungone_series.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..225fc6f --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/__pycache__ diff --git a/check_reverts.py b/check_reverts.py new file mode 100644 index 0000000..f8598d3 --- /dev/null +++ b/check_reverts.py @@ -0,0 +1,58 @@ +import patchstate as ps +import io, os, sys, subprocess, tempfile + +def main(args): + args = iter(args) + arg0 = next(args) + + @ps.argparse_all(args) + def path(arg): + raise RuntimeError(f'Invalid argument: {arg!r}') + + [repo_path] = path + + repo = ps.Repository(repo_path) + + with io.open(os.path.join(repo.path, 'series'), 'r') as f: + pcur = ps.Series.parse(f.read()) + + ok = True + reverted = {} + for pi in pcur.info: + mode = pi.mode + p = pi.get_patch(repo) + if mode == 'reverted': + pid = p.id + if pid in reverted: + raise RuntimeError(f'reverted twice: {pid!r}') + reverted[pid] = p + continue + if not mode.startswith('reverts '): + continue + pid = mode[8:] + revp = reverted.pop(pid) + + d1 = ps.fmt_diff(revp.files, munge=True) + d2 = ps.fmt_diff(p.revert().files, munge=True) + if d1 != d2: + ok = False + print(f'Bad diff for {pid} {revp.title}:') + if 0: + with tempfile.NamedTemporaryFile('w+b') as t1, tempfile.NamedTemporaryFile('w+b') as t2: + t1.write(d1) + t1.flush() + t2.write(d2) + t2.flush() + diff = subprocess.run(['diff', t1.name, t2.name], capture_output=True, check=False) + diff = diff.stdout.decode('utf-8') + for line in diff.splitlines(): + print(f'\t{line}') + + for p in reverted.values(): + print(f'Not reverted: {pid} {revp.title}') + ok = False + + return 0 if ok else 1 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/export_series.py b/export_series.py new file mode 100644 index 0000000..a4ebc8f --- /dev/null +++ b/export_series.py @@ -0,0 +1,154 @@ +import patchstate as ps +import patchtheory as pt +import os, sys + +class PatchSeq: + __slots__ = ('upstreamed', 'diff', 'reverted', 'late') + + def __init__(self, *, rebase=pt.Diff()): + self.upstreamed = -rebase + self.diff = pt.Diff() + self.reverted = [] + + def flush(self): + # We move all late patches back through the reverted + try: + late_kind,late = self.late + except AttributeError: + return + del self.late + + print('Flushing', file=sys.stderr) + + late = sum(late, pt.Diff()) + + for rev in reversed(self.reverted): + (late, rev[1]) = pt.Diff.commute(rev[1], late) + + if late_kind == 'apply': + self.diff += late + return + + (late, self.diff) = pt.Diff.commute(self.diff, late) + + if late_kind == 'upstreamed': + self.upstreamed += late + return + + raise AssertionError + + def _push_late(self, kind, diff): + try: + lkind,late = self.late + except AttributeError: + self.late = (kind, (late := [])) + else: + if lkind != kind: + self.flush() + self.late = (kind, (late := [])) + late.append(diff) + + def _push_reverted(self, by_pid, diff): + self.flush() + self.reverted.append([by_pid, diff]) + + def _push_revert(self, pid, diff): + self.flush() + + for i,rev in enumerate(reverted := self.reverted): + if rev[0] == pid: + break + else: + raise KeyError(f'patch {pid} not marked as reverted') + + if reverted.pop(i) is not rev: + raise AssertionError + + rdiff = rev[1] + for rev in reverted[i:]: + rev[1],rdiff = pt.Diff.commute(rdiff, rev[1]) + + if rdiff + diff: + raise KeyError(f'revert of {pid} not exact') + + def push(self, patch_info, patch): + diff = patch.diff + mode = patch_info.mode.split(' ', 1) + kind = mode[0] + + if kind in {'apply', 'upstreamed'}: + self._push_late(kind, diff) + return + + if kind == 'subst': + with open(mode[1], 'rb') as diff2: + diff2 = diff2.read() + diff2 = pt.Diff.parse(diff2) + self._push_late('apply', diff2) + self._push_reverted(None, (-diff2) + diff) + return + + if kind == 'reverted': + self._push_reverted(patch_info.patch_id, diff) + return + + if kind in {'skip', 'replaced'}: + self._push_reverted(None, diff) + return + + if kind == 'reverts': + self._push_revert(mode[1], diff) + return + + raise TypeError(f'bad patch kind: {kind!r}') + + def finish(self, prb): + self.flush() + + for pid,_ in self.reverted: + if pid is not None: + raise RuntimeError(f'missing revert for {pid!r}') + + return prb + pt.Diff.commute(-prb + self.upstreamed, self.diff)[0] + +def main(args): + args = iter(args) + arg0 = next(args) + + @ps.argparse_all(args) + def path(arg): + raise RuntimeError(f'Invalid argument: {arg!r}') + + repo_path = path.pop(0) + + if path: + with open(path.pop(0), 'rb') as rebase_patch: + rebase_patch = rebase_patch.read() + rebase_patch = pt.Diff.parse(rebase_patch) + else: + rebase_patch = pt.Diff() + + if path: + with open(path.pop(0), 'rb') as prb_patch: + prb_patch = prb_patch.read() + prb_patch = pt.Diff.parse(prb_patch) + else: + prb_patch = pt.Diff() + + assert not path + + repo = ps.Repository(repo_path) + + with open(os.path.join(repo.path, 'series'), 'r') as f: + pcur = ps.Series.parse(f.read()) + + seq = PatchSeq(rebase=rebase_patch) + for pi in pcur.info: + p = pi.get_patch(repo) + print('Applying', p.title, file=sys.stderr) + seq.push(pi, p) + seq.finish(prb_patch).write(sys.stdout.buffer.write) + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/import_series.py b/import_series.py new file mode 100644 index 0000000..800f0c9 --- /dev/null +++ b/import_series.py @@ -0,0 +1,86 @@ +import patchstate as ps +import io, os, sys + +def lcs(a, b, *, key): + a = [*a] + b = [*b] + if not a or not b: + return [] + + kas = [*map(key, a)] + + m = [(0,)] + for v in a: + m.append((m[-1][0] + 1, v, None, m[-1])) + for v in b: + kb = key(v) + + mi = iter(m) + mv = next(mi) + m2 = [(mv[0] + 1, None, v, mv)] + for ia,mv in enumerate(mi): + if kas[ia] == kb: + mv = m[ia] + m2.append((mv[0], a[ia], v, mv)) + continue + mv1 = m2[-1] + if mv[0] <= mv1[0]: + m2.append((mv[0] + 1, None, v, mv)) + else: + m2.append((mv1[0] + 1, a[ia], None, mv1)) + del mv1 + del ia + del mi, mv + m = m2 + + m = m[-1] + del v, a, b, kas + m2 = [] + while len(m) == 4: + _,a,b,m = m + m2.append((a,b)) + assert m == (0,) + m2.reverse() + return m2 + +def main(args): + args = iter(args) + arg0 = next(args) + + @ps.argparse_all(args) + def path(arg): + raise RuntimeError(f'Invalid argument: {arg!r}') + + [repo_path, import_path] = path + + repo = ps.Repository(repo_path) + + pnew = repo.import_dir(import_path) + + with io.open(os.path.join(repo.path, 'series'), 'r') as f: + pcur = ps.Series.parse(f.read()) + + r = [] + for p1,p2 in lcs(pcur.info, pnew.info, key=lambda p: p.patch_id): + if p1 is None: + r.append(p2) + continue + + if p2 is None: + r.append(p1.update(new_mode='gone')) + continue + + r.append(p2.update(mode=p1.mode, info=p1.info)) + + pnew.info = r + + pnew_data = pnew.fmt() + with io.open(os.path.join(repo.path, 'series'), 'w') as f: + f.write(pnew_data) + + repo.gc({p.patch_hash for p in pnew.info}) + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/linux.sh b/linux.sh new file mode 100755 index 0000000..11b9fe7 --- /dev/null +++ b/linux.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +ver=5.13 + +DIR="$(realpath $(dirname "$BASH_SOURCE"))" +cd "$DIR" + +rm -rf "incoming-$ver" +mkdir "incoming-$ver" + +( + cd "$HOME/sw/linux" + base="$(git merge-base "linux-stable/linux-$ver.y" "raspberrypi/rpi-$ver.y")" + git format-patch -o "$DIR/incoming-$ver" "$base".."raspberrypi/rpi-$ver.y" + git diff "$base".."linux-stable/linux-$ver.y" > "$DIR/stable-$ver.patch" +) + +python3 import_series.py "rpi-$ver" "incoming-$ver" + +python3 export_series.py "rpi-$ver" "stable-$ver.patch" "prb-$ver.patch" > "rpi-$ver.patch" +cp "rpi-$ver.patch" ~/sw/fedora/hvenev-kernel/patches/9000-rpi.patch diff --git a/migrate_series.py b/migrate_series.py new file mode 100644 index 0000000..c09dea1 --- /dev/null +++ b/migrate_series.py @@ -0,0 +1,47 @@ +import patchstate as ps +import io, os, sys + +def main(args): + args = iter(args) + arg0 = next(args) + + @ps.argparse_all(args) + def path(arg): + raise RuntimeError(f'Invalid argument: {arg!r}') + + [repo_path] = path + + repo = ps.Repository(repo_path) + + with io.open(os.path.join(repo.path, 'series'), 'r') as f: + pcur = ps.Series.parse(f.read()) + + idmap = {} + for i,pi in enumerate(pcur.info): + h = pi.patch_hash + try: + h = repo.aliases[h] + except KeyError: + pass + p = repo.patches[h] + if pi.patch_id in idmap: + raise RuntimeError(f'duplicate patch id: {pi.patch_id!r}') + idmap[pi.patch_id] = p.id + pcur.info[i] = pi.update(patch=p) + + print(idmap) + + for i,pi in enumerate(pcur.info): + m = pi.mode + if m.startswith('reverts '): + m = f'reverts {idmap[m[8:]]}' + pcur.info[i] = pi.update(mode=m) + + pcur = pcur.fmt() + with io.open(os.path.join(repo.path, 'series'), 'w') as f: + f.write(pcur) + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/patchstate.py b/patchstate.py new file mode 100644 index 0000000..79600e0 --- /dev/null +++ b/patchstate.py @@ -0,0 +1,383 @@ +import io, sys, os, operator, re, base64, hashlib +import patchtheory + +def strip_subject(s): + if not s.startswith(b'[PATCH'): + raise ValueError(f'Invalid subject: {s!r}') + i = s.find(b'] ') + if i == -1: + raise ValueError(f'Invalid subject: {s!r}') + return s[i+2:] + +rem_patch = re.compile('patch ([0-9a-z]{16})').fullmatch +rem_author = re.compile('[^\0-\x1f]+ <[^\0-\x1f]+@[^\0-\x1f]+>').fullmatch +rem_date = re.compile('[^\0-\x1f]+').fullmatch +rem_title = re.compile('[^\0-\x1f]+').fullmatch + +class Patch: + __slots__ = ('author', 'date', 'title', 'body', 'diff', 'data', 'hash', 'id') + + def __init__(self, /, *, author, date, title, body, diff): + assert type(author) is str and rem_author(author) is not None + assert type(date) is str and rem_date(date) is not None + assert type(title) is str and rem_title(title) is not None + assert type(body) is str + assert type(diff) is patchtheory.Diff + + self.author = author + self.date = date + self.title = title + self.body = body + self.diff = diff + + self._set_data() + self._set_id() + + @classmethod + def parse(tp, body): + lines = iter(body.split(b'\n')) + line = lines.__next__ + + l = line() + if l.startswith(b'From '): + l = line() + + headers = {} + hdr = None + while True: + if not l: + break + if hdr is not None and l.startswith(b' '): + headers[hdr] += l + else: + hdr,sep,val = l.partition(b':') + if not sep: + raise ValueError(f'Expected header line: {l!r}') + headers[hdr] = val.lstrip() + l = line() + + author = headers.pop(b'From').decode('utf-8') + date = headers.pop(b'Date').decode('utf-8') + title = strip_subject(headers.pop(b'Subject')).decode('utf-8') + + headers.pop(b'MIME-Version', None) + headers.pop(b'Content-Type', None) + headers.pop(b'Content-Transfer-Encoding', None) + + body = [] + for l in lines: + if l.startswith(b'diff --git'): + break + body.append(l) + else: + raise RuntimeError('unexpected EOF') + while not body[-1]: + del body[-1] + while body[-1].startswith(b' '): + del body[-1] + if body[-1] != b'---': + raise RuntimeError + del body[-1] + while body and not body[-1]: + del body[-1] + + body = b'\n'.join(body).decode('utf-8') + + return Patch( + author = author, + date = date, + title = title, + body = body, + diff = patchtheory.Diff.parse_l(l, line), + ) + + def _set_id(self): + out = io.BytesIO() + w = out.write + w(self.author.encode('utf-8')) + w(b'\n') + w(self.date.encode('utf-8')) + w(b'\n') + w(self.title.encode('utf-8')) + w(b'\n') + self.diff.write_munged(w) + + out = out.getvalue() + dgst = hashlib.blake2b(out).digest() + self.id = base64.b32encode(dgst[:10]).lower().decode('ascii') + + def as_filename(self): + return '-'.join(re.findall('[0-9A-Za-z]+', self.title))[:72] + + def _set_data(self): + out = io.BytesIO() + w = out.write + + w(b'From: ') + w(self.author.encode('utf-8')) + w(b'\nDate: ') + w(self.date.encode('utf-8')) + w(b'\nSubject: [PATCH] ') + w(self.title.encode('utf-8')) + w(b'\n\n') + w(self.body.encode('utf-8')) + w(b'\n---\n\n') + self.diff.write(w) + w(b'-- \n0.0.0 patchstate\n\n') + + self.data = data = out.getvalue() + + self.hash = base64.b32encode(hashlib.blake2b(data).digest()[:20]).lower().decode('ascii') + + def revert(self): + r = Patch( + author = self.author, + date = self.date, + title = f'Revert "{self.title}"', + body = self.body, + diff = -self.diff, + ) + + r._set_data() + r._set_id() + return r + +class PatchInfo: + __slots__ = ('patch_id', '_body') + + def __init__(self, patch_id, info): + self.patch_id = patch_id + self._body = info + + def get_patch(self, repo): + p = repo.patches[self.patch_hash] + if p.id != self.patch_id: + raise KeyError(f'patch id mismatch: {self.patch_id!r} -> {p.id!r}') + if p.title != self.patch_title: + raise KeyError(f'patch title mismatch: {self.patch_title!r} -> {p.title!r}') + return p + + @property + def patch_hash(self): + return self._body[0] + + @property + def patch_title(self): + return self._body[1] + + @property + def mode(self): + return self._body[2] + + @property + def info(self): + return self._body[3:] + + def update(self, *, patch=None, patch_id=None, patch_hash=None, patch_title=None, mode=None, new_mode=None, info=None): + if patch_title is None: + patch_title = self.patch_title + + if patch is not None: + assert patch_id is None + assert patch_hash is None + if patch_title != patch.title: + raise ValueError('patch title change: {patch_title!r} -> {patch.title!r}') + patch_id = patch.id + patch_hash = patch.hash + + else: + if patch_id is None: + patch_id = self.patch_id + + if patch_hash is None: + patch_hash = self.patch_hash + + if info is None: + info = self.info + + if mode is None: + mode = self.mode + + if new_mode is not None and new_mode != mode: + info.insert(0, f'(was {mode})') + mode = new_mode + + return PatchInfo(patch_id, [patch_hash, patch_title, mode, *info]) + +class Series: + __slots__ = ('info',) + + def __init__(self, /, info=None): + if info is None: + info = [] + self.info = info + + @classmethod + def parse(tp, data): + inf = [] + + lines = iter(data.split('\n')) + l = next(lines) + while True: + if not l: + try: + l = next(lines) + except StopIteration: + break + continue + m = rem_patch(l) + if m is None: + raise ValueError(f'Invalid header: {l!r}') + pid = m.group(1) + ldata = [] + while True: + l = next(lines) + if not l.startswith('\t'): + break + ldata.append(l[1:]) + if len(ldata) < 3: + raise ValueError(f'Expected data after {f"patch {pid}"!r}') + + while ldata and not ldata[-1]: + del ldata[-1] + + inf.append(PatchInfo(pid, ldata)) + + return tp(info=inf) + + def fmt(self): + out = io.StringIO() + w = out.write + + for pi in self.info: + w(f'patch {pi.patch_id}') + for line in pi._body: + w('\n\t') + w(line) + w('\n') + + return out.getvalue() + +class Repository: + __slots__ = ('path', 'patches', 'aliases', 'by_id') + + def __init__(self, path): + self.path = path + self.patches = {} + self.by_id = {} + + aliases = {} + for fn in os.listdir(path): + if not fn.endswith('.patch'): + continue + fp = os.path.join(path, fn) + with io.open(fp, 'rb') as f: + data = f.read() + + try: + p = Patch.parse(data) + except ValueError as exc: + print(f'Failed to parse {fn!r}: {exc}', file=sys.stderr) + os.unlink(fp) + continue + + h = p.hash + if fn != f'{h}.patch': + aliases[fn[:-6]] = (h,p) + else: + self.patches[h] = p + self.by_id.setdefault(p.id, []).append(p) + + for h,(h2,p) in aliases.items(): + assert h != h2 + assert h not in self.patches + if h2 in self.patches: + continue + print(f'Migrating patch: {h!r} -> {h2!r}', file=sys.stderr) + self.patches[h2] = p + with io.open(os.path.join(self.path, f'{h2}.patch'), 'xb') as f: + f.write(p.data) + + self.aliases = {h: h2 for h,(h2,p) in aliases.items()} + + def hash_data(self, data): + data = hashlib.blake2b(data).digest() + return base64.b32encode(data[:20]).lower().decode() + + def insert_patch(self, patch): + p = self.patches.setdefault(patch.hash, patch) + if p is patch: + pid = p.id + try: + pd = self.by_id[pid] + except KeyError: + self.by_id[pid] = [p] + else: + print(f'Duplicate patch ID: {pid!r} {p.title}', file=sys.stderr) + pd.append(p) + + with io.open(os.path.join(self.path, f'{p.hash}.patch'), 'xb') as f: + f.write(p.data) + + return p + + def import_dir(self, path): + r = Series() + r.info = inf = [] + + for fn in sorted(os.listdir(path)): + if not fn.endswith('.patch'): + print(f'Skipping {fn!r}', file=sys.stderr) + continue + with io.open(os.path.join(path, fn), 'rb') as f: + patch = Patch.parse(f.read()) + + patch = self.insert_patch(patch) + inf.append(PatchInfo(patch.id, [patch.hash, patch.title, 'new'])) + + return r + + def gc(self, keep): + for h in self.patches.keys() - keep: + del self.patches[h] + print(f'Removing patch: {h!r}', file=sys.stderr) + os.unlink(os.path.join(self.path, f'{h}.patch')) + +def argparse_next(args, func=...): + def do_argparse(func): + while True: + try: + arg = next(args) + except StopIteration: + return None + if arg == '--': + return next(args, None) + if not arg.startswith('-'): + return arg + arg = func(arg) + if arg is not None: + return arg + if func is not ...: + return do_argparse(func) + return do_argparse + +def argparse_all(args, func=...): + def do_argparse(func): + r = [] + while True: + try: + arg = next(args) + except StopIteration: + return r + if arg == '--': + r.extend(args) + return r + if not arg.startswith('-'): + r.append(arg) + continue + arg = func(arg) + if arg is not None: + r.extend(arg) + if func is not ...: + return do_argparse(func) + return do_argparse diff --git a/patchtheory.py b/patchtheory.py new file mode 100644 index 0000000..21b3e29 --- /dev/null +++ b/patchtheory.py @@ -0,0 +1,960 @@ +import re, operator, io, sys + +rem_range = re.compile(b'@@ -([0-9]+)(?:,([0-9]+))? \\+([0-9]+)(?:,([0-9]+))? @@(?: .*)?').fullmatch +rem_diffgit = re.compile('diff --git a/([^ ]*) b/([^ ]*)').fullmatch + +INFINITY = 2**64 + +class PatchError(Exception): + __slots__ = ('a', 'b') + def __init__(self, a, b, /, fn=None): + super().__init__(fn) + self.a = a + self.b = b + + def __str__(self, /): + fn = self.args[0] + r = [f' in {fn!r}:' if fn is not None else ':'] + + def tabmangle(p): + o = io.BytesIO() + p.write(o.write) + p = o.getvalue().decode('utf-8', 'replace').splitlines() + return [' ' + l for l in p] + + r.append('') + r += tabmangle(self.a) + + r.append('') + r += tabmangle(self.b) + + return '\n'.join(r) + +class ConflictError(PatchError): + __slots__ = () + + def __str__(self, /): + return 'Conflict' + PatchError.__str__(self) + +class MismatchError(PatchError): + __slots__ = () + + def __str__(self, /): + return 'Mismatch' + PatchError.__str__(self) + +class Hunk: + __slots__ = ('lhs_line', 'rhs_line', 'pre_context', 'lhs', 'rhs', 'post_context') + + def __init__(self, /, *, lhs_line, rhs_line, pre_context, lhs, rhs, post_context): + assert lhs_line > 0 + assert rhs_line > 0 + + self.lhs_line = lhs_line + self.rhs_line = rhs_line + self.pre_context = pre_context + self.lhs = lhs + self.rhs = rhs + self.post_context = post_context + + @property + def lhs_size(self, /): + return len(self.pre_context) + len(self.lhs) + len(self.post_context) + + @property + def rhs_size(self, /): + return len(self.pre_context) + len(self.rhs) + len(self.post_context) + + def write(self, w, /, munge=False, *, lhs_line=None, rhs_line=None, lhs_nl=True, rhs_nl=True): + if lhs_line is None: + lhs_line = self.lhs_line + if rhs_line is None: + rhs_line = self.rhs_line + pre_context = self.pre_context + lhs = self.lhs + rhs = self.rhs + post_context = self.post_context + clen = len(pre_context) + len(post_context) + llen = len(lhs) + clen + rlen = len(rhs) + clen + + if munge: + w(f'@@ -xxx,{llen} +yyy,{rlen} @@'.encode('utf-8')) + else: + w(f'@@ -{lhs_line},{llen} +{rhs_line},{rlen} @@'.encode('utf-8')) + + for l in pre_context: + w(b'\n ') + w(l) + + for l in lhs: + w(b'\n-') + w(l) + + for l in rhs: + w(b'\n+') + w(l) + + for l in post_context: + w(b'\n ') + w(l) + + w(b'\n') + + @classmethod + def parse_l(tp, l, line, /): + m = rem_range(l) + if m is None: + raise ValueError(f'Invalid patch: bad range info {l!r}') + lhs_line,n2,rhs_line,n4 = m.groups() + lhs_line = 1 if lhs_line == b'0' else int(lhs_line) + rhs_line = 1 if rhs_line == b'0' else int(rhs_line) + n2 = int(n2) if n2 else 1 + n4 = int(n4) if n4 else 1 + + hunk = [] + prev = -1 + lhs_nl = True + rhs_nl = True + post_nl = True + while True: + l = line() + if l.startswith(b'\\ '): + if prev == 0: + lhs_nl = False + elif prev == 1: + rhs_nl = False + elif prev == -1: + post_nl = False + else: + raise ValueError('Invalid patch: bad "No newline at end of file" flag') + prev = -2 + continue + if not n2 and not n4: + break + k = l[0] + if k == b' '[0]: + if not post_nl or not lhs_nl or not rhs_nl: + raise ValueError('Invalid patch: bad "No newline at end of file" flag') + if not n2 or not n4: + raise ValueError('Invalid patch: bad line count') + n2 -= 1 + n4 -= 1 + prev = -1 + elif k == b'-'[0]: + if not post_nl or not lhs_nl: + raise ValueError('Invalid patch: bad "No newline at end of file" flag') + if not n2: + raise ValueError('Invalid patch: bad line count') + n2 -= 1 + prev = 0 + elif k == b'+'[0]: + if not post_nl or not rhs_nl: + raise ValueError('Invalid patch: bad "No newline at end of file" flag') + if not n4: + raise ValueError('Invalid patch: bad line count') + n4 -= 1 + prev = 1 + else: + raise ValueError(f'Invalid patch: bad prefix character {chr(k)!r}') + assert False + hunk.append(l) + lret = l + + strip1 = operator.itemgetter(slice(1,None)) + + for i,l in enumerate(hunk): + if l[0] != b' '[0]: + break + else: + raise ValueError('Invalid patch: no changed lines in hunk') + pre_context = (*map(strip1, hunk[:i]),) + del hunk[:i] + + j = -1 + while True: + l = hunk[j] + if l[0] != b' '[0]: + break + j -= 1 + j += len(hunk) + 1 + if not post_nl: + hunk[j] += b'\n\\ No newline at end of file' + post_context = (*map(strip1, hunk[j:]),) + del hunk[j:] + + lhs = [] + rhs = [] + for l in hunk: + l0 = l[0] + if l0 != b'+'[0]: + lhs.append(l) + if l0 != b'-'[0]: + rhs.append(l) + if not lhs_nl: + lhs[-1] += b'\n\\ No newline at end of file' + if not rhs_nl: + rhs[-1] += b'\n\\ No newline at end of file' + + return lret, tp( + lhs_line = lhs_line, + rhs_line = rhs_line, + pre_context = pre_context, + lhs = (*map(strip1, lhs),), + rhs = (*map(strip1, rhs),), + post_context = post_context, + ) + + def reduce_context(self, n=3): + pre = self.pre_context + lhs = self.lhs + rhs = self.rhs + post = self.post_context + + i = 0 + j = -1 + m = min(len(lhs), len(rhs)) + while i - j <= m: + if lhs[i] == rhs[i]: + i += 1 + if i - j > m: + break + if lhs[j] == rhs[j]: + j -= 1 + else: + if lhs[j] != rhs[j]: + break + j -= 1 + j += 1 + jl = len(lhs) + j + jr = len(rhs) + j + + assert lhs[:i] == rhs[:i] + assert lhs[jl:] == rhs[jr:] + pre = pre + lhs[:i] + post = lhs[jl:] + post + lhs = lhs[i:jl] + rhs = rhs[i:jr] + + if not lhs and not rhs: + return None + + assert lhs != rhs + assert not lhs or not rhs or lhs[0] != rhs[0] and lhs[-1] != rhs[-1] + + lhs_line = self.lhs_line + rhs_line = self.rhs_line + + if n is None: + pass + elif not n: + d = len(pre) + lhs_line += d + rhs_line += d + pre = () + post = () + else: + if len(pre) > n: + d = len(pre) - n + lhs_line += d + rhs_line += d + pre = pre[d:] + + if len(post) > n: + post = post[:n] + + return Hunk( + lhs_line = lhs_line, + rhs_line = rhs_line, + pre_context = pre, + lhs = lhs, + rhs = rhs, + post_context = post, + ) + +def join_hunks(hs_a, hs_b): + hs_a = iter(hs_a) + hs_b = iter(hs_b) + + need_a = True + need_b = True + have_cur = False + + def make_cur(n): + if not have_cur: + return None + return Hunk( + lhs_line = cur_lhs_line, + rhs_line = cur_rhs_line, + pre_context = (), + lhs = tuple(cur_lhs), + rhs = tuple(cur_rhs), + post_context = (), + ).reduce_context(n) + + delta = 0 + r = [] + + while True: + if need_a: + need_a = False + a = next(hs_a, None) + + if need_b: + need_b = False + b = next(hs_b, None) + + if not have_cur: + if a is not None and (b is None or a.rhs_line <= b.lhs_line): + cur_lhs_line = a.lhs_line + cur_rhs_line = a.lhs_line + delta + cur_lhs = [*a.pre_context, *a.lhs, *a.post_context] + cur_rhs = [*a.pre_context, *a.rhs, *a.post_context] + need_a = True + elif b is not None: + cur_lhs_line = b.rhs_line - delta + cur_rhs_line = b.rhs_line + cur_lhs = [*b.pre_context, *b.lhs, *b.post_context] + cur_rhs = [*b.pre_context, *b.rhs, *b.post_context] + need_b = True + else: + return (*r,) + have_cur = True + continue + + cur_lhs_end = cur_lhs_line + len(cur_lhs) + cur_rhs_end = cur_rhs_line + len(cur_rhs) + + if a is not None and a.lhs_line <= cur_lhs_end: + lhs = [*a.pre_context, *a.lhs, *a.post_context] + rhs = [*a.pre_context, *a.rhs, *a.post_context] + + i = a.lhs_line - cur_lhs_line + k = i + len(rhs) + + if len(cur_lhs) >= k: + if cur_lhs[i:k] != rhs: + raise MismatchError(make_cur(None), a) + cur_lhs[i:k] = lhs + else: + j = len(cur_lhs) - i + if cur_lhs[i:] != rhs[:j]: + raise MismatchError(make_cur(None), a) + cur_lhs[i:] = lhs + cur_rhs.extend(rhs[j:]) + need_a = True + continue + + if b is not None and b.rhs_line <= cur_rhs_end: + lhs = [*b.pre_context, *b.lhs, *b.post_context] + rhs = [*b.pre_context, *b.rhs, *b.post_context] + i = b.rhs_line - cur_rhs_line + k = i + len(lhs) + + if len(cur_rhs) >= k: + if cur_rhs[i:k] != lhs: + raise MismatchError(make_cur(None), b) + cur_rhs[i:k] = rhs + else: + j = len(cur_rhs) - i + if cur_rhs[i:] != lhs[:j]: + raise MismatchError(make_cur(None), b) + cur_rhs[i:] = rhs + cur_lhs.extend(lhs[j:]) + need_b = True + continue + + cur = make_cur(3) + have_cur = False + if cur is None: + continue + r.append(cur) + delta += len(cur.rhs) - len(cur.lhs) + +def commute_hunks(hs_a, hs_b): + hs_a = iter(hs_a) + hs_b = iter(hs_b) + + need_a = True + need_b = True + + delta_a = 0 + delta_b = 0 + ra = [] + rb = [] + wa = ra.append + wb = rb.append + + while True: + if need_a: + need_a = False + try: + a = next(hs_a) + except StopIteration: + if b is None and not need_b: + break + a = None + a_begin = INFINITY + a_pre = () + a_post = () + a_end = INFINITY + else: + a_pre = a.pre_context + a_post = a.post_context + a_begin = a.rhs_line + len(a_pre) + a_end = a_begin + len(a.rhs) + + if need_b: + need_b = False + try: + b = next(hs_b) + except StopIteration: + if a is None: + break + b = None + b_begin = INFINITY + b_pre = () + b_post = () + b_end = INFINITY + else: + b_pre = b.pre_context + b_post = b.post_context + b_begin = b.lhs_line + len(b_pre) + b_end = b_begin + len(b.lhs) + + if a_end <= b_begin: + gap = b_begin - a_end + + if (n := len(a_post) + len(b_pre) - gap) > 0: + assert (a_pre + a.rhs + a_post)[-n:] == (b_pre + b.lhs + b_post)[:n] + if (n := len(a_post) - gap) > 0: + a_post = a_post[:gap] + (b.rhs + b_post)[:n] + if (n := len(b_pre) - gap) > 0: + b_pre = (a_pre + a.lhs)[-n:] + b_pre[n:] + + wa(Hunk( + lhs_line = a.lhs_line + delta_b, + rhs_line = a.rhs_line + delta_b, + pre_context = a_pre, + lhs = a.lhs, + rhs = a.rhs, + post_context = a_post, + )) + delta_a += len(a.rhs) - len(a.lhs) + need_a = True + continue + + if b_end <= a_begin: + gap = a_begin - b_end + + if (n := len(b_post) + len(a_pre) - gap) > 0: + assert (b_pre + b.lhs + b_post)[-n:] == (a_pre + a.rhs + a_post)[:n] + if (n := len(b_post) - gap) > 0: + b_post = b_post[:gap] + (a.lhs + a_post)[:n] + if (n := len(a_pre) - gap) > 0: + a_pre = (b_pre + b.rhs)[-n:] + a_pre[n:] + + wb(Hunk( + lhs_line = b.lhs_line - delta_a, + rhs_line = b.rhs_line - delta_a, + pre_context = b_pre, + lhs = b.lhs, + rhs = b.rhs, + post_context = b_post, + )) + delta_b += len(b.rhs) - len(b.lhs) + need_b = True + continue + + raise ConflictError(a, b) + + return (*rb,), (*ra,) + +class FileDiff: + __slots__ = ('src_name', 'dst_name', 'src_mode', 'dst_mode', 'hunks',) + + def __init__(self, /, *, src_name, dst_name, src_mode, dst_mode, hunks): + assert (src_mode is ...) == (dst_mode is ...) + assert (src_name is None) == (src_mode is None) + assert (dst_name is None) == (dst_mode is None) + + self.src_name = src_name + self.dst_name = dst_name + if src_mode == dst_mode: + self.src_mode = ... + self.dst_mode = ... + else: + self.src_mode = src_mode + self.dst_mode = dst_mode + self.hunks = hunks + + @property + def op_kind_name(self): + if self.src_name is None: + return 'create' + if self.dst_name is None: + return 'delete' + is_chmod = self.dst_mode != self.dst_mode + if self.dst_name != self.dst_name: + return 'chmod+rename' if is_chmod else 'rename' + if self.hunks: + return 'chmod+modify' if is_chmod else 'modify' + return 'chmod' if is_chmod else 'noop' + + def write(self, w): + src_name = self.src_name + dst_name = self.dst_name + + if src_name is None: + assert dst_name is not None + w(f'diff --git a/{dst_name} b/{dst_name}\n'.encode('utf-8')) + elif dst_name is None: + w(f'diff --git a/{src_name} b/{src_name}\n'.encode('utf-8')) + else: + w(f'diff --git a/{src_name} b/{dst_name}\n'.encode('utf-8')) + + if src_name is not None and dst_name is not None and src_name != dst_name: + w(f'rename from {src_name}\nrename to {dst_name}\n'.encode('utf-8')) + + m = self.src_mode + if dst_name is None: + assert m is not None + w(f'deleted file mode {m}\n'.encode('utf-8')) + elif m not in (None, ...): + w(f'old mode {m}\n'.encode('utf-8')) + + m = self.dst_mode + if src_name is None: + assert m is not None + w(f'new file mode {m}\n'.encode('utf-8')) + elif m not in (None, ...): + w(f'new mode {m}\n'.encode('utf-8')) + + h = self.hunks + if h: + if src_name is None: + src_name = '/dev/null' + lhs_line = 0 + else: + src_name = f'a/{src_name}' + lhs_line = None + if dst_name is None: + dst_name = '/dev/null' + rhs_line = 0 + else: + dst_name = f'b/{dst_name}' + rhs_line = None + + w(f'--- {src_name}\n+++ {dst_name}\n'.encode('utf-8')) + + for h in h: + h.write(w, lhs_line=lhs_line, rhs_line=rhs_line) + + def __neg__(self, /): + tp = type(self) + +class Diff: + __slots__ = ('files',) + + def __init__(self): + self.files = () + + def __bool__(self): + return not not self.files + + @classmethod + def _sort_files(tp, files): + r = object.__new__(tp) + r.files = (*sorted(files, key=lambda f: (f.src_name or '', f.dst_name or '')),) + return r + + @classmethod + def parse_l(tp, l, line): + files = [] + + while True: + l = l.decode('utf-8').rstrip() + if l == '--' or not l: + break + m = rem_diffgit(l) + if m is None: + raise ValueError(f'Invalid patch: expected \'diff --git ...\' header, got {l!r}') + + header = l + + src_name,dst_name = m.groups() + src_exists = ... + dst_exists = ... + + if not src_exists and not dst_exists: + raise RuntimeError('wtf') + if (not src_exists or not dst_exists) and src_name != dst_name: + raise RuntimeError('wtf') + + has_diff = False + has_chmod = False + has_rename = False + dst_mode = ... + src_mode = ... + + while True: + l = line() + if l.startswith(b'--- a/'): + if src_exists is False: raise RuntimeError('wtf') + src_exists = True + if l[6:].decode('utf-8') != src_name: + raise RuntimeError('wtf') + has_diff = True + continue + if l == b'--- /dev/null': + if src_exists is True: raise RuntimeError('wtf') + src_exists = False + has_diff = True + continue + if l.startswith(b'+++ b/'): + if dst_exists is False: raise RuntimeError('wtf') + dst_exists = True + if l[6:].decode('utf-8') != dst_name: + raise RuntimeError('wtf') + has_diff = True + continue + if l == b'+++ /dev/null': + if dst_exists is True: raise RuntimeError('wtf') + dst_exists = False + has_diff = True + continue + if l.startswith(b'index '): + continue + if l.startswith(b'new file mode '): + if src_exists is True: raise RuntimeError('wtf') + if dst_exists is False: raise RuntimeError('wtf') + src_exists = False + dst_exists=True + dst_mode = l[14:].decode('ascii') + src_mode = None + continue + if l.startswith(b'new mode '): + dst_mode = l[9:].decode('ascii') + has_chmod = True + continue + if l.startswith(b'deleted file mode '): + if src_exists is False: raise RuntimeError('wtf') + if dst_exists is True: raise RuntimeError('wtf') + src_exists = True + dst_exists = False + src_mode = l[18:].decode('ascii') + dst_mode = None + continue + if l.startswith(b'old mode '): + if dst_exists is False: + raise RuntimeError('File deleted') + src_mode = l[9:].decode('ascii') + has_chmod = True + dst_exists = True + continue + if l.startswith(b'similarity index '): + has_rename = True + continue + if l.startswith(b'rename from '): + has_rename = True + rename_from = l[12:].decode('utf-8') + continue + if l.startswith(b'rename to '): + rename_to = l[10:].decode('utf-8') + has_rename = True + continue + break + + if has_chmod or has_rename: + if src_exists is False: raise RuntimeError('wtf') + if dst_exists is False: raise RuntimeError('wtf') + src_exists = True + dst_exists = True + + if has_chmod: + assert src_mode is not ... + assert dst_mode is not ... + + assert src_exists is not ... + if not src_exists: + assert dst_mode is not ... + + assert dst_exists is not ... + if not dst_exists: + assert src_mode is not ... + + if has_rename: + if src_name == dst_name or not src_exists or not dst_exists: + raise RuntimeError(f'Bad rename: {src_name!r} -> {dst_name!r}') + assert rename_from == src_name + assert rename_to == dst_name + + if not src_exists: src_name = None + if not dst_exists: dst_name = None + + assert (src_mode is ...) == (dst_mode is ...) + assert has_diff or has_chmod or has_rename or not dst_exists + + #want_header = fmt_diff_git(src_name, dst_name) + #if header != want_header: + # raise ValueError(f'Invalid patch: expected {want_header!r} header, got {header!r}') + + if has_diff: + hunks = [] + + while True: + l,hunk = Hunk.parse_l(l, line) + hunk = hunk.reduce_context() + if hunk is not None: + hunks.append(hunk) + if not l.startswith(b'@@ '): + break + + if src_name is None: + assert len(hunks) == 1 + assert len(hunks[0].pre_context) == 0 + assert len(hunks[0].post_context) == 0 + assert len(hunks[0].lhs) == 0 + assert hunks[0].lhs_line == 1 + hunks[0].lhs_line = 1 + + if dst_name is None: + assert len(hunks) == 1 + assert len(hunks[0].pre_context) == 0 + assert len(hunks[0].post_context) == 0 + assert len(hunks[0].rhs) == 0 + assert hunks[0].rhs_line == 1 + + hunks = (*hunks,) + + else: + hunks = () + + files.append(FileDiff( + src_name = src_name, + dst_name = dst_name, + src_mode = src_mode, + dst_mode = dst_mode, + hunks = hunks, + )) + + try: + l = line().rstrip() + except StopIteration: + pass + else: + if l[0] not in b'0123456789': + l = l.decode('utf-8') + raise ValueError('Invalid patch: expected git version, got {l!r}') + + + while True: + try: + l = line() + except StopIteration: + break + if l.strip(): + l = l.decode('utf-8') + raise ValueError('Invalid patch: got trailing garbage {l!r}') + + return tp._sort_files(files) + + @classmethod + def parse(tp, body): + line = iter(body.split(b'\n')).__next__ + return tp.parse_l(line(), line) + + def write(self, w, /): + for f in self.files: + f.write(w) + + def write_munged(self, w, /): + for f in self.files: + w((f.src_name or '').encode('utf-8')) + w(b'\n') + w((f.dst_name or '').encode('utf-8')) + w(b'\n') + for h in f.hunks: + h.write(w, True) + w(b'\n') + + def __neg__(self): + return Diff._sort_files((*(FileDiff( + src_name = f.dst_name, + dst_name = f.src_name, + src_mode = f.dst_mode, + dst_mode = f.src_mode, + hunks = (*(Hunk( + lhs_line = h.rhs_line, + rhs_line = h.lhs_line, + pre_context = h.pre_context, + lhs = h.rhs, + rhs = h.lhs, + post_context = h.post_context, + ) for h in f.hunks),), + ) for f in self.files),)) + + def __add__(a, b): + tp = type(a) + if type(b) is not tp: + raise TypeError + + r = [] + w = r.append + + fdiff = FileDiff + def join(fa, fb): + src_name = fa.src_name + dst_name = fb.dst_name + src_mode = fa.src_mode + dst_mode = fb.dst_mode + if src_mode is ...: + if dst_mode is not ...: + src_mode = fb.src_mode + elif dst_mode is ...: + dst_mode = fa.dst_mode + elif fa.dst_mode != fb.src_mode: + raise ValueError('mismatch') + + hunks = (*join_hunks(fa.hunks, fb.hunks),) + + if src_name is None and dst_name is None: + if hunks: + raise ValueError('mismatch') + return + + if src_name == dst_name and src_mode == dst_mode and not hunks: + return + + w(fdiff( + src_name = src_name, + dst_name = dst_name, + src_mode = src_mode, + dst_mode = dst_mode, + hunks = hunks, + )) + + _pair_diffs(a, b, on_lhs=w, on_rhs=w, on_pair=join, on_re=join) + return tp._sort_files(r) + + @classmethod + def commute(tp, a, b): + if type(a) is not tp or type(b) is not tp: + raise TypeError + + ra = [] + rb = [] + wa = ra.append + wb = rb.append + fdiff = FileDiff + + def join(fa, fb): + name1 = fa.src_name + name2 = fa.dst_name + name3 = fb.dst_name + if name1 == name2: + name2 = name3 + elif name2 == name3: + name2 = name1 + else: + raise ValueError('cannot commute {name2!r}: {fa.op_kind_name} / {fb.op_kind_name}') + + mode1 = fa.src_mode + mode3 = fb.dst_mode + if mode1 is ...: + if mode3 is ...: + mode2 = ... + else: + mode1 = fb.src_mode + mode2 = mode3 + + elif mode3 is ...: + mode3 = fa.dst_mode + mode2 = mode1 + + else: + mode2 = fb.dst_mode + if fa.src_mode != mode2: + raise ValueError('mismatch') + if mode1 == mode2: + mode2 = mode3 + elif mode2 == mode3: + mode2 = mode1 + else: + raise ValueError('cannot commute {name2!r}: chmod/chmod conflict') + + try: + hb,ha = commute_hunks(fa.hunks, fb.hunks) + except PatchError as err: + err.args = (name1, name2, name3) + raise + + wa(fdiff( + src_name = name1, + dst_name = name2, + src_mode = mode1, + dst_mode = mode2, + hunks = ha, + )) + wb(fdiff( + src_name = name2, + dst_name = name3, + src_mode = mode2, + dst_mode = mode3, + hunks = hb, + )) + + _pair_diffs(a, b, on_lhs=wa, on_rhs=wb, on_re=join, on_pair=join) + return tp._sort_files(rb), tp._sort_files(ra) + +def _pair_diffs(a, b, *, on_lhs, on_rhs, on_pair, on_re): + a_del = {} + b_new = {} + + a_dst = {} + b_dst = set() + for f in a.files: + dst = f.dst_name + if dst is None: + a_del[f.src_name] = f + continue + a_dst[dst] = f + + a = a_dst.pop + for f in b.files: + dst = f.dst_name + if dst is not None: + b_dst.add(dst) + + src = f.src_name + if src is None: + b_new[dst] = f + continue + + try: + f2 = a(src) + except KeyError: + pass + else: + on_pair(f2, f) + continue + on_rhs(f) + + for dst in a_dst.keys() & b_dst: + raise ValueError(f'mismatch for {dst!r}') + + b = b_new.pop + for src,f in a_del.items(): + try: + f2 = b(src) + except KeyError: + pass + else: + on_re(f, f2) + continue + on_lhs(f) + + for f in b_new.values(): + on_rhs(f) + + for f in a_dst.values(): + on_lhs(f) diff --git a/prune_aliases.py b/prune_aliases.py new file mode 100644 index 0000000..24c5392 --- /dev/null +++ b/prune_aliases.py @@ -0,0 +1,21 @@ +import patchstate as ps +import io, os, sys + +def main(args): + args = iter(args) + arg0 = next(args) + + @ps.argparse_all(args) + def path(arg): + raise RuntimeError(f'Invalid argument: {arg!r}') + + [repo_path] = path + + repo = ps.Repository(repo_path) + for h in repo.aliases: + os.unlink(os.path.join(repo.path, f'{h}.patch')) + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/ungone_series.py b/ungone_series.py new file mode 100644 index 0000000..065ed01 --- /dev/null +++ b/ungone_series.py @@ -0,0 +1,70 @@ +import patchstate as ps +import io, os, sys +from import_series import lcs + +def parse_prev(info): + if not info.startswith('(was ') or not info.endswith(')'): + raise ValueError + return info[5:-1] + +def main(args): + args = iter(args) + arg0 = next(args) + + @ps.argparse_all(args) + def path(arg): + raise RuntimeError(f'Invalid argument: {arg!r}') + + [repo_path] = path + + repo = ps.Repository(repo_path) + + with io.open(os.path.join(repo.path, 'series'), 'r') as f: + pcur = ps.Series.parse(f.read()) + + gone_seq = [] + new_seq = [] + updated = {} + + def fix_seq(): + s = lcs(gone_seq, new_seq, key=lambda p: p.patch_title) + del gone_seq[:] + del new_seq[:] + + for p_gone, p_new in s: + if p_gone is None or p_new is None: + continue + updated[p_new] = parse_prev(p_gone.info[0]) + updated[p_gone] = None + + for i,pi in enumerate(pcur.info): + m = pi.mode + if m == 'gone': + gone_seq.append(pi) + elif m == 'new': + new_seq.append(pi) + else: + fix_seq() + fix_seq() + + inf = [] + for pi in pcur.info: + try: + mode = updated[pi] + except KeyError: + pass + else: + if mode is None: + continue + assert pi.mode == 'new' + pi = pi.update(mode=mode) + inf.append(pi) + + pnew = ps.Series(inf).fmt() + with io.open(os.path.join(repo.path, 'series'), 'w') as f: + f.write(pnew) + + return 0 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) -- cgit