From a4ef3e1a5ae48457e228e343127e73d8fa2388ac Mon Sep 17 00:00:00 2001 From: Frans Kaashoek Date: Sun, 2 Oct 2022 09:09:11 -0400 Subject: [PATCH 1/2] cow lab --- .gitignore | 9 + Makefile | 259 ++++++++++++++++++++- conf/lab.mk | 1 + grade-lab-cow | 55 +++++ gradelib.py | 611 +++++++++++++++++++++++++++++++++++++++++++++++++ user/cowtest.c | 197 ++++++++++++++++ 6 files changed, 1121 insertions(+), 11 deletions(-) create mode 100644 conf/lab.mk create mode 100755 grade-lab-cow create mode 100644 gradelib.py create mode 100644 user/cowtest.c diff --git a/.gitignore b/.gitignore index 07216f3..3d25221 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,12 @@ mkfs kernel/kernel user/usys.S .gdbinit +myapi.key +*-handin.tar.gz +xv6.out* +.vagrant/ +submissions/ +ph +barrier +/lab-*.json +.DS_Store \ No newline at end of file diff --git a/Makefile b/Makefile index 39a99d7..ded5bc2 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,16 @@ + +# To compile and run with a lab solution, set the lab name in conf/lab.mk +# (e.g., LAB=util). Run make grade to test solution with the lab's +# grade script (e.g., grade-lab-util). + +-include conf/lab.mk + K=kernel U=user OBJS = \ $K/entry.o \ - $K/start.o \ - $K/console.o \ - $K/printf.o \ - $K/uart.o \ $K/kalloc.o \ - $K/spinlock.o \ $K/string.o \ $K/main.o \ $K/vm.o \ @@ -30,6 +32,34 @@ OBJS = \ $K/plic.o \ $K/virtio_disk.o +OBJS_KCSAN = \ + $K/start.o \ + $K/console.o \ + $K/printf.o \ + $K/uart.o \ + $K/spinlock.o + +ifdef KCSAN +OBJS_KCSAN += \ + $K/kcsan.o +endif + +ifeq ($(LAB),$(filter $(LAB), lock)) +OBJS += \ + $K/stats.o\ + $K/sprintf.o +endif + + +ifeq ($(LAB),net) +OBJS += \ + $K/e1000.o \ + $K/net.o \ + $K/sysnet.o \ + $K/pci.o +endif + + # riscv64-unknown-elf- or riscv64-linux-gnu- # perhaps in /opt/riscv/bin #TOOLPREFIX = @@ -57,12 +87,28 @@ OBJCOPY = $(TOOLPREFIX)objcopy OBJDUMP = $(TOOLPREFIX)objdump CFLAGS = -Wall -Werror -O -fno-omit-frame-pointer -ggdb -gdwarf-2 + +ifdef LAB +LABUPPER = $(shell echo $(LAB) | tr a-z A-Z) +XCFLAGS += -DSOL_$(LABUPPER) -DLAB_$(LABUPPER) +endif + +CFLAGS += $(XCFLAGS) CFLAGS += -MD CFLAGS += -mcmodel=medany CFLAGS += -ffreestanding -fno-common -nostdlib -mno-relax CFLAGS += -I. CFLAGS += $(shell $(CC) -fno-stack-protector -E -x c /dev/null >/dev/null 2>&1 && echo -fno-stack-protector) +ifeq ($(LAB),net) +CFLAGS += -DNET_TESTS_PORT=$(SERVERPORT) +endif + +ifdef KCSAN +CFLAGS += -DKCSAN +KCSANFLAG = -fsanitize=thread +endif + # Disable PIE when possible (for Ubuntu 16.10 toolchain) ifneq ($(shell $(CC) -dumpspecs 2>/dev/null | grep -e '[^f]no-pie'),) CFLAGS += -fno-pie -no-pie @@ -73,11 +119,17 @@ endif LDFLAGS = -z max-page-size=4096 -$K/kernel: $(OBJS) $K/kernel.ld $U/initcode - $(LD) $(LDFLAGS) -T $K/kernel.ld -o $K/kernel $(OBJS) +$K/kernel: $(OBJS) $(OBJS_KCSAN) $K/kernel.ld $U/initcode + $(LD) $(LDFLAGS) -T $K/kernel.ld -o $K/kernel $(OBJS) $(OBJS_KCSAN) $(OBJDUMP) -S $K/kernel > $K/kernel.asm $(OBJDUMP) -t $K/kernel | sed '1,/SYMBOL TABLE/d; s/ .* / /; /^$$/d' > $K/kernel.sym +$(OBJS): EXTRAFLAG := $(KCSANFLAG) + +$K/%.o: $K/%.c + $(CC) $(CFLAGS) $(EXTRAFLAG) -c -o $@ $< + + $U/initcode: $U/initcode.S $(CC) $(CFLAGS) -march=rv64g -nostdinc -I. -Ikernel -c $U/initcode.S -o $U/initcode.o $(LD) $(LDFLAGS) -N -e start -Ttext 0 -o $U/initcode.out $U/initcode.o @@ -89,6 +141,10 @@ tags: $(OBJS) _init ULIB = $U/ulib.o $U/usys.o $U/printf.o $U/umalloc.o +ifeq ($(LAB),$(filter $(LAB), lock)) +ULIB += $U/statistics.o +endif + _%: %.o $(ULIB) $(LD) $(LDFLAGS) -T $U/user.ld -o $@ $^ $(OBJDUMP) -S $@ > $*.asm @@ -107,7 +163,7 @@ $U/_forktest: $U/forktest.o $(ULIB) $(OBJDUMP) -S $U/_forktest > $U/forktest.asm mkfs/mkfs: mkfs/mkfs.c $K/fs.h $K/param.h - gcc -Werror -Wall -I. -o mkfs/mkfs mkfs/mkfs.c + gcc $(XCFLAGS) -Werror -Wall -I. -o mkfs/mkfs mkfs/mkfs.c # Prevent deletion of intermediate files, e.g. cat.o, after first build, so # that disk image changes after first build are persistent until clean. More @@ -133,8 +189,79 @@ UPROGS=\ $U/_wc\ $U/_zombie\ -fs.img: mkfs/mkfs README $(UPROGS) - mkfs/mkfs fs.img README $(UPROGS) + + + +ifeq ($(LAB),$(filter $(LAB), lock)) +UPROGS += \ + $U/_stats +endif + +ifeq ($(LAB),traps) +UPROGS += \ + $U/_call\ + $U/_bttest +endif + +ifeq ($(LAB),lazy) +UPROGS += \ + $U/_lazytests +endif + +ifeq ($(LAB),cow) +UPROGS += \ + $U/_cowtest +endif + +ifeq ($(LAB),thread) +UPROGS += \ + $U/_uthread + +$U/uthread_switch.o : $U/uthread_switch.S + $(CC) $(CFLAGS) -c -o $U/uthread_switch.o $U/uthread_switch.S + +$U/_uthread: $U/uthread.o $U/uthread_switch.o $(ULIB) + $(LD) $(LDFLAGS) -N -e main -Ttext 0 -o $U/_uthread $U/uthread.o $U/uthread_switch.o $(ULIB) + $(OBJDUMP) -S $U/_uthread > $U/uthread.asm + +ph: notxv6/ph.c + gcc -o ph -g -O2 $(XCFLAGS) notxv6/ph.c -pthread + +barrier: notxv6/barrier.c + gcc -o barrier -g -O2 $(XCFLAGS) notxv6/barrier.c -pthread +endif + +ifeq ($(LAB),pgtbl) +UPROGS += \ + $U/_pgtbltest +endif + +ifeq ($(LAB),lock) +UPROGS += \ + $U/_kalloctest\ + $U/_bcachetest +endif + +ifeq ($(LAB),fs) +UPROGS += \ + $U/_bigfile +endif + + + +ifeq ($(LAB),net) +UPROGS += \ + $U/_nettests +endif + +UEXTRA= +ifeq ($(LAB),util) + UEXTRA += user/xargstest.sh +endif + + +fs.img: mkfs/mkfs README $(UEXTRA) $(UPROGS) + mkfs/mkfs fs.img README $(UEXTRA) $(UPROGS) -include kernel/*.d user/*.d @@ -144,7 +271,8 @@ clean: $U/initcode $U/initcode.out $K/kernel fs.img \ mkfs/mkfs .gdbinit \ $U/usys.S \ - $(UPROGS) + $(UPROGS) \ + ph barrier # try to generate a unique GDB port GDBPORT = $(shell expr `id -u` % 5000 + 25000) @@ -155,12 +283,22 @@ QEMUGDB = $(shell if $(QEMU) -help | grep -q '^-gdb'; \ ifndef CPUS CPUS := 3 endif +ifeq ($(LAB),fs) +CPUS := 1 +endif + +FWDPORT = $(shell expr `id -u` % 5000 + 25999) QEMUOPTS = -machine virt -bios none -kernel $K/kernel -m 128M -smp $(CPUS) -nographic QEMUOPTS += -global virtio-mmio.force-legacy=false QEMUOPTS += -drive file=fs.img,if=none,format=raw,id=x0 QEMUOPTS += -device virtio-blk-device,drive=x0,bus=virtio-mmio-bus.0 +ifeq ($(LAB),net) +QEMUOPTS += -netdev user,id=net0,hostfwd=udp::$(FWDPORT)-:2000 -object filter-dump,id=net0,netdev=net0,file=packets.pcap +QEMUOPTS += -device e1000,netdev=net0,bus=pcie.0 +endif + qemu: $K/kernel fs.img $(QEMU) $(QEMUOPTS) @@ -171,3 +309,102 @@ qemu-gdb: $K/kernel .gdbinit fs.img @echo "*** Now run 'gdb' in another window." 1>&2 $(QEMU) $(QEMUOPTS) -S $(QEMUGDB) +ifeq ($(LAB),net) +# try to generate a unique port for the echo server +SERVERPORT = $(shell expr `id -u` % 5000 + 25099) + +server: + python3 server.py $(SERVERPORT) + +ping: + python3 ping.py $(FWDPORT) +endif + +## +## FOR testing lab grading script +## + +ifneq ($(V),@) +GRADEFLAGS += -v +endif + +print-gdbport: + @echo $(GDBPORT) + +grade: + @echo $(MAKE) clean + @$(MAKE) clean || \ + (echo "'make clean' failed. HINT: Do you have another running instance of xv6?" && exit 1) + ./grade-lab-$(LAB) $(GRADEFLAGS) + +## +## FOR web handin +## + + +WEBSUB := https://6828.scripts.mit.edu/2022/handin.py + +handin: tarball-pref myapi.key + @SUF=$(LAB); \ + curl -f -F file=@lab-$$SUF-handin.tar.gz -F key=\ /dev/null || { \ + echo ; \ + echo Submit seems to have failed.; \ + echo Please go to $(WEBSUB)/ and upload the tarball manually.; } + +handin-check: + @if ! test -d .git; then \ + echo No .git directory, is this a git repository?; \ + false; \ + fi + @if test "$$(git symbolic-ref HEAD)" != refs/heads/$(LAB); then \ + git branch; \ + read -p "You are not on the $(LAB) branch. Hand-in the current branch? [y/N] " r; \ + test "$$r" = y; \ + fi + @if ! git diff-files --quiet || ! git diff-index --quiet --cached HEAD; then \ + git status -s; \ + echo; \ + echo "You have uncomitted changes. Please commit or stash them."; \ + false; \ + fi + @if test -n "`git status -s`"; then \ + git status -s; \ + read -p "Untracked files will not be handed in. Continue? [y/N] " r; \ + test "$$r" = y; \ + fi + +UPSTREAM := $(shell git remote -v | grep -m 1 "xv6-labs-2022" | awk '{split($$0,a," "); print a[1]}') + +tarball: handin-check + git archive --format=tar HEAD | gzip > lab-$(LAB)-handin.tar.gz + +tarball-pref: handin-check + @SUF=$(LAB); \ + git archive --format=tar HEAD > lab-$$SUF-handin.tar; \ + git diff $(UPSTREAM)/$(LAB) > /tmp/lab-$$SUF-diff.patch; \ + tar -rf lab-$$SUF-handin.tar /tmp/lab-$$SUF-diff.patch; \ + gzip -c lab-$$SUF-handin.tar > lab-$$SUF-handin.tar.gz; \ + rm lab-$$SUF-handin.tar; \ + rm /tmp/lab-$$SUF-diff.patch; \ + +myapi.key: + @echo Get an API key for yourself by visiting $(WEBSUB)/ + @read -p "Please enter your API key: " k; \ + if test `echo "$$k" |tr -d '\n' |wc -c` = 32 ; then \ + TF=`mktemp -t tmp.XXXXXX`; \ + if test "x$$TF" != "x" ; then \ + echo "$$k" |tr -d '\n' > $$TF; \ + mv -f $$TF $@; \ + else \ + echo mktemp failed; \ + false; \ + fi; \ + else \ + echo Bad API key: $$k; \ + echo An API key should be 32 characters long.; \ + false; \ + fi; + + +.PHONY: handin tarball tarball-pref clean grade handin-check diff --git a/conf/lab.mk b/conf/lab.mk new file mode 100644 index 0000000..629f978 --- /dev/null +++ b/conf/lab.mk @@ -0,0 +1 @@ +LAB=cow diff --git a/grade-lab-cow b/grade-lab-cow new file mode 100755 index 0000000..08b6990 --- /dev/null +++ b/grade-lab-cow @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +import re +from gradelib import * + +r = Runner(save("xv6.out")) + +@test(0, "running cowtest") +def test_cowtest(): + r.run_qemu(shell_script([ + 'cowtest' + ])) + +@test(30, "simple", parent=test_cowtest) +def test_simple(): + matches = re.findall("^simple: ok$", r.qemu.output, re.M) + assert_equal(len(matches), 2, "Number of appearances of 'simple: ok'") + +@test(30, "three", parent=test_cowtest) +def test_three(): + matches = re.findall("^three: ok$", r.qemu.output, re.M) + assert_equal(len(matches), 3, "Number of appearances of 'three: ok'") + +@test(20, "file", parent=test_cowtest) +def test_file(): + r.match('^file: ok$') + +@test(0, "usertests") +def test_usertests(): + r.run_qemu(shell_script([ + 'usertests -q' + ]), timeout=1000) + r.match('^ALL TESTS PASSED$') + +def usertest_check(testcase, nextcase, output): + if not re.search(r'\ntest {}: [\s\S]*OK\ntest {}'.format(testcase, nextcase), output): + raise AssertionError('Failed ' + testcase) + +@test(5, "usertests: copyin", parent=test_usertests) +def test_sbrkbugs(): + usertest_check("copyin", "copyout", r.qemu.output) + +@test(5, "usertests: copyout", parent=test_usertests) +def test_sbrkbugs(): + usertest_check("copyout", "copyinstr1", r.qemu.output) + +@test(19, "usertests: all tests", parent=test_usertests) +def test_usertests_all(): + r.match('^ALL TESTS PASSED$') + +@test(1, "time") +def test_time(): + check_time() + +run_tests() diff --git a/gradelib.py b/gradelib.py new file mode 100644 index 0000000..e8d7814 --- /dev/null +++ b/gradelib.py @@ -0,0 +1,611 @@ +from __future__ import print_function + +import sys, os, re, time, socket, select, subprocess, errno, shutil, random, string +from subprocess import check_call, Popen +from optparse import OptionParser + +__all__ = [] + +################################################################## +# Test structure +# + +__all__ += ["test", "end_part", "run_tests", "get_current_test"] + +TESTS = [] +TOTAL = POSSIBLE = 0 +PART_TOTAL = PART_POSSIBLE = 0 +CURRENT_TEST = None + +def test(points, title=None, parent=None): + """Decorator for declaring test functions. If title is None, the + title of the test will be derived from the function name by + stripping the leading "test_" and replacing underscores with + spaces.""" + + def register_test(fn, title=title): + if not title: + assert fn.__name__.startswith("test_") + title = fn.__name__[5:].replace("_", " ") + if parent: + title = " " + title + + def run_test(): + global TOTAL, POSSIBLE, CURRENT_TEST + + # Handle test dependencies + if run_test.complete: + return run_test.ok + run_test.complete = True + parent_failed = False + if parent: + parent_failed = not parent() + + # Run the test + fail = None + start = time.time() + CURRENT_TEST = run_test + sys.stdout.write("== Test %s == " % title) + if parent: + sys.stdout.write("\n") + sys.stdout.flush() + try: + if parent_failed: + raise AssertionError('Parent failed: %s' % parent.__name__) + fn() + except AssertionError as e: + fail = str(e) + + # Display and handle test result + POSSIBLE += points + if points: + print("%s: %s" % (title, \ + (color("red", "FAIL") if fail else color("green", "OK"))), end=' ') + if time.time() - start > 0.1: + print("(%.1fs)" % (time.time() - start), end=' ') + print() + if fail: + print(" %s" % fail.replace("\n", "\n ")) + else: + TOTAL += points + for callback in run_test.on_finish: + callback(fail) + CURRENT_TEST = None + + run_test.ok = not fail + return run_test.ok + + # Record test metadata on the test wrapper function + run_test.__name__ = fn.__name__ + run_test.title = title + run_test.complete = False + run_test.ok = False + run_test.on_finish = [] + TESTS.append(run_test) + return run_test + return register_test + +def end_part(name): + def show_part(): + global PART_TOTAL, PART_POSSIBLE + print("Part %s score: %d/%d" % \ + (name, TOTAL - PART_TOTAL, POSSIBLE - PART_POSSIBLE)) + print() + PART_TOTAL, PART_POSSIBLE = TOTAL, POSSIBLE + show_part.title = "" + TESTS.append(show_part) + +def run_tests(): + """Set up for testing and run the registered test functions.""" + + # Handle command line + global options + parser = OptionParser(usage="usage: %prog [-v] [filters...]") + parser.add_option("-v", "--verbose", action="store_true", + help="print commands") + parser.add_option("--color", choices=["never", "always", "auto"], + default="auto", help="never, always, or auto") + (options, args) = parser.parse_args() + + # Start with a full build to catch build errors + make() + + # Clean the file system if there is one + reset_fs() + + # Run tests + limit = list(map(str.lower, args)) + try: + for test in TESTS: + if not limit or any(l in test.title.lower() for l in limit): + test() + if not limit: + print("Score: %d/%d" % (TOTAL, POSSIBLE)) + except KeyboardInterrupt: + pass + if TOTAL < POSSIBLE: + sys.exit(1) + +def get_current_test(): + if not CURRENT_TEST: + raise RuntimeError("No test is running") + return CURRENT_TEST + +################################################################## +# Assertions +# + +__all__ += ["assert_equal", "assert_lines_match"] + +def assert_equal(got, expect, msg=""): + if got == expect: + return + if msg: + msg += "\n" + raise AssertionError("%sgot:\n %s\nexpected:\n %s" % + (msg, str(got).replace("\n", "\n "), + str(expect).replace("\n", "\n "))) + +def assert_lines_match(text, *regexps, **kw): + """Assert that all of regexps match some line in text. If a 'no' + keyword argument is given, it must be a list of regexps that must + *not* match any line in text.""" + + def assert_lines_match_kw(no=[]): + return no + no = assert_lines_match_kw(**kw) + + # Check text against regexps + lines = text.splitlines() + good = set() + bad = set() + for i, line in enumerate(lines): + if any(re.match(r, line) for r in regexps): + good.add(i) + regexps = [r for r in regexps if not re.match(r, line)] + if any(re.match(r, line) for r in no): + bad.add(i) + + if not regexps and not bad: + return + + # We failed; construct an informative failure message + show = set() + for lineno in good.union(bad): + for offset in range(-2, 3): + show.add(lineno + offset) + if regexps: + show.update(n for n in range(len(lines) - 5, len(lines))) + + msg = [] + last = -1 + for lineno in sorted(show): + if 0 <= lineno < len(lines): + if lineno != last + 1: + msg.append("...") + last = lineno + msg.append("%s %s" % (color("red", "BAD ") if lineno in bad else + color("green", "GOOD") if lineno in good + else " ", + lines[lineno])) + if last != len(lines) - 1: + msg.append("...") + if bad: + msg.append("unexpected lines in output") + for r in regexps: + msg.append(color("red", "MISSING") + " '%s'" % r) + raise AssertionError("\n".join(msg)) + +################################################################## +# Utilities +# + +__all__ += ["make", "maybe_unlink", "reset_fs", "color", "random_str", "check_time", "check_answers"] + +MAKE_TIMESTAMP = 0 + +def pre_make(): + """Delay prior to running make to ensure file mtimes change.""" + while int(time.time()) == MAKE_TIMESTAMP: + time.sleep(0.1) + +def post_make(): + """Record the time after make completes so that the next run of + make can be delayed if needed.""" + global MAKE_TIMESTAMP + MAKE_TIMESTAMP = int(time.time()) + +def make(*target): + pre_make() + if Popen(("make",) + target).wait(): + sys.exit(1) + post_make() + +def show_command(cmd): + from pipes import quote + print("\n$", " ".join(map(quote, cmd))) + +def maybe_unlink(*paths): + for path in paths: + try: + os.unlink(path) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + +COLORS = {"default": "\033[0m", "red": "\033[31m", "green": "\033[32m"} + +def color(name, text): + if options.color == "always" or (options.color == "auto" and os.isatty(1)): + return COLORS[name] + text + COLORS["default"] + return text + +def reset_fs(): + if os.path.exists("obj/fs/clean-fs.img"): + shutil.copyfile("obj/fs/clean-fs.img", "obj/fs/fs.img") + +def random_str(n=8): + letters = string.ascii_letters + string.digits + return ''.join(random.choice(letters) for _ in range(n)) + +def check_time(): + try: + print("") + with open('time.txt') as f: + d = f.read().strip() + if not re.match(r'^\d+$', d): + raise AssertionError('time.txt does not contain a single integer (number of hours spent on the lab)') + except IOError: + raise AssertionError('Cannot read time.txt') + +def check_answers(file, n=10): + try: + with open(file) as f: + d = f.read().strip() + if len(d) < n: + raise AssertionError('%s does not seem to contain enough text' % file) + except IOError: + raise AssertionError('Cannot read %s' % file) + + +################################################################## +# Controllers +# + +__all__ += ["QEMU", "GDBClient"] + +class QEMU(object): + _GDBPORT = None + + def __init__(self, *make_args): + # Check that QEMU is not currently running + try: + GDBClient(self.get_gdb_port(), timeout=0).close() + except socket.error: + pass + else: + print("""\ +GDB stub found on port %d. +QEMU appears to already be running. Please exit it if possible or use +'killall qemu' or 'killall qemu.real'.""" % self.get_gdb_port(), file=sys.stderr) + sys.exit(1) + + if options.verbose: + show_command(("make",) + make_args) + cmd = ("make", "-s", "--no-print-directory") + make_args + self.proc = Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.PIPE) + # Accumulated output as a string + self.output = "" + # Accumulated output as a bytearray + self.outbytes = bytearray() + self.on_output = [] + + @staticmethod + def get_gdb_port(): + if QEMU._GDBPORT is None: + p = Popen(["make", "-s", "--no-print-directory", "print-gdbport"], + stdout=subprocess.PIPE) + (out, _) = p.communicate() + if p.returncode: + raise RuntimeError( + "Failed to get gdbport: make exited with %d" % + p.returncode) + QEMU._GDBPORT = int(out) + return QEMU._GDBPORT + + def fileno(self): + if self.proc: + return self.proc.stdout.fileno() + + def handle_read(self): + buf = os.read(self.proc.stdout.fileno(), 4096) + self.outbytes.extend(buf) + self.output = self.outbytes.decode("utf-8", "replace") + for callback in self.on_output: + callback(buf) + if buf == b"": + self.wait() + return + + def write(self, buf): + if isinstance(buf, str): + buf = buf.encode('utf-8') + self.proc.stdin.write(buf) + self.proc.stdin.flush() + + def wait(self): + if self.proc: + self.proc.wait() + self.proc = None + + def kill(self): + if self.proc: + self.proc.terminate() + +class GDBClient(object): + def __init__(self, port, timeout=15): + start = time.time() + while True: + self.sock = socket.socket() + try: + self.sock.settimeout(1) + self.sock.connect(("localhost", port)) + break + except socket.error: + if time.time() >= start + timeout: + raise + self.__buf = "" + + def fileno(self): + if self.sock: + return self.sock.fileno() + + def handle_read(self): + try: + data = self.sock.recv(4096).decode("ascii", "replace") + except socket.error: + data = "" + if data == "": + self.sock.close() + self.sock = None + return + self.__buf += data + + while True: + m = re.search(r"\$([^#]*)#[0-9a-zA-Z]{2}", self.__buf) + if not m: + break + pkt = m.group(1) + self.__buf = self.__buf[m.end():] + + if pkt.startswith("T05"): + # Breakpoint + raise TerminateTest + + def __send(self, cmd): + packet = "$%s#%02x" % (cmd, sum(map(ord, cmd)) % 256) + self.sock.sendall(packet.encode("ascii")) + + def __send_break(self): + self.sock.sendall(b"\x03") + + def close(self): + if self.sock: + self.sock.close() + self.sock = None + + def cont(self): + self.__send("c") + + def breakpoint(self, addr): + self.__send("Z1,%x,1" % addr) + + +################################################################## +# QEMU test runner +# + +__all__ += ["TerminateTest", "Runner"] + +class TerminateTest(Exception): + pass + +class Runner(): + def __init__(self, *default_monitors): + self.__default_monitors = default_monitors + + def run_qemu(self, *monitors, **kw): + """Run a QEMU-based test. monitors should functions that will + be called with this Runner instance once QEMU and GDB are + started. Typically, they should register callbacks that throw + TerminateTest when stop events occur. The target_base + argument gives the make target to run. The make_args argument + should be a list of additional arguments to pass to make. The + timeout argument bounds how long to run before returning.""" + + def run_qemu_kw(target_base="qemu", make_args=[], timeout=30): + return target_base, make_args, timeout + target_base, make_args, timeout = run_qemu_kw(**kw) + + # Start QEMU + pre_make() + self.qemu = QEMU(target_base + "-gdb", *make_args) + self.gdb = None + + try: + # Wait for QEMU to start or make to fail. This will set + # self.gdb if QEMU starts. + self.qemu.on_output = [self.__monitor_start] + self.__react([self.qemu], timeout=90) + self.qemu.on_output = [] + if self.gdb is None: + print("Failed to connect to QEMU; output:") + print(self.qemu.output) + sys.exit(1) + post_make() + + # QEMU and GDB are up + self.reactors = [self.qemu, self.gdb] + + # Start monitoring + for m in self.__default_monitors + monitors: + m(self) + + # Run and react + self.gdb.cont() + self.__react(self.reactors, timeout) + finally: + # Shutdown QEMU + try: + if self.gdb is None: + sys.exit(1) + self.qemu.kill() + self.__react(self.reactors, 5) + self.gdb.close() + self.qemu.wait() + except: + print("""\ +Failed to shutdown QEMU. You might need to 'killall qemu' or +'killall qemu.real'. +""") + raise + + def __monitor_start(self, output): + if b"\n" in output: + try: + self.gdb = GDBClient(self.qemu.get_gdb_port(), timeout=2) + raise TerminateTest + except socket.error: + pass + if not len(output): + raise TerminateTest + + def __react(self, reactors, timeout): + deadline = time.time() + timeout + try: + while True: + timeleft = deadline - time.time() + if timeleft < 0: + sys.stdout.write("Timeout! ") + sys.stdout.flush() + return + + rset = [r for r in reactors if r.fileno() is not None] + if not rset: + return + + rset, _, _ = select.select(rset, [], [], timeleft) + for reactor in rset: + reactor.handle_read() + except TerminateTest: + pass + + def user_test(self, binary, *monitors, **kw): + """Run a user test using the specified binary. Monitors and + keyword arguments are as for run_qemu. This runs on a disk + snapshot unless the keyword argument 'snapshot' is False.""" + + maybe_unlink("obj/kern/init.o", "obj/kern/kernel") + if kw.pop("snapshot", True): + kw.setdefault("make_args", []).append("QEMUEXTRA+=-snapshot") + self.run_qemu(target_base="run-%s" % binary, *monitors, **kw) + + def match(self, *args, **kwargs): + """Shortcut to call assert_lines_match on the most recent QEMU + output.""" + + assert_lines_match(self.qemu.output, *args, **kwargs) + +################################################################## +# Monitors +# + +__all__ += ["save", "stop_breakpoint", "call_on_line", "stop_on_line", "shell_script"] + +def save(path): + """Return a monitor that writes QEMU's output to path. If the + test fails, copy the output to path.test-name.""" + + def setup_save(runner): + f.seek(0) + f.truncate() + runner.qemu.on_output.append(f.write) + get_current_test().on_finish.append(save_on_finish) + + def save_on_finish(fail): + f.flush() + save_path = path + "." + get_current_test().__name__[5:] + if fail: + shutil.copyfile(path, save_path) + print(" QEMU output saved to %s" % save_path) + elif os.path.exists(save_path): + os.unlink(save_path) + print(" (Old %s failure log removed)" % save_path) + + f = open(path, "wb") + return setup_save + +def stop_breakpoint(addr): + """Returns a monitor that stops when addr is reached. addr may be + a number or the name of a symbol.""" + + def setup_breakpoint(runner): + if isinstance(addr, str): + addrs = [int(sym[:16], 16) for sym in open("kernel/kernel.sym") + if sym[17:].strip() == addr] + assert len(addrs), "Symbol %s not found" % addr + runner.gdb.breakpoint(addrs[0]) + else: + runner.gdb.breakpoint(addr) + return setup_breakpoint + +def call_on_line(regexp, callback): + """Returns a monitor that calls 'callback' when QEMU prints a line + matching 'regexp'.""" + + def setup_call_on_line(runner): + buf = bytearray() + def handle_output(output): + buf.extend(output) + while b"\n" in buf: + line, buf[:] = buf.split(b"\n", 1) + line = line.decode("utf-8", "replace") + if re.match(regexp, line): + callback(line) + runner.qemu.on_output.append(handle_output) + return setup_call_on_line + +def stop_on_line(regexp): + """Returns a monitor that stops when QEMU prints a line matching + 'regexp'.""" + + def stop(line): + raise TerminateTest + return call_on_line(regexp, stop) + +def shell_script(script, terminate_match=None): + """Returns a monitor that plays the script, and stops when the script is + done executing.""" + + def setup_call_on_line(runner): + class context: + n = 0 + buf = bytearray() + def handle_output(output): + context.buf.extend(output) + if terminate_match is not None: + if re.match(terminate_match, context.buf.decode('utf-8', 'replace')): + raise TerminateTest + if b'$ ' in context.buf: + context.buf = bytearray() + if context.n < len(script): + runner.qemu.write(script[context.n]) + runner.qemu.write('\n') + context.n += 1 + else: + if terminate_match is None: + raise TerminateTest + runner.qemu.on_output.append(handle_output) + return setup_call_on_line diff --git a/user/cowtest.c b/user/cowtest.c new file mode 100644 index 0000000..29b918f --- /dev/null +++ b/user/cowtest.c @@ -0,0 +1,197 @@ +// +// tests for copy-on-write fork() assignment. +// + +#include "kernel/types.h" +#include "kernel/memlayout.h" +#include "user/user.h" + +// allocate more than half of physical memory, +// then fork. this will fail in the default +// kernel, which does not support copy-on-write. +void +simpletest() +{ + uint64 phys_size = PHYSTOP - KERNBASE; + int sz = (phys_size / 3) * 2; + + printf("simple: "); + + char *p = sbrk(sz); + if(p == (char*)0xffffffffffffffffL){ + printf("sbrk(%d) failed\n", sz); + exit(-1); + } + + for(char *q = p; q < p + sz; q += 4096){ + *(int*)q = getpid(); + } + + int pid = fork(); + if(pid < 0){ + printf("fork() failed\n"); + exit(-1); + } + + if(pid == 0) + exit(0); + + wait(0); + + if(sbrk(-sz) == (char*)0xffffffffffffffffL){ + printf("sbrk(-%d) failed\n", sz); + exit(-1); + } + + printf("ok\n"); +} + +// three processes all write COW memory. +// this causes more than half of physical memory +// to be allocated, so it also checks whether +// copied pages are freed. +void +threetest() +{ + uint64 phys_size = PHYSTOP - KERNBASE; + int sz = phys_size / 4; + int pid1, pid2; + + printf("three: "); + + char *p = sbrk(sz); + if(p == (char*)0xffffffffffffffffL){ + printf("sbrk(%d) failed\n", sz); + exit(-1); + } + + pid1 = fork(); + if(pid1 < 0){ + printf("fork failed\n"); + exit(-1); + } + if(pid1 == 0){ + pid2 = fork(); + if(pid2 < 0){ + printf("fork failed"); + exit(-1); + } + if(pid2 == 0){ + for(char *q = p; q < p + (sz/5)*4; q += 4096){ + *(int*)q = getpid(); + } + for(char *q = p; q < p + (sz/5)*4; q += 4096){ + if(*(int*)q != getpid()){ + printf("wrong content\n"); + exit(-1); + } + } + exit(-1); + } + for(char *q = p; q < p + (sz/2); q += 4096){ + *(int*)q = 9999; + } + exit(0); + } + + for(char *q = p; q < p + sz; q += 4096){ + *(int*)q = getpid(); + } + + wait(0); + + sleep(1); + + for(char *q = p; q < p + sz; q += 4096){ + if(*(int*)q != getpid()){ + printf("wrong content\n"); + exit(-1); + } + } + + if(sbrk(-sz) == (char*)0xffffffffffffffffL){ + printf("sbrk(-%d) failed\n", sz); + exit(-1); + } + + printf("ok\n"); +} + +char junk1[4096]; +int fds[2]; +char junk2[4096]; +char buf[4096]; +char junk3[4096]; + +// test whether copyout() simulates COW faults. +void +filetest() +{ + printf("file: "); + + buf[0] = 99; + + for(int i = 0; i < 4; i++){ + if(pipe(fds) != 0){ + printf("pipe() failed\n"); + exit(-1); + } + int pid = fork(); + if(pid < 0){ + printf("fork failed\n"); + exit(-1); + } + if(pid == 0){ + sleep(1); + if(read(fds[0], buf, sizeof(i)) != sizeof(i)){ + printf("error: read failed\n"); + exit(1); + } + sleep(1); + int j = *(int*)buf; + if(j != i){ + printf("error: read the wrong value\n"); + exit(1); + } + exit(0); + } + if(write(fds[1], &i, sizeof(i)) != sizeof(i)){ + printf("error: write failed\n"); + exit(-1); + } + } + + int xstatus = 0; + for(int i = 0; i < 4; i++) { + wait(&xstatus); + if(xstatus != 0) { + exit(1); + } + } + + if(buf[0] != 99){ + printf("error: child overwrote parent\n"); + exit(1); + } + + printf("ok\n"); +} + +int +main(int argc, char *argv[]) +{ + simpletest(); + + // check that the first simpletest() freed the physical memory. + simpletest(); + + threetest(); + threetest(); + threetest(); + + filetest(); + + printf("ALL COW TESTS PASSED\n"); + + exit(0); +} From 95754426e9c7d2f7647406a7e72b473bdb2edd7b Mon Sep 17 00:00:00 2001 From: the-die Date: Tue, 7 Feb 2023 00:38:58 +0800 Subject: [PATCH 2/2] cow: copy-on-write fork --- .gitignore | 3 +- compile_commands.json | 702 ++++++++++++++++++++++++++++++++++++++++++ kernel/defs.h | 5 + kernel/kalloc.c | 47 ++- kernel/proc.c | 2 +- kernel/riscv.h | 1 + kernel/trap.c | 42 ++- kernel/vm.c | 33 +- 8 files changed, 816 insertions(+), 19 deletions(-) create mode 100644 compile_commands.json diff --git a/.gitignore b/.gitignore index 3d25221..b28aa9a 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ submissions/ ph barrier /lab-*.json -.DS_Store \ No newline at end of file +.DS_Store +.cache diff --git a/compile_commands.json b/compile_commands.json new file mode 100644 index 0000000..17c9a9c --- /dev/null +++ b/compile_commands.json @@ -0,0 +1,702 @@ +[ + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-c", + "-o", + "kernel/entry.o", + "kernel/entry.S" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/entry.S", + "output": "/home/hong/projects/xv6-labs-2022/kernel/entry.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/kalloc.o", + "kernel/kalloc.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/kalloc.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/kalloc.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/string.o", + "kernel/string.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/string.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/string.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/main.o", + "kernel/main.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/main.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/main.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/vm.o", + "kernel/vm.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/vm.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/vm.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/proc.o", + "kernel/proc.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/proc.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/proc.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-c", + "-o", + "kernel/swtch.o", + "kernel/swtch.S" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/swtch.S", + "output": "/home/hong/projects/xv6-labs-2022/kernel/swtch.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-c", + "-o", + "kernel/trampoline.o", + "kernel/trampoline.S" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/trampoline.S", + "output": "/home/hong/projects/xv6-labs-2022/kernel/trampoline.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/trap.o", + "kernel/trap.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/trap.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/trap.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/syscall.o", + "kernel/syscall.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/syscall.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/syscall.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/sysproc.o", + "kernel/sysproc.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/sysproc.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/sysproc.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/bio.o", + "kernel/bio.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/bio.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/bio.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/fs.o", + "kernel/fs.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/fs.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/fs.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/log.o", + "kernel/log.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/log.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/log.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/sleeplock.o", + "kernel/sleeplock.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/sleeplock.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/sleeplock.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/file.o", + "kernel/file.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/file.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/file.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/pipe.o", + "kernel/pipe.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/pipe.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/pipe.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/exec.o", + "kernel/exec.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/exec.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/exec.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/sysfile.o", + "kernel/sysfile.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/sysfile.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/sysfile.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-c", + "-o", + "kernel/kernelvec.o", + "kernel/kernelvec.S" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/kernelvec.S", + "output": "/home/hong/projects/xv6-labs-2022/kernel/kernelvec.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/plic.o", + "kernel/plic.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/plic.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/plic.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/virtio_disk.o", + "kernel/virtio_disk.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/virtio_disk.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/virtio_disk.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/start.o", + "kernel/start.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/start.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/start.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/console.o", + "kernel/console.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/console.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/console.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/printf.o", + "kernel/printf.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/printf.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/printf.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/uart.o", + "kernel/uart.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/uart.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/uart.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-c", + "-o", + "kernel/spinlock.o", + "kernel/spinlock.c" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/kernel/spinlock.c", + "output": "/home/hong/projects/xv6-labs-2022/kernel/spinlock.o" + }, + { + "arguments": [ + "/usr/bin/riscv64-linux-gnu-gcc", + "-Wall", + "-Werror", + "-O", + "-fno-omit-frame-pointer", + "-ggdb", + "-gdwarf-2", + "-DSOL_COW", + "-DLAB_COW", + "-mcmodel=medany", + "-ffreestanding", + "-fno-common", + "-mno-relax", + "-I.", + "-fno-stack-protector", + "-fno-pie", + "-march=rv64g", + "-nostdinc", + "-I.", + "-Ikernel", + "-c", + "-o", + "user/initcode.o", + "user/initcode.S" + ], + "directory": "/home/hong/projects/xv6-labs-2022", + "file": "/home/hong/projects/xv6-labs-2022/user/initcode.S", + "output": "/home/hong/projects/xv6-labs-2022/user/initcode.o" + } +] diff --git a/kernel/defs.h b/kernel/defs.h index a3c962b..187a660 100644 --- a/kernel/defs.h +++ b/kernel/defs.h @@ -63,6 +63,10 @@ void ramdiskrw(struct buf*); void* kalloc(void); void kfree(void *); void kinit(void); +void kref_lock(); +void kref_unlock(); +uint64 inc_refcnt(uint64 pa); +uint64 dec_refcnt(uint64 pa); // log.c void initlog(int, struct superblock*); @@ -147,6 +151,7 @@ void trapinit(void); void trapinithart(void); extern struct spinlock tickslock; void usertrapret(void); +int cowtrap(uint64 va); // uart.c void uartinit(void); diff --git a/kernel/kalloc.c b/kernel/kalloc.c index 0699e7e..990d3f5 100644 --- a/kernel/kalloc.c +++ b/kernel/kalloc.c @@ -9,6 +9,8 @@ #include "riscv.h" #include "defs.h" +#define REF_CNT_IDX(pa) (((pa) - KERNBASE) / PGSIZE) + void freerange(void *pa_start, void *pa_end); extern char end[]; // first address after kernel. @@ -23,10 +25,42 @@ struct { struct run *freelist; } kmem; +static struct { + struct spinlock lock; + int cnt[REF_CNT_IDX(PHYSTOP)]; +} kref; + +void kref_lock() { + acquire(&kref.lock); +} + +void kref_unlock() { + release(&kref.lock); +} + +static void set_refcnt(uint64 pa, int cnt) { + kref.cnt[REF_CNT_IDX(pa)] = cnt; +} + +uint64 inc_refcnt(uint64 pa) { + return ++kref.cnt[REF_CNT_IDX(pa)]; +} + +uint64 dec_refcnt(uint64 pa) { + return --kref.cnt[REF_CNT_IDX(pa)]; +} + +static void init_ref_cnt() { + for (int i = 0; i < REF_CNT_IDX(PHYSTOP); ++i) + kref.cnt[i] = 1; +} + void kinit() { initlock(&kmem.lock, "kmem"); + initlock(&kref.lock, "kref"); + init_ref_cnt(); freerange(end, (void*)PHYSTOP); } @@ -51,6 +85,13 @@ kfree(void *pa) if(((uint64)pa % PGSIZE) != 0 || (char*)pa < end || (uint64)pa >= PHYSTOP) panic("kfree"); + acquire(&kref.lock); + if (dec_refcnt((uint64)pa) > 0) { + release(&kref.lock); + return; + } + release(&kref.lock); + // Fill with junk to catch dangling refs. memset(pa, 1, PGSIZE); @@ -72,11 +113,13 @@ kalloc(void) acquire(&kmem.lock); r = kmem.freelist; - if(r) + if (r) kmem.freelist = r->next; release(&kmem.lock); - if(r) + if (r) { + set_refcnt((uint64)r, 1); memset((char*)r, 5, PGSIZE); // fill with junk + } return (void*)r; } diff --git a/kernel/proc.c b/kernel/proc.c index 959b778..6d7006d 100644 --- a/kernel/proc.c +++ b/kernel/proc.c @@ -33,7 +33,7 @@ void proc_mapstacks(pagetable_t kpgtbl) { struct proc *p; - + for(p = proc; p < &proc[NPROC]; p++) { char *pa = kalloc(); if(pa == 0) diff --git a/kernel/riscv.h b/kernel/riscv.h index 20a01db..720df2d 100644 --- a/kernel/riscv.h +++ b/kernel/riscv.h @@ -343,6 +343,7 @@ typedef uint64 *pagetable_t; // 512 PTEs #define PTE_W (1L << 2) #define PTE_X (1L << 3) #define PTE_U (1L << 4) // user can access +#define PTE_COW (1L << 8) // shift a physical address to the right place for a PTE. #define PA2PTE(pa) ((((uint64)pa) >> 12) << 10) diff --git a/kernel/trap.c b/kernel/trap.c index 512c850..aa682b0 100644 --- a/kernel/trap.c +++ b/kernel/trap.c @@ -29,6 +29,32 @@ trapinithart(void) w_stvec((uint64)kernelvec); } +int cowtrap(uint64 va) { + if (va >= MAXVA) + return -1; + + struct proc* p = myproc(); + pte_t* pte = walk(p->pagetable, va, 0); + if (pte == 0) + return -1; + + uint flags = PTE_FLAGS(*pte); + if ((flags & PTE_COW) == 0) + return -1; + + void* newpa = kalloc(); + if (newpa == 0) + return -1; + + flags |= PTE_W; + flags &= ~PTE_COW; + void* oldpa = (void*)PTE2PA(*pte); + memmove(newpa, oldpa, PGSIZE); + kfree(oldpa); + *pte = PA2PTE(newpa) | flags; + return 0; +} + // // handle an interrupt, exception, or system call from user space. // called from trampoline.S @@ -46,11 +72,11 @@ usertrap(void) w_stvec((uint64)kernelvec); struct proc *p = myproc(); - + // save user program counter. p->trapframe->epc = r_sepc(); - - if(r_scause() == 8){ + + if (r_scause() == 8) { // Environment call from U-mode // system call if(killed(p)) @@ -65,7 +91,13 @@ usertrap(void) intr_on(); syscall(); - } else if((which_dev = devintr()) != 0){ + } else if(r_scause() == 15) { // Store/AMO Page Fault + if (killed(p)) + exit(-1); + + if (cowtrap(r_stval()) < 0) + setkilled(p); + } else if((which_dev = devintr()) != 0) { // ok } else { printf("usertrap(): unexpected scause %p pid=%d\n", r_scause(), p->pid); @@ -131,7 +163,7 @@ usertrapret(void) // interrupts and exceptions from kernel code go here via kernelvec, // on whatever the current kernel stack is. -void +void kerneltrap() { int which_dev = 0; diff --git a/kernel/vm.c b/kernel/vm.c index 9f69783..e8ca195 100644 --- a/kernel/vm.c +++ b/kernel/vm.c @@ -45,7 +45,7 @@ kvmmake(void) // allocate and map a kernel stack for each process. proc_mapstacks(kpgtbl); - + return kpgtbl; } @@ -147,7 +147,7 @@ mappages(pagetable_t pagetable, uint64 va, uint64 size, uint64 pa, int perm) if(size == 0) panic("mappages: size"); - + a = PGROUNDDOWN(va); last = PGROUNDDOWN(va + size - 1); for(;;){ @@ -308,27 +308,30 @@ uvmcopy(pagetable_t old, pagetable_t new, uint64 sz) pte_t *pte; uint64 pa, i; uint flags; - char *mem; for(i = 0; i < sz; i += PGSIZE){ if((pte = walk(old, i, 0)) == 0) panic("uvmcopy: pte should exist"); if((*pte & PTE_V) == 0) panic("uvmcopy: page not present"); + if (*pte & PTE_W) { + *pte &= ~PTE_W; + *pte |= PTE_COW; + } pa = PTE2PA(*pte); flags = PTE_FLAGS(*pte); - if((mem = kalloc()) == 0) - goto err; - memmove(mem, (char*)pa, PGSIZE); - if(mappages(new, i, PGSIZE, (uint64)mem, flags) != 0){ - kfree(mem); + if(mappages(new, i, PGSIZE, pa, flags) != 0){ goto err; } + kref_lock(); + inc_refcnt(pa); + kref_unlock(); } return 0; err: - uvmunmap(new, 0, i / PGSIZE, 1); + // There is no need to release these pages, because the parent process will still use them. + uvmunmap(new, 0, i / PGSIZE, 0); return -1; } @@ -338,7 +341,7 @@ void uvmclear(pagetable_t pagetable, uint64 va) { pte_t *pte; - + pte = walk(pagetable, va, 0); if(pte == 0) panic("uvmclear"); @@ -361,6 +364,16 @@ copyout(pagetable_t pagetable, uint64 dstva, char *src, uint64 len) n = PGSIZE - (dstva - va0); if(n > len) n = len; + + pte_t* pte = walk(pagetable, va0, 0); + if (pte == 0) + return -1; + if (*pte & PTE_COW) { + if (cowtrap(va0) < 0) + return -1; + pa0 = walkaddr(pagetable, va0); + } + memmove((void *)(pa0 + (dstva - va0)), src, n); len -= n;