Merge branch 'dev' into cosi

.github/workflows/README.md

@@ -1,25 +1,20 @@
 # GitHub Actions Tests
 
-**Important Note:** GitHub Actions does *not* recommend self-hosted runners for public repos b/c a PR could trigger arbitrary code execution on the hosted server. However, our tests run in a temporary container that is removed after the test completes, so this mitigates *much* of the risk.
-
-
-## Ideal Design:
-On PR/push:
-    If dockerfile unmodified and code (*.c, *.cpp, *.h, Makefile) changed: pull container from dockerhub, copy in source, rebuild with label `$UID` -> Kick off PR test suite
-    If dockerfile modified: rebuild container from scratch with label `$UID` -> Kick off PR test suite
-    If nothing modified: -> No tests
-
-Test suite: Given `$UID` for a docker container with code built
-    For each arch: qemu checks
-    For each arch: taint unit tests
-
-On push to `master`:
-    Rebuild container, push to dockerhub
-
-
 ## Current Design:
 On PR/push:
   Rebuild container from source. Once container is built, run all test suites in parallel
 
 On push to `master`
   Rebuild container (again), push to dockerhub
+
+
+## Testing locally
+
+We recommend using [act](https://github.com/nektos/act) to locally run CI tests.
+This is much easier than repeatedly pushing to a branch/PR if you have a CI failure.
+
+Running with the `-b` flag seems to be required, but then your directory will be owned by `root`:
+
+```
+act -b -j local_build_container; sudo chown -R $USER:$USER . .git
+```

.github/workflows/local_tests.yml

@@ -0,0 +1,18 @@
+# Run directly with act - standard repo-name guards are disabled and runs without self-hosted.
+# See .github/workflows/README.md for more details
+#
+# Note that this action never runs automatically
+
+name: Local
+
+jobs:
+  local_build_container:
+    runs-on: ubuntu:22.04
+    steps:
+    - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory
+
+    - name: Build docker container from project root
+      run: echo $GITHUB_WORKSPACE; cd $GITHUB_WORKSPACE && DOCKER_BUILDKIT=1 docker build --progress=plain --target developer -t panda_local_${{ github.sha }} .
+
+    - name: Minimal test of built container # Just test to see if one of our binaries is built
+      run: docker run --rm "panda_local_${{ github.sha }}" /bin/bash -c 'exit $(/panda/build/arm-softmmu/panda-system-arm -help | grep -q "usage. panda-system-arm")'

.github/workflows/parallel_tests.yml

@@ -17,9 +17,15 @@ on:
 jobs:
 
   test_installer: # test install_ubuntu.sh
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-20.04 # Note 22.04 would work, but it requires docker > 20.10.7 which is not on our CI box (yet)
     steps:
     - uses: actions/checkout@v2 # Clones to $GITHUB_WORKSPACE. NOTE: this requires git > 2.18 (not on ubuntu 18.04 by default) to get .git directory
+    - name: Lint PyPANDA with flake8
+      run: |
+       python -m pip install --upgrade pip
+       python -m pip install flake8
+       python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --select=E9,F63,F7,F82 --show-source --statistics
+       # python -m flake8 $GITHUB_WORKSPACE/panda/python/core/pandare/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Run install_ubuntu.sh
       run: cd $GITHUB_WORKSPACE && ./panda/scripts/install_ubuntu.sh
 

.gitignore

@@ -114,6 +114,7 @@ venv
 /docs/version.texi
 /panda/plugins/*/docs/
 /panda/plugins/*/target/
+**/target
 *.tps
 .stgit-*
 .git-submodule-status
@@ -152,6 +153,7 @@ pwc_log
 
 # Recordings and pandalogs
 *-rr-*
+*.rr2
 *.plog
 
 # Kerenels/qcows

Dockerfile

@@ -1,4 +1,6 @@
 ARG BASE_IMAGE="ubuntu:20.04"
+# Note PANDA supports ubuntu:22.04, but docker versions <= 20.10.7 can't run 22.04 containers
+
 ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu,aarch64-softmmu"
 
 ### BASE IMAGE
@@ -7,7 +9,7 @@ ARG BASE_IMAGE
 
 # Copy dependencies lists into container. Note this
 #  will rarely change so caching should still work well
-COPY ./panda/dependencies/${BASE_IMAGE}*.txt /tmp/
+COPY ./panda/dependencies/${BASE_IMAGE}*.txt /tmp/ 
 
 # Base image just needs runtime dependencies
 RUN [ -e /tmp/${BASE_IMAGE}_base.txt ] && \
@@ -26,19 +28,27 @@ RUN [ -e /tmp/${BASE_IMAGE}_build.txt ] && \
     apt-get clean && \
     python3 -m pip install --upgrade --no-cache-dir pip && \
     python3 -m pip install --upgrade --no-cache-dir "cffi>1.14.3" && \
+    python3 -m pip install --upgrade --no-cache-dir "capstone" && \
     curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal
 
 # Then install capstone from source
 RUN cd /tmp && \
-    curl -o cap.tgz -L https://github.com/aquynh/capstone/archive/4.0.2.tar.gz && \
-    tar xvf cap.tgz && cd capstone-4.0.2/ && ./make.sh && make install && cd /tmp && \
-    rm -rf /tmp/capstone-4.0.2
+    git clone https://github.com/capstone-engine/capstone/ -b 4.0.2 && \
+    cd capstone/ && ./make.sh && make install && cd /tmp && \
+    rm -rf /tmp/capstone && ldconfig
 
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 # Sanity check to ensure cargo is installed
 RUN cargo --help
 
+# install libosi
+RUN cd /tmp && \
+    git clone https://github.com/panda-re/libosi && \
+    mkdir /tmp/libosi/build && cd /tmp/libosi/build && \
+    cmake -GNinja .. && ninja && ninja package && dpkg -i libosi*.deb && \
+    cd /tmp && rm -rf libosi/ && ldconfig 
+
 # Build and install panda
 # Copy repo root directory to /panda, note we explicitly copy in .git directory
 # Note .dockerignore file keeps us from copying things we don't need
@@ -70,13 +80,19 @@ RUN  make -C /panda/build install
 # Install pypanda
 RUN cd /panda/panda/python/core && \
     python3 setup.py install
+RUN python3 -m pip install --ignore-install pycparser && python3 -m pip install --force-reinstall --no-binary :all: cffi
+
+# BUG: PANDA sometimes fails to generate all the necessary files for PyPANDA. This is a temporary fix to detect and fail when this occurs
+RUN ls -alt $(pip show pandare | grep Location: | awk '{print $2}')/pandare/autogen/
+RUN bash -c "ls $(pip show pandare | grep Location: | awk '{print $2}')/pandare/autogen/panda_{aarch64_64,arm_32,mips64_64,mips_32,mipsel_32,ppc_32,ppc_64,x86_64_64,i386_32}.py"
 
 ### Copy files for panda+pypanda from installer  - Stage 5
 FROM base as panda
 
-# Copy panda + libcapstone.so*
+# Copy panda + libcapstone.so* + libosi libraries
 COPY --from=installer /usr/local /usr/local
-COPY --from=installer /lib/x86_64-linux-gnu/libcapstone* /lib/x86_64-linux-gnu/
+COPY --from=installer /usr/lib/libcapstone* /usr/lib/
+COPY --from=installer /lib/libosi.so /lib/libiohal.so /lib/liboffset.so /lib/
 
 # Workaround issue #901 - ensure LD_LIBRARY_PATH contains the panda plugins directories
 #ARG TARGET_LIST="x86_64-softmmu,i386-softmmu,arm-softmmu,ppc-softmmu,mips-softmmu,mipsel-softmmu"

MAINTAINERS

@@ -1123,6 +1123,7 @@ e1000e
 M: Dmitry Fleytman <[email protected]>
 S: Maintained
 F: hw/net/e1000e*
+F: tests/qtest/fuzz-e1000e-test.c
 
 Generic Loader
 M: Alistair Francis <[email protected]>

Makefile.target

@@ -276,6 +276,7 @@ endif
 		echo $$f; \
 		$(INSTALL_PROG) $$f "$(DESTDIR)$(qemu_datadir)/scripts"; \
 	done
+	$(INSTALL_DATA) $(SRC_PATH)/docs/qemupciserial.inf "$(DESTDIR)$(qemu_datadir)/qemupciserial.inf"
 # N.B. the new rpath must not be longer than the old rpath or chrpath will fail
 	for p in $(PANDA_PLUGINS) $(EXTRA_PANDA_PLUGINS); do \
 		echo $$p; \

block/sheepdog.c

@@ -1472,7 +1472,7 @@ static int reload_inode(BDRVSheepdogState *s, uint32_t snapid, const char *tag)
         return -EIO;
     }
 
-    inode = g_malloc(SD_INODE_HEADER_SIZE);
+    inode = g_malloc(SD_INODE_SIZE); // Was INODE_HEADER_SIZE but GCC (incorrectly?) thought there would be an OOB read
 
     ret = find_vdi_name(s, s->name, snapid, tag, &vid, false, &local_err);
     if (ret) {

block/vpc.c

@@ -213,7 +213,7 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
     QemuOpts *opts = NULL;
     Error *local_err = NULL;
     bool use_chs;
-    uint8_t buf[HEADER_SIZE];
+    uint8_t dyndisk_header_buf[1024];
     uint32_t checksum;
     uint64_t computed_size;
     uint64_t pagetable_size;
@@ -332,14 +332,14 @@ static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
     }
 
     if (disk_type == VHD_DYNAMIC) {
-        ret = bdrv_pread(bs->file, be64_to_cpu(footer->data_offset), buf,
-                         HEADER_SIZE);
+        ret = bdrv_pread(bs->file, be64_to_cpu(footer->data_offset),
+                         dyndisk_header_buf, 1024);
         if (ret < 0) {
             error_setg(errp, "Error reading dynamic VHD header");
             goto fail;
         }
 
-        dyndisk_header = (VHDDynDiskHeader *) buf;
+        dyndisk_header = (VHDDynDiskHeader *)dyndisk_header_buf;
 
         if (strncmp(dyndisk_header->magic, "cxsparse", 8)) {
             error_setg(errp, "Invalid header magic");

block/vvfat.c

@@ -413,28 +413,19 @@ static void init_mbr(BDRVVVFATState *s, int cyls, int heads, int secs)
 
 /* direntry functions */
 
-/* dest is assumed to hold 258 bytes, and pads with 0xffff up to next multiple of 26 */
-static inline int short2long_name(char* dest,const char* src)
+static direntry_t *create_long_filename(BDRVVVFATState *s, const char *filename)
 {
-    int i;
-    int len;
-    for(i=0;i<129 && src[i];i++) {
-        dest[2*i]=src[i];
-	dest[2*i+1]=0;
+    int number_of_entries, i;
+    glong length;
+    direntry_t *entry;
+
+    gunichar2 *longname = g_utf8_to_utf16(filename, -1, NULL, &length, NULL);
+    if (!longname) {
+        fprintf(stderr, "vvfat: invalid UTF-8 name: %s\n", filename);
+        return NULL;
     }
-    len=2*i;
-    dest[2*i]=dest[2*i+1]=0;
-    for(i=2*i+2;(i%26);i++)
-	dest[i]=0xff;
-    return len;
-}
 
-static inline direntry_t* create_long_filename(BDRVVVFATState* s,const char* filename)
-{
-    char buffer[258];
-    int length=short2long_name(buffer,filename),
-        number_of_entries=(length+25)/26,i;
-    direntry_t* entry;
+    number_of_entries = (length * 2 + 25) / 26;
 
     for(i=0;i<number_of_entries;i++) {
 	entry=array_get_next(&(s->directory));
@@ -444,13 +435,20 @@ static inline direntry_t* create_long_filename(BDRVVVFATState* s,const char* fil
 	entry->name[0]=(number_of_entries-i)|(i==0?0x40:0);
     }
     for(i=0;i<26*number_of_entries;i++) {
-	int offset=(i%26);
-	if(offset<10) offset=1+offset;
-	else if(offset<22) offset=14+offset-10;
-	else offset=28+offset-22;
-	entry=array_get(&(s->directory),s->directory.next-1-(i/26));
-	entry->name[offset]=buffer[i];
+        int offset=(i%26);
+        if(offset<10) offset=1+offset;
+        else if(offset<22) offset=14+offset-10;
+        else offset=28+offset-22;
+        entry=array_get(&(s->directory),s->directory.next-1-(i/26));
+        if (i >= 2 * length + 2) {
+            entry->name[offset] = 0xff;
+        } else if (i % 2 == 0) {
+            entry->name[offset] = longname[i / 2] & 0xff;
+        } else {
+            entry->name[offset] = longname[i / 2] >> 8;
+        }
     }
+    g_free(longname);
     return array_get(&(s->directory),s->directory.next-number_of_entries);
 }
 

build.sh

@@ -35,7 +35,11 @@ fi
 if [ $# -ge 1 ]; then if [ "$1" = "small" ]; then
         TARGET_LIST="i386-softmmu"
     else
-        TARGET_LIST="$1"
+        if [[ "$1" == *"-softmmu" ]]; then
+            TARGET_LIST="$1"
+        else
+            TARGET_LIST="$1-softmmu"
+        fi
     fi
     echo "Building PANDA for target(s): $TARGET_LIST"
     shift
@@ -65,8 +69,9 @@ gcc --version | awk '/gcc/ && ($3+0)<7.1{print "Fatal error: GCC too old"; exit
 g++ --version | awk '/g\+\+/ && ($3+0)<7.1{print "Fatal error: G++ too old"; exit 1}' || exit 1
 
 # Untested GCC - it's probably going to have some warnings - Just disable Werror and hope it works
-gcc --version | awk '/gcc/   && ($3+0)>9.3{print "WARNING: Your GCC is too new: disabling -Werror and hoping this builds"; exit 1}' || COMPILER_CONFIG="--extra-cflags=-Wno-error"
-g++ --version | awk '/g\+\+/ && ($3+0)>9.3{print "WARNING: Your G++ is too new: disabling -Werror and hoping this builds"; exit 1}' ||  COMPILER_CONFIG="--extra-cxxflags=-Wno-error"
+COMPILER_CONFIG=""
+gcc --version | awk '/gcc/   && ($3+0)>11.2{print "WARNING: Your GCC is too new: disabling -Werror and hoping this builds"; exit 1}' || COMPILER_CONFIG+="--extra-cflags=-Wno-error"
+g++ --version | awk '/g\+\+/ && ($3+0)>11.2{print "WARNING: Your G++ is too new: disabling -Werror and hoping this builds"; exit 1}' ||  COMPILER_CONFIG+=" --extra-cxxflags=-Wno-error"
 
 ### Check for protobuf v2.
 if ! pkg-config --exists protobuf; then

configure

@@ -304,7 +304,7 @@ vde=""
 vnc_sasl=""
 vnc_jpeg=""
 vnc_png=""
-xen=""
+xen="no" # Disabled for PANDA
 xen_ctrl_version=""
 xen_pv_domain_build="no"
 xen_pci_passthrough=""
@@ -3254,6 +3254,31 @@ if ! compile_prog "$glib_cflags -Werror" "$glib_libs" ; then
     fi
 fi
 
+##########################################
+# libarchive support probe
+if $pkg_config --modversion libarchive > /dev/null 2>&1 ; then
+    archive_libs=`$pkg_config --libs libarchive`
+    LIBS="$archive_libs $LIBS"
+else
+    echo "libarchive is required to compile PANDA"
+    exit 1
+fi
+
+##########################################
+# openssl support probe
+if $pkg_config --modversion libssl > /dev/null 2>&1; then
+    if $pkg_config --modversion libcrypto > /dev/null 2>&1; then
+        ssl_libs=`$pkg_config --libs libssl libcrypto`
+        LIBS="$ssl_libs $LIBS"
+    else
+        echo "libcrypto is required to compile PANDA"
+        exit1
+    fi
+else
+    echo "libssl (openssl) is required to compile PANDA"
+    exit 1
+fi
+
 ##########################################
 # SHA command probe for modules
 if test "$modules" = yes; then

cpu-exec.c

@@ -523,9 +523,14 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
             cpu->exception_index = -1;
             return true;
         } else {
+            int32_t exception = cpu->exception_index;
 
             cpu->exception_index = panda_callbacks_before_handle_exception(cpu, cpu->exception_index);
 
+            if (exception != cpu->exception_index){
+                return cpu_handle_exception(cpu, ret);
+            }
+
 #if defined(CONFIG_USER_ONLY)
             /* if user mode only, we simulate a fake exception
                which will be handled outside the cpu execution

cputlb.c

@@ -36,6 +36,9 @@
 #include "panda/rr/rr_log.h"
 #include "panda/callbacks/cb-support.h"
 
+// less frustrating than importing common.h
+extern target_ulong panda_current_pc(CPUState *cpu);
+
 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
 /* #define DEBUG_TLB */
 /* #define DEBUG_TLB_LOG */

disas/arm-a64.cc

@@ -17,10 +17,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-extern "C" {
 #include "qemu/osdep.h"
 #include "disas/bfd.h"
-}
 
 #include "vixl/a64/disasm-a64.h"