Minix 3のSMP対応

更新：qemuの起動方法とカーネルソースを弄ったら`/usr/tests/minix-posix`のテストが１〜４０・４４〜６９までパスするようになり、ネットワークも使えるようになったので、その手順を記事末尾に新たに追加する(2022.03)

[Minix 3](https://ja.wikipedia.org/wiki/MINIX#MINIX_3) は SMP (symmetric multiprocessing) に対応していることになっているが、[実際にはそもそもコンパイルが通らない](https://github.com/Stichting-MINIX-Research-Foundation/minix/pull/292)ので、とりあえず「login:」プロンプトが出るようにするまでの手順です。

# JBrouronのSMP修正をそのまま使う手順
1. http://download.minix3.org/iso/snapshot/ からバージョン3.4rc6をダウンロードする
1. https://wiki.minix3.org/doku.php?id=usersguide:runningonqemu の手順に従ってQEMU上にインストールする
1. QEMUのハードディスクから再起動したら、 `pkgin update` と `pkgin_sets` でカーネルコンパイルに必要なアプリをインストールする
1. `cd /usr; git clone -b smpfix https://github.com/JBouron/minix.git src` でSMP修正版のソースを /usr/src に展開する
1. `cd /usr/src; NOCLANGERROR=yes NOGCCERROR=yes make -k build` でヘッダーファイルやツールを再構築する
1. `cd /usr/src/releasetools; CONFIB_SMP=y CONFIG_MAX_CPUS=4 make clean hdboot` でSMP版minixをコンパイルしインストールする。`CONFIG_MAX_CPUS`はCPUの最大数である。
1. このディスクイメージは `qemu-system-i386 -machine pc -cpu Conroe-v1 -accel kvm -smp sockets=4 -m 1920M -net none -drive if=virtio,file=minix-current.qcow2 -serial vc -serial vc -serial vc -serial vc  -boot c` などのQEMUコマンドで起動できる。ネットワーク無しにしないとログインプロンプトまでたどり着かなかった。　`/bin/update_bootcfg` を用いてMinixへの起動オプションに`acpi=yes no_apic=0` を含める必要がある

上記の手順で作ったQEMUイメージファイルが http://p1437140-ipoe.ipoe.ocn.ne.jp:63873/minixsmp/ に置いてある

# NetBSD-currentの最新版を導入した上で独自のSMP修正を使う手順

* virtioはSMPと相性が悪いので使わない
* AHCIのドライバは出来が悪いので使わない。そのため、qemuの`q35`マシンは使えない

## QEMUディスクイメージ作成
`qemu-img create -f qcow2 -o preallocation=off,lazy_refcounts=on,extended_l2=on minix-current.qcow2 255G` でイメージファイルを作る。QEMUのバージョンが古いと`extended_l2`が無いので外す。

## QEMUの起動方法
`qemu-system-i386 -machine pc -cpu host -accel kvm -smp cores=8 -m 1920M -net user -net nic,model=rtl8139 -drive discard=unmap,detect-zeroes=unmap,if=ide,file=minix-current.qcow2 -serial vc -serial vc -serial vc -serial vc  -boot d  -cdrom minix_R3.4.0rc6-d5e4fc0.iso`
ハードディスクイメージにインストールできたら `-boot d` 以降を消して再起動する。

## NetBSD-current を取り込んだソースの取得
`pkgin update` と `pkgin_sets` で必要なツールをインストールした後に `cd /usr; git clone -b make-netbsdcurrent git://github.com/marceloalencar/minix src` でソースをインストールする。これに対応するプルリクエストは https://github.com/Stichting-MINIX-Research-Foundation/minix/pull/319

## パッチを当てる
この記事末尾のパッチを当てる。同じものは http://p1437140-ipoe.ipoe.ocn.ne.jp:63873/minixsmp/ にもある。

## カーネルコンパイルの前の全リビルド
カーネルをコンパイルする前に `cd /usr/src; MAKEVERBOSE=2 NOGCCERROR=yes NOCLANGERROR=yes make build` でいろいろなものを再コンパイルする。

## カーネルコンパイル
`cd /usr/src/releasetools &&
MAKEVERBOSE=2 NOGCCERROR=yes NOCLANGERROR=yes CONFIG_SMP=y CONFIG_MAX_CPUS=8 make -k clean hdboot 2>&1   &&
/bin/update_bootcfg` でカーネルを再コンパイルする。SMP有効で起動するためにはカーネル起動オプションに `acpi=yes no_apic=0 no_smp=0` が必要である。ここまでやったディスクイメージが http://p1437140-ipoe.ipoe.ocn.ne.jp:63873/minixsmp/ にある

# JBouronのSMP修正を元にしたSMP修正パッチ

```diff
Only in src/minix/drivers/storage/ramdisk: etc
diff -ur src-netbsd-current202104/minix/include/minix/bitmap.h src/minix/include/minix/bitmap.h
--- src-netbsd-current202104/minix/include/minix/bitmap.h	Fri Mar 11 03:54:35 2022
+++ src/minix/include/minix/bitmap.h	Thu Mar 10 01:38:47 2022
@@ -17,8 +17,14 @@
 #define SET_BIT(map,bit) ( MAP_CHUNK(map,bit) |= (1 << CHUNK_OFFSET(bit) ))
 #define UNSET_BIT(map,bit) ( MAP_CHUNK(map,bit) &= ~(1 << CHUNK_OFFSET(bit) ))
 
-#if defined(CONFIG_SMP) && defined(__GNUC__)
+#if defined(CONFIG_SMP)
 #ifndef __ASSEMBLY__
+
+#define __minix 3
+#define _NETBSD_SOURCE 1
+#include <sys/types.h>
+#include <limits.h>
+
 static inline void bits_fill(bitchunk_t * chunks, unsigned bits)
 {
 	unsigned c, cnt;
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/Makefile.inc src/minix/kernel/arch/i386/Makefile.inc
--- src-netbsd-current202104/minix/kernel/arch/i386/Makefile.inc	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/Makefile.inc	Thu Mar 10 01:44:52 2022
@@ -84,6 +84,8 @@
 
 .ifdef CONFIG_SMP
 SRCS += arch_smp.c trampoline.S
+trampoline.o: trampoline.S
+	${CC} -E ${AFLAGS} ${AFLAGS.${<:T}} ${CPPFLAGS} ${.IMPSRC} | ${AS} -o ${.TARGET}
 .endif
 
 .if ${USE_ACPI} != "no"
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/apic.c src/minix/kernel/arch/i386/apic.c
--- src-netbsd-current202104/minix/kernel/arch/i386/apic.c	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/apic.c	Thu Mar 10 07:09:48 2022
@@ -164,7 +164,7 @@
  * arch specific cpulocals. As this variable is write-once-read-only it is ok to
  * have at as an array until we resolve the cpulocals properly
  */
-static u32_t lapic_bus_freq[CONFIG_MAX_CPUS];
+static u64_t lapic_bus_freq[CONFIG_MAX_CPUS];
 /* the probe period will be roughly 100ms */
 #define PROBE_TICKS	(system_hz / 10)
 
@@ -442,7 +442,8 @@
 
 static void apic_calibrate_clocks(unsigned cpu)
 {
-	u32_t lvtt, val, lapic_delta;
+	u32_t lvtt, val;
+	u64_t lapic_delta;
 	u64_t tsc_delta;
 	u64_t cpu_freq;
 
@@ -515,7 +516,7 @@
 	tsc_delta = tsc1 - tsc0;
 
 	lapic_bus_freq[cpuid] = system_hz * lapic_delta / (PROBE_TICKS - 1);
-	BOOT_VERBOSE(printf("APIC bus freq %u MHz\n",
+	BOOT_VERBOSE(printf("APIC bus freq %llu MHz\n",
 				lapic_bus_freq[cpuid] / 1000000));
 	cpu_freq = (tsc_delta / (PROBE_TICKS - 1)) * make64(system_hz, 0);
 	cpu_set_freq(cpuid, cpu_freq);
@@ -848,7 +849,7 @@
 #endif
 
 #ifdef APIC_DEBUG
-static void lapic_set_dummy_handlers(void)
+void lapic_set_dummy_handlers(void)
 {
 	char * handler;
 	int vect = 32; /* skip the reserved vectors */
@@ -884,7 +885,9 @@
 #ifdef APIC_DEBUG
 	if (is_bsp)
 		printf("APIC debugging is enabled\n");
+#ifdef CONFIG_SMP
 	lapic_set_dummy_handlers();
+#endif
 #endif
 
 	/* Build descriptors for interrupt gates in IDT. */
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/arch_clock.c src/minix/kernel/arch/i386/arch_clock.c
--- src-netbsd-current202104/minix/kernel/arch/i386/arch_clock.c	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/arch_clock.c	Thu Mar 10 01:40:02 2022
@@ -436,5 +436,5 @@
 
 	/* TODO: make this inter-CPU safe! */
 	for (i = 0; i < CPUSTATES; i++)
-		ticks[i] = tsc_per_state[cpu][i] / tsc_per_tick[cpu];
+		ticks[i] = tsc_per_tick[cpu] ? tsc_per_state[cpu][i] / tsc_per_tick[cpu] : 0;
 }
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/arch_smp.c src/minix/kernel/arch/i386/arch_smp.c
--- src-netbsd-current202104/minix/kernel/arch/i386/arch_smp.c	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/arch_smp.c	Thu Mar 10 07:05:29 2022
@@ -13,6 +13,7 @@
 #include <machine/cmos.h>
 #include <machine/bios.h>
 
+#include "kernel/kernel.h"
 #include "kernel/spinlock.h"
 #include "kernel/smp.h"
 #include "apic.h"
@@ -218,6 +219,9 @@
 	/* inform the world of our presence. */
 	ap_cpu_ready = cpu;
 
+	/* Set up sysenter/syscall. */
+	setup_sysenter_syscall();
+
 	/*
 	 * Finish processor initialisation.  CPUs must be excluded from running.
 	 * lapic timer calibration locks and unlocks the BKL because of the
@@ -298,6 +302,7 @@
 	ioapic_enabled = 0;
 
 	tss_init_all();
+	setup_sysenter_syscall();
 
 	/* 
 	 * we still run on the boot stack and we cannot use cpuid as its value
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/include/arch_proto.h src/minix/kernel/arch/i386/include/arch_proto.h
--- src-netbsd-current202104/minix/kernel/arch/i386/include/arch_proto.h	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/include/arch_proto.h	Thu Mar 10 06:38:12 2022
@@ -249,6 +249,11 @@
  */
 int tss_init(unsigned cpu, void * kernel_stack);
 
+/*
+ * Set up MSRs for sysenter or syscall for the current cpu.
+ * Assumes that tss_init has been called before for that cpu ! */
+void setup_sysenter_syscall(void);
+
 void int_gate_idt(unsigned vec_nr, vir_bytes offset, unsigned dpl_type);
 
 void __copy_msg_from_user_end(void);
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/include/arch_smp.h src/minix/kernel/arch/i386/include/arch_smp.h
--- src-netbsd-current202104/minix/kernel/arch/i386/include/arch_smp.h	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/include/arch_smp.h	Thu Mar 10 06:38:12 2022
@@ -16,6 +16,7 @@
  */
 #define smp_single_cpu_fallback() do {		\
 	  tss_init(0, get_k_stack_top(0));	\
+	  setup_sysenter_syscall();		\
 	  bsp_cpu_id = 0;			\
 	  ncpus = 1;				\
 	  bsp_finish_booting();			\
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/memory.c src/minix/kernel/arch/i386/memory.c
--- src-netbsd-current202104/minix/kernel/arch/i386/memory.c	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/memory.c	Thu Mar 10 03:35:00 2022
@@ -965,7 +965,7 @@
 #if CONFIG_SMP
 	barrier();
 
-	wait_for_APs_to_finish_booting();
+	//wait_for_APs_to_finish_booting();
 #endif
 #endif
 
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/mpx.S src/minix/kernel/arch/i386/mpx.S
--- src-netbsd-current202104/minix/kernel/arch/i386/mpx.S	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/mpx.S	Thu Mar 10 06:35:23 2022
@@ -405,6 +405,7 @@
 
         /* restore PSW */
         movl    PSWREG(%ebp), %edi      /* load desired PSW to EDI */
+        andl    $0xfffffdff, %edi	/* Clear IF to avoid receiving int. */
         push    %edi
         popf
 
@@ -421,6 +422,7 @@
 
         /* restore PSW (before we switch to user stack!) */
         movl    PSWREG(%ebp), %edi      /* load desired PSW to EDI */
+        andl    $0xfffffdff, %edi	/* Clear IF to avoid receiving int. */
         push    %edi
         popf
 
@@ -429,6 +431,7 @@
 	mov	AXREG(%ebp), %eax	/* trap return value */
 	mov	BXREG(%ebp), %ebx	/* secondary return value */
 
+	sti
 	sysret				/* jump to EIP in user */
 
 ENTRY(restore_user_context_int)
diff -ur src-netbsd-current202104/minix/kernel/arch/i386/protect.c src/minix/kernel/arch/i386/protect.c
--- src-netbsd-current202104/minix/kernel/arch/i386/protect.c	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/arch/i386/protect.c	Thu Mar 10 06:35:23 2022
@@ -151,35 +151,10 @@
 	{ NULL, 0, 0}
 };
 
-int tss_init(unsigned cpu, void * kernel_stack)
+void setup_sysenter_syscall(void)
 {
-	struct tss_s * t = &tss[cpu];
-	int index = TSS_INDEX(cpu);
-	struct segdesc_s *tssgdt;
-
-	tssgdt = &gdt[index];
-  
-	init_param_dataseg(tssgdt, (phys_bytes) t,
-			sizeof(struct tss_s), INTR_PRIVILEGE);
-	tssgdt->access = PRESENT | (INTR_PRIVILEGE << DPL_SHIFT) | TSS_TYPE;
-
-	/* Build TSS. */
-	memset(t, 0, sizeof(*t));
-	t->ds = t->es = t->fs = t->gs = t->ss0 = KERN_DS_SELECTOR;
-	t->cs = KERN_CS_SELECTOR;
-	t->iobase = sizeof(struct tss_s);	/* empty i/o permissions map */
-
-	/* 
-	 * make space for process pointer and cpu id and point to the first
-	 * usable word
-	 */
-	k_percpu_stacks[cpu] = t->sp0 = ((unsigned) kernel_stack) - X86_STACK_TOP_RESERVED;
-	/* 
-	 * set the cpu id at the top of the stack so we know on which cpu is
-	 * this stack in use when we trap to kernel
-	 */
-	*((reg_t *)(t->sp0 + 1 * sizeof(reg_t))) = cpu;
-
+	const int cpu = cpuid;
+	struct tss_s const *const t = &tss[cpu];
 	/* Set up Intel SYSENTER support if available. */
 	if(minix_feature_flags & MKF_I386_INTEL_SYSENTER) {
 	  ia32_msr_write(INTEL_MSR_SYSENTER_CS, 0, KERN_CS_SELECTOR);
@@ -211,7 +186,36 @@
 		set_star_cpu(7);
 		assert(CONFIG_MAX_CPUS <= 8);
   	}
+}
 
+int tss_init(unsigned cpu, void * kernel_stack)
+{
+	struct tss_s * t = &tss[cpu];
+	int index = TSS_INDEX(cpu);
+	struct segdesc_s *tssgdt;
+
+	tssgdt = &gdt[index];
+  
+	init_param_dataseg(tssgdt, (phys_bytes) t,
+			sizeof(struct tss_s), INTR_PRIVILEGE);
+	tssgdt->access = PRESENT | (INTR_PRIVILEGE << DPL_SHIFT) | TSS_TYPE;
+
+	/* Build TSS. */
+	memset(t, 0, sizeof(*t));
+	t->ds = t->es = t->fs = t->gs = t->ss0 = KERN_DS_SELECTOR;
+	t->cs = KERN_CS_SELECTOR;
+	t->iobase = sizeof(struct tss_s);	/* empty i/o permissions map */
+
+	/* 
+	 * make space for process pointer and cpu id and point to the first
+	 * usable word
+	 */
+	k_percpu_stacks[cpu] = t->sp0 = ((unsigned) kernel_stack) - X86_STACK_TOP_RESERVED;
+	/* 
+	 * set the cpu id at the top of the stack so we know on which cpu is
+	 * this stack in use when we trap to kernel
+	 */
+	*((reg_t *)(t->sp0 + 1 * sizeof(reg_t))) = cpu;
 	return SEG_SELECTOR(index);
 }
 
Only in src/minix/kernel: arch_smp.o
diff -ur src-netbsd-current202104/minix/kernel/main.c src/minix/kernel/main.c
--- src-netbsd-current202104/minix/kernel/main.c	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/main.c	Thu Mar 10 06:37:09 2022
@@ -42,6 +42,7 @@
   sprofiling = 0;      /* we're not profiling until instructed to */
 #endif /* SPROFILE */
 
+  setup_sysenter_syscall();
   cpu_identify();
 
   vm_running = 0;
@@ -103,6 +104,9 @@
 
   /* Kernel may no longer use bits of memory as VM will be running soon */
   kernel_may_alloc = 0;
+#ifdef CONFIG_SMP
+  wait_for_APs_to_finish_booting();
+#endif
 
   switch_to_user();
   NOT_REACHABLE;
diff -ur src-netbsd-current202104/minix/kernel/proc.c src/minix/kernel/proc.c
--- src-netbsd-current202104/minix/kernel/proc.c	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/proc.c	Fri Mar 11 03:12:43 2022
@@ -58,7 +58,6 @@
 static int try_one(endpoint_t receive_e, struct proc *src_ptr,
 	struct proc *dst_ptr);
 static struct proc * pick_proc(void);
-static void enqueue_head(struct proc *rp);
 
 /* all idles share the same idle_priv structure */
 static struct priv idle_priv;
@@ -166,7 +165,8 @@
 	 * when the CPU wakes up the kernel is mapped and no surprises happen.
 	 * This is only a problem if more than 1 cpus are available
 	 */
-	switch_address_space(proc_addr(VM_PROC_NR));
+	//if (cpuid == bsp_cpu_id)
+		switch_address_space(proc_addr(VM_PROC_NR));
 #endif
 }
 
@@ -303,9 +303,10 @@
 	 */
 	struct proc * p;
 #ifdef CONFIG_SMP
-	int tlb_must_refresh = 0;
+	int tlb_must_refresh;
+not_runnable_pick_new:
+	tlb_must_refresh = 0;
 #endif
-
 	p = get_cpulocal_var(proc_ptr);
 	/*
 	 * if the current process is still runnable check the misc flags and let
@@ -318,7 +319,9 @@
 	 * need to pick a new one here and start from scratch. Also if the
 	 * current process wasn't runnable, we pick a new one here
 	 */
+#ifndef CONFIG_SMP
 not_runnable_pick_new:
+#endif
 	if (proc_is_preempted(p)) {
 		p->p_rts_flags &= ~RTS_PREEMPTED;
 		if (proc_is_runnable(p)) {
@@ -356,7 +359,13 @@
 		(MF_KCALL_RESUME | MF_DELIVERMSG |
 		 MF_SC_DEFER | MF_SC_TRACE | MF_SC_ACTIVE)) {
 
+#ifdef CONFIG_SMP
+		if (!proc_is_runnable(p))
+			// something crazy happened!
+			goto not_runnable_pick_new;
+#else
 		assert(proc_is_runnable(p));
+#endif
 		if (p->p_misc_flags & MF_KCALL_RESUME) {
 			kernel_call_resume(p);
 		}
@@ -1667,7 +1676,7 @@
  * process on a run queue. We have to put this process back at the fron to be
  * fair
  */
-static void enqueue_head(struct proc *rp)
+void enqueue_head(struct proc *rp)
 {
   const int q = rp->p_priority;	 		/* scheduling queue to use */
 
Only in src/minix/kernel: proc.c_semiorig
diff -ur src-netbsd-current202104/minix/kernel/proc.h src/minix/kernel/proc.h
--- src-netbsd-current202104/minix/kernel/proc.h	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/proc.h	Thu Mar 10 12:17:51 2022
@@ -219,7 +219,7 @@
 		rts = (rp)->p_rts_flags;				\
 		(rp)->p_rts_flags &= ~(f);				\
 		if(!rts_f_is_runnable(rts) && proc_is_runnable(rp)) {	\
-			enqueue(rp);					\
+		(rp)->p_cpu_time_left > 1 ? enqueue(rp) : enqueue(rp);\
 		}							\
 	} while(0)
 
diff -ur src-netbsd-current202104/minix/kernel/proto.h src/minix/kernel/proto.h
--- src-netbsd-current202104/minix/kernel/proto.h	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/proto.h	Thu Mar 10 09:55:42 2022
@@ -67,6 +67,7 @@
         const vir_bytes linaddr, const vir_bytes len, const int type,
         const int writeflag);
 void enqueue(struct proc *rp);
+void enqueue_head(struct proc *rp);
 void dequeue(struct proc *rp);
 void switch_to_user(void);
 void arch_proc_reset(struct proc *rp);
Only in src/minix/kernel: smp.o
diff -ur src-netbsd-current202104/minix/kernel/spinlock.h src/minix/kernel/spinlock.h
--- src-netbsd-current202104/minix/kernel/spinlock.h	Fri Mar 11 03:54:35 2022
+++ src/minix/kernel/spinlock.h	Thu Mar 10 05:04:30 2022
@@ -1,8 +1,6 @@
 #ifndef __SPINLOCK_H__
 #define __SPINLOCK_H__
 
-#include "kernel/kernel.h"
-
 typedef struct spinlock {
 	atomic_t val;
 } spinlock_t;
Only in src/minix/kernel: trampoline.o
diff -ur src-netbsd-current202104/minix/lib/libc/sys/close.c src/minix/lib/libc/sys/close.c
--- src-netbsd-current202104/minix/lib/libc/sys/close.c	Fri Mar 11 03:54:35 2022
+++ src/minix/lib/libc/sys/close.c	Thu Mar 10 12:06:03 2022
@@ -12,7 +12,7 @@
 
 	memset(&m, 0, sizeof(m));
 	m.m_lc_vfs_close.fd = fd;
-	m.m_lc_vfs_close.nblock = 0;
+	m.m_lc_vfs_close.nblock = 1;
 
 	return _syscall(VFS_PROC_NR, VFS_CLOSE, &m);
 }
diff -ur src-netbsd-current202104/minix/servers/sched/schedule.c src/minix/servers/sched/schedule.c
--- src-netbsd-current202104/minix/servers/sched/schedule.c	Fri Mar 11 03:54:35 2022
+++ src/minix/servers/sched/schedule.c	Fri Mar 11 03:18:04 2022
@@ -29,11 +29,6 @@
 		SCHEDULE_CHANGE_CPU		\
 		)
 
-#define schedule_process_local(p)	\
-	schedule_process(p, SCHEDULE_CHANGE_PRIO | SCHEDULE_CHANGE_QUANTUM)
-#define schedule_process_migrate(p)	\
-	schedule_process(p, SCHEDULE_CHANGE_CPU)
-
 #define CPU_DEAD	-1
 
 #define cpu_is_available(c)	(cpu_proc[c] >= 0)
@@ -41,42 +36,46 @@
 #define DEFAULT_USER_TIME_SLICE 200
 
 /* processes created by RS are sysytem processes */
-#define is_system_proc(p)	((p)->parent == RS_PROC_NR)
+#define is_system_proc(p)	((p)->parent == RS_PROC_NR || (p)->endpoint <= LAST_SPECIAL_PROC_NR)
 
 static unsigned cpu_proc[CONFIG_MAX_CPUS];
 
-static void pick_cpu(struct schedproc * proc)
+static void allow_all_cpus(struct schedproc *proc)
 {
+	int i;
+	for(i=0;i<BITMAP_CHUNKS(CONFIG_MAX_CPUS);++i) {
+		bit_fill(proc->cpu_mask[i]);
+	}
+}
+
+static int next_cpu = 0;
+static int pick_cpu(struct schedproc * proc)
+{
 #ifdef CONFIG_SMP
 	unsigned cpu, c;
-	unsigned cpu_load = (unsigned) -1;
 	
-	if (machine.processors_count == 1) {
-		proc->cpu = machine.bsp_id;
-		return;
-	}
+	if (machine.processors_count == 1)
+		return machine.bsp_id;
+	if (is_system_proc(proc))
+		return machine.bsp_id;
 
-	/* schedule sysytem processes only on the boot cpu */
-	if (is_system_proc(proc)) {
-		proc->cpu = machine.bsp_id;
-		return;
-	}
-
 	/* if no other cpu available, try BSP */
 	cpu = machine.bsp_id;
+	unsigned cpu_load = 1 + cpu_proc[machine.bsp_id];
 	for (c = 0; c < machine.processors_count; c++) {
 		/* skip dead cpus */
 		if (!cpu_is_available(c))
 			continue;
-		if (c != machine.bsp_id && cpu_load > cpu_proc[c]) {
+		if(!GET_BIT(proc->cpu_mask,c))
+			continue;
+		if (c != machine.bsp_id && cpu_load >= cpu_proc[c]) {
 			cpu_load = cpu_proc[c];
 			cpu = c;
 		}
 	}
-	proc->cpu = cpu;
-	cpu_proc[cpu]++;
+	return cpu;
 #else
-	proc->cpu = 0;
+	return 0;
 #endif
 }
 
@@ -100,7 +99,8 @@
 		rmp->priority += 1; /* lower priority */
 	}
 
-	if ((rv = schedule_process_local(rmp)) != OK) {
+	rv = schedule_process(rmp,SCHEDULE_CHANGE_PRIO|SCHEDULE_CHANGE_QUANTUM);
+	if (rv != OK) {
 		return rv;
 	}
 	return OK;
@@ -126,10 +126,8 @@
 	}
 
 	rmp = &schedproc[proc_nr_n];
-#ifdef CONFIG_SMP
-	cpu_proc[rmp->cpu]--;
-#endif
 	rmp->flags = 0; /*&= ~IN_USE;*/
+	cpu_proc[rmp->cpu]--;
 
 	return OK;
 }
@@ -141,6 +139,7 @@
 {
 	register struct schedproc *rmp;
 	int rv, proc_nr_n, parent_nr_n;
+	int cpu_chosen;
 	
 	/* we can handle two kinds of messages here */
 	assert(m_ptr->m_type == SCHEDULING_START || 
@@ -165,6 +164,9 @@
 		return EINVAL;
 	}
 
+	allow_all_cpus(rmp);
+
+	cpu_chosen = 0;
 	/* Inherit current priority and time slice from parent. Since there
 	 * is currently only one scheduler scheduling the whole system, this
 	 * value is local and we assert that the parent endpoint is valid */
@@ -183,6 +185,7 @@
 #ifdef CONFIG_SMP
 		rmp->cpu = machine.bsp_id;
 		/* FIXME set the cpu mask */
+		cpu_chosen = 1;
 #endif
 	}
 	
@@ -223,12 +226,11 @@
 	rmp->flags = IN_USE;
 
 	/* Schedule the process, giving it some quantum */
-	pick_cpu(rmp);
-	while ((rv = schedule_process(rmp, SCHEDULE_CHANGE_ALL)) == EBADCPU) {
-		/* don't try this CPU ever again */
-		cpu_proc[rmp->cpu] = CPU_DEAD;
-		pick_cpu(rmp);
-	}
+	if(!cpu_chosen)
+		rmp->cpu = pick_cpu(rmp);
+	cpu_proc[rmp->cpu]++;
+	if((rv = schedule_process(rmp, SCHEDULE_CHANGE_ALL)) == EBADCPU)
+		panic("EBADCPU\n");
 
 	if (rv != OK) {
 		printf("Sched: Error while scheduling process, kernel replied %d\n",
@@ -281,7 +283,8 @@
 	/* Update the proc entry and reschedule the process */
 	rmp->max_priority = rmp->priority = new_q;
 
-	if ((rv = schedule_process_local(rmp)) != OK) {
+	rv = schedule_process(rmp,SCHEDULE_CHANGE_PRIO|SCHEDULE_CHANGE_QUANTUM);
+	if (rv != OK) {
 		/* Something went wrong when rescheduling the process, roll
 		 * back the changes to proc struct */
 		rmp->priority     = old_q;
@@ -299,8 +302,6 @@
 	int err;
 	int new_prio, new_quantum, new_cpu, niced;
 
-	pick_cpu(rmp);
-
 	if (flags & SCHEDULE_CHANGE_PRIO)
 		new_prio = rmp->priority;
 	else
@@ -359,7 +360,11 @@
 		if (rmp->flags & IN_USE) {
 			if (rmp->priority > rmp->max_priority) {
 				rmp->priority -= 1; /* increase priority */
-				schedule_process_local(rmp);
+				/* Balance the load, select new cpu. */
+				cpu_proc[rmp->cpu]--;
+				rmp->cpu = pick_cpu(rmp);
+				cpu_proc[rmp->cpu]++;
+				schedule_process(rmp,SCHEDULE_CHANGE_ALL);
 			}
 		}
 	}
Only in src/minix/servers/sched: schedule.c-ok4
Only in src/minix/servers/sched: schedule.c_new
Only in src/minix/servers/sched: schedule.c_ok1
Only in src/minix/servers/sched: schedule.c_ok2
Only in src/minix/servers/sched: schedule.c_ok3
Only in src/minix/servers/sched: schedule.c_orig
diff -ur src-netbsd-current202104/minix/servers/vfs/const.h src/minix/servers/vfs/const.h
--- src-netbsd-current202104/minix/servers/vfs/const.h	Fri Mar 11 03:54:35 2022
+++ src/minix/servers/vfs/const.h	Thu Mar 10 11:00:47 2022
@@ -6,7 +6,7 @@
 #define NR_LOCKS           8	/* # slots in the file locking table */
 #define NR_MNTS           16 	/* # slots in mount table */
 #define NR_VNODES       1024	/* # slots in vnode table */
-#define NR_WTHREADS	   9	/* # slots in worker thread table */
+#define NR_WTHREADS	  64	/* # slots in worker thread table */
 #define NR_SOCKDEVS	   8	/* # slots in smap table */
 
 #define NR_NONEDEVS	NR_MNTS	/* # slots in nonedev bitmap */
diff -ur src-netbsd-current202104/minix/servers/vfs/open.c src/minix/servers/vfs/open.c
--- src-netbsd-current202104/minix/servers/vfs/open.c	Fri Mar 11 03:54:35 2022
+++ src/minix/servers/vfs/open.c	Thu Mar 10 12:09:49 2022
@@ -679,7 +679,8 @@
   fd = job_m_in.m_lc_vfs_close.fd;
   nblock = job_m_in.m_lc_vfs_close.nblock;
 
-  return close_fd(fp, fd, !nblock /*may_suspend*/);
+//  return close_fd(fp, fd, !nblock /*may_suspend*/);
+  return close_fd(fp, fd, 0);
 }
 
 
diff -ur src-netbsd-current202104/etc/boot.cfg.default src/etc/boot.cfg.default
--- src-netbsd-current202104/etc/boot.cfg.default	Fri Mar 11 03:54:27 2022
+++ src/etc/boot.cfg.default	Thu Mar 10 01:36:07 2022
@@ -2,7 +2,7 @@
 timeout=5
 default=2
 menu=Start MINIX 3:load_mods /boot/minix_default/mod*;multiboot /boot/minix_default/kernel rootdevname=$rootdevname $args
-menu=Start latest MINIX 3:load_mods /boot/minix_latest/mod*;multiboot /boot/minix_latest/kernel rootdevname=$rootdevname $args
+menu=Start latest MINIX 3:load_mods /boot/minix_latest/mod*;multiboot /boot/minix_latest/kernel rootdevname=$rootdevname $args acpi=yes cttyline=0 no_smp=1 no_apic=1
 menu=Start latest MINIX 3 in single user mode:load_mods /boot/minix_latest/mod*;multiboot /boot/minix_latest/kernel rootdevname=$rootdevname bootopts=-s $args
 menu=Edit menu option:edit
 menu=Drop to boot prompt:prompt
Only in src/etc/defaults: rc_conf
Only in src/etc: etc-release
Only in src/etc/mtree: NetBSD.dist
Only in src/etc: obsolete.dir
diff -ur src-netbsd-current202104/etc/ttys src/etc/ttys
--- src-netbsd-current202104/etc/ttys	Fri Mar 11 03:54:27 2022
+++ src/etc/ttys	Thu Mar 10 01:36:03 2022
@@ -7,8 +7,10 @@
 ttyc1		"/usr/libexec/getty default"	minix	on secure
 ttyc2		"/usr/libexec/getty default"	minix	on secure
 ttyc3		"/usr/libexec/getty default"	minix	on secure
-tty00		""		unknown	off secure
-tty01		""		unknown off secure
+tty00		"/usr/libexec/getty default"	ansi	on secure
+tty01		"/usr/libexec/getty default"	ansi	on secure
+tty02		"/usr/libexec/getty default"	ansi	on secure
+tty03		"/usr/libexec/getty default"	ansi	on secure
 
 ttyp0		""		network	off 
 ttyp1		""		network	off
```