kern/arch/x86/uaccess.h

/* Copyright (c) 2015 Google Inc
 * Davide Libenzi <dlibenzi@google.com>
 * See LICENSE for details.
 *
 * Part of this code coming from a Linux kernel file:
 *
 * linux/arch/x86/include/asm/uaccess.h
 *
 * Which, even though missing specific copyright, it is supposed to be
 * ruled by the overall Linux copyright.
 */

#pragma once

#include <ros/errno.h>
#include <compiler.h>
#include <stdint.h>
#include <umem.h>
#include <arch/fixup.h>

#define __m(x) *(x)

struct extable_ip_fixup {
	uint64_t insn;
	uint64_t fixup;
};

#define __read_msr_asm(eax, edx, addr, err, errret)						\
	asm volatile(ASM_STAC "\n"											\
	             "1:		rdmsr\n"									\
	             "			mfence\n"									\
	             "2: " ASM_CLAC "\n"									\
	             ".section .fixup,\"ax\"\n"								\
	             "3:		mov %4,%0\n"								\
	             "	jmp 2b\n"											\
	             ".previous\n"											\
	             _ASM_EXTABLE(1b, 3b)									\
	             : "=r" (err), "=d" (edx), "=a" (eax)					\
	             : "c" (addr), "i" (errret), "0" (err))

#define __write_msr_asm(val, addr, err, errret)							\
	asm volatile(ASM_STAC "\n"											\
				 "1:		wrmsr\n"									\
				 "2: " ASM_CLAC "\n"									\
				 ".section .fixup,\"ax\"\n"								\
				 "3:		mov %4,%0\n"								\
				 "	jmp 2b\n"											\
				 ".previous\n"											\
				 _ASM_EXTABLE(1b, 3b)									\
				 : "=r" (err)											\
				 : "d" ((uint32_t) (val >> 32)),						\
				   "a" ((uint32_t) (val & 0xffffffff)), "c" (addr),		\
				   "i" (errret), "0" (err))

#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)		\
	asm volatile(ASM_STAC "\n"											\
				 "1:		mov"itype" %"rtype"1,%2\n"					\
				 "2: " ASM_CLAC "\n"									\
				 ".section .fixup,\"ax\"\n"								\
				 "3:		mov %3,%0\n"								\
				 "	jmp 2b\n"											\
				 ".previous\n"											\
				 _ASM_EXTABLE(1b, 3b)									\
				 : "=r"(err)											\
				 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))

#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
	asm volatile(ASM_STAC "\n"										\
				 "1:		mov"itype" %2,%"rtype"1\n"				\
				 "2: " ASM_CLAC "\n"								\
				 ".section .fixup,\"ax\"\n"							\
				 "3:		mov %3,%0\n"							\
				 "	xor"itype" %"rtype"1,%"rtype"1\n"				\
				 "	jmp 2b\n"										\
				 ".previous\n"										\
				 _ASM_EXTABLE(1b, 3b)								\
				 : "=r" (err), ltype(x)								\
				 : "m" (__m(addr)), "i" (errret), "0" (err))

#define __user_memcpy(dst, src, count, err, errret)						\
	asm volatile(ASM_STAC "\n"											\
				 "  		cld\n"										\
				 "1:		rep movsb\n"								\
				 "2: " ASM_CLAC "\n"									\
				 ".section .fixup,\"ax\"\n"								\
				 "3:		mov %4,%0\n"								\
				 "	jmp 2b\n"											\
				 ".previous\n"											\
				 _ASM_EXTABLE(1b, 3b)									\
				 : "=r"(err), "+D" (dst), "+S" (src), "+c" (count)		\
				 : "i" (errret), "0" (err)								\
				 : "memory")

static inline int __put_user(void *dst, const void *src, unsigned int count)
{
	int err = 0;

	switch (count) {
	case 1:
		__put_user_asm(*(const uint8_t *) src, (uint8_t *) dst, err, "b",
		               "b", "iq", -EFAULT);
		break;
	case 2:
		__put_user_asm(*(const uint16_t *) src, (uint16_t *) dst, err, "w",
		               "w", "ir", -EFAULT);
		break;
	case 4:
		__put_user_asm(*(const uint32_t *) src, (uint32_t *) dst, err, "l",
		               "k", "ir", -EFAULT);
		break;
	case 8:
		__put_user_asm(*(const uint64_t *) src, (uint64_t *) dst, err, "q",
		               "", "er", -EFAULT);
		break;
	default:
		__user_memcpy(dst, src, count, err, -EFAULT);
	}

	return err;
}

static inline int copy_to_user(void *dst, const void *src, unsigned int count)
{
	int err = 0;

	if (unlikely(!is_user_rwaddr(dst, count))) {
		err = -EFAULT;
	} else if (!__builtin_constant_p(count)) {
		__user_memcpy(dst, src, count, err, -EFAULT);
	} else {
		err = __put_user(dst, src, count);
	}

	return err;
}

static inline int __get_user(void *dst, const void *src, unsigned int count)
{
	int err = 0;

	switch (count) {
	case 1:
		__get_user_asm(*(uint8_t *) dst, (const uint8_t *) src, err, "b",
		               "b", "=q", -EFAULT);
		break;
	case 2:
		__get_user_asm(*(uint16_t *) dst, (const uint16_t *) src, err, "w",
		               "w", "=r", -EFAULT);
		break;
	case 4:
		__get_user_asm(*(uint32_t *) dst, (const uint32_t *) src, err, "l",
		               "k", "=r", -EFAULT);
		break;
	case 8:
		__get_user_asm(*(uint64_t *) dst, (const uint64_t *) src, err, "q",
		               "", "=r", -EFAULT);
		break;
	default:
		__user_memcpy(dst, src, count, err, -EFAULT);
	}

	return err;
}

static inline int copy_from_user(void *dst, const void *src,
                                 unsigned int count)
{
	int err = 0;

	if (unlikely(!is_user_raddr((void *) src, count))) {
		err = -EFAULT;
	} else if (!__builtin_constant_p(count)) {
		__user_memcpy(dst, src, count, err, -EFAULT);
	} else {
		err = __get_user(dst, src, count);
	}

	return err;
}

static inline int read_msr_safe(uint32_t addr, uint64_t *value)
{
	int err = 0;
	uint32_t edx, eax;

	__read_msr_asm(eax, edx, addr, err, -EFAULT);
	if (likely(err == 0))
		*value = ((uint64_t) edx << 32) | eax;

	return err;
}

static inline int write_msr_safe(uint32_t addr, uint64_t value)
{
	int err = 0;

	__write_msr_asm(value, addr, err, -EFAULT);

	return err;
}

static inline uintptr_t ex_insn_addr(const struct extable_ip_fixup *x)
{
	return (uintptr_t) &x->insn + x->insn;
}

static inline uintptr_t ex_fixup_addr(const struct extable_ip_fixup *x)
{
	return (uintptr_t) &x->fixup + x->fixup;
}