From b3f163527ab238f96e484d40d3741e8a6c5ede56 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:40:01 -0500 Subject: [PATCH 001/213] merge: aufs-kbuild Signed-off-by: Robert Nelson --- fs/Kconfig | 1 + fs/Makefile | 1 + 2 files changed, 2 insertions(+) diff --git a/fs/Kconfig b/fs/Kconfig index ac474a61be379..284cee954591e 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -255,6 +255,7 @@ source "fs/pstore/Kconfig" source "fs/sysv/Kconfig" source "fs/ufs/Kconfig" source "fs/exofs/Kconfig" +source "fs/aufs/Kconfig" endif # MISC_FILESYSTEMS diff --git a/fs/Makefile b/fs/Makefile index 293733f61594b..12d19d0de07a6 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -128,3 +128,4 @@ obj-y += exofs/ # Multiple modules obj-$(CONFIG_CEPH_FS) += ceph/ obj-$(CONFIG_PSTORE) += pstore/ obj-$(CONFIG_EFIVAR_FS) += efivarfs/ +obj-$(CONFIG_AUFS_FS) += aufs/ From 73078e0e283a7877d415d251f7d7c60c15dcced2 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:40:02 -0500 Subject: [PATCH 002/213] merge: aufs-base Signed-off-by: Robert Nelson --- MAINTAINERS | 13 +++++++++++++ drivers/block/loop.c | 18 ++++++++++++++++++ fs/dcache.c | 2 +- fs/fcntl.c | 4 +++- fs/inode.c | 2 +- fs/namespace.c | 6 ++++++ fs/read_write.c | 22 ++++++++++++++++++++++ fs/splice.c | 10 +++++----- fs/sync.c | 2 +- include/linux/fs.h | 10 ++++++++++ include/linux/lockdep.h | 3 +++ include/linux/mnt_namespace.h | 3 +++ include/linux/splice.h | 6 ++++++ kernel/locking/lockdep.c | 3 ++- 14 files changed, 94 insertions(+), 10 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 8ef6fedbc8da9..5de967b327049 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2606,6 +2606,19 @@ F: include/linux/audit.h F: include/uapi/linux/audit.h F: kernel/audit* +AUFS (advanced multi layered unification filesystem) FILESYSTEM +M: "J. R. Okajima" +L: aufs-users@lists.sourceforge.net (members only) +L: linux-unionfs@vger.kernel.org +W: http://aufs.sourceforge.net +T: git://github.com/sfjro/aufs4-linux.git +S: Supported +F: Documentation/filesystems/aufs/ +F: Documentation/ABI/testing/debugfs-aufs +F: Documentation/ABI/testing/sysfs-aufs +F: fs/aufs/ +F: include/uapi/linux/aufs_type.h + AUXILIARY DISPLAY DRIVERS M: Miguel Ojeda Sandonis S: Maintained diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 9cd231a27328e..39c2c53e26de9 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -749,6 +749,24 @@ static int loop_change_fd(struct loop_device *lo, struct block_device *bdev, return error; } +/* + * for AUFS + * no get/put for file. + */ +struct file *loop_backing_file(struct super_block *sb) +{ + struct file *ret; + struct loop_device *l; + + ret = NULL; + if (MAJOR(sb->s_dev) == LOOP_MAJOR) { + l = sb->s_bdev->bd_disk->private_data; + ret = l->lo_backing_file; + } + return ret; +} +EXPORT_SYMBOL_GPL(loop_backing_file); + /* loop sysfs attributes */ static ssize_t loop_attr_show(struct device *dev, char *page, diff --git a/fs/dcache.c b/fs/dcache.c index 6e0022326afe3..3bd53094ac3df 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1234,7 +1234,7 @@ enum d_walk_ret { * * The @enter() callbacks are called with d_lock held. */ -static void d_walk(struct dentry *parent, void *data, +void d_walk(struct dentry *parent, void *data, enum d_walk_ret (*enter)(void *, struct dentry *)) { struct dentry *this_parent; diff --git a/fs/fcntl.c b/fs/fcntl.c index 4137d96534a6c..d4da10be555c9 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -32,7 +32,7 @@ #define SETFL_MASK (O_APPEND | O_NONBLOCK | O_NDELAY | O_DIRECT | O_NOATIME) -static int setfl(int fd, struct file * filp, unsigned long arg) +int setfl(int fd, struct file *filp, unsigned long arg) { struct inode * inode = file_inode(filp); int error = 0; @@ -63,6 +63,8 @@ static int setfl(int fd, struct file * filp, unsigned long arg) if (filp->f_op->check_flags) error = filp->f_op->check_flags(arg); + if (!error && filp->f_op->setfl) + error = filp->f_op->setfl(filp, arg); if (error) return error; diff --git a/fs/inode.c b/fs/inode.c index 5c63693326bb4..43046d7223e43 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1657,7 +1657,7 @@ EXPORT_SYMBOL(generic_update_time); * This does the actual work of updating an inodes time or version. Must have * had called mnt_want_write() before calling this. */ -static int update_time(struct inode *inode, struct timespec64 *time, int flags) +int update_time(struct inode *inode, struct timespec64 *time, int flags) { int (*update_time)(struct inode *, struct timespec64 *, int); diff --git a/fs/namespace.c b/fs/namespace.c index 1fce41ba35357..fbd7edd49a2f4 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -770,6 +770,12 @@ static inline int check_mnt(struct mount *mnt) return mnt->mnt_ns == current->nsproxy->mnt_ns; } +/* for aufs, CONFIG_AUFS_BR_FUSE */ +int is_current_mnt_ns(struct vfsmount *mnt) +{ + return check_mnt(real_mount(mnt)); +} + /* * vfsmount lock must be held for write */ diff --git a/fs/read_write.c b/fs/read_write.c index 2195380620d02..8cf040de0a658 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -489,6 +489,28 @@ ssize_t __vfs_write(struct file *file, const char __user *p, size_t count, return -EINVAL; } +vfs_readf_t vfs_readf(struct file *file) +{ + const struct file_operations *fop = file->f_op; + + if (fop->read) + return fop->read; + if (fop->read_iter) + return new_sync_read; + return ERR_PTR(-ENOSYS); /* doesn't have ->read(|_iter)() op */ +} + +vfs_writef_t vfs_writef(struct file *file) +{ + const struct file_operations *fop = file->f_op; + + if (fop->write) + return fop->write; + if (fop->write_iter) + return new_sync_write; + return ERR_PTR(-ENOSYS); /* doesn't have ->write(|_iter)() op */ +} + ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos) { mm_segment_t old_fs; diff --git a/fs/splice.c b/fs/splice.c index fd28c7da3c837..959f2655c2168 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -838,8 +838,8 @@ EXPORT_SYMBOL(generic_splice_sendpage); /* * Attempt to initiate a splice from pipe to file. */ -static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, - loff_t *ppos, size_t len, unsigned int flags) +long do_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) { ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); @@ -855,9 +855,9 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, /* * Attempt to initiate a splice from a file to a pipe. */ -static long do_splice_to(struct file *in, loff_t *ppos, - struct pipe_inode_info *pipe, size_t len, - unsigned int flags) +long do_splice_to(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) { ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); diff --git a/fs/sync.c b/fs/sync.c index b54e0541ad899..28607828e96f2 100644 --- a/fs/sync.c +++ b/fs/sync.c @@ -28,7 +28,7 @@ * wait == 1 case since in that case write_inode() functions do * sync_dirty_buffer() and thus effectively write one block at a time. */ -static int __sync_filesystem(struct super_block *sb, int wait) +int __sync_filesystem(struct super_block *sb, int wait) { if (wait) sync_inodes_sb(sb); diff --git a/include/linux/fs.h b/include/linux/fs.h index 92420009b9bcb..c7328e183f779 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1295,6 +1295,7 @@ extern void fasync_free(struct fasync_struct *); /* can be called from interrupts */ extern void kill_fasync(struct fasync_struct **, int, int); +extern int setfl(int fd, struct file *filp, unsigned long arg); extern void __f_setown(struct file *filp, struct pid *, enum pid_type, int force); extern int f_setown(struct file *filp, unsigned long arg, int force); extern void f_delown(struct file *filp); @@ -1759,6 +1760,7 @@ struct file_operations { ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); int (*check_flags)(int); + int (*setfl)(struct file *, unsigned long); int (*flock) (struct file *, int, struct file_lock *); ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); @@ -1830,6 +1832,12 @@ ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, struct iovec *fast_pointer, struct iovec **ret_pointer); +typedef ssize_t (*vfs_readf_t)(struct file *, char __user *, size_t, loff_t *); +typedef ssize_t (*vfs_writef_t)(struct file *, const char __user *, size_t, + loff_t *); +vfs_readf_t vfs_readf(struct file *file); +vfs_writef_t vfs_writef(struct file *file); + extern ssize_t __vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t vfs_read(struct file *, char __user *, size_t, loff_t *); extern ssize_t vfs_write(struct file *, const char __user *, size_t, loff_t *); @@ -2255,6 +2263,7 @@ extern int current_umask(void); extern void ihold(struct inode * inode); extern void iput(struct inode *); extern int generic_update_time(struct inode *, struct timespec64 *, int); +extern int update_time(struct inode *, struct timespec64 *, int); /* /sys/fs */ extern struct kobject *fs_kobj; @@ -2542,6 +2551,7 @@ static inline bool sb_is_blkdev_sb(struct super_block *sb) return false; } #endif +extern int __sync_filesystem(struct super_block *, int); extern int sync_filesystem(struct super_block *); extern const struct file_operations def_blk_fops; extern const struct file_operations def_chr_fops; diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index b0d0b51c4d850..f73ffaa0199ee 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -313,6 +313,8 @@ static inline int lockdep_match_key(struct lockdep_map *lock, return lock->key == key; } +struct lock_class *lockdep_hlock_class(struct held_lock *hlock); + /* * Acquire a lock. * @@ -439,6 +441,7 @@ struct lockdep_map { }; #define lockdep_depth(tsk) (0) +#define lockdep_is_held(lock) (1) #define lockdep_is_held_type(l, r) (1) #define lockdep_assert_held(l) do { (void)(l); } while (0) diff --git a/include/linux/mnt_namespace.h b/include/linux/mnt_namespace.h index 35942084cd40d..24f5fd1a789de 100644 --- a/include/linux/mnt_namespace.h +++ b/include/linux/mnt_namespace.h @@ -6,11 +6,14 @@ struct mnt_namespace; struct fs_struct; struct user_namespace; +struct vfsmount; extern struct mnt_namespace *copy_mnt_ns(unsigned long, struct mnt_namespace *, struct user_namespace *, struct fs_struct *); extern void put_mnt_ns(struct mnt_namespace *ns); +extern int is_current_mnt_ns(struct vfsmount *mnt); + extern const struct file_operations proc_mounts_operations; extern const struct file_operations proc_mountinfo_operations; extern const struct file_operations proc_mountstats_operations; diff --git a/include/linux/splice.h b/include/linux/splice.h index 74b4911ac16dd..19789fbea5675 100644 --- a/include/linux/splice.h +++ b/include/linux/splice.h @@ -87,4 +87,10 @@ extern void splice_shrink_spd(struct splice_pipe_desc *); extern const struct pipe_buf_operations page_cache_pipe_buf_ops; extern const struct pipe_buf_operations default_pipe_buf_ops; + +extern long do_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags); +extern long do_splice_to(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags); #endif diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 1e272f6a01e73..b14c20deb6a70 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -140,7 +140,7 @@ static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES]; unsigned long nr_lock_classes; static struct lock_class lock_classes[MAX_LOCKDEP_KEYS]; -static inline struct lock_class *hlock_class(struct held_lock *hlock) +inline struct lock_class *lockdep_hlock_class(struct held_lock *hlock) { if (!hlock->class_idx) { /* @@ -151,6 +151,7 @@ static inline struct lock_class *hlock_class(struct held_lock *hlock) } return lock_classes + hlock->class_idx - 1; } +#define hlock_class(hlock) lockdep_hlock_class(hlock) #ifdef CONFIG_LOCK_STAT static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], cpu_lock_stats); From 1248d8a63a9ee192003de3ccb1ffb5bf44d60989 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:40:02 -0500 Subject: [PATCH 003/213] merge: aufs-mmap Signed-off-by: Robert Nelson --- fs/proc/base.c | 2 +- fs/proc/nommu.c | 5 ++- fs/proc/task_mmu.c | 7 +++- fs/proc/task_nommu.c | 5 ++- include/linux/mm.h | 22 ++++++++++ include/linux/mm_types.h | 2 + kernel/fork.c | 2 +- mm/Makefile | 2 +- mm/filemap.c | 2 +- mm/mmap.c | 33 +++++++++++---- mm/nommu.c | 10 ++--- mm/prfile.c | 86 ++++++++++++++++++++++++++++++++++++++++ 12 files changed, 157 insertions(+), 21 deletions(-) create mode 100644 mm/prfile.c diff --git a/fs/proc/base.c b/fs/proc/base.c index 3b9b726b1a6ca..5c2ddf46ebff8 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -2036,7 +2036,7 @@ static int map_files_get_link(struct dentry *dentry, struct path *path) rc = -ENOENT; vma = find_exact_vma(mm, vm_start, vm_end); if (vma && vma->vm_file) { - *path = vma->vm_file->f_path; + *path = vma_pr_or_file(vma)->f_path; path_get(path); rc = 0; } diff --git a/fs/proc/nommu.c b/fs/proc/nommu.c index 3b63be64e4364..fb9913bf3d10d 100644 --- a/fs/proc/nommu.c +++ b/fs/proc/nommu.c @@ -45,7 +45,10 @@ static int nommu_region_show(struct seq_file *m, struct vm_region *region) file = region->vm_file; if (file) { - struct inode *inode = file_inode(region->vm_file); + struct inode *inode; + + file = vmr_pr_or_file(region); + inode = file_inode(file); dev = inode->i_sb->s_dev; ino = inode->i_ino; } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index efa6273c00067..75dbd27826f72 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -309,7 +309,10 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) const char *name = NULL; if (file) { - struct inode *inode = file_inode(vma->vm_file); + struct inode *inode; + + file = vma_pr_or_file(vma); + inode = file_inode(file); dev = inode->i_sb->s_dev; ino = inode->i_ino; pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; @@ -1768,7 +1771,7 @@ static int show_numa_map(struct seq_file *m, void *v) struct proc_maps_private *proc_priv = &numa_priv->proc_maps; struct vm_area_struct *vma = v; struct numa_maps *md = &numa_priv->md; - struct file *file = vma->vm_file; + struct file *file = vma_pr_or_file(vma); struct mm_struct *mm = vma->vm_mm; struct mm_walk walk = { .hugetlb_entry = gather_hugetlb_stats, diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c index 5161894a6d623..b6d13cce45f3e 100644 --- a/fs/proc/task_nommu.c +++ b/fs/proc/task_nommu.c @@ -155,7 +155,10 @@ static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) file = vma->vm_file; if (file) { - struct inode *inode = file_inode(vma->vm_file); + struct inode *inode; + + file = vma_pr_or_file(vma); + inode = file_inode(file); dev = inode->i_sb->s_dev; ino = inode->i_ino; pgoff = (loff_t)vma->vm_pgoff << PAGE_SHIFT; diff --git a/include/linux/mm.h b/include/linux/mm.h index 45f10f5896b7b..f311952672417 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1448,6 +1448,28 @@ static inline void unmap_shared_mapping_range(struct address_space *mapping, unmap_mapping_range(mapping, holebegin, holelen, 0); } +extern void vma_do_file_update_time(struct vm_area_struct *, const char[], int); +extern struct file *vma_do_pr_or_file(struct vm_area_struct *, const char[], + int); +extern void vma_do_get_file(struct vm_area_struct *, const char[], int); +extern void vma_do_fput(struct vm_area_struct *, const char[], int); + +#define vma_file_update_time(vma) vma_do_file_update_time(vma, __func__, \ + __LINE__) +#define vma_pr_or_file(vma) vma_do_pr_or_file(vma, __func__, \ + __LINE__) +#define vma_get_file(vma) vma_do_get_file(vma, __func__, __LINE__) +#define vma_fput(vma) vma_do_fput(vma, __func__, __LINE__) + +#ifndef CONFIG_MMU +extern struct file *vmr_do_pr_or_file(struct vm_region *, const char[], int); +extern void vmr_do_fput(struct vm_region *, const char[], int); + +#define vmr_pr_or_file(region) vmr_do_pr_or_file(region, __func__, \ + __LINE__) +#define vmr_fput(region) vmr_do_fput(region, __func__, __LINE__) +#endif /* !CONFIG_MMU */ + extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags); extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 3a9a996af229e..eaaad59a6b94a 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -244,6 +244,7 @@ struct vm_region { unsigned long vm_top; /* region allocated to here */ unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ struct file *vm_file; /* the backing file or NULL */ + struct file *vm_prfile; /* the virtual backing file or NULL */ int vm_usage; /* region usage count (access under nommu_region_sem) */ bool vm_icache_flushed : 1; /* true if the icache has been flushed for @@ -318,6 +319,7 @@ struct vm_area_struct { unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ struct file * vm_file; /* File we map to (can be NULL). */ + struct file *vm_prfile; /* shadow of vm_file */ void * vm_private_data; /* was vm_pte (shared mem) */ atomic_long_t swap_readahead_info; diff --git a/kernel/fork.c b/kernel/fork.c index 8cb5cd7c97e19..0b13b4f1c896d 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -500,7 +500,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm, struct inode *inode = file_inode(file); struct address_space *mapping = file->f_mapping; - get_file(file); + vma_get_file(tmp); if (tmp->vm_flags & VM_DENYWRITE) atomic_dec(&inode->i_writecount); i_mmap_lock_write(mapping); diff --git a/mm/Makefile b/mm/Makefile index 26ef77a3883b5..b2869af1ef08e 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -39,7 +39,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ mm_init.o mmu_context.o percpu.o slab_common.o \ compaction.o vmacache.o \ interval_tree.o list_lru.o workingset.o \ - debug.o $(mmu-y) + prfile.o debug.o $(mmu-y) obj-y += init-mm.o diff --git a/mm/filemap.c b/mm/filemap.c index 45f1c6d73b5b0..41e4c3c51ec8a 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2723,7 +2723,7 @@ vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf) vm_fault_t ret = VM_FAULT_LOCKED; sb_start_pagefault(inode->i_sb); - file_update_time(vmf->vma->vm_file); + vma_file_update_time(vmf->vma); lock_page(page); if (page->mapping != inode->i_mapping) { unlock_page(page); diff --git a/mm/mmap.c b/mm/mmap.c index a98f09b830197..4c930b0ccbd9f 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -175,7 +175,7 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) if (vma->vm_ops && vma->vm_ops->close) vma->vm_ops->close(vma); if (vma->vm_file) - fput(vma->vm_file); + vma_fput(vma); mpol_put(vma_policy(vma)); vm_area_free(vma); return next; @@ -900,7 +900,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start, if (remove_next) { if (file) { uprobe_munmap(next, next->vm_start, next->vm_end); - fput(file); + vma_fput(vma); } if (next->anon_vma) anon_vma_merge(vma, next); @@ -1816,8 +1816,8 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return addr; unmap_and_free_vma: + vma_fput(vma); vma->vm_file = NULL; - fput(file); /* Undo any partial mapping done by a device driver. */ unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end); @@ -2639,7 +2639,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, goto out_free_mpol; if (new->vm_file) - get_file(new->vm_file); + vma_get_file(new); if (new->vm_ops && new->vm_ops->open) new->vm_ops->open(new); @@ -2658,7 +2658,7 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma, if (new->vm_ops && new->vm_ops->close) new->vm_ops->close(new); if (new->vm_file) - fput(new->vm_file); + vma_fput(new); unlink_anon_vmas(new); out_free_mpol: mpol_put(vma_policy(new)); @@ -2820,7 +2820,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, struct vm_area_struct *vma; unsigned long populate = 0; unsigned long ret = -EINVAL; - struct file *file; + struct file *file, *prfile; pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.rst.\n", current->comm, current->pid); @@ -2895,10 +2895,27 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, } } - file = get_file(vma->vm_file); + vma_get_file(vma); + file = vma->vm_file; + prfile = vma->vm_prfile; ret = do_mmap_pgoff(vma->vm_file, start, size, prot, flags, pgoff, &populate, NULL); + if (!IS_ERR_VALUE(ret) && file && prfile) { + struct vm_area_struct *new_vma; + + new_vma = find_vma(mm, ret); + if (!new_vma->vm_prfile) + new_vma->vm_prfile = prfile; + if (new_vma != vma) + get_file(prfile); + } + /* + * two fput()s instead of vma_fput(vma), + * coz vma may not be available anymore. + */ fput(file); + if (prfile) + fput(prfile); out: up_write(&mm->mmap_sem); if (populate) @@ -3204,7 +3221,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (anon_vma_clone(new_vma, vma)) goto out_free_mempol; if (new_vma->vm_file) - get_file(new_vma->vm_file); + vma_get_file(new_vma); if (new_vma->vm_ops && new_vma->vm_ops->open) new_vma->vm_ops->open(new_vma); vma_link(mm, new_vma, prev, rb_link, rb_parent); diff --git a/mm/nommu.c b/mm/nommu.c index 1d63ecfc98c5d..15eafddeb9446 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -625,7 +625,7 @@ static void __put_nommu_region(struct vm_region *region) up_write(&nommu_region_sem); if (region->vm_file) - fput(region->vm_file); + vmr_fput(region); /* IO memory and memory shared directly out of the pagecache * from ramfs/tmpfs mustn't be released here */ @@ -763,7 +763,7 @@ static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) if (vma->vm_ops && vma->vm_ops->close) vma->vm_ops->close(vma); if (vma->vm_file) - fput(vma->vm_file); + vma_fput(vma); put_nommu_region(vma->vm_region); vm_area_free(vma); } @@ -1286,7 +1286,7 @@ unsigned long do_mmap(struct file *file, goto error_just_free; } } - fput(region->vm_file); + vmr_fput(region); kmem_cache_free(vm_region_jar, region); region = pregion; result = start; @@ -1361,10 +1361,10 @@ unsigned long do_mmap(struct file *file, up_write(&nommu_region_sem); error: if (region->vm_file) - fput(region->vm_file); + vmr_fput(region); kmem_cache_free(vm_region_jar, region); if (vma->vm_file) - fput(vma->vm_file); + vma_fput(vma); vm_area_free(vma); return ret; diff --git a/mm/prfile.c b/mm/prfile.c new file mode 100644 index 0000000000000..00d51187c3250 --- /dev/null +++ b/mm/prfile.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Mainly for aufs which mmap(2) different file and wants to print different + * path in /proc/PID/maps. + * Call these functions via macros defined in linux/mm.h. + * + * See Documentation/filesystems/aufs/design/06mmap.txt + * + * Copyright (c) 2014-2020 Junjro R. Okajima + * Copyright (c) 2014 Ian Campbell + */ + +#include +#include +#include + +/* #define PRFILE_TRACE */ +static inline void prfile_trace(struct file *f, struct file *pr, + const char func[], int line, const char func2[]) +{ +#ifdef PRFILE_TRACE + if (pr) + pr_info("%s:%d: %s, %pD2\n", func, line, func2, f); +#endif +} + +void vma_do_file_update_time(struct vm_area_struct *vma, const char func[], + int line) +{ + struct file *f = vma->vm_file, *pr = vma->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + file_update_time(f); + if (f && pr) + file_update_time(pr); +} + +struct file *vma_do_pr_or_file(struct vm_area_struct *vma, const char func[], + int line) +{ + struct file *f = vma->vm_file, *pr = vma->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + return (f && pr) ? pr : f; +} + +void vma_do_get_file(struct vm_area_struct *vma, const char func[], int line) +{ + struct file *f = vma->vm_file, *pr = vma->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + get_file(f); + if (f && pr) + get_file(pr); +} + +void vma_do_fput(struct vm_area_struct *vma, const char func[], int line) +{ + struct file *f = vma->vm_file, *pr = vma->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + fput(f); + if (f && pr) + fput(pr); +} + +#ifndef CONFIG_MMU +struct file *vmr_do_pr_or_file(struct vm_region *region, const char func[], + int line) +{ + struct file *f = region->vm_file, *pr = region->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + return (f && pr) ? pr : f; +} + +void vmr_do_fput(struct vm_region *region, const char func[], int line) +{ + struct file *f = region->vm_file, *pr = region->vm_prfile; + + prfile_trace(f, pr, func, line, __func__); + fput(f); + if (f && pr) + fput(pr); +} +#endif /* !CONFIG_MMU */ From 31023744e5d8904be267ea565d3d57b6603f12b5 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:40:03 -0500 Subject: [PATCH 004/213] merge: aufs-standalone Signed-off-by: Robert Nelson --- fs/dcache.c | 2 ++ fs/exec.c | 1 + fs/fcntl.c | 1 + fs/file_table.c | 3 +++ fs/inode.c | 1 + fs/namespace.c | 3 +++ fs/notify/group.c | 3 +++ fs/notify/mark.c | 4 ++++ fs/open.c | 1 + fs/read_write.c | 4 ++++ fs/splice.c | 2 ++ fs/sync.c | 1 + fs/xattr.c | 1 + kernel/locking/lockdep.c | 1 + kernel/task_work.c | 1 + security/device_cgroup.c | 1 + security/security.c | 9 +++++++++ 17 files changed, 39 insertions(+) diff --git a/fs/dcache.c b/fs/dcache.c index 3bd53094ac3df..d6b2f7a994f4c 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1339,6 +1339,7 @@ void d_walk(struct dentry *parent, void *data, seq = 1; goto again; } +EXPORT_SYMBOL_GPL(d_walk); struct check_mount { struct vfsmount *mnt; @@ -2835,6 +2836,7 @@ void d_exchange(struct dentry *dentry1, struct dentry *dentry2) write_sequnlock(&rename_lock); } +EXPORT_SYMBOL_GPL(d_exchange); /** * d_ancestor - search for an ancestor diff --git a/fs/exec.c b/fs/exec.c index 561ea64829ece..60b920fcc4730 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -109,6 +109,7 @@ bool path_noexec(const struct path *path) return (path->mnt->mnt_flags & MNT_NOEXEC) || (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); } +EXPORT_SYMBOL_GPL(path_noexec); #ifdef CONFIG_USELIB /* diff --git a/fs/fcntl.c b/fs/fcntl.c index d4da10be555c9..fbc70ac63462f 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -85,6 +85,7 @@ int setfl(int fd, struct file *filp, unsigned long arg) out: return error; } +EXPORT_SYMBOL_GPL(setfl); static void f_modown(struct file *filp, struct pid *pid, enum pid_type type, int force) diff --git a/fs/file_table.c b/fs/file_table.c index e49af4caf15d9..569020fd1fb38 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -161,6 +161,7 @@ struct file *alloc_empty_file(int flags, const struct cred *cred) } return ERR_PTR(-ENFILE); } +EXPORT_SYMBOL_GPL(alloc_empty_file); /* * Variant of alloc_empty_file() that doesn't check and modify nr_files. @@ -323,6 +324,7 @@ void flush_delayed_fput(void) { delayed_fput(NULL); } +EXPORT_SYMBOL_GPL(flush_delayed_fput); static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput); @@ -365,6 +367,7 @@ void __fput_sync(struct file *file) } EXPORT_SYMBOL(fput); +EXPORT_SYMBOL_GPL(__fput_sync); void __init files_init(void) { diff --git a/fs/inode.c b/fs/inode.c index 43046d7223e43..36146c757aa29 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -1666,6 +1666,7 @@ int update_time(struct inode *inode, struct timespec64 *time, int flags) return update_time(inode, time, flags); } +EXPORT_SYMBOL_GPL(update_time); /** * touch_atime - update the access time diff --git a/fs/namespace.c b/fs/namespace.c index fbd7edd49a2f4..d6eca814d9fc4 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -437,6 +437,7 @@ void __mnt_drop_write(struct vfsmount *mnt) mnt_dec_writers(real_mount(mnt)); preempt_enable(); } +EXPORT_SYMBOL_GPL(__mnt_drop_write); /** * mnt_drop_write - give up write access to a mount @@ -775,6 +776,7 @@ int is_current_mnt_ns(struct vfsmount *mnt) { return check_mnt(real_mount(mnt)); } +EXPORT_SYMBOL_GPL(is_current_mnt_ns); /* * vfsmount lock must be held for write @@ -1844,6 +1846,7 @@ int iterate_mounts(int (*f)(struct vfsmount *, void *), void *arg, } return 0; } +EXPORT_SYMBOL_GPL(iterate_mounts); static void cleanup_group_ids(struct mount *mnt, struct mount *end) { diff --git a/fs/notify/group.c b/fs/notify/group.c index c03b836628769..94d210ca384a6 100644 --- a/fs/notify/group.c +++ b/fs/notify/group.c @@ -112,6 +112,7 @@ void fsnotify_get_group(struct fsnotify_group *group) { refcount_inc(&group->refcnt); } +EXPORT_SYMBOL_GPL(fsnotify_get_group); /* * Drop a reference to a group. Free it if it's through. @@ -121,6 +122,7 @@ void fsnotify_put_group(struct fsnotify_group *group) if (refcount_dec_and_test(&group->refcnt)) fsnotify_final_destroy_group(group); } +EXPORT_SYMBOL_GPL(fsnotify_put_group); /* * Create a new fsnotify_group and hold a reference for the group returned. @@ -150,6 +152,7 @@ struct fsnotify_group *fsnotify_alloc_group(const struct fsnotify_ops *ops) return group; } +EXPORT_SYMBOL_GPL(fsnotify_alloc_group); int fsnotify_fasync(int fd, struct file *file, int on) { diff --git a/fs/notify/mark.c b/fs/notify/mark.c index 09535f6423fc5..e9401ec71fc75 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -285,6 +285,7 @@ void fsnotify_put_mark(struct fsnotify_mark *mark) queue_delayed_work(system_unbound_wq, &reaper_work, FSNOTIFY_REAPER_DELAY); } +EXPORT_SYMBOL_GPL(fsnotify_put_mark); /* * Get mark reference when we found the mark via lockless traversal of object @@ -439,6 +440,7 @@ void fsnotify_destroy_mark(struct fsnotify_mark *mark, mutex_unlock(&group->mark_mutex); fsnotify_free_mark(mark); } +EXPORT_SYMBOL_GPL(fsnotify_destroy_mark); /* * Sorting function for lists of fsnotify marks. @@ -654,6 +656,7 @@ int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp, mutex_unlock(&group->mark_mutex); return ret; } +EXPORT_SYMBOL_GPL(fsnotify_add_mark); /* * Given a list of marks, find the mark associated with given group. If found @@ -777,6 +780,7 @@ void fsnotify_init_mark(struct fsnotify_mark *mark, fsnotify_get_group(group); mark->group = group; } +EXPORT_SYMBOL_GPL(fsnotify_init_mark); /* * Destroy all marks in destroy_list, waits for SRCU period to finish before diff --git a/fs/open.c b/fs/open.c index 878478745924b..ab755f585f293 100644 --- a/fs/open.c +++ b/fs/open.c @@ -64,6 +64,7 @@ int do_truncate(struct dentry *dentry, loff_t length, unsigned int time_attrs, inode_unlock(dentry->d_inode); return ret; } +EXPORT_SYMBOL_GPL(do_truncate); long vfs_truncate(const struct path *path, loff_t length) { diff --git a/fs/read_write.c b/fs/read_write.c index 8cf040de0a658..57d97612ca407 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -459,6 +459,7 @@ ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) return ret; } +EXPORT_SYMBOL_GPL(vfs_read); static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos) { @@ -499,6 +500,7 @@ vfs_readf_t vfs_readf(struct file *file) return new_sync_read; return ERR_PTR(-ENOSYS); /* doesn't have ->read(|_iter)() op */ } +EXPORT_SYMBOL_GPL(vfs_readf); vfs_writef_t vfs_writef(struct file *file) { @@ -510,6 +512,7 @@ vfs_writef_t vfs_writef(struct file *file) return new_sync_write; return ERR_PTR(-ENOSYS); /* doesn't have ->write(|_iter)() op */ } +EXPORT_SYMBOL_GPL(vfs_writef); ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos) { @@ -579,6 +582,7 @@ ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_ return ret; } +EXPORT_SYMBOL_GPL(vfs_write); static inline loff_t file_pos_read(struct file *file) { diff --git a/fs/splice.c b/fs/splice.c index 959f2655c2168..a17d3a6207c55 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -851,6 +851,7 @@ long do_splice_from(struct pipe_inode_info *pipe, struct file *out, return splice_write(pipe, out, ppos, len, flags); } +EXPORT_SYMBOL_GPL(do_splice_from); /* * Attempt to initiate a splice from a file to a pipe. @@ -880,6 +881,7 @@ long do_splice_to(struct file *in, loff_t *ppos, return splice_read(in, ppos, pipe, len, flags); } +EXPORT_SYMBOL_GPL(do_splice_to); /** * splice_direct_to_actor - splices data directly between two non-pipes diff --git a/fs/sync.c b/fs/sync.c index 28607828e96f2..ffd7ea43831ea 100644 --- a/fs/sync.c +++ b/fs/sync.c @@ -39,6 +39,7 @@ int __sync_filesystem(struct super_block *sb, int wait) sb->s_op->sync_fs(sb, wait); return __sync_blockdev(sb->s_bdev, wait); } +EXPORT_SYMBOL_GPL(__sync_filesystem); /* * Write out and wait upon all dirty data associated with this diff --git a/fs/xattr.c b/fs/xattr.c index 0d6a6a4af8616..7ce4701b7289d 100644 --- a/fs/xattr.c +++ b/fs/xattr.c @@ -295,6 +295,7 @@ vfs_getxattr_alloc(struct dentry *dentry, const char *name, char **xattr_value, *xattr_value = value; return error; } +EXPORT_SYMBOL_GPL(vfs_getxattr_alloc); ssize_t __vfs_getxattr(struct dentry *dentry, struct inode *inode, const char *name, diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index b14c20deb6a70..1f7c761cb47cb 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -151,6 +151,7 @@ inline struct lock_class *lockdep_hlock_class(struct held_lock *hlock) } return lock_classes + hlock->class_idx - 1; } +EXPORT_SYMBOL_GPL(lockdep_hlock_class); #define hlock_class(hlock) lockdep_hlock_class(hlock) #ifdef CONFIG_LOCK_STAT diff --git a/kernel/task_work.c b/kernel/task_work.c index 0fef395662a6e..83fb1ecfc33de 100644 --- a/kernel/task_work.c +++ b/kernel/task_work.c @@ -116,3 +116,4 @@ void task_work_run(void) } while (work); } } +EXPORT_SYMBOL_GPL(task_work_run); diff --git a/security/device_cgroup.c b/security/device_cgroup.c index dc28914fa72e0..c2ddfea2b2803 100644 --- a/security/device_cgroup.c +++ b/security/device_cgroup.c @@ -824,3 +824,4 @@ int __devcgroup_check_permission(short type, u32 major, u32 minor, return 0; } +EXPORT_SYMBOL_GPL(__devcgroup_check_permission); diff --git a/security/security.c b/security/security.c index 5ce2448f3a454..3997af3462bc9 100644 --- a/security/security.c +++ b/security/security.c @@ -542,6 +542,7 @@ int security_path_rmdir(const struct path *dir, struct dentry *dentry) return 0; return call_int_hook(path_rmdir, 0, dir, dentry); } +EXPORT_SYMBOL_GPL(security_path_rmdir); int security_path_unlink(const struct path *dir, struct dentry *dentry) { @@ -558,6 +559,7 @@ int security_path_symlink(const struct path *dir, struct dentry *dentry, return 0; return call_int_hook(path_symlink, 0, dir, dentry, old_name); } +EXPORT_SYMBOL_GPL(security_path_symlink); int security_path_link(struct dentry *old_dentry, const struct path *new_dir, struct dentry *new_dentry) @@ -566,6 +568,7 @@ int security_path_link(struct dentry *old_dentry, const struct path *new_dir, return 0; return call_int_hook(path_link, 0, old_dentry, new_dir, new_dentry); } +EXPORT_SYMBOL_GPL(security_path_link); int security_path_rename(const struct path *old_dir, struct dentry *old_dentry, const struct path *new_dir, struct dentry *new_dentry, @@ -593,6 +596,7 @@ int security_path_truncate(const struct path *path) return 0; return call_int_hook(path_truncate, 0, path); } +EXPORT_SYMBOL_GPL(security_path_truncate); int security_path_chmod(const struct path *path, umode_t mode) { @@ -600,6 +604,7 @@ int security_path_chmod(const struct path *path, umode_t mode) return 0; return call_int_hook(path_chmod, 0, path, mode); } +EXPORT_SYMBOL_GPL(security_path_chmod); int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid) { @@ -607,6 +612,7 @@ int security_path_chown(const struct path *path, kuid_t uid, kgid_t gid) return 0; return call_int_hook(path_chown, 0, path, uid, gid); } +EXPORT_SYMBOL_GPL(security_path_chown); int security_path_chroot(const struct path *path) { @@ -707,6 +713,7 @@ int security_inode_permission(struct inode *inode, int mask) return 0; return call_int_hook(inode_permission, 0, inode, mask); } +EXPORT_SYMBOL_GPL(security_inode_permission); int security_inode_setattr(struct dentry *dentry, struct iattr *attr) { @@ -878,6 +885,7 @@ int security_file_permission(struct file *file, int mask) return fsnotify_perm(file, mask); } +EXPORT_SYMBOL_GPL(security_file_permission); int security_file_alloc(struct file *file) { @@ -937,6 +945,7 @@ int security_mmap_file(struct file *file, unsigned long prot, return ret; return ima_file_mmap(file, prot); } +EXPORT_SYMBOL_GPL(security_mmap_file); int security_mmap_addr(unsigned long addr) { From b94a18b01fa47926eb5b1ed427b347a99d12c885 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:40:04 -0500 Subject: [PATCH 005/213] merge: aufs https://github.com/sfjro/aufs4-standalone/commit/ccb15898ed1eb7fe2e914439aca993e903e8ed51 Signed-off-by: Robert Nelson --- Documentation/ABI/testing/debugfs-aufs | 55 + Documentation/ABI/testing/sysfs-aufs | 31 + Documentation/filesystems/aufs/README | 404 ++++ .../filesystems/aufs/design/01intro.txt | 171 ++ .../filesystems/aufs/design/02struct.txt | 258 +++ .../filesystems/aufs/design/03atomic_open.txt | 85 + .../filesystems/aufs/design/03lookup.txt | 113 + .../filesystems/aufs/design/04branch.txt | 74 + .../filesystems/aufs/design/05wbr_policy.txt | 64 + .../filesystems/aufs/design/06dirren.dot | 31 + .../filesystems/aufs/design/06dirren.txt | 102 + .../filesystems/aufs/design/06fhsm.txt | 120 + .../filesystems/aufs/design/06mmap.txt | 72 + .../filesystems/aufs/design/06xattr.txt | 96 + .../filesystems/aufs/design/07export.txt | 58 + .../filesystems/aufs/design/08shwh.txt | 52 + .../filesystems/aufs/design/10dynop.txt | 47 + fs/aufs/Kconfig | 199 ++ fs/aufs/Makefile | 46 + fs/aufs/aufs.h | 62 + fs/aufs/branch.c | 1422 ++++++++++++ fs/aufs/branch.h | 366 +++ fs/aufs/conf.mk | 40 + fs/aufs/cpup.c | 1448 ++++++++++++ fs/aufs/cpup.h | 100 + fs/aufs/dbgaufs.c | 526 +++++ fs/aufs/dbgaufs.h | 53 + fs/aufs/dcsub.c | 225 ++ fs/aufs/dcsub.h | 137 ++ fs/aufs/debug.c | 441 ++++ fs/aufs/debug.h | 226 ++ fs/aufs/dentry.c | 1154 ++++++++++ fs/aufs/dentry.h | 268 +++ fs/aufs/dinfo.c | 554 +++++ fs/aufs/dir.c | 763 +++++++ fs/aufs/dir.h | 134 ++ fs/aufs/dirren.c | 1316 +++++++++++ fs/aufs/dirren.h | 140 ++ fs/aufs/dynop.c | 370 ++++ fs/aufs/dynop.h | 75 + fs/aufs/export.c | 838 +++++++ fs/aufs/f_op.c | 819 +++++++ fs/aufs/fhsm.c | 427 ++++ fs/aufs/file.c | 863 ++++++++ fs/aufs/file.h | 342 +++ fs/aufs/finfo.c | 149 ++ fs/aufs/fstype.h | 401 ++++ fs/aufs/hbl.h | 65 + fs/aufs/hfsnotify.c | 289 +++ fs/aufs/hfsplus.c | 60 + fs/aufs/hnotify.c | 715 ++++++ fs/aufs/i_op.c | 1505 +++++++++++++ fs/aufs/i_op_add.c | 936 ++++++++ fs/aufs/i_op_del.c | 513 +++++ fs/aufs/i_op_ren.c | 1250 +++++++++++ fs/aufs/iinfo.c | 286 +++ fs/aufs/inode.c | 529 +++++ fs/aufs/inode.h | 698 ++++++ fs/aufs/ioctl.c | 220 ++ fs/aufs/lcnt.h | 186 ++ fs/aufs/loop.c | 148 ++ fs/aufs/loop.h | 55 + fs/aufs/magic.mk | 31 + fs/aufs/module.c | 273 +++ fs/aufs/module.h | 166 ++ fs/aufs/mvdown.c | 705 ++++++ fs/aufs/opts.c | 1880 ++++++++++++++++ fs/aufs/opts.h | 225 ++ fs/aufs/plink.c | 516 +++++ fs/aufs/poll.c | 51 + fs/aufs/posix_acl.c | 105 + fs/aufs/procfs.c | 171 ++ fs/aufs/rdu.c | 382 ++++ fs/aufs/rwsem.h | 73 + fs/aufs/sbinfo.c | 314 +++ fs/aufs/super.c | 1049 +++++++++ fs/aufs/super.h | 589 +++++ fs/aufs/sysaufs.c | 93 + fs/aufs/sysaufs.h | 102 + fs/aufs/sysfs.c | 373 ++++ fs/aufs/sysrq.c | 149 ++ fs/aufs/vdir.c | 896 ++++++++ fs/aufs/vfsub.c | 902 ++++++++ fs/aufs/vfsub.h | 354 +++ fs/aufs/wbr_policy.c | 830 +++++++ fs/aufs/whout.c | 1062 +++++++++ fs/aufs/whout.h | 86 + fs/aufs/wkq.c | 372 ++++ fs/aufs/wkq.h | 89 + fs/aufs/xattr.c | 356 +++ fs/aufs/xino.c | 1966 +++++++++++++++++ include/uapi/linux/aufs_type.h | 452 ++++ 92 files changed, 37804 insertions(+) create mode 100644 Documentation/ABI/testing/debugfs-aufs create mode 100644 Documentation/ABI/testing/sysfs-aufs create mode 100644 Documentation/filesystems/aufs/README create mode 100644 Documentation/filesystems/aufs/design/01intro.txt create mode 100644 Documentation/filesystems/aufs/design/02struct.txt create mode 100644 Documentation/filesystems/aufs/design/03atomic_open.txt create mode 100644 Documentation/filesystems/aufs/design/03lookup.txt create mode 100644 Documentation/filesystems/aufs/design/04branch.txt create mode 100644 Documentation/filesystems/aufs/design/05wbr_policy.txt create mode 100644 Documentation/filesystems/aufs/design/06dirren.dot create mode 100644 Documentation/filesystems/aufs/design/06dirren.txt create mode 100644 Documentation/filesystems/aufs/design/06fhsm.txt create mode 100644 Documentation/filesystems/aufs/design/06mmap.txt create mode 100644 Documentation/filesystems/aufs/design/06xattr.txt create mode 100644 Documentation/filesystems/aufs/design/07export.txt create mode 100644 Documentation/filesystems/aufs/design/08shwh.txt create mode 100644 Documentation/filesystems/aufs/design/10dynop.txt create mode 100644 fs/aufs/Kconfig create mode 100644 fs/aufs/Makefile create mode 100644 fs/aufs/aufs.h create mode 100644 fs/aufs/branch.c create mode 100644 fs/aufs/branch.h create mode 100644 fs/aufs/conf.mk create mode 100644 fs/aufs/cpup.c create mode 100644 fs/aufs/cpup.h create mode 100644 fs/aufs/dbgaufs.c create mode 100644 fs/aufs/dbgaufs.h create mode 100644 fs/aufs/dcsub.c create mode 100644 fs/aufs/dcsub.h create mode 100644 fs/aufs/debug.c create mode 100644 fs/aufs/debug.h create mode 100644 fs/aufs/dentry.c create mode 100644 fs/aufs/dentry.h create mode 100644 fs/aufs/dinfo.c create mode 100644 fs/aufs/dir.c create mode 100644 fs/aufs/dir.h create mode 100644 fs/aufs/dirren.c create mode 100644 fs/aufs/dirren.h create mode 100644 fs/aufs/dynop.c create mode 100644 fs/aufs/dynop.h create mode 100644 fs/aufs/export.c create mode 100644 fs/aufs/f_op.c create mode 100644 fs/aufs/fhsm.c create mode 100644 fs/aufs/file.c create mode 100644 fs/aufs/file.h create mode 100644 fs/aufs/finfo.c create mode 100644 fs/aufs/fstype.h create mode 100644 fs/aufs/hbl.h create mode 100644 fs/aufs/hfsnotify.c create mode 100644 fs/aufs/hfsplus.c create mode 100644 fs/aufs/hnotify.c create mode 100644 fs/aufs/i_op.c create mode 100644 fs/aufs/i_op_add.c create mode 100644 fs/aufs/i_op_del.c create mode 100644 fs/aufs/i_op_ren.c create mode 100644 fs/aufs/iinfo.c create mode 100644 fs/aufs/inode.c create mode 100644 fs/aufs/inode.h create mode 100644 fs/aufs/ioctl.c create mode 100644 fs/aufs/lcnt.h create mode 100644 fs/aufs/loop.c create mode 100644 fs/aufs/loop.h create mode 100644 fs/aufs/magic.mk create mode 100644 fs/aufs/module.c create mode 100644 fs/aufs/module.h create mode 100644 fs/aufs/mvdown.c create mode 100644 fs/aufs/opts.c create mode 100644 fs/aufs/opts.h create mode 100644 fs/aufs/plink.c create mode 100644 fs/aufs/poll.c create mode 100644 fs/aufs/posix_acl.c create mode 100644 fs/aufs/procfs.c create mode 100644 fs/aufs/rdu.c create mode 100644 fs/aufs/rwsem.h create mode 100644 fs/aufs/sbinfo.c create mode 100644 fs/aufs/super.c create mode 100644 fs/aufs/super.h create mode 100644 fs/aufs/sysaufs.c create mode 100644 fs/aufs/sysaufs.h create mode 100644 fs/aufs/sysfs.c create mode 100644 fs/aufs/sysrq.c create mode 100644 fs/aufs/vdir.c create mode 100644 fs/aufs/vfsub.c create mode 100644 fs/aufs/vfsub.h create mode 100644 fs/aufs/wbr_policy.c create mode 100644 fs/aufs/whout.c create mode 100644 fs/aufs/whout.h create mode 100644 fs/aufs/wkq.c create mode 100644 fs/aufs/wkq.h create mode 100644 fs/aufs/xattr.c create mode 100644 fs/aufs/xino.c create mode 100644 include/uapi/linux/aufs_type.h diff --git a/Documentation/ABI/testing/debugfs-aufs b/Documentation/ABI/testing/debugfs-aufs new file mode 100644 index 0000000000000..4a6694194ba69 --- /dev/null +++ b/Documentation/ABI/testing/debugfs-aufs @@ -0,0 +1,55 @@ +What: /debug/aufs/si_/ +Date: March 2009 +Contact: J. R. Okajima +Description: + Under /debug/aufs, a directory named si_ is created + per aufs mount, where is a unique id generated + internally. + +What: /debug/aufs/si_/plink +Date: Apr 2013 +Contact: J. R. Okajima +Description: + It has three lines and shows the information about the + pseudo-link. The first line is a single number + representing a number of buckets. The second line is a + number of pseudo-links per buckets (separated by a + blank). The last line is a single number representing a + total number of psedo-links. + When the aufs mount option 'noplink' is specified, it + will show "1\n0\n0\n". + +What: /debug/aufs/si_/xib +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the consumed blocks by xib (External Inode Number + Bitmap), its block size and file size. + When the aufs mount option 'noxino' is specified, it + will be empty. About XINO files, see the aufs manual. + +What: /debug/aufs/si_/xi0, xi1 ... xiN and xiN-N +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the consumed blocks by xino (External Inode Number + Translation Table), its link count, block size and file + size. + Due to the file size limit, there may exist multiple + xino files per branch. In this case, "-N" is added to + the filename and it corresponds to the index of the + internal xino array. "-0" is omitted. + When the aufs mount option 'noxino' is specified, Those + entries won't exist. About XINO files, see the aufs + manual. + +What: /debug/aufs/si_/xigen +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the consumed blocks by xigen (External Inode + Generation Table), its block size and file size. + If CONFIG_AUFS_EXPORT is disabled, this entry will not + be created. + When the aufs mount option 'noxino' is specified, it + will be empty. About XINO files, see the aufs manual. diff --git a/Documentation/ABI/testing/sysfs-aufs b/Documentation/ABI/testing/sysfs-aufs new file mode 100644 index 0000000000000..82f9518495eae --- /dev/null +++ b/Documentation/ABI/testing/sysfs-aufs @@ -0,0 +1,31 @@ +What: /sys/fs/aufs/si_/ +Date: March 2009 +Contact: J. R. Okajima +Description: + Under /sys/fs/aufs, a directory named si_ is created + per aufs mount, where is a unique id generated + internally. + +What: /sys/fs/aufs/si_/br0, br1 ... brN +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the abolute path of a member directory (which + is called branch) in aufs, and its permission. + +What: /sys/fs/aufs/si_/brid0, brid1 ... bridN +Date: July 2013 +Contact: J. R. Okajima +Description: + It shows the id of a member directory (which is called + branch) in aufs. + +What: /sys/fs/aufs/si_/xi_path +Date: March 2009 +Contact: J. R. Okajima +Description: + It shows the abolute path of XINO (External Inode Number + Bitmap, Translation Table and Generation Table) file + even if it is the default path. + When the aufs mount option 'noxino' is specified, it + will be empty. About XINO files, see the aufs manual. diff --git a/Documentation/filesystems/aufs/README b/Documentation/filesystems/aufs/README new file mode 100644 index 0000000000000..e7291a3c66a63 --- /dev/null +++ b/Documentation/filesystems/aufs/README @@ -0,0 +1,404 @@ + +Aufs4 -- advanced multi layered unification filesystem version 4.x +http://aufs.sf.net +Junjiro R. Okajima + + +0. Introduction +---------------------------------------- +In the early days, aufs was entirely re-designed and re-implemented +Unionfs Version 1.x series. Adding many original ideas, approaches, +improvements and implementations, it becomes totally different from +Unionfs while keeping the basic features. +Recently, Unionfs Version 2.x series begin taking some of the same +approaches to aufs1's. +Unionfs is being developed by Professor Erez Zadok at Stony Brook +University and his team. + +Aufs4 supports linux-4.0 and later, and for linux-3.x series try aufs3. +If you want older kernel version support, try aufs2-2.6.git or +aufs2-standalone.git repository, aufs1 from CVS on SourceForge. + +Note: it becomes clear that "Aufs was rejected. Let's give it up." + According to Christoph Hellwig, linux rejects all union-type + filesystems but UnionMount. + + +PS. Al Viro seems have a plan to merge aufs as well as overlayfs and + UnionMount, and he pointed out an issue around a directory mutex + lock and aufs addressed it. But it is still unsure whether aufs will + be merged (or any other union solution). + + + +1. Features +---------------------------------------- +- unite several directories into a single virtual filesystem. The member + directory is called as a branch. +- you can specify the permission flags to the branch, which are 'readonly', + 'readwrite' and 'whiteout-able.' +- by upper writable branch, internal copyup and whiteout, files/dirs on + readonly branch are modifiable logically. +- dynamic branch manipulation, add, del. +- etc... + +Also there are many enhancements in aufs, such as: +- test only the highest one for the directory permission (dirperm1) +- copyup on open (coo=) +- 'move' policy for copy-up between two writable branches, after + checking free space. +- xattr, acl +- readdir(3) in userspace. +- keep inode number by external inode number table +- keep the timestamps of file/dir in internal copyup operation +- seekable directory, supporting NFS readdir. +- whiteout is hardlinked in order to reduce the consumption of inodes + on branch +- do not copyup, nor create a whiteout when it is unnecessary +- revert a single systemcall when an error occurs in aufs +- remount interface instead of ioctl +- maintain /etc/mtab by an external command, /sbin/mount.aufs. +- loopback mounted filesystem as a branch +- kernel thread for removing the dir who has a plenty of whiteouts +- support copyup sparse file (a file which has a 'hole' in it) +- default permission flags for branches +- selectable permission flags for ro branch, whether whiteout can + exist or not +- export via NFS. +- support /fs/aufs and /aufs. +- support multiple writable branches, some policies to select one + among multiple writable branches. +- a new semantics for link(2) and rename(2) to support multiple + writable branches. +- no glibc changes are required. +- pseudo hardlink (hardlink over branches) +- allow a direct access manually to a file on branch, e.g. bypassing aufs. + including NFS or remote filesystem branch. +- userspace wrapper for pathconf(3)/fpathconf(3) with _PC_LINK_MAX. +- and more... + +Currently these features are dropped temporary from aufs4. +See design/08plan.txt in detail. +- nested mount, i.e. aufs as readonly no-whiteout branch of another aufs + (robr) +- statistics of aufs thread (/sys/fs/aufs/stat) + +Features or just an idea in the future (see also design/*.txt), +- reorder the branch index without del/re-add. +- permanent xino files for NFSD +- an option for refreshing the opened files after add/del branches +- light version, without branch manipulation. (unnecessary?) +- copyup in userspace +- inotify in userspace +- readv/writev + + +2. Download +---------------------------------------- +There are three GIT trees for aufs4, aufs4-linux.git, +aufs4-standalone.git, and aufs-util.git. Note that there is no "4" in +"aufs-util.git." +While the aufs-util is always necessary, you need either of aufs4-linux +or aufs4-standalone. + +The aufs4-linux tree includes the whole linux mainline GIT tree, +git://git.kernel.org/.../torvalds/linux.git. +And you cannot select CONFIG_AUFS_FS=m for this version, eg. you cannot +build aufs4 as an external kernel module. +Several extra patches are not included in this tree. Only +aufs4-standalone tree contains them. They are described in the later +section "Configuration and Compilation." + +On the other hand, the aufs4-standalone tree has only aufs source files +and necessary patches, and you can select CONFIG_AUFS_FS=m. +But you need to apply all aufs patches manually. + +You will find GIT branches whose name is in form of "aufs4.x" where "x" +represents the linux kernel version, "linux-4.x". For instance, +"aufs4.0" is for linux-4.0. For latest "linux-4.x-rcN", use +"aufs4.x-rcN" branch. + +o aufs4-linux tree +$ git clone --reference /your/linux/git/tree \ + git://github.com/sfjro/aufs4-linux.git aufs4-linux.git +- if you don't have linux GIT tree, then remove "--reference ..." +$ cd aufs4-linux.git +$ git checkout origin/aufs4.0 + +Or You may want to directly git-pull aufs into your linux GIT tree, and +leave the patch-work to GIT. +$ cd /your/linux/git/tree +$ git remote add aufs4 git://github.com/sfjro/aufs4-linux.git +$ git fetch aufs4 +$ git checkout -b my4.0 v4.0 +$ (add your local change...) +$ git pull aufs4 aufs4.0 +- now you have v4.0 + your_changes + aufs4.0 in you my4.0 branch. +- you may need to solve some conflicts between your_changes and + aufs4.0. in this case, git-rerere is recommended so that you can + solve the similar conflicts automatically when you upgrade to 4.1 or + later in the future. + +o aufs4-standalone tree +$ git clone git://github.com/sfjro/aufs4-standalone.git aufs4-standalone.git +$ cd aufs4-standalone.git +$ git checkout origin/aufs4.0 + +o aufs-util tree +$ git clone git://git.code.sf.net/p/aufs/aufs-util aufs-util.git +- note that the public aufs-util.git is on SourceForge instead of + GitHUB. +$ cd aufs-util.git +$ git checkout origin/aufs4.0 + +Note: The 4.x-rcN branch is to be used with `rc' kernel versions ONLY. +The minor version number, 'x' in '4.x', of aufs may not always +follow the minor version number of the kernel. +Because changes in the kernel that cause the use of a new +minor version number do not always require changes to aufs-util. + +Since aufs-util has its own minor version number, you may not be +able to find a GIT branch in aufs-util for your kernel's +exact minor version number. +In this case, you should git-checkout the branch for the +nearest lower number. + +For (an unreleased) example: +If you are using "linux-4.10" and the "aufs4.10" branch +does not exist in aufs-util repository, then "aufs4.9", "aufs4.8" +or something numerically smaller is the branch for your kernel. + +Also you can view all branches by + $ git branch -a + + +3. Configuration and Compilation +---------------------------------------- +Make sure you have git-checkout'ed the correct branch. + +For aufs4-linux tree, +- enable CONFIG_AUFS_FS. +- set other aufs configurations if necessary. + +For aufs4-standalone tree, +There are several ways to build. + +1. +- apply ./aufs4-kbuild.patch to your kernel source files. +- apply ./aufs4-base.patch too. +- apply ./aufs4-mmap.patch too. +- apply ./aufs4-standalone.patch too, if you have a plan to set + CONFIG_AUFS_FS=m. otherwise you don't need ./aufs4-standalone.patch. +- copy ./{Documentation,fs,include/uapi/linux/aufs_type.h} files to your + kernel source tree. Never copy $PWD/include/uapi/linux/Kbuild. +- enable CONFIG_AUFS_FS, you can select either + =m or =y. +- and build your kernel as usual. +- install the built kernel. + Note: Since linux-3.9, every filesystem module requires an alias + "fs-". You should make sure that "fs-aufs" is listed in your + modules.aliases file if you set CONFIG_AUFS_FS=m. +- install the header files too by "make headers_install" to the + directory where you specify. By default, it is $PWD/usr. + "make help" shows a brief note for headers_install. +- and reboot your system. + +2. +- module only (CONFIG_AUFS_FS=m). +- apply ./aufs4-base.patch to your kernel source files. +- apply ./aufs4-mmap.patch too. +- apply ./aufs4-standalone.patch too. +- build your kernel, don't forget "make headers_install", and reboot. +- edit ./config.mk and set other aufs configurations if necessary. + Note: You should read $PWD/fs/aufs/Kconfig carefully which describes + every aufs configurations. +- build the module by simple "make". + Note: Since linux-3.9, every filesystem module requires an alias + "fs-". You should make sure that "fs-aufs" is listed in your + modules.aliases file. +- you can specify ${KDIR} make variable which points to your kernel + source tree. +- install the files + + run "make install" to install the aufs module, or copy the built + $PWD/aufs.ko to /lib/modules/... and run depmod -a (or reboot simply). + + run "make install_headers" (instead of headers_install) to install + the modified aufs header file (you can specify DESTDIR which is + available in aufs standalone version's Makefile only), or copy + $PWD/usr/include/linux/aufs_type.h to /usr/include/linux or wherever + you like manually. By default, the target directory is $PWD/usr. +- no need to apply aufs4-kbuild.patch, nor copying source files to your + kernel source tree. + +Note: The header file aufs_type.h is necessary to build aufs-util + as well as "make headers_install" in the kernel source tree. + headers_install is subject to be forgotten, but it is essentially + necessary, not only for building aufs-util. + You may not meet problems without headers_install in some older + version though. + +And then, +- read README in aufs-util, build and install it +- note that your distribution may contain an obsoleted version of + aufs_type.h in /usr/include/linux or something. When you build aufs + utilities, make sure that your compiler refers the correct aufs header + file which is built by "make headers_install." +- if you want to use readdir(3) in userspace or pathconf(3) wrapper, + then run "make install_ulib" too. And refer to the aufs manual in + detail. + +There several other patches in aufs4-standalone.git. They are all +optional. When you meet some problems, they will help you. +- aufs4-loopback.patch + Supports a nested loopback mount in a branch-fs. This patch is + unnecessary until aufs produces a message like "you may want to try + another patch for loopback file". +- proc_mounts.patch + When there are many mountpoints and many mount(2)/umount(2) are + running, then /proc/mounts may not show the all mountpoints. This + patch makes /proc/mounts always show the full mountpoints list. + If you don't want to apply this patch and meet such problem, then you + need to increase the value of 'ProcMounts_Times' make-variable in + aufs-util.git as a second best solution. +- vfs-ino.patch + Modifies a system global kernel internal function get_next_ino() in + order to stop assigning 0 for an inode-number. Not directly related to + aufs, but recommended generally. +- tmpfs-idr.patch + Keeps the tmpfs inode number as the lowest value. Effective to reduce + the size of aufs XINO files for tmpfs branch. Also it prevents the + duplication of inode number, which is important for backup tools and + other utilities. When you find aufs XINO files for tmpfs branch + growing too much, try this patch. +- lockdep-debug.patch + Because aufs is not only an ordinary filesystem (callee of VFS), but + also a caller of VFS functions for branch filesystems, subclassing of + the internal locks for LOCKDEP is necessary. LOCKDEP is a debugging + feature of linux kernel. If you enable CONFIG_LOCKDEP, then you will + need to apply this debug patch to expand several constant values. + If you don't know what LOCKDEP is, then you don't have apply this + patch. + + +4. Usage +---------------------------------------- +At first, make sure aufs-util are installed, and please read the aufs +manual, aufs.5 in aufs-util.git tree. +$ man -l aufs.5 + +And then, +$ mkdir /tmp/rw /tmp/aufs +# mount -t aufs -o br=/tmp/rw:${HOME} none /tmp/aufs + +Here is another example. The result is equivalent. +# mount -t aufs -o br=/tmp/rw=rw:${HOME}=ro none /tmp/aufs + Or +# mount -t aufs -o br:/tmp/rw none /tmp/aufs +# mount -o remount,append:${HOME} /tmp/aufs + +Then, you can see whole tree of your home dir through /tmp/aufs. If +you modify a file under /tmp/aufs, the one on your home directory is +not affected, instead the same named file will be newly created under +/tmp/rw. And all of your modification to a file will be applied to +the one under /tmp/rw. This is called the file based Copy on Write +(COW) method. +Aufs mount options are described in aufs.5. +If you run chroot or something and make your aufs as a root directory, +then you need to customize the shutdown script. See the aufs manual in +detail. + +Additionally, there are some sample usages of aufs which are a +diskless system with network booting, and LiveCD over NFS. +See sample dir in CVS tree on SourceForge. + + +5. Contact +---------------------------------------- +When you have any problems or strange behaviour in aufs, please let me +know with: +- /proc/mounts (instead of the output of mount(8)) +- /sys/module/aufs/* +- /sys/fs/aufs/* (if you have them) +- /debug/aufs/* (if you have them) +- linux kernel version + if your kernel is not plain, for example modified by distributor, + the url where i can download its source is necessary too. +- aufs version which was printed at loading the module or booting the + system, instead of the date you downloaded. +- configuration (define/undefine CONFIG_AUFS_xxx) +- kernel configuration or /proc/config.gz (if you have it) +- LSM (linux security module, if you are using) +- behaviour which you think to be incorrect +- actual operation, reproducible one is better +- mailto: aufs-users at lists.sourceforge.net + +Usually, I don't watch the Public Areas(Bugs, Support Requests, Patches, +and Feature Requests) on SourceForge. Please join and write to +aufs-users ML. + + +6. Acknowledgements +---------------------------------------- +Thanks to everyone who have tried and are using aufs, whoever +have reported a bug or any feedback. + +Especially donators: +Tomas Matejicek(slax.org) made a donation (much more than once). + Since Apr 2010, Tomas M (the author of Slax and Linux Live + scripts) is making "doubling" donations. + Unfortunately I cannot list all of the donators, but I really + appreciate. + It ends Aug 2010, but the ordinary donation URL is still available. + +Dai Itasaka made a donation (2007/8). +Chuck Smith made a donation (2008/4, 10 and 12). +Henk Schoneveld made a donation (2008/9). +Chih-Wei Huang, ASUS, CTC donated Eee PC 4G (2008/10). +Francois Dupoux made a donation (2008/11). +Bruno Cesar Ribas and Luis Carlos Erpen de Bona, C3SL serves public + aufs2 GIT tree (2009/2). +William Grant made a donation (2009/3). +Patrick Lane made a donation (2009/4). +The Mail Archive (mail-archive.com) made donations (2009/5). +Nippy Networks (Ed Wildgoose) made a donation (2009/7). +New Dream Network, LLC (www.dreamhost.com) made a donation (2009/11). +Pavel Pronskiy made a donation (2011/2). +Iridium and Inmarsat satellite phone retailer (www.mailasail.com), Nippy + Networks (Ed Wildgoose) made a donation for hardware (2011/3). +Max Lekomcev (DOM-TV project) made a donation (2011/7, 12, 2012/3, 6 and +11). +Sam Liddicott made a donation (2011/9). +Era Scarecrow made a donation (2013/4). +Bor Ratajc made a donation (2013/4). +Alessandro Gorreta made a donation (2013/4). +POIRETTE Marc made a donation (2013/4). +Alessandro Gorreta made a donation (2013/4). +lauri kasvandik made a donation (2013/5). +"pemasu from Finland" made a donation (2013/7). +The Parted Magic Project made a donation (2013/9 and 11). +Pavel Barta made a donation (2013/10). +Nikolay Pertsev made a donation (2014/5). +James B made a donation (2014/7 and 2015/7). +Stefano Di Biase made a donation (2014/8). +Daniel Epellei made a donation (2015/1). +OmegaPhil made a donation (2016/1, 2018/4). +Tomasz Szewczyk made a donation (2016/4). +James Burry made a donation (2016/12). +Carsten Rose made a donation (2018/9). +Porteus Kiosk made a donation (2018/10). + +Thank you very much. +Donations are always, including future donations, very important and +helpful for me to keep on developing aufs. + + +7. +---------------------------------------- +If you are an experienced user, no explanation is needed. Aufs is +just a linux filesystem. + + +Enjoy! + +# Local variables: ; +# mode: text; +# End: ; diff --git a/Documentation/filesystems/aufs/design/01intro.txt b/Documentation/filesystems/aufs/design/01intro.txt new file mode 100644 index 0000000000000..fa2c8f634c325 --- /dev/null +++ b/Documentation/filesystems/aufs/design/01intro.txt @@ -0,0 +1,171 @@ + +# Copyright (C) 2005-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Introduction +---------------------------------------- + +aufs [ei ju: ef es] | /ey-yoo-ef-es/ | [a u f s] +1. abbrev. for "advanced multi-layered unification filesystem". +2. abbrev. for "another unionfs". +3. abbrev. for "auf das" in German which means "on the" in English. + Ex. "Butter aufs Brot"(G) means "butter onto bread"(E). + But "Filesystem aufs Filesystem" is hard to understand. +4. abbrev. for "African Urban Fashion Show". + +AUFS is a filesystem with features: +- multi layered stackable unification filesystem, the member directory + is called as a branch. +- branch permission and attribute, 'readonly', 'real-readonly', + 'readwrite', 'whiteout-able', 'link-able whiteout', etc. and their + combination. +- internal "file copy-on-write". +- logical deletion, whiteout. +- dynamic branch manipulation, adding, deleting and changing permission. +- allow bypassing aufs, user's direct branch access. +- external inode number translation table and bitmap which maintains the + persistent aufs inode number. +- seekable directory, including NFS readdir. +- file mapping, mmap and sharing pages. +- pseudo-link, hardlink over branches. +- loopback mounted filesystem as a branch. +- several policies to select one among multiple writable branches. +- revert a single systemcall when an error occurs in aufs. +- and more... + + +Multi Layered Stackable Unification Filesystem +---------------------------------------------------------------------- +Most people already knows what it is. +It is a filesystem which unifies several directories and provides a +merged single directory. When users access a file, the access will be +passed/re-directed/converted (sorry, I am not sure which English word is +correct) to the real file on the member filesystem. The member +filesystem is called 'lower filesystem' or 'branch' and has a mode +'readonly' and 'readwrite.' And the deletion for a file on the lower +readonly branch is handled by creating 'whiteout' on the upper writable +branch. + +On LKML, there have been discussions about UnionMount (Jan Blunck, +Bharata B Rao and Valerie Aurora) and Unionfs (Erez Zadok). They took +different approaches to implement the merged-view. +The former tries putting it into VFS, and the latter implements as a +separate filesystem. +(If I misunderstand about these implementations, please let me know and +I shall correct it. Because it is a long time ago when I read their +source files last time). + +UnionMount's approach will be able to small, but may be hard to share +branches between several UnionMount since the whiteout in it is +implemented in the inode on branch filesystem and always +shared. According to Bharata's post, readdir does not seems to be +finished yet. +There are several missing features known in this implementations such as +- for users, the inode number may change silently. eg. copy-up. +- link(2) may break by copy-up. +- read(2) may get an obsoleted filedata (fstat(2) too). +- fcntl(F_SETLK) may be broken by copy-up. +- unnecessary copy-up may happen, for example mmap(MAP_PRIVATE) after + open(O_RDWR). + +In linux-3.18, "overlay" filesystem (formerly known as "overlayfs") was +merged into mainline. This is another implementation of UnionMount as a +separated filesystem. All the limitations and known problems which +UnionMount are equally inherited to "overlay" filesystem. + +Unionfs has a longer history. When I started implementing a stackable +filesystem (Aug 2005), it already existed. It has virtual super_block, +inode, dentry and file objects and they have an array pointing lower +same kind objects. After contributing many patches for Unionfs, I +re-started my project AUFS (Jun 2006). + +In AUFS, the structure of filesystem resembles to Unionfs, but I +implemented my own ideas, approaches and enhancements and it became +totally different one. + +Comparing DM snapshot and fs based implementation +- the number of bytes to be copied between devices is much smaller. +- the type of filesystem must be one and only. +- the fs must be writable, no readonly fs, even for the lower original + device. so the compression fs will not be usable. but if we use + loopback mount, we may address this issue. + for instance, + mount /cdrom/squashfs.img /sq + losetup /sq/ext2.img + losetup /somewhere/cow + dmsetup "snapshot /dev/loop0 /dev/loop1 ..." +- it will be difficult (or needs more operations) to extract the + difference between the original device and COW. +- DM snapshot-merge may help a lot when users try merging. in the + fs-layer union, users will use rsync(1). + +You may want to read my old paper "Filesystems in LiveCD" +(http://aufs.sourceforge.net/aufs2/report/sq/sq.pdf). + + +Several characters/aspects/persona of aufs +---------------------------------------------------------------------- + +Aufs has several characters, aspects or persona. +1. a filesystem, callee of VFS helper +2. sub-VFS, caller of VFS helper for branches +3. a virtual filesystem which maintains persistent inode number +4. reader/writer of files on branches such like an application + +1. Callee of VFS Helper +As an ordinary linux filesystem, aufs is a callee of VFS. For instance, +unlink(2) from an application reaches sys_unlink() kernel function and +then vfs_unlink() is called. vfs_unlink() is one of VFS helper and it +calls filesystem specific unlink operation. Actually aufs implements the +unlink operation but it behaves like a redirector. + +2. Caller of VFS Helper for Branches +aufs_unlink() passes the unlink request to the branch filesystem as if +it were called from VFS. So the called unlink operation of the branch +filesystem acts as usual. As a caller of VFS helper, aufs should handle +every necessary pre/post operation for the branch filesystem. +- acquire the lock for the parent dir on a branch +- lookup in a branch +- revalidate dentry on a branch +- mnt_want_write() for a branch +- vfs_unlink() for a branch +- mnt_drop_write() for a branch +- release the lock on a branch + +3. Persistent Inode Number +One of the most important issue for a filesystem is to maintain inode +numbers. This is particularly important to support exporting a +filesystem via NFS. Aufs is a virtual filesystem which doesn't have a +backend block device for its own. But some storage is necessary to +keep and maintain the inode numbers. It may be a large space and may not +suit to keep in memory. Aufs rents some space from its first writable +branch filesystem (by default) and creates file(s) on it. These files +are created by aufs internally and removed soon (currently) keeping +opened. +Note: Because these files are removed, they are totally gone after + unmounting aufs. It means the inode numbers are not persistent + across unmount or reboot. I have a plan to make them really + persistent which will be important for aufs on NFS server. + +4. Read/Write Files Internally (copy-on-write) +Because a branch can be readonly, when you write a file on it, aufs will +"copy-up" it to the upper writable branch internally. And then write the +originally requested thing to the file. Generally kernel doesn't +open/read/write file actively. In aufs, even a single write may cause a +internal "file copy". This behaviour is very similar to cp(1) command. + +Some people may think it is better to pass such work to user space +helper, instead of doing in kernel space. Actually I am still thinking +about it. But currently I have implemented it in kernel space. diff --git a/Documentation/filesystems/aufs/design/02struct.txt b/Documentation/filesystems/aufs/design/02struct.txt new file mode 100644 index 0000000000000..e1e531a97b1a7 --- /dev/null +++ b/Documentation/filesystems/aufs/design/02struct.txt @@ -0,0 +1,258 @@ + +# Copyright (C) 2005-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Basic Aufs Internal Structure + +Superblock/Inode/Dentry/File Objects +---------------------------------------------------------------------- +As like an ordinary filesystem, aufs has its own +superblock/inode/dentry/file objects. All these objects have a +dynamically allocated array and store the same kind of pointers to the +lower filesystem, branch. +For example, when you build a union with one readwrite branch and one +readonly, mounted /au, /rw and /ro respectively. +- /au = /rw + /ro +- /ro/fileA exists but /rw/fileA + +Aufs lookup operation finds /ro/fileA and gets dentry for that. These +pointers are stored in a aufs dentry. The array in aufs dentry will be, +- [0] = NULL (because /rw/fileA doesn't exist) +- [1] = /ro/fileA + +This style of an array is essentially same to the aufs +superblock/inode/dentry/file objects. + +Because aufs supports manipulating branches, ie. add/delete/change +branches dynamically, these objects has its own generation. When +branches are changed, the generation in aufs superblock is +incremented. And a generation in other object are compared when it is +accessed. When a generation in other objects are obsoleted, aufs +refreshes the internal array. + + +Superblock +---------------------------------------------------------------------- +Additionally aufs superblock has some data for policies to select one +among multiple writable branches, XIB files, pseudo-links and kobject. +See below in detail. +About the policies which supports copy-down a directory, see +wbr_policy.txt too. + + +Branch and XINO(External Inode Number Translation Table) +---------------------------------------------------------------------- +Every branch has its own xino (external inode number translation table) +file. The xino file is created and unlinked by aufs internally. When two +members of a union exist on the same filesystem, they share the single +xino file. +The struct of a xino file is simple, just a sequence of aufs inode +numbers which is indexed by the lower inode number. +In the above sample, assume the inode number of /ro/fileA is i111 and +aufs assigns the inode number i999 for fileA. Then aufs writes 999 as +4(8) bytes at 111 * 4(8) bytes offset in the xino file. + +When the inode numbers are not contiguous, the xino file will be sparse +which has a hole in it and doesn't consume as much disk space as it +might appear. If your branch filesystem consumes disk space for such +holes, then you should specify 'xino=' option at mounting aufs. + +Aufs has a mount option to free the disk blocks for such holes in XINO +files on tmpfs or ramdisk. But it is not so effective actually. If you +meet a problem of disk shortage due to XINO files, then you should try +"tmpfs-ino.patch" (and "vfs-ino.patch" too) in aufs4-standalone.git. +The patch localizes the assignment inumbers per tmpfs-mount and avoid +the holes in XINO files. + +Also a writable branch has three kinds of "whiteout bases". All these +are existed when the branch is joined to aufs, and their names are +whiteout-ed doubly, so that users will never see their names in aufs +hierarchy. +1. a regular file which will be hardlinked to all whiteouts. +2. a directory to store a pseudo-link. +3. a directory to store an "orphan"-ed file temporary. + +1. Whiteout Base + When you remove a file on a readonly branch, aufs handles it as a + logical deletion and creates a whiteout on the upper writable branch + as a hardlink of this file in order not to consume inode on the + writable branch. +2. Pseudo-link Dir + See below, Pseudo-link. +3. Step-Parent Dir + When "fileC" exists on the lower readonly branch only and it is + opened and removed with its parent dir, and then user writes + something into it, then aufs copies-up fileC to this + directory. Because there is no other dir to store fileC. After + creating a file under this dir, the file is unlinked. + +Because aufs supports manipulating branches, ie. add/delete/change +dynamically, a branch has its own id. When the branch order changes, +aufs finds the new index by searching the branch id. + + +Pseudo-link +---------------------------------------------------------------------- +Assume "fileA" exists on the lower readonly branch only and it is +hardlinked to "fileB" on the branch. When you write something to fileA, +aufs copies-up it to the upper writable branch. Additionally aufs +creates a hardlink under the Pseudo-link Directory of the writable +branch. The inode of a pseudo-link is kept in aufs super_block as a +simple list. If fileB is read after unlinking fileA, aufs returns +filedata from the pseudo-link instead of the lower readonly +branch. Because the pseudo-link is based upon the inode, to keep the +inode number by xino (see above) is essentially necessary. + +All the hardlinks under the Pseudo-link Directory of the writable branch +should be restored in a proper location later. Aufs provides a utility +to do this. The userspace helpers executed at remounting and unmounting +aufs by default. +During this utility is running, it puts aufs into the pseudo-link +maintenance mode. In this mode, only the process which began the +maintenance mode (and its child processes) is allowed to operate in +aufs. Some other processes which are not related to the pseudo-link will +be allowed to run too, but the rest have to return an error or wait +until the maintenance mode ends. If a process already acquires an inode +mutex (in VFS), it has to return an error. + + +XIB(external inode number bitmap) +---------------------------------------------------------------------- +Addition to the xino file per a branch, aufs has an external inode number +bitmap in a superblock object. It is also an internal file such like a +xino file. +It is a simple bitmap to mark whether the aufs inode number is in-use or +not. +To reduce the file I/O, aufs prepares a single memory page to cache xib. + +As well as XINO files, aufs has a feature to truncate/refresh XIB to +reduce the number of consumed disk blocks for these files. + + +Virtual or Vertical Dir, and Readdir in Userspace +---------------------------------------------------------------------- +In order to support multiple layers (branches), aufs readdir operation +constructs a virtual dir block on memory. For readdir, aufs calls +vfs_readdir() internally for each dir on branches, merges their entries +with eliminating the whiteout-ed ones, and sets it to file (dir) +object. So the file object has its entry list until it is closed. The +entry list will be updated when the file position is zero and becomes +obsoleted. This decision is made in aufs automatically. + +The dynamically allocated memory block for the name of entries has a +unit of 512 bytes (by default) and stores the names contiguously (no +padding). Another block for each entry is handled by kmem_cache too. +During building dir blocks, aufs creates hash list and judging whether +the entry is whiteouted by its upper branch or already listed. +The merged result is cached in the corresponding inode object and +maintained by a customizable life-time option. + +Some people may call it can be a security hole or invite DoS attack +since the opened and once readdir-ed dir (file object) holds its entry +list and becomes a pressure for system memory. But I'd say it is similar +to files under /proc or /sys. The virtual files in them also holds a +memory page (generally) while they are opened. When an idea to reduce +memory for them is introduced, it will be applied to aufs too. +For those who really hate this situation, I've developed readdir(3) +library which operates this merging in userspace. You just need to set +LD_PRELOAD environment variable, and aufs will not consume no memory in +kernel space for readdir(3). + + +Workqueue +---------------------------------------------------------------------- +Aufs sometimes requires privilege access to a branch. For instance, +in copy-up/down operation. When a user process is going to make changes +to a file which exists in the lower readonly branch only, and the mode +of one of ancestor directories may not be writable by a user +process. Here aufs copy-up the file with its ancestors and they may +require privilege to set its owner/group/mode/etc. +This is a typical case of a application character of aufs (see +Introduction). + +Aufs uses workqueue synchronously for this case. It creates its own +workqueue. The workqueue is a kernel thread and has privilege. Aufs +passes the request to call mkdir or write (for example), and wait for +its completion. This approach solves a problem of a signal handler +simply. +If aufs didn't adopt the workqueue and changed the privilege of the +process, then the process may receive the unexpected SIGXFSZ or other +signals. + +Also aufs uses the system global workqueue ("events" kernel thread) too +for asynchronous tasks, such like handling inotify/fsnotify, re-creating a +whiteout base and etc. This is unrelated to a privilege. +Most of aufs operation tries acquiring a rw_semaphore for aufs +superblock at the beginning, at the same time waits for the completion +of all queued asynchronous tasks. + + +Whiteout +---------------------------------------------------------------------- +The whiteout in aufs is very similar to Unionfs's. That is represented +by its filename. UnionMount takes an approach of a file mode, but I am +afraid several utilities (find(1) or something) will have to support it. + +Basically the whiteout represents "logical deletion" which stops aufs to +lookup further, but also it represents "dir is opaque" which also stop +further lookup. + +In aufs, rmdir(2) and rename(2) for dir uses whiteout alternatively. +In order to make several functions in a single systemcall to be +revertible, aufs adopts an approach to rename a directory to a temporary +unique whiteouted name. +For example, in rename(2) dir where the target dir already existed, aufs +renames the target dir to a temporary unique whiteouted name before the +actual rename on a branch, and then handles other actions (make it opaque, +update the attributes, etc). If an error happens in these actions, aufs +simply renames the whiteouted name back and returns an error. If all are +succeeded, aufs registers a function to remove the whiteouted unique +temporary name completely and asynchronously to the system global +workqueue. + + +Copy-up +---------------------------------------------------------------------- +It is a well-known feature or concept. +When user modifies a file on a readonly branch, aufs operate "copy-up" +internally and makes change to the new file on the upper writable branch. +When the trigger systemcall does not update the timestamps of the parent +dir, aufs reverts it after copy-up. + + +Move-down (aufs3.9 and later) +---------------------------------------------------------------------- +"Copy-up" is one of the essential feature in aufs. It copies a file from +the lower readonly branch to the upper writable branch when a user +changes something about the file. +"Move-down" is an opposite action of copy-up. Basically this action is +ran manually instead of automatically and internally. +For desgin and implementation, aufs has to consider these issues. +- whiteout for the file may exist on the lower branch. +- ancestor directories may not exist on the lower branch. +- diropq for the ancestor directories may exist on the upper branch. +- free space on the lower branch will reduce. +- another access to the file may happen during moving-down, including + UDBA (see "Revalidate Dentry and UDBA"). +- the file should not be hard-linked nor pseudo-linked. they should be + handled by auplink utility later. + +Sometimes users want to move-down a file from the upper writable branch +to the lower readonly or writable branch. For instance, +- the free space of the upper writable branch is going to run out. +- create a new intermediate branch between the upper and lower branch. +- etc. + +For this purpose, use "aumvdown" command in aufs-util.git. diff --git a/Documentation/filesystems/aufs/design/03atomic_open.txt b/Documentation/filesystems/aufs/design/03atomic_open.txt new file mode 100644 index 0000000000000..8e3914edfe122 --- /dev/null +++ b/Documentation/filesystems/aufs/design/03atomic_open.txt @@ -0,0 +1,85 @@ + +# Copyright (C) 2015-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Support for a branch who has its ->atomic_open() +---------------------------------------------------------------------- +The filesystems who implement its ->atomic_open() are not majority. For +example NFSv4 does, and aufs should call NFSv4 ->atomic_open, +particularly for open(O_CREAT|O_EXCL, 0400) case. Other than +->atomic_open(), NFSv4 returns an error for this open(2). While I am not +sure whether all filesystems who have ->atomic_open() behave like this, +but NFSv4 surely returns the error. + +In order to support ->atomic_open() for aufs, there are a few +approaches. + +A. Introduce aufs_atomic_open() + - calls one of VFS:do_last(), lookup_open() or atomic_open() for + branch fs. +B. Introduce aufs_atomic_open() calling create, open and chmod. this is + an aufs user Pip Cet's approach + - calls aufs_create(), VFS finish_open() and notify_change(). + - pass fake-mode to finish_open(), and then correct the mode by + notify_change(). +C. Extend aufs_open() to call branch fs's ->atomic_open() + - no aufs_atomic_open(). + - aufs_lookup() registers the TID to an aufs internal object. + - aufs_create() does nothing when the matching TID is registered, but + registers the mode. + - aufs_open() calls branch fs's ->atomic_open() when the matching + TID is registered. +D. Extend aufs_open() to re-try branch fs's ->open() with superuser's + credential + - no aufs_atomic_open(). + - aufs_create() registers the TID to an internal object. this info + represents "this process created this file just now." + - when aufs gets EACCES from branch fs's ->open(), then confirm the + registered TID and re-try open() with superuser's credential. + +Pros and cons for each approach. + +A. + - straightforward but highly depends upon VFS internal. + - the atomic behavaiour is kept. + - some of parameters such as nameidata are hard to reproduce for + branch fs. + - large overhead. +B. + - easy to implement. + - the atomic behavaiour is lost. +C. + - the atomic behavaiour is kept. + - dirty and tricky. + - VFS checks whether the file is created correctly after calling + ->create(), which means this approach doesn't work. +D. + - easy to implement. + - the atomic behavaiour is lost. + - to open a file with superuser's credential and give it to a user + process is a bad idea, since the file object keeps the credential + in it. It may affect LSM or something. This approach doesn't work + either. + +The approach A is ideal, but it hard to implement. So here is a +variation of A, which is to be implemented. + +A-1. Introduce aufs_atomic_open() + - calls branch fs ->atomic_open() if exists. otherwise calls + vfs_create() and finish_open(). + - the demerit is that the several checks after branch fs + ->atomic_open() are lost. in the ordinary case, the checks are + done by VFS:do_last(), lookup_open() and atomic_open(). some can + be implemented in aufs, but not all I am afraid. diff --git a/Documentation/filesystems/aufs/design/03lookup.txt b/Documentation/filesystems/aufs/design/03lookup.txt new file mode 100644 index 0000000000000..9e08050245f06 --- /dev/null +++ b/Documentation/filesystems/aufs/design/03lookup.txt @@ -0,0 +1,113 @@ + +# Copyright (C) 2005-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Lookup in a Branch +---------------------------------------------------------------------- +Since aufs has a character of sub-VFS (see Introduction), it operates +lookup for branches as VFS does. It may be a heavy work. But almost all +lookup operation in aufs is the simplest case, ie. lookup only an entry +directly connected to its parent. Digging down the directory hierarchy +is unnecessary. VFS has a function lookup_one_len() for that use, and +aufs calls it. + +When a branch is a remote filesystem, aufs basically relies upon its +->d_revalidate(), also aufs forces the hardest revalidate tests for +them. +For d_revalidate, aufs implements three levels of revalidate tests. See +"Revalidate Dentry and UDBA" in detail. + + +Test Only the Highest One for the Directory Permission (dirperm1 option) +---------------------------------------------------------------------- +Let's try case study. +- aufs has two branches, upper readwrite and lower readonly. + /au = /rw + /ro +- "dirA" exists under /ro, but /rw. and its mode is 0700. +- user invoked "chmod a+rx /au/dirA" +- the internal copy-up is activated and "/rw/dirA" is created and its + permission bits are set to world readable. +- then "/au/dirA" becomes world readable? + +In this case, /ro/dirA is still 0700 since it exists in readonly branch, +or it may be a natively readonly filesystem. If aufs respects the lower +branch, it should not respond readdir request from other users. But user +allowed it by chmod. Should really aufs rejects showing the entries +under /ro/dirA? + +To be honest, I don't have a good solution for this case. So aufs +implements 'dirperm1' and 'nodirperm1' mount options, and leave it to +users. +When dirperm1 is specified, aufs checks only the highest one for the +directory permission, and shows the entries. Otherwise, as usual, checks +every dir existing on all branches and rejects the request. + +As a side effect, dirperm1 option improves the performance of aufs +because the number of permission check is reduced when the number of +branch is many. + + +Revalidate Dentry and UDBA (User's Direct Branch Access) +---------------------------------------------------------------------- +Generally VFS helpers re-validate a dentry as a part of lookup. +0. digging down the directory hierarchy. +1. lock the parent dir by its i_mutex. +2. lookup the final (child) entry. +3. revalidate it. +4. call the actual operation (create, unlink, etc.) +5. unlock the parent dir + +If the filesystem implements its ->d_revalidate() (step 3), then it is +called. Actually aufs implements it and checks the dentry on a branch is +still valid. +But it is not enough. Because aufs has to release the lock for the +parent dir on a branch at the end of ->lookup() (step 2) and +->d_revalidate() (step 3) while the i_mutex of the aufs dir is still +held by VFS. +If the file on a branch is changed directly, eg. bypassing aufs, after +aufs released the lock, then the subsequent operation may cause +something unpleasant result. + +This situation is a result of VFS architecture, ->lookup() and +->d_revalidate() is separated. But I never say it is wrong. It is a good +design from VFS's point of view. It is just not suitable for sub-VFS +character in aufs. + +Aufs supports such case by three level of revalidation which is +selectable by user. +1. Simple Revalidate + Addition to the native flow in VFS's, confirm the child-parent + relationship on the branch just after locking the parent dir on the + branch in the "actual operation" (step 4). When this validation + fails, aufs returns EBUSY. ->d_revalidate() (step 3) in aufs still + checks the validation of the dentry on branches. +2. Monitor Changes Internally by Inotify/Fsnotify + Addition to above, in the "actual operation" (step 4) aufs re-lookup + the dentry on the branch, and returns EBUSY if it finds different + dentry. + Additionally, aufs sets the inotify/fsnotify watch for every dir on branches + during it is in cache. When the event is notified, aufs registers a + function to kernel 'events' thread by schedule_work(). And the + function sets some special status to the cached aufs dentry and inode + private data. If they are not cached, then aufs has nothing to + do. When the same file is accessed through aufs (step 0-3) later, + aufs will detect the status and refresh all necessary data. + In this mode, aufs has to ignore the event which is fired by aufs + itself. +3. No Extra Validation + This is the simplest test and doesn't add any additional revalidation + test, and skip the revalidation in step 4. It is useful and improves + aufs performance when system surely hide the aufs branches from user, + by over-mounting something (or another method). diff --git a/Documentation/filesystems/aufs/design/04branch.txt b/Documentation/filesystems/aufs/design/04branch.txt new file mode 100644 index 0000000000000..d17f405c656dc --- /dev/null +++ b/Documentation/filesystems/aufs/design/04branch.txt @@ -0,0 +1,74 @@ + +# Copyright (C) 2005-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Branch Manipulation + +Since aufs supports dynamic branch manipulation, ie. add/remove a branch +and changing its permission/attribute, there are a lot of works to do. + + +Add a Branch +---------------------------------------------------------------------- +o Confirm the adding dir exists outside of aufs, including loopback + mount, and its various attributes. +o Initialize the xino file and whiteout bases if necessary. + See struct.txt. + +o Check the owner/group/mode of the directory + When the owner/group/mode of the adding directory differs from the + existing branch, aufs issues a warning because it may impose a + security risk. + For example, when a upper writable branch has a world writable empty + top directory, a malicious user can create any files on the writable + branch directly, like copy-up and modify manually. If something like + /etc/{passwd,shadow} exists on the lower readonly branch but the upper + writable branch, and the writable branch is world-writable, then a + malicious guy may create /etc/passwd on the writable branch directly + and the infected file will be valid in aufs. + I am afraid it can be a security issue, but aufs can do nothing except + producing a warning. + + +Delete a Branch +---------------------------------------------------------------------- +o Confirm the deleting branch is not busy + To be general, there is one merit to adopt "remount" interface to + manipulate branches. It is to discard caches. At deleting a branch, + aufs checks the still cached (and connected) dentries and inodes. If + there are any, then they are all in-use. An inode without its + corresponding dentry can be alive alone (for example, inotify/fsnotify case). + + For the cached one, aufs checks whether the same named entry exists on + other branches. + If the cached one is a directory, because aufs provides a merged view + to users, as long as one dir is left on any branch aufs can show the + dir to users. In this case, the branch can be removed from aufs. + Otherwise aufs rejects deleting the branch. + + If any file on the deleting branch is opened by aufs, then aufs + rejects deleting. + + +Modify the Permission of a Branch +---------------------------------------------------------------------- +o Re-initialize or remove the xino file and whiteout bases if necessary. + See struct.txt. + +o rw --> ro: Confirm the modifying branch is not busy + Aufs rejects the request if any of these conditions are true. + - a file on the branch is mmap-ed. + - a regular file on the branch is opened for write and there is no + same named entry on the upper branch. diff --git a/Documentation/filesystems/aufs/design/05wbr_policy.txt b/Documentation/filesystems/aufs/design/05wbr_policy.txt new file mode 100644 index 0000000000000..85ac715776591 --- /dev/null +++ b/Documentation/filesystems/aufs/design/05wbr_policy.txt @@ -0,0 +1,64 @@ + +# Copyright (C) 2005-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Policies to Select One among Multiple Writable Branches +---------------------------------------------------------------------- +When the number of writable branch is more than one, aufs has to decide +the target branch for file creation or copy-up. By default, the highest +writable branch which has the parent (or ancestor) dir of the target +file is chosen (top-down-parent policy). +By user's request, aufs implements some other policies to select the +writable branch, for file creation several policies, round-robin, +most-free-space, and other policies. For copy-up, top-down-parent, +bottom-up-parent, bottom-up and others. + +As expected, the round-robin policy selects the branch in circular. When +you have two writable branches and creates 10 new files, 5 files will be +created for each branch. mkdir(2) systemcall is an exception. When you +create 10 new directories, all will be created on the same branch. +And the most-free-space policy selects the one which has most free +space among the writable branches. The amount of free space will be +checked by aufs internally, and users can specify its time interval. + +The policies for copy-up is more simple, +top-down-parent is equivalent to the same named on in create policy, +bottom-up-parent selects the writable branch where the parent dir +exists and the nearest upper one from the copyup-source, +bottom-up selects the nearest upper writable branch from the +copyup-source, regardless the existence of the parent dir. + +There are some rules or exceptions to apply these policies. +- If there is a readonly branch above the policy-selected branch and + the parent dir is marked as opaque (a variation of whiteout), or the + target (creating) file is whiteout-ed on the upper readonly branch, + then the result of the policy is ignored and the target file will be + created on the nearest upper writable branch than the readonly branch. +- If there is a writable branch above the policy-selected branch and + the parent dir is marked as opaque or the target file is whiteouted + on the branch, then the result of the policy is ignored and the target + file will be created on the highest one among the upper writable + branches who has diropq or whiteout. In case of whiteout, aufs removes + it as usual. +- link(2) and rename(2) systemcalls are exceptions in every policy. + They try selecting the branch where the source exists as possible + since copyup a large file will take long time. If it can't be, + ie. the branch where the source exists is readonly, then they will + follow the copyup policy. +- There is an exception for rename(2) when the target exists. + If the rename target exists, aufs compares the index of the branches + where the source and the target exists and selects the higher + one. If the selected branch is readonly, then aufs follows the + copyup policy. diff --git a/Documentation/filesystems/aufs/design/06dirren.dot b/Documentation/filesystems/aufs/design/06dirren.dot new file mode 100644 index 0000000000000..2d62bb6dd55fd --- /dev/null +++ b/Documentation/filesystems/aufs/design/06dirren.dot @@ -0,0 +1,31 @@ + +// to view this graph, run dot(1) command in GRAPHVIZ. + +digraph G { +node [shape=box]; +whinfo [label="detailed info file\n(lower_brid_root-hinum, h_inum, namelen, old name)"]; + +node [shape=oval]; + +aufs_rename -> whinfo [label="store/remove"]; + +node [shape=oval]; +inode_list [label="h_inum list in branch\ncache"]; + +node [shape=box]; +whinode [label="h_inum list file"]; + +node [shape=oval]; +brmgmt [label="br_add/del/mod/umount"]; + +brmgmt -> inode_list [label="create/remove"]; +brmgmt -> whinode [label="load/store"]; + +inode_list -> whinode [style=dashed,dir=both]; + +aufs_rename -> inode_list [label="add/del"]; + +aufs_lookup -> inode_list [label="search"]; + +aufs_lookup -> whinfo [label="load/remove"]; +} diff --git a/Documentation/filesystems/aufs/design/06dirren.txt b/Documentation/filesystems/aufs/design/06dirren.txt new file mode 100644 index 0000000000000..38ae77b2c842d --- /dev/null +++ b/Documentation/filesystems/aufs/design/06dirren.txt @@ -0,0 +1,102 @@ + +# Copyright (C) 2017-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Special handling for renaming a directory (DIRREN) +---------------------------------------------------------------------- +First, let's assume we have a simple usecase. + +- /u = /rw + /ro +- /rw/dirA exists +- /ro/dirA and /ro/dirA/file exist too +- there is no dirB on both branches +- a user issues rename("dirA", "dirB") + +Now, what should aufs behave against this rename(2)? +There are a few possible cases. + +A. returns EROFS. + since dirA exists on a readonly branch which cannot be renamed. +B. returns EXDEV. + it is possible to copy-up dirA (only the dir itself), but the child + entries ("file" in this case) should not be. it must be a bad + approach to copy-up recursively. +C. returns a success. + even the branch /ro is readonly, aufs tries renaming it. Obviously it + is a violation of aufs' policy. +D. construct an extra information which indicates that /ro/dirA should + be handled as the name of dirB. + overlayfs has a similar feature called REDIRECT. + +Until now, aufs implements the case B only which returns EXDEV, and +expects the userspace application behaves like mv(1) which tries +issueing rename(2) recursively. + +A new aufs feature called DIRREN is introduced which implements the case +D. There are several "extra information" added. + +1. detailed info per renamed directory + path: /rw/dirB/$AUFS_WH_DR_INFO_PFX. +2. the inode-number list of directories on a branch + path: /rw/dirB/$AUFS_WH_DR_BRHINO + +The filename of "detailed info per directory" represents the lower +branch, and its format is +- a type of the branch id + one of these. + + uuid (not implemented yet) + + fsid + + dev +- the inode-number of the branch root dir + +And it contains these info in a single regular file. +- magic number +- branch's inode-number of the logically renamed dir +- the name of the before-renamed dir + +The "detailed info per directory" file is created in aufs rename(2), and +loaded in any lookup. +The info is considered in lookup for the matching case only. Here +"matching" means that the root of branch (in the info filename) is same +to the current looking-up branch. After looking-up the before-renamed +name, the inode-number is compared. And the matched dentry is used. + +The "inode-number list of directories" is a regular file which contains +simply the inode-numbers on the branch. The file is created or updated +in removing the branch, and loaded in adding the branch. Its lifetime is +equal to the branch. +The list is refered in lookup, and when the current target inode is +found in the list, the aufs tries loading the "detailed info per +directory" and get the changed and valid name of the dir. + +Theoretically these "extra informaiton" may be able to be put into XATTR +in the dir inode. But aufs doesn't choose this way because +1. XATTR may not be supported by the branch (or its configuration) +2. XATTR may have its size limit. +3. XATTR may be less easy to convert than a regular file, when the + format of the info is changed in the future. +At the same time, I agree that the regular file approach is much slower +than XATTR approach. So, in the future, aufs may take the XATTR or other +better approach. + +This DIRREN feature is enabled by aufs configuration, and is activated +by a new mount option. + +For the more complicated case, there is a work with UDBA option, which +is to dected the direct access to the branches (by-passing aufs) and to +maintain the cashes in aufs. Since a single cached aufs dentry may +contains two names, before- and after-rename, the name comparision in +UDBA handler may not work correctly. In this case, the behaviour will be +equivalen to udba=reval case. diff --git a/Documentation/filesystems/aufs/design/06fhsm.txt b/Documentation/filesystems/aufs/design/06fhsm.txt new file mode 100644 index 0000000000000..e3870dfdd28fe --- /dev/null +++ b/Documentation/filesystems/aufs/design/06fhsm.txt @@ -0,0 +1,120 @@ + +# Copyright (C) 2011-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +File-based Hierarchical Storage Management (FHSM) +---------------------------------------------------------------------- +Hierarchical Storage Management (or HSM) is a well-known feature in the +storage world. Aufs provides this feature as file-based with multiple +writable branches, based upon the principle of "Colder, the Lower". +Here the word "colder" means that the less used files, and "lower" means +that the position in the order of the stacked branches vertically. +These multiple writable branches are prioritized, ie. the topmost one +should be the fastest drive and be used heavily. + +o Characters in aufs FHSM story +- aufs itself and a new branch attribute. +- a new ioctl interface to move-down and to establish a connection with + the daemon ("move-down" is a converse of "copy-up"). +- userspace tool and daemon. + +The userspace daemon establishes a connection with aufs and waits for +the notification. The notified information is very similar to struct +statfs containing the number of consumed blocks and inodes. +When the consumed blocks/inodes of a branch exceeds the user-specified +upper watermark, the daemon activates its move-down process until the +consumed blocks/inodes reaches the user-specified lower watermark. + +The actual move-down is done by aufs based upon the request from +user-space since we need to maintain the inode number and the internal +pointer arrays in aufs. + +Currently aufs FHSM handles the regular files only. Additionally they +must not be hard-linked nor pseudo-linked. + + +o Cowork of aufs and the user-space daemon + During the userspace daemon established the connection, aufs sends a + small notification to it whenever aufs writes something into the + writable branch. But it may cost high since aufs issues statfs(2) + internally. So user can specify a new option to cache the + info. Actually the notification is controlled by these factors. + + the specified cache time. + + classified as "force" by aufs internally. + Until the specified time expires, aufs doesn't send the info + except the forced cases. When aufs decide forcing, the info is always + notified to userspace. + For example, the number of free inodes is generally large enough and + the shortage of it happens rarely. So aufs doesn't force the + notification when creating a new file, directory and others. This is + the typical case which aufs doesn't force. + When aufs writes the actual filedata and the files consumes any of new + blocks, the aufs forces notifying. + + +o Interfaces in aufs +- New branch attribute. + + fhsm + Specifies that the branch is managed by FHSM feature. In other word, + participant in the FHSM. + When nofhsm is set to the branch, it will not be the source/target + branch of the move-down operation. This attribute is set + independently from coo and moo attributes, and if you want full + FHSM, you should specify them as well. +- New mount option. + + fhsm_sec + Specifies a second to suppress many less important info to be + notified. +- New ioctl. + + AUFS_CTL_FHSM_FD + create a new file descriptor which userspace can read the notification + (a subset of struct statfs) from aufs. +- Module parameter 'brs' + It has to be set to 1. Otherwise the new mount option 'fhsm' will not + be set. +- mount helpers /sbin/mount.aufs and /sbin/umount.aufs + When there are two or more branches with fhsm attributes, + /sbin/mount.aufs invokes the user-space daemon and /sbin/umount.aufs + terminates it. As a result of remounting and branch-manipulation, the + number of branches with fhsm attribute can be one. In this case, + /sbin/mount.aufs will terminate the user-space daemon. + + +Finally the operation is done as these steps in kernel-space. +- make sure that, + + no one else is using the file. + + the file is not hard-linked. + + the file is not pseudo-linked. + + the file is a regular file. + + the parent dir is not opaqued. +- find the target writable branch. +- make sure the file is not whiteout-ed by the upper (than the target) + branch. +- make the parent dir on the target branch. +- mutex lock the inode on the branch. +- unlink the whiteout on the target branch (if exists). +- lookup and create the whiteout-ed temporary name on the target branch. +- copy the file as the whiteout-ed temporary name on the target branch. +- rename the whiteout-ed temporary name to the original name. +- unlink the file on the source branch. +- maintain the internal pointer array and the external inode number + table (XINO). +- maintain the timestamps and other attributes of the parent dir and the + file. + +And of course, in every step, an error may happen. So the operation +should restore the original file state after an error happens. diff --git a/Documentation/filesystems/aufs/design/06mmap.txt b/Documentation/filesystems/aufs/design/06mmap.txt new file mode 100644 index 0000000000000..e63b044204ea6 --- /dev/null +++ b/Documentation/filesystems/aufs/design/06mmap.txt @@ -0,0 +1,72 @@ + +# Copyright (C) 2005-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +mmap(2) -- File Memory Mapping +---------------------------------------------------------------------- +In aufs, the file-mapped pages are handled by a branch fs directly, no +interaction with aufs. It means aufs_mmap() calls the branch fs's +->mmap(). +This approach is simple and good, but there is one problem. +Under /proc, several entries show the mmapped files by its path (with +device and inode number), and the printed path will be the path on the +branch fs's instead of virtual aufs's. +This is not a problem in most cases, but some utilities lsof(1) (and its +user) may expect the path on aufs. + +To address this issue, aufs adds a new member called vm_prfile in struct +vm_area_struct (and struct vm_region). The original vm_file points to +the file on the branch fs in order to handle everything correctly as +usual. The new vm_prfile points to a virtual file in aufs, and the +show-functions in procfs refers to vm_prfile if it is set. +Also we need to maintain several other places where touching vm_file +such like +- fork()/clone() copies vma and the reference count of vm_file is + incremented. +- merging vma maintains the ref count too. + +This is not a good approach. It just fakes the printed path. But it +leaves all behaviour around f_mapping unchanged. This is surely an +advantage. +Actually aufs had adopted another complicated approach which calls +generic_file_mmap() and handles struct vm_operations_struct. In this +approach, aufs met a hard problem and I could not solve it without +switching the approach. + +There may be one more another approach which is +- bind-mount the branch-root onto the aufs-root internally +- grab the new vfsmount (ie. struct mount) +- lazy-umount the branch-root internally +- in open(2) the aufs-file, open the branch-file with the hidden + vfsmount (instead of the original branch's vfsmount) +- ideally this "bind-mount and lazy-umount" should be done atomically, + but it may be possible from userspace by the mount helper. + +Adding the internal hidden vfsmount and using it in opening a file, the +file path under /proc will be printed correctly. This approach looks +smarter, but is not possible I am afraid. +- aufs-root may be bind-mount later. when it happens, another hidden + vfsmount will be required. +- it is hard to get the chance to bind-mount and lazy-umount + + in kernel-space, FS can have vfsmount in open(2) via + file->f_path, and aufs can know its vfsmount. But several locks are + already acquired, and if aufs tries to bind-mount and lazy-umount + here, then it may cause a deadlock. + + in user-space, bind-mount doesn't invoke the mount helper. +- since /proc shows dev and ino, aufs has to give vma these info. it + means a new member vm_prinode will be necessary. this is essentially + equivalent to vm_prfile described above. + +I have to give up this "looks-smater" approach. diff --git a/Documentation/filesystems/aufs/design/06xattr.txt b/Documentation/filesystems/aufs/design/06xattr.txt new file mode 100644 index 0000000000000..d0f6aedfe2d00 --- /dev/null +++ b/Documentation/filesystems/aufs/design/06xattr.txt @@ -0,0 +1,96 @@ + +# Copyright (C) 2014-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Listing XATTR/EA and getting the value +---------------------------------------------------------------------- +For the inode standard attributes (owner, group, timestamps, etc.), aufs +shows the values from the topmost existing file. This behaviour is good +for the non-dir entries since the bahaviour exactly matches the shown +information. But for the directories, aufs considers all the same named +entries on the lower branches. Which means, if one of the lower entry +rejects readdir call, then aufs returns an error even if the topmost +entry allows it. This behaviour is necessary to respect the branch fs's +security, but can make users confused since the user-visible standard +attributes don't match the behaviour. +To address this issue, aufs has a mount option called dirperm1 which +checks the permission for the topmost entry only, and ignores the lower +entry's permission. + +A similar issue can happen around XATTR. +getxattr(2) and listxattr(2) families behave as if dirperm1 option is +always set. Otherwise these very unpleasant situation would happen. +- listxattr(2) may return the duplicated entries. +- users may not be able to remove or reset the XATTR forever, + + +XATTR/EA support in the internal (copy,move)-(up,down) +---------------------------------------------------------------------- +Generally the extended attributes of inode are categorized as these. +- "security" for LSM and capability. +- "system" for posix ACL, 'acl' mount option is required for the branch + fs generally. +- "trusted" for userspace, CAP_SYS_ADMIN is required. +- "user" for userspace, 'user_xattr' mount option is required for the + branch fs generally. + +Moreover there are some other categories. Aufs handles these rather +unpopular categories as the ordinary ones, ie. there is no special +condition nor exception. + +In copy-up, the support for XATTR on the dst branch may differ from the +src branch. In this case, the copy-up operation will get an error and +the original user operation which triggered the copy-up will fail. It +can happen that even all copy-up will fail. +When both of src and dst branches support XATTR and if an error occurs +during copying XATTR, then the copy-up should fail obviously. That is a +good reason and aufs should return an error to userspace. But when only +the src branch support that XATTR, aufs should not return an error. +For example, the src branch supports ACL but the dst branch doesn't +because the dst branch may natively un-support it or temporary +un-support it due to "noacl" mount option. Of course, the dst branch fs +may NOT return an error even if the XATTR is not supported. It is +totally up to the branch fs. + +Anyway when the aufs internal copy-up gets an error from the dst branch +fs, then aufs tries removing the just copied entry and returns the error +to the userspace. The worst case of this situation will be all copy-up +will fail. + +For the copy-up operation, there two basic approaches. +- copy the specified XATTR only (by category above), and return the + error unconditionally if it happens. +- copy all XATTR, and ignore the error on the specified category only. + +In order to support XATTR and to implement the correct behaviour, aufs +chooses the latter approach and introduces some new branch attributes, +"icexsec", "icexsys", "icextr", "icexusr", and "icexoth". +They correspond to the XATTR namespaces (see above). Additionally, to be +convenient, "icex" is also provided which means all "icex*" attributes +are set (here the word "icex" stands for "ignore copy-error on XATTR"). + +The meaning of these attributes is to ignore the error from setting +XATTR on that branch. +Note that aufs tries copying all XATTR unconditionally, and ignores the +error from the dst branch according to the specified attributes. + +Some XATTR may have its default value. The default value may come from +the parent dir or the environment. If the default value is set at the +file creating-time, it will be overwritten by copy-up. +Some contradiction may happen I am afraid. +Do we need another attribute to stop copying XATTR? I am unsure. For +now, aufs implements the branch attributes to ignore the error. diff --git a/Documentation/filesystems/aufs/design/07export.txt b/Documentation/filesystems/aufs/design/07export.txt new file mode 100644 index 0000000000000..82204d7ca6849 --- /dev/null +++ b/Documentation/filesystems/aufs/design/07export.txt @@ -0,0 +1,58 @@ + +# Copyright (C) 2005-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Export Aufs via NFS +---------------------------------------------------------------------- +Here is an approach. +- like xino/xib, add a new file 'xigen' which stores aufs inode + generation. +- iget_locked(): initialize aufs inode generation for a new inode, and + store it in xigen file. +- destroy_inode(): increment aufs inode generation and store it in xigen + file. it is necessary even if it is not unlinked, because any data of + inode may be changed by UDBA. +- encode_fh(): for a root dir, simply return FILEID_ROOT. otherwise + build file handle by + + branch id (4 bytes) + + superblock generation (4 bytes) + + inode number (4 or 8 bytes) + + parent dir inode number (4 or 8 bytes) + + inode generation (4 bytes)) + + return value of exportfs_encode_fh() for the parent on a branch (4 + bytes) + + file handle for a branch (by exportfs_encode_fh()) +- fh_to_dentry(): + + find the index of a branch from its id in handle, and check it is + still exist in aufs. + + 1st level: get the inode number from handle and search it in cache. + + 2nd level: if not found in cache, get the parent inode number from + the handle and search it in cache. and then open the found parent + dir, find the matching inode number by vfs_readdir() and get its + name, and call lookup_one_len() for the target dentry. + + 3rd level: if the parent dir is not cached, call + exportfs_decode_fh() for a branch and get the parent on a branch, + build a pathname of it, convert it a pathname in aufs, call + path_lookup(). now aufs gets a parent dir dentry, then handle it as + the 2nd level. + + to open the dir, aufs needs struct vfsmount. aufs keeps vfsmount + for every branch, but not itself. to get this, (currently) aufs + searches in current->nsproxy->mnt_ns list. it may not be a good + idea, but I didn't get other approach. + + test the generation of the gotten inode. +- every inode operation: they may get EBUSY due to UDBA. in this case, + convert it into ESTALE for NFSD. +- readdir(): call lockdep_on/off() because filldir in NFSD calls + lookup_one_len(), vfs_getattr(), encode_fh() and others. diff --git a/Documentation/filesystems/aufs/design/08shwh.txt b/Documentation/filesystems/aufs/design/08shwh.txt new file mode 100644 index 0000000000000..9cef85013945f --- /dev/null +++ b/Documentation/filesystems/aufs/design/08shwh.txt @@ -0,0 +1,52 @@ + +# Copyright (C) 2005-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Show Whiteout Mode (shwh) +---------------------------------------------------------------------- +Generally aufs hides the name of whiteouts. But in some cases, to show +them is very useful for users. For instance, creating a new middle layer +(branch) by merging existing layers. + +(borrowing aufs1 HOW-TO from a user, Michael Towers) +When you have three branches, +- Bottom: 'system', squashfs (underlying base system), read-only +- Middle: 'mods', squashfs, read-only +- Top: 'overlay', ram (tmpfs), read-write + +The top layer is loaded at boot time and saved at shutdown, to preserve +the changes made to the system during the session. +When larger changes have been made, or smaller changes have accumulated, +the size of the saved top layer data grows. At this point, it would be +nice to be able to merge the two overlay branches ('mods' and 'overlay') +and rewrite the 'mods' squashfs, clearing the top layer and thus +restoring save and load speed. + +This merging is simplified by the use of another aufs mount, of just the +two overlay branches using the 'shwh' option. +# mount -t aufs -o ro,shwh,br:/livesys/overlay=ro+wh:/livesys/mods=rr+wh \ + aufs /livesys/merge_union + +A merged view of these two branches is then available at +/livesys/merge_union, and the new feature is that the whiteouts are +visible! +Note that in 'shwh' mode the aufs mount must be 'ro', which will disable +writing to all branches. Also the default mode for all branches is 'ro'. +It is now possible to save the combined contents of the two overlay +branches to a new squashfs, e.g.: +# mksquashfs /livesys/merge_union /path/to/newmods.squash + +This new squashfs archive can be stored on the boot device and the +initramfs will use it to replace the old one at the next boot. diff --git a/Documentation/filesystems/aufs/design/10dynop.txt b/Documentation/filesystems/aufs/design/10dynop.txt new file mode 100644 index 0000000000000..d55cae285dff3 --- /dev/null +++ b/Documentation/filesystems/aufs/design/10dynop.txt @@ -0,0 +1,47 @@ + +# Copyright (C) 2010-2020 Junjiro R. Okajima +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +Dynamically customizable FS operations +---------------------------------------------------------------------- +Generally FS operations (struct inode_operations, struct +address_space_operations, struct file_operations, etc.) are defined as +"static const", but it never means that FS have only one set of +operation. Some FS have multiple sets of them. For instance, ext2 has +three sets, one for XIP, for NOBH, and for normal. +Since aufs overrides and redirects these operations, sometimes aufs has +to change its behaviour according to the branch FS type. More importantly +VFS acts differently if a function (member in the struct) is set or +not. It means aufs should have several sets of operations and select one +among them according to the branch FS definition. + +In order to solve this problem and not to affect the behaviour of VFS, +aufs defines these operations dynamically. For instance, aufs defines +dummy direct_IO function for struct address_space_operations, but it may +not be set to the address_space_operations actually. When the branch FS +doesn't have it, aufs doesn't set it to its address_space_operations +while the function definition itself is still alive. So the behaviour +itself will not change, and it will return an error when direct_IO is +not set. + +The lifetime of these dynamically generated operation object is +maintained by aufs branch object. When the branch is removed from aufs, +the reference counter of the object is decremented. When it reaches +zero, the dynamically generated operation object will be freed. + +This approach is designed to support AIO (io_submit), Direct I/O and +XIP (DAX) mainly. +Currently this approach is applied to address_space_operations for +regular files only. diff --git a/fs/aufs/Kconfig b/fs/aufs/Kconfig new file mode 100644 index 0000000000000..9f436425716a4 --- /dev/null +++ b/fs/aufs/Kconfig @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: GPL-2.0 +config AUFS_FS + tristate "Aufs (Advanced multi layered unification filesystem) support" + help + Aufs is a stackable unification filesystem such as Unionfs, + which unifies several directories and provides a merged single + directory. + In the early days, aufs was entirely re-designed and + re-implemented Unionfs Version 1.x series. Introducing many + original ideas, approaches and improvements, it becomes totally + different from Unionfs while keeping the basic features. + +if AUFS_FS +choice + prompt "Maximum number of branches" + default AUFS_BRANCH_MAX_127 + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +config AUFS_BRANCH_MAX_127 + bool "127" + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +config AUFS_BRANCH_MAX_511 + bool "511" + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +config AUFS_BRANCH_MAX_1023 + bool "1023" + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +config AUFS_BRANCH_MAX_32767 + bool "32767" + help + Specifies the maximum number of branches (or member directories) + in a single aufs. The larger value consumes more system + resources and has a minor impact to performance. +endchoice + +config AUFS_SBILIST + bool + depends on AUFS_MAGIC_SYSRQ || PROC_FS + default y + help + Automatic configuration for internal use. + When aufs supports Magic SysRq or /proc, enabled automatically. + +config AUFS_HNOTIFY + bool "Detect direct branch access (bypassing aufs)" + help + If you want to modify files on branches directly, eg. bypassing aufs, + and want aufs to detect the changes of them fully, then enable this + option and use 'udba=notify' mount option. + Currently there is only one available configuration, "fsnotify". + It will have a negative impact to the performance. + See detail in aufs.5. + +choice + prompt "method" if AUFS_HNOTIFY + default AUFS_HFSNOTIFY +config AUFS_HFSNOTIFY + bool "fsnotify" + select FSNOTIFY +endchoice + +config AUFS_EXPORT + bool "NFS-exportable aufs" + depends on EXPORTFS + help + If you want to export your mounted aufs via NFS, then enable this + option. There are several requirements for this configuration. + See detail in aufs.5. + +config AUFS_INO_T_64 + bool + depends on AUFS_EXPORT + depends on 64BIT && !(ALPHA || S390) + default y + help + Automatic configuration for internal use. + /* typedef unsigned long/int __kernel_ino_t */ + /* alpha and s390x are int */ + +config AUFS_XATTR + bool "support for XATTR/EA (including Security Labels)" + help + If your branch fs supports XATTR/EA and you want to make them + available in aufs too, then enable this opsion and specify the + branch attributes for EA. + See detail in aufs.5. + +config AUFS_FHSM + bool "File-based Hierarchical Storage Management" + help + Hierarchical Storage Management (or HSM) is a well-known feature + in the storage world. Aufs provides this feature as file-based. + with multiple branches. + These multiple branches are prioritized, ie. the topmost one + should be the fastest drive and be used heavily. + +config AUFS_RDU + bool "Readdir in userspace" + help + Aufs has two methods to provide a merged view for a directory, + by a user-space library and by kernel-space natively. The latter + is always enabled but sometimes large and slow. + If you enable this option, install the library in aufs2-util + package, and set some environment variables for your readdir(3), + then the work will be handled in user-space which generally + shows better performance in most cases. + See detail in aufs.5. + +config AUFS_DIRREN + bool "Workaround for rename(2)-ing a directory" + help + By default, aufs returns EXDEV error in renameing a dir who has + his child on the lower branch, since it is a bad idea to issue + rename(2) internally for every lower branch. But user may not + accept this behaviour. So here is a workaround to allow such + rename(2) and store some extra infromation on the writable + branch. Obviously this costs high (and I don't like it). + To use this feature, you need to enable this configuration AND + to specify the mount option `dirren.' + See details in aufs.5 and the design documents. + +config AUFS_SHWH + bool "Show whiteouts" + help + If you want to make the whiteouts in aufs visible, then enable + this option and specify 'shwh' mount option. Although it may + sounds like philosophy or something, but in technically it + simply shows the name of whiteout with keeping its behaviour. + +config AUFS_BR_RAMFS + bool "Ramfs (initramfs/rootfs) as an aufs branch" + help + If you want to use ramfs as an aufs branch fs, then enable this + option. Generally tmpfs is recommended. + Aufs prohibited them to be a branch fs by default, because + initramfs becomes unusable after switch_root or something + generally. If you sets initramfs as an aufs branch and boot your + system by switch_root, you will meet a problem easily since the + files in initramfs may be inaccessible. + Unless you are going to use ramfs as an aufs branch fs without + switch_root or something, leave it N. + +config AUFS_BR_FUSE + bool "Fuse fs as an aufs branch" + depends on FUSE_FS + select AUFS_POLL + help + If you want to use fuse-based userspace filesystem as an aufs + branch fs, then enable this option. + It implements the internal poll(2) operation which is + implemented by fuse only (curretnly). + +config AUFS_POLL + bool + help + Automatic configuration for internal use. + +config AUFS_BR_HFSPLUS + bool "Hfsplus as an aufs branch" + depends on HFSPLUS_FS + default y + help + If you want to use hfsplus fs as an aufs branch fs, then enable + this option. This option introduces a small overhead at + copying-up a file on hfsplus. + +config AUFS_BDEV_LOOP + bool + depends on BLK_DEV_LOOP + default y + help + Automatic configuration for internal use. + Convert =[ym] into =y. + +config AUFS_DEBUG + bool "Debug aufs" + help + Enable this to compile aufs internal debug code. + It will have a negative impact to the performance. + +config AUFS_MAGIC_SYSRQ + bool + depends on AUFS_DEBUG && MAGIC_SYSRQ + default y + help + Automatic configuration for internal use. + When aufs supports Magic SysRq, enabled automatically. +endif diff --git a/fs/aufs/Makefile b/fs/aufs/Makefile new file mode 100644 index 0000000000000..2c819a64935e6 --- /dev/null +++ b/fs/aufs/Makefile @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0 + +include ${src}/magic.mk +ifeq (${CONFIG_AUFS_FS},m) +include ${src}/conf.mk +endif +-include ${src}/priv_def.mk + +# cf. include/linux/kernel.h +# enable pr_debug +ccflags-y += -DDEBUG +# sparse requires the full pathname +ifdef M +ccflags-y += -include ${M}/../../include/uapi/linux/aufs_type.h +else +ccflags-y += -include ${srctree}/include/uapi/linux/aufs_type.h +endif + +obj-$(CONFIG_AUFS_FS) += aufs.o +aufs-y := module.o sbinfo.o super.o branch.o xino.o sysaufs.o opts.o \ + wkq.o vfsub.o dcsub.o \ + cpup.o whout.o wbr_policy.o \ + dinfo.o dentry.o \ + dynop.o \ + finfo.o file.o f_op.o \ + dir.o vdir.o \ + iinfo.o inode.o i_op.o i_op_add.o i_op_del.o i_op_ren.o \ + mvdown.o ioctl.o + +# all are boolean +aufs-$(CONFIG_PROC_FS) += procfs.o plink.o +aufs-$(CONFIG_SYSFS) += sysfs.o +aufs-$(CONFIG_DEBUG_FS) += dbgaufs.o +aufs-$(CONFIG_AUFS_BDEV_LOOP) += loop.o +aufs-$(CONFIG_AUFS_HNOTIFY) += hnotify.o +aufs-$(CONFIG_AUFS_HFSNOTIFY) += hfsnotify.o +aufs-$(CONFIG_AUFS_EXPORT) += export.o +aufs-$(CONFIG_AUFS_XATTR) += xattr.o +aufs-$(CONFIG_FS_POSIX_ACL) += posix_acl.o +aufs-$(CONFIG_AUFS_DIRREN) += dirren.o +aufs-$(CONFIG_AUFS_FHSM) += fhsm.o +aufs-$(CONFIG_AUFS_POLL) += poll.o +aufs-$(CONFIG_AUFS_RDU) += rdu.o +aufs-$(CONFIG_AUFS_BR_HFSPLUS) += hfsplus.o +aufs-$(CONFIG_AUFS_DEBUG) += debug.o +aufs-$(CONFIG_AUFS_MAGIC_SYSRQ) += sysrq.o diff --git a/fs/aufs/aufs.h b/fs/aufs/aufs.h new file mode 100644 index 0000000000000..9a573445876f4 --- /dev/null +++ b/fs/aufs/aufs.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * all header files + */ + +#ifndef __AUFS_H__ +#define __AUFS_H__ + +#ifdef __KERNEL__ + +#define AuStub(type, name, body, ...) \ + static inline type name(__VA_ARGS__) { body; } + +#define AuStubVoid(name, ...) \ + AuStub(void, name, , __VA_ARGS__) +#define AuStubInt0(name, ...) \ + AuStub(int, name, return 0, __VA_ARGS__) + +#include "debug.h" + +#include "branch.h" +#include "cpup.h" +#include "dcsub.h" +#include "dbgaufs.h" +#include "dentry.h" +#include "dir.h" +#include "dirren.h" +#include "dynop.h" +#include "file.h" +#include "fstype.h" +#include "hbl.h" +#include "inode.h" +#include "lcnt.h" +#include "loop.h" +#include "module.h" +#include "opts.h" +#include "rwsem.h" +#include "super.h" +#include "sysaufs.h" +#include "vfsub.h" +#include "whout.h" +#include "wkq.h" + +#endif /* __KERNEL__ */ +#endif /* __AUFS_H__ */ diff --git a/fs/aufs/branch.c b/fs/aufs/branch.c new file mode 100644 index 0000000000000..48edd56b19e6f --- /dev/null +++ b/fs/aufs/branch.c @@ -0,0 +1,1422 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * branch management + */ + +#include +#include +#include "aufs.h" + +/* + * free a single branch + */ +static void au_br_do_free(struct au_branch *br) +{ + int i; + struct au_wbr *wbr; + struct au_dykey **key; + + au_hnotify_fin_br(br); + /* always, regardless the mount option */ + au_dr_hino_free(&br->br_dirren); + au_xino_put(br); + + AuLCntZero(au_lcnt_read(&br->br_nfiles, /*do_rev*/0)); + au_lcnt_fin(&br->br_nfiles, /*do_sync*/0); + AuLCntZero(au_lcnt_read(&br->br_count, /*do_rev*/0)); + au_lcnt_fin(&br->br_count, /*do_sync*/0); + + wbr = br->br_wbr; + if (wbr) { + for (i = 0; i < AuBrWh_Last; i++) + dput(wbr->wbr_wh[i]); + AuDebugOn(atomic_read(&wbr->wbr_wh_running)); + AuRwDestroy(&wbr->wbr_wh_rwsem); + } + + if (br->br_fhsm) { + au_br_fhsm_fin(br->br_fhsm); + au_kfree_try_rcu(br->br_fhsm); + } + + key = br->br_dykey; + for (i = 0; i < AuBrDynOp; i++, key++) + if (*key) + au_dy_put(*key); + else + break; + + /* recursive lock, s_umount of branch's */ + /* synchronize_rcu(); */ /* why? */ + lockdep_off(); + path_put(&br->br_path); + lockdep_on(); + au_kfree_rcu(wbr); + au_lcnt_wait_for_fin(&br->br_nfiles); + au_lcnt_wait_for_fin(&br->br_count); + /* I don't know why, but percpu_refcount requires this */ + /* synchronize_rcu(); */ + au_kfree_rcu(br); +} + +/* + * frees all branches + */ +void au_br_free(struct au_sbinfo *sbinfo) +{ + aufs_bindex_t bmax; + struct au_branch **br; + + AuRwMustWriteLock(&sbinfo->si_rwsem); + + bmax = sbinfo->si_bbot + 1; + br = sbinfo->si_branch; + while (bmax--) + au_br_do_free(*br++); +} + +/* + * find the index of a branch which is specified by @br_id. + */ +int au_br_index(struct super_block *sb, aufs_bindex_t br_id) +{ + aufs_bindex_t bindex, bbot; + + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) + if (au_sbr_id(sb, bindex) == br_id) + return bindex; + return -1; +} + +/* ---------------------------------------------------------------------- */ + +/* + * add a branch + */ + +static int test_overlap(struct super_block *sb, struct dentry *h_adding, + struct dentry *h_root) +{ + if (unlikely(h_adding == h_root + || au_test_loopback_overlap(sb, h_adding))) + return 1; + if (h_adding->d_sb != h_root->d_sb) + return 0; + return au_test_subdir(h_adding, h_root) + || au_test_subdir(h_root, h_adding); +} + +/* + * returns a newly allocated branch. @new_nbranch is a number of branches + * after adding a branch. + */ +static struct au_branch *au_br_alloc(struct super_block *sb, int new_nbranch, + int perm) +{ + struct au_branch *add_branch; + struct dentry *root; + struct inode *inode; + int err; + + err = -ENOMEM; + add_branch = kzalloc(sizeof(*add_branch), GFP_NOFS); + if (unlikely(!add_branch)) + goto out; + add_branch->br_xino = au_xino_alloc(/*nfile*/1); + if (unlikely(!add_branch->br_xino)) + goto out_br; + err = au_hnotify_init_br(add_branch, perm); + if (unlikely(err)) + goto out_xino; + + if (au_br_writable(perm)) { + /* may be freed separately at changing the branch permission */ + add_branch->br_wbr = kzalloc(sizeof(*add_branch->br_wbr), + GFP_NOFS); + if (unlikely(!add_branch->br_wbr)) + goto out_hnotify; + } + + if (au_br_fhsm(perm)) { + err = au_fhsm_br_alloc(add_branch); + if (unlikely(err)) + goto out_wbr; + } + + root = sb->s_root; + err = au_sbr_realloc(au_sbi(sb), new_nbranch, /*may_shrink*/0); + if (!err) + err = au_di_realloc(au_di(root), new_nbranch, /*may_shrink*/0); + if (!err) { + inode = d_inode(root); + err = au_hinode_realloc(au_ii(inode), new_nbranch, + /*may_shrink*/0); + } + if (!err) + return add_branch; /* success */ + +out_wbr: + au_kfree_rcu(add_branch->br_wbr); +out_hnotify: + au_hnotify_fin_br(add_branch); +out_xino: + au_xino_put(add_branch); +out_br: + au_kfree_rcu(add_branch); +out: + return ERR_PTR(err); +} + +/* + * test if the branch permission is legal or not. + */ +static int test_br(struct inode *inode, int brperm, char *path) +{ + int err; + + err = (au_br_writable(brperm) && IS_RDONLY(inode)); + if (!err) + goto out; + + err = -EINVAL; + pr_err("write permission for readonly mount or inode, %s\n", path); + +out: + return err; +} + +/* + * returns: + * 0: success, the caller will add it + * plus: success, it is already unified, the caller should ignore it + * minus: error + */ +static int test_add(struct super_block *sb, struct au_opt_add *add, int remount) +{ + int err; + aufs_bindex_t bbot, bindex; + struct dentry *root, *h_dentry; + struct inode *inode, *h_inode; + + root = sb->s_root; + bbot = au_sbbot(sb); + if (unlikely(bbot >= 0 + && au_find_dbindex(root, add->path.dentry) >= 0)) { + err = 1; + if (!remount) { + err = -EINVAL; + pr_err("%s duplicated\n", add->pathname); + } + goto out; + } + + err = -ENOSPC; /* -E2BIG; */ + if (unlikely(AUFS_BRANCH_MAX <= add->bindex + || AUFS_BRANCH_MAX - 1 <= bbot)) { + pr_err("number of branches exceeded %s\n", add->pathname); + goto out; + } + + err = -EDOM; + if (unlikely(add->bindex < 0 || bbot + 1 < add->bindex)) { + pr_err("bad index %d\n", add->bindex); + goto out; + } + + inode = d_inode(add->path.dentry); + err = -ENOENT; + if (unlikely(!inode->i_nlink)) { + pr_err("no existence %s\n", add->pathname); + goto out; + } + + err = -EINVAL; + if (unlikely(inode->i_sb == sb)) { + pr_err("%s must be outside\n", add->pathname); + goto out; + } + + if (unlikely(au_test_fs_unsuppoted(inode->i_sb))) { + pr_err("unsupported filesystem, %s (%s)\n", + add->pathname, au_sbtype(inode->i_sb)); + goto out; + } + + if (unlikely(inode->i_sb->s_stack_depth)) { + pr_err("already stacked, %s (%s)\n", + add->pathname, au_sbtype(inode->i_sb)); + goto out; + } + + err = test_br(d_inode(add->path.dentry), add->perm, add->pathname); + if (unlikely(err)) + goto out; + + if (bbot < 0) + return 0; /* success */ + + err = -EINVAL; + for (bindex = 0; bindex <= bbot; bindex++) + if (unlikely(test_overlap(sb, add->path.dentry, + au_h_dptr(root, bindex)))) { + pr_err("%s is overlapped\n", add->pathname); + goto out; + } + + err = 0; + if (au_opt_test(au_mntflags(sb), WARN_PERM)) { + h_dentry = au_h_dptr(root, 0); + h_inode = d_inode(h_dentry); + if ((h_inode->i_mode & S_IALLUGO) != (inode->i_mode & S_IALLUGO) + || !uid_eq(h_inode->i_uid, inode->i_uid) + || !gid_eq(h_inode->i_gid, inode->i_gid)) + pr_warn("uid/gid/perm %s %u/%u/0%o, %u/%u/0%o\n", + add->pathname, + i_uid_read(inode), i_gid_read(inode), + (inode->i_mode & S_IALLUGO), + i_uid_read(h_inode), i_gid_read(h_inode), + (h_inode->i_mode & S_IALLUGO)); + } + +out: + return err; +} + +/* + * initialize or clean the whiteouts for an adding branch + */ +static int au_br_init_wh(struct super_block *sb, struct au_branch *br, + int new_perm) +{ + int err, old_perm; + aufs_bindex_t bindex; + struct inode *h_inode; + struct au_wbr *wbr; + struct au_hinode *hdir; + struct dentry *h_dentry; + + err = vfsub_mnt_want_write(au_br_mnt(br)); + if (unlikely(err)) + goto out; + + wbr = br->br_wbr; + old_perm = br->br_perm; + br->br_perm = new_perm; + hdir = NULL; + h_inode = NULL; + bindex = au_br_index(sb, br->br_id); + if (0 <= bindex) { + hdir = au_hi(d_inode(sb->s_root), bindex); + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + } else { + h_dentry = au_br_dentry(br); + h_inode = d_inode(h_dentry); + inode_lock_nested(h_inode, AuLsc_I_PARENT); + } + if (!wbr) + err = au_wh_init(br, sb); + else { + wbr_wh_write_lock(wbr); + err = au_wh_init(br, sb); + wbr_wh_write_unlock(wbr); + } + if (hdir) + au_hn_inode_unlock(hdir); + else + inode_unlock(h_inode); + vfsub_mnt_drop_write(au_br_mnt(br)); + br->br_perm = old_perm; + + if (!err && wbr && !au_br_writable(new_perm)) { + au_kfree_rcu(wbr); + br->br_wbr = NULL; + } + +out: + return err; +} + +static int au_wbr_init(struct au_branch *br, struct super_block *sb, + int perm) +{ + int err; + struct kstatfs kst; + struct au_wbr *wbr; + + wbr = br->br_wbr; + au_rw_init(&wbr->wbr_wh_rwsem); + atomic_set(&wbr->wbr_wh_running, 0); + + /* + * a limit for rmdir/rename a dir + * cf. AUFS_MAX_NAMELEN in include/uapi/linux/aufs_type.h + */ + err = vfs_statfs(&br->br_path, &kst); + if (unlikely(err)) + goto out; + err = -EINVAL; + if (kst.f_namelen >= NAME_MAX) + err = au_br_init_wh(sb, br, perm); + else + pr_err("%pd(%s), unsupported namelen %ld\n", + au_br_dentry(br), + au_sbtype(au_br_dentry(br)->d_sb), kst.f_namelen); + +out: + return err; +} + +/* initialize a new branch */ +static int au_br_init(struct au_branch *br, struct super_block *sb, + struct au_opt_add *add) +{ + int err; + struct au_branch *brbase; + struct file *xf; + struct inode *h_inode; + + err = 0; + br->br_perm = add->perm; + br->br_path = add->path; /* set first, path_get() later */ + spin_lock_init(&br->br_dykey_lock); + au_lcnt_init(&br->br_nfiles, /*release*/NULL); + au_lcnt_init(&br->br_count, /*release*/NULL); + br->br_id = au_new_br_id(sb); + AuDebugOn(br->br_id < 0); + + /* always, regardless the given option */ + err = au_dr_br_init(sb, br, &add->path); + if (unlikely(err)) + goto out_err; + + if (au_br_writable(add->perm)) { + err = au_wbr_init(br, sb, add->perm); + if (unlikely(err)) + goto out_err; + } + + if (au_opt_test(au_mntflags(sb), XINO)) { + brbase = au_sbr(sb, 0); + xf = au_xino_file(brbase->br_xino, /*idx*/-1); + AuDebugOn(!xf); + h_inode = d_inode(add->path.dentry); + err = au_xino_init_br(sb, br, h_inode->i_ino, &xf->f_path); + if (unlikely(err)) { + AuDebugOn(au_xino_file(br->br_xino, /*idx*/-1)); + goto out_err; + } + } + + sysaufs_br_init(br); + path_get(&br->br_path); + goto out; /* success */ + +out_err: + memset(&br->br_path, 0, sizeof(br->br_path)); +out: + return err; +} + +static void au_br_do_add_brp(struct au_sbinfo *sbinfo, aufs_bindex_t bindex, + struct au_branch *br, aufs_bindex_t bbot, + aufs_bindex_t amount) +{ + struct au_branch **brp; + + AuRwMustWriteLock(&sbinfo->si_rwsem); + + brp = sbinfo->si_branch + bindex; + memmove(brp + 1, brp, sizeof(*brp) * amount); + *brp = br; + sbinfo->si_bbot++; + if (unlikely(bbot < 0)) + sbinfo->si_bbot = 0; +} + +static void au_br_do_add_hdp(struct au_dinfo *dinfo, aufs_bindex_t bindex, + aufs_bindex_t bbot, aufs_bindex_t amount) +{ + struct au_hdentry *hdp; + + AuRwMustWriteLock(&dinfo->di_rwsem); + + hdp = au_hdentry(dinfo, bindex); + memmove(hdp + 1, hdp, sizeof(*hdp) * amount); + au_h_dentry_init(hdp); + dinfo->di_bbot++; + if (unlikely(bbot < 0)) + dinfo->di_btop = 0; +} + +static void au_br_do_add_hip(struct au_iinfo *iinfo, aufs_bindex_t bindex, + aufs_bindex_t bbot, aufs_bindex_t amount) +{ + struct au_hinode *hip; + + AuRwMustWriteLock(&iinfo->ii_rwsem); + + hip = au_hinode(iinfo, bindex); + memmove(hip + 1, hip, sizeof(*hip) * amount); + au_hinode_init(hip); + iinfo->ii_bbot++; + if (unlikely(bbot < 0)) + iinfo->ii_btop = 0; +} + +static void au_br_do_add(struct super_block *sb, struct au_branch *br, + aufs_bindex_t bindex) +{ + struct dentry *root, *h_dentry; + struct inode *root_inode, *h_inode; + aufs_bindex_t bbot, amount; + + root = sb->s_root; + root_inode = d_inode(root); + bbot = au_sbbot(sb); + amount = bbot + 1 - bindex; + h_dentry = au_br_dentry(br); + au_sbilist_lock(); + au_br_do_add_brp(au_sbi(sb), bindex, br, bbot, amount); + au_br_do_add_hdp(au_di(root), bindex, bbot, amount); + au_br_do_add_hip(au_ii(root_inode), bindex, bbot, amount); + au_set_h_dptr(root, bindex, dget(h_dentry)); + h_inode = d_inode(h_dentry); + au_set_h_iptr(root_inode, bindex, au_igrab(h_inode), /*flags*/0); + au_sbilist_unlock(); +} + +int au_br_add(struct super_block *sb, struct au_opt_add *add, int remount) +{ + int err; + aufs_bindex_t bbot, add_bindex; + struct dentry *root, *h_dentry; + struct inode *root_inode; + struct au_branch *add_branch; + + root = sb->s_root; + root_inode = d_inode(root); + IMustLock(root_inode); + IiMustWriteLock(root_inode); + err = test_add(sb, add, remount); + if (unlikely(err < 0)) + goto out; + if (err) { + err = 0; + goto out; /* success */ + } + + bbot = au_sbbot(sb); + add_branch = au_br_alloc(sb, bbot + 2, add->perm); + err = PTR_ERR(add_branch); + if (IS_ERR(add_branch)) + goto out; + + err = au_br_init(add_branch, sb, add); + if (unlikely(err)) { + au_br_do_free(add_branch); + goto out; + } + + add_bindex = add->bindex; + sysaufs_brs_del(sb, add_bindex); /* remove successors */ + au_br_do_add(sb, add_branch, add_bindex); + sysaufs_brs_add(sb, add_bindex); /* append successors */ + dbgaufs_brs_add(sb, add_bindex, /*topdown*/0); /* rename successors */ + + h_dentry = add->path.dentry; + if (!add_bindex) { + au_cpup_attr_all(root_inode, /*force*/1); + sb->s_maxbytes = h_dentry->d_sb->s_maxbytes; + } else + au_add_nlink(root_inode, d_inode(h_dentry)); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +static unsigned long long au_farray_cb(struct super_block *sb, void *a, + unsigned long long max __maybe_unused, + void *arg) +{ + unsigned long long n; + struct file **p, *f; + struct hlist_bl_head *files; + struct hlist_bl_node *pos; + struct au_finfo *finfo; + + n = 0; + p = a; + files = &au_sbi(sb)->si_files; + hlist_bl_lock(files); + hlist_bl_for_each_entry(finfo, pos, files, fi_hlist) { + f = finfo->fi_file; + if (file_count(f) + && !special_file(file_inode(f)->i_mode)) { + get_file(f); + *p++ = f; + n++; + AuDebugOn(n > max); + } + } + hlist_bl_unlock(files); + + return n; +} + +static struct file **au_farray_alloc(struct super_block *sb, + unsigned long long *max) +{ + struct au_sbinfo *sbi; + + sbi = au_sbi(sb); + *max = au_lcnt_read(&sbi->si_nfiles, /*do_rev*/1); + return au_array_alloc(max, au_farray_cb, sb, /*arg*/NULL); +} + +static void au_farray_free(struct file **a, unsigned long long max) +{ + unsigned long long ull; + + for (ull = 0; ull < max; ull++) + if (a[ull]) + fput(a[ull]); + kvfree(a); +} + +/* ---------------------------------------------------------------------- */ + +/* + * delete a branch + */ + +/* to show the line number, do not make it inlined function */ +#define AuVerbose(do_info, fmt, ...) do { \ + if (do_info) \ + pr_info(fmt, ##__VA_ARGS__); \ +} while (0) + +static int au_test_ibusy(struct inode *inode, aufs_bindex_t btop, + aufs_bindex_t bbot) +{ + return (inode && !S_ISDIR(inode->i_mode)) || btop == bbot; +} + +static int au_test_dbusy(struct dentry *dentry, aufs_bindex_t btop, + aufs_bindex_t bbot) +{ + return au_test_ibusy(d_inode(dentry), btop, bbot); +} + +/* + * test if the branch is deletable or not. + */ +static int test_dentry_busy(struct dentry *root, aufs_bindex_t bindex, + unsigned int sigen, const unsigned int verbose) +{ + int err, i, j, ndentry; + aufs_bindex_t btop, bbot; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry *d; + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_dcsub_pages(&dpages, root, NULL, NULL); + if (unlikely(err)) + goto out_dpages; + + for (i = 0; !err && i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + ndentry = dpage->ndentry; + for (j = 0; !err && j < ndentry; j++) { + d = dpage->dentries[j]; + AuDebugOn(au_dcount(d) <= 0); + if (!au_digen_test(d, sigen)) { + di_read_lock_child(d, AuLock_IR); + if (unlikely(au_dbrange_test(d))) { + di_read_unlock(d, AuLock_IR); + continue; + } + } else { + di_write_lock_child(d); + if (unlikely(au_dbrange_test(d))) { + di_write_unlock(d); + continue; + } + err = au_reval_dpath(d, sigen); + if (!err) + di_downgrade_lock(d, AuLock_IR); + else { + di_write_unlock(d); + break; + } + } + + /* AuDbgDentry(d); */ + btop = au_dbtop(d); + bbot = au_dbbot(d); + if (btop <= bindex + && bindex <= bbot + && au_h_dptr(d, bindex) + && au_test_dbusy(d, btop, bbot)) { + err = -EBUSY; + AuVerbose(verbose, "busy %pd\n", d); + AuDbgDentry(d); + } + di_read_unlock(d, AuLock_IR); + } + } + +out_dpages: + au_dpages_free(&dpages); +out: + return err; +} + +static int test_inode_busy(struct super_block *sb, aufs_bindex_t bindex, + unsigned int sigen, const unsigned int verbose) +{ + int err; + unsigned long long max, ull; + struct inode *i, **array; + aufs_bindex_t btop, bbot; + + array = au_iarray_alloc(sb, &max); + err = PTR_ERR(array); + if (IS_ERR(array)) + goto out; + + err = 0; + AuDbg("b%d\n", bindex); + for (ull = 0; !err && ull < max; ull++) { + i = array[ull]; + if (unlikely(!i)) + break; + if (i->i_ino == AUFS_ROOT_INO) + continue; + + /* AuDbgInode(i); */ + if (au_iigen(i, NULL) == sigen) + ii_read_lock_child(i); + else { + ii_write_lock_child(i); + err = au_refresh_hinode_self(i); + au_iigen_dec(i); + if (!err) + ii_downgrade_lock(i); + else { + ii_write_unlock(i); + break; + } + } + + btop = au_ibtop(i); + bbot = au_ibbot(i); + if (btop <= bindex + && bindex <= bbot + && au_h_iptr(i, bindex) + && au_test_ibusy(i, btop, bbot)) { + err = -EBUSY; + AuVerbose(verbose, "busy i%lu\n", i->i_ino); + AuDbgInode(i); + } + ii_read_unlock(i); + } + au_iarray_free(array, max); + +out: + return err; +} + +static int test_children_busy(struct dentry *root, aufs_bindex_t bindex, + const unsigned int verbose) +{ + int err; + unsigned int sigen; + + sigen = au_sigen(root->d_sb); + DiMustNoWaiters(root); + IiMustNoWaiters(d_inode(root)); + di_write_unlock(root); + err = test_dentry_busy(root, bindex, sigen, verbose); + if (!err) + err = test_inode_busy(root->d_sb, bindex, sigen, verbose); + di_write_lock_child(root); /* aufs_write_lock() calls ..._child() */ + + return err; +} + +static int test_dir_busy(struct file *file, aufs_bindex_t br_id, + struct file **to_free, int *idx) +{ + int err; + unsigned char matched, root; + aufs_bindex_t bindex, bbot; + struct au_fidir *fidir; + struct au_hfile *hfile; + + err = 0; + root = IS_ROOT(file->f_path.dentry); + if (root) { + get_file(file); + to_free[*idx] = file; + (*idx)++; + goto out; + } + + matched = 0; + fidir = au_fi(file)->fi_hdir; + AuDebugOn(!fidir); + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); bindex <= bbot; bindex++) { + hfile = fidir->fd_hfile + bindex; + if (!hfile->hf_file) + continue; + + if (hfile->hf_br->br_id == br_id) { + matched = 1; + break; + } + } + if (matched) + err = -EBUSY; + +out: + return err; +} + +static int test_file_busy(struct super_block *sb, aufs_bindex_t br_id, + struct file **to_free, int opened) +{ + int err, idx; + unsigned long long ull, max; + aufs_bindex_t btop; + struct file *file, **array; + struct dentry *root; + struct au_hfile *hfile; + + array = au_farray_alloc(sb, &max); + err = PTR_ERR(array); + if (IS_ERR(array)) + goto out; + + err = 0; + idx = 0; + root = sb->s_root; + di_write_unlock(root); + for (ull = 0; ull < max; ull++) { + file = array[ull]; + if (unlikely(!file)) + break; + + /* AuDbg("%pD\n", file); */ + fi_read_lock(file); + btop = au_fbtop(file); + if (!d_is_dir(file->f_path.dentry)) { + hfile = &au_fi(file)->fi_htop; + if (hfile->hf_br->br_id == br_id) + err = -EBUSY; + } else + err = test_dir_busy(file, br_id, to_free, &idx); + fi_read_unlock(file); + if (unlikely(err)) + break; + } + di_write_lock_child(root); + au_farray_free(array, max); + AuDebugOn(idx > opened); + +out: + return err; +} + +static void br_del_file(struct file **to_free, unsigned long long opened, + aufs_bindex_t br_id) +{ + unsigned long long ull; + aufs_bindex_t bindex, btop, bbot, bfound; + struct file *file; + struct au_fidir *fidir; + struct au_hfile *hfile; + + for (ull = 0; ull < opened; ull++) { + file = to_free[ull]; + if (unlikely(!file)) + break; + + /* AuDbg("%pD\n", file); */ + AuDebugOn(!d_is_dir(file->f_path.dentry)); + bfound = -1; + fidir = au_fi(file)->fi_hdir; + AuDebugOn(!fidir); + fi_write_lock(file); + btop = au_fbtop(file); + bbot = au_fbbot_dir(file); + for (bindex = btop; bindex <= bbot; bindex++) { + hfile = fidir->fd_hfile + bindex; + if (!hfile->hf_file) + continue; + + if (hfile->hf_br->br_id == br_id) { + bfound = bindex; + break; + } + } + AuDebugOn(bfound < 0); + au_set_h_fptr(file, bfound, NULL); + if (bfound == btop) { + for (btop++; btop <= bbot; btop++) + if (au_hf_dir(file, btop)) { + au_set_fbtop(file, btop); + break; + } + } + fi_write_unlock(file); + } +} + +static void au_br_do_del_brp(struct au_sbinfo *sbinfo, + const aufs_bindex_t bindex, + const aufs_bindex_t bbot) +{ + struct au_branch **brp, **p; + + AuRwMustWriteLock(&sbinfo->si_rwsem); + + brp = sbinfo->si_branch + bindex; + if (bindex < bbot) + memmove(brp, brp + 1, sizeof(*brp) * (bbot - bindex)); + sbinfo->si_branch[0 + bbot] = NULL; + sbinfo->si_bbot--; + + p = au_krealloc(sbinfo->si_branch, sizeof(*p) * bbot, AuGFP_SBILIST, + /*may_shrink*/1); + if (p) + sbinfo->si_branch = p; + /* harmless error */ +} + +static void au_br_do_del_hdp(struct au_dinfo *dinfo, const aufs_bindex_t bindex, + const aufs_bindex_t bbot) +{ + struct au_hdentry *hdp, *p; + + AuRwMustWriteLock(&dinfo->di_rwsem); + + hdp = au_hdentry(dinfo, bindex); + if (bindex < bbot) + memmove(hdp, hdp + 1, sizeof(*hdp) * (bbot - bindex)); + /* au_h_dentry_init(au_hdentry(dinfo, bbot); */ + dinfo->di_bbot--; + + p = au_krealloc(dinfo->di_hdentry, sizeof(*p) * bbot, AuGFP_SBILIST, + /*may_shrink*/1); + if (p) + dinfo->di_hdentry = p; + /* harmless error */ +} + +static void au_br_do_del_hip(struct au_iinfo *iinfo, const aufs_bindex_t bindex, + const aufs_bindex_t bbot) +{ + struct au_hinode *hip, *p; + + AuRwMustWriteLock(&iinfo->ii_rwsem); + + hip = au_hinode(iinfo, bindex); + if (bindex < bbot) + memmove(hip, hip + 1, sizeof(*hip) * (bbot - bindex)); + /* au_hinode_init(au_hinode(iinfo, bbot)); */ + iinfo->ii_bbot--; + + p = au_krealloc(iinfo->ii_hinode, sizeof(*p) * bbot, AuGFP_SBILIST, + /*may_shrink*/1); + if (p) + iinfo->ii_hinode = p; + /* harmless error */ +} + +static void au_br_do_del(struct super_block *sb, aufs_bindex_t bindex, + struct au_branch *br) +{ + aufs_bindex_t bbot; + struct au_sbinfo *sbinfo; + struct dentry *root, *h_root; + struct inode *inode, *h_inode; + struct au_hinode *hinode; + + SiMustWriteLock(sb); + + root = sb->s_root; + inode = d_inode(root); + sbinfo = au_sbi(sb); + bbot = sbinfo->si_bbot; + + h_root = au_h_dptr(root, bindex); + hinode = au_hi(inode, bindex); + h_inode = au_igrab(hinode->hi_inode); + au_hiput(hinode); + + au_sbilist_lock(); + au_br_do_del_brp(sbinfo, bindex, bbot); + au_br_do_del_hdp(au_di(root), bindex, bbot); + au_br_do_del_hip(au_ii(inode), bindex, bbot); + au_sbilist_unlock(); + + /* ignore an error */ + au_dr_br_fin(sb, br); /* always, regardless the mount option */ + + dput(h_root); + iput(h_inode); + au_br_do_free(br); +} + +static unsigned long long empty_cb(struct super_block *sb, void *array, + unsigned long long max, void *arg) +{ + return max; +} + +int au_br_del(struct super_block *sb, struct au_opt_del *del, int remount) +{ + int err, rerr, i; + unsigned long long opened; + unsigned int mnt_flags; + aufs_bindex_t bindex, bbot, br_id; + unsigned char do_wh, verbose; + struct au_branch *br; + struct au_wbr *wbr; + struct dentry *root; + struct file **to_free; + + err = 0; + opened = 0; + to_free = NULL; + root = sb->s_root; + bindex = au_find_dbindex(root, del->h_path.dentry); + if (bindex < 0) { + if (remount) + goto out; /* success */ + err = -ENOENT; + pr_err("%s no such branch\n", del->pathname); + goto out; + } + AuDbg("bindex b%d\n", bindex); + + err = -EBUSY; + mnt_flags = au_mntflags(sb); + verbose = !!au_opt_test(mnt_flags, VERBOSE); + bbot = au_sbbot(sb); + if (unlikely(!bbot)) { + AuVerbose(verbose, "no more branches left\n"); + goto out; + } + + br = au_sbr(sb, bindex); + AuDebugOn(!path_equal(&br->br_path, &del->h_path)); + if (unlikely(au_lcnt_read(&br->br_count, /*do_rev*/1))) { + AuVerbose(verbose, "br %pd2 is busy now\n", del->h_path.dentry); + goto out; + } + + br_id = br->br_id; + opened = au_lcnt_read(&br->br_nfiles, /*do_rev*/1); + if (unlikely(opened)) { + to_free = au_array_alloc(&opened, empty_cb, sb, NULL); + err = PTR_ERR(to_free); + if (IS_ERR(to_free)) + goto out; + + err = test_file_busy(sb, br_id, to_free, opened); + if (unlikely(err)) { + AuVerbose(verbose, "%llu file(s) opened\n", opened); + goto out; + } + } + + wbr = br->br_wbr; + do_wh = wbr && (wbr->wbr_whbase || wbr->wbr_plink || wbr->wbr_orph); + if (do_wh) { + /* instead of WbrWhMustWriteLock(wbr) */ + SiMustWriteLock(sb); + for (i = 0; i < AuBrWh_Last; i++) { + dput(wbr->wbr_wh[i]); + wbr->wbr_wh[i] = NULL; + } + } + + err = test_children_busy(root, bindex, verbose); + if (unlikely(err)) { + if (do_wh) + goto out_wh; + goto out; + } + + err = 0; + if (to_free) { + /* + * now we confirmed the branch is deletable. + * let's free the remaining opened dirs on the branch. + */ + di_write_unlock(root); + br_del_file(to_free, opened, br_id); + di_write_lock_child(root); + } + + sysaufs_brs_del(sb, bindex); /* remove successors */ + dbgaufs_xino_del(br); /* remove one */ + au_br_do_del(sb, bindex, br); + sysaufs_brs_add(sb, bindex); /* append successors */ + dbgaufs_brs_add(sb, bindex, /*topdown*/1); /* rename successors */ + + if (!bindex) { + au_cpup_attr_all(d_inode(root), /*force*/1); + sb->s_maxbytes = au_sbr_sb(sb, 0)->s_maxbytes; + } else + au_sub_nlink(d_inode(root), d_inode(del->h_path.dentry)); + if (au_opt_test(mnt_flags, PLINK)) + au_plink_half_refresh(sb, br_id); + + goto out; /* success */ + +out_wh: + /* revert */ + rerr = au_br_init_wh(sb, br, br->br_perm); + if (rerr) + pr_warn("failed re-creating base whiteout, %s. (%d)\n", + del->pathname, rerr); +out: + if (to_free) + au_farray_free(to_free, opened); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_ibusy(struct super_block *sb, struct aufs_ibusy __user *arg) +{ + int err; + aufs_bindex_t btop, bbot; + struct aufs_ibusy ibusy; + struct inode *inode, *h_inode; + + err = -EPERM; + if (unlikely(!capable(CAP_SYS_ADMIN))) + goto out; + + err = copy_from_user(&ibusy, arg, sizeof(ibusy)); + if (!err) + err = !access_ok(VERIFY_WRITE, &arg->h_ino, sizeof(arg->h_ino)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out; + } + + err = -EINVAL; + si_read_lock(sb, AuLock_FLUSH); + if (unlikely(ibusy.bindex < 0 || ibusy.bindex > au_sbbot(sb))) + goto out_unlock; + + err = 0; + ibusy.h_ino = 0; /* invalid */ + inode = ilookup(sb, ibusy.ino); + if (!inode + || inode->i_ino == AUFS_ROOT_INO + || au_is_bad_inode(inode)) + goto out_unlock; + + ii_read_lock_child(inode); + btop = au_ibtop(inode); + bbot = au_ibbot(inode); + if (btop <= ibusy.bindex && ibusy.bindex <= bbot) { + h_inode = au_h_iptr(inode, ibusy.bindex); + if (h_inode && au_test_ibusy(inode, btop, bbot)) + ibusy.h_ino = h_inode->i_ino; + } + ii_read_unlock(inode); + iput(inode); + +out_unlock: + si_read_unlock(sb); + if (!err) { + err = __put_user(ibusy.h_ino, &arg->h_ino); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + } + } +out: + return err; +} + +long au_ibusy_ioctl(struct file *file, unsigned long arg) +{ + return au_ibusy(file->f_path.dentry->d_sb, (void __user *)arg); +} + +#ifdef CONFIG_COMPAT +long au_ibusy_compat_ioctl(struct file *file, unsigned long arg) +{ + return au_ibusy(file->f_path.dentry->d_sb, compat_ptr(arg)); +} +#endif + +/* ---------------------------------------------------------------------- */ + +/* + * change a branch permission + */ + +static void au_warn_ima(void) +{ +#ifdef CONFIG_IMA + /* since it doesn't support mark_files_ro() */ + AuWarn1("RW -> RO makes IMA to produce wrong message\n"); +#endif +} + +static int do_need_sigen_inc(int a, int b) +{ + return au_br_whable(a) && !au_br_whable(b); +} + +static int need_sigen_inc(int old, int new) +{ + return do_need_sigen_inc(old, new) + || do_need_sigen_inc(new, old); +} + +static int au_br_mod_files_ro(struct super_block *sb, aufs_bindex_t bindex) +{ + int err, do_warn; + unsigned int mnt_flags; + unsigned long long ull, max; + aufs_bindex_t br_id; + unsigned char verbose, writer; + struct file *file, *hf, **array; + struct au_hfile *hfile; + + mnt_flags = au_mntflags(sb); + verbose = !!au_opt_test(mnt_flags, VERBOSE); + + array = au_farray_alloc(sb, &max); + err = PTR_ERR(array); + if (IS_ERR(array)) + goto out; + + do_warn = 0; + br_id = au_sbr_id(sb, bindex); + for (ull = 0; ull < max; ull++) { + file = array[ull]; + if (unlikely(!file)) + break; + + /* AuDbg("%pD\n", file); */ + fi_read_lock(file); + if (unlikely(au_test_mmapped(file))) { + err = -EBUSY; + AuVerbose(verbose, "mmapped %pD\n", file); + AuDbgFile(file); + FiMustNoWaiters(file); + fi_read_unlock(file); + goto out_array; + } + + hfile = &au_fi(file)->fi_htop; + hf = hfile->hf_file; + if (!d_is_reg(file->f_path.dentry) + || !(file->f_mode & FMODE_WRITE) + || hfile->hf_br->br_id != br_id + || !(hf->f_mode & FMODE_WRITE)) + array[ull] = NULL; + else { + do_warn = 1; + get_file(file); + } + + FiMustNoWaiters(file); + fi_read_unlock(file); + fput(file); + } + + err = 0; + if (do_warn) + au_warn_ima(); + + for (ull = 0; ull < max; ull++) { + file = array[ull]; + if (!file) + continue; + + /* todo: already flushed? */ + /* + * fs/super.c:mark_files_ro() is gone, but aufs keeps its + * approach which resets f_mode and calls mnt_drop_write() and + * file_release_write() for each file, because the branch + * attribute in aufs world is totally different from the native + * fs rw/ro mode. + */ + /* fi_read_lock(file); */ + hfile = &au_fi(file)->fi_htop; + hf = hfile->hf_file; + /* fi_read_unlock(file); */ + spin_lock(&hf->f_lock); + writer = !!(hf->f_mode & FMODE_WRITER); + hf->f_mode &= ~(FMODE_WRITE | FMODE_WRITER); + spin_unlock(&hf->f_lock); + if (writer) { + put_write_access(file_inode(hf)); + __mnt_drop_write(hf->f_path.mnt); + } + } + +out_array: + au_farray_free(array, max); +out: + AuTraceErr(err); + return err; +} + +int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount, + int *do_refresh) +{ + int err, rerr; + aufs_bindex_t bindex; + struct dentry *root; + struct au_branch *br; + struct au_br_fhsm *bf; + + root = sb->s_root; + bindex = au_find_dbindex(root, mod->h_root); + if (bindex < 0) { + if (remount) + return 0; /* success */ + err = -ENOENT; + pr_err("%s no such branch\n", mod->path); + goto out; + } + AuDbg("bindex b%d\n", bindex); + + err = test_br(d_inode(mod->h_root), mod->perm, mod->path); + if (unlikely(err)) + goto out; + + br = au_sbr(sb, bindex); + AuDebugOn(mod->h_root != au_br_dentry(br)); + if (br->br_perm == mod->perm) + return 0; /* success */ + + /* pre-allocate for non-fhsm --> fhsm */ + bf = NULL; + if (!au_br_fhsm(br->br_perm) && au_br_fhsm(mod->perm)) { + err = au_fhsm_br_alloc(br); + if (unlikely(err)) + goto out; + bf = br->br_fhsm; + br->br_fhsm = NULL; + } + + if (au_br_writable(br->br_perm)) { + /* remove whiteout base */ + err = au_br_init_wh(sb, br, mod->perm); + if (unlikely(err)) + goto out_bf; + + if (!au_br_writable(mod->perm)) { + /* rw --> ro, file might be mmapped */ + DiMustNoWaiters(root); + IiMustNoWaiters(d_inode(root)); + di_write_unlock(root); + err = au_br_mod_files_ro(sb, bindex); + /* aufs_write_lock() calls ..._child() */ + di_write_lock_child(root); + + if (unlikely(err)) { + rerr = -ENOMEM; + br->br_wbr = kzalloc(sizeof(*br->br_wbr), + GFP_NOFS); + if (br->br_wbr) + rerr = au_wbr_init(br, sb, br->br_perm); + if (unlikely(rerr)) { + AuIOErr("nested error %d (%d)\n", + rerr, err); + br->br_perm = mod->perm; + } + } + } + } else if (au_br_writable(mod->perm)) { + /* ro --> rw */ + err = -ENOMEM; + br->br_wbr = kzalloc(sizeof(*br->br_wbr), GFP_NOFS); + if (br->br_wbr) { + err = au_wbr_init(br, sb, mod->perm); + if (unlikely(err)) { + au_kfree_rcu(br->br_wbr); + br->br_wbr = NULL; + } + } + } + if (unlikely(err)) + goto out_bf; + + if (au_br_fhsm(br->br_perm)) { + if (!au_br_fhsm(mod->perm)) { + /* fhsm --> non-fhsm */ + au_br_fhsm_fin(br->br_fhsm); + au_kfree_rcu(br->br_fhsm); + br->br_fhsm = NULL; + } + } else if (au_br_fhsm(mod->perm)) + /* non-fhsm --> fhsm */ + br->br_fhsm = bf; + + *do_refresh |= need_sigen_inc(br->br_perm, mod->perm); + br->br_perm = mod->perm; + goto out; /* success */ + +out_bf: + au_kfree_try_rcu(bf); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_br_stfs(struct au_branch *br, struct aufs_stfs *stfs) +{ + int err; + struct kstatfs kstfs; + + err = vfs_statfs(&br->br_path, &kstfs); + if (!err) { + stfs->f_blocks = kstfs.f_blocks; + stfs->f_bavail = kstfs.f_bavail; + stfs->f_files = kstfs.f_files; + stfs->f_ffree = kstfs.f_ffree; + } + + return err; +} diff --git a/fs/aufs/branch.h b/fs/aufs/branch.h new file mode 100644 index 0000000000000..594c8bd674b2a --- /dev/null +++ b/fs/aufs/branch.h @@ -0,0 +1,366 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * branch filesystems and xino for them + */ + +#ifndef __AUFS_BRANCH_H__ +#define __AUFS_BRANCH_H__ + +#ifdef __KERNEL__ + +#include +#include "dirren.h" +#include "dynop.h" +#include "lcnt.h" +#include "rwsem.h" +#include "super.h" + +/* ---------------------------------------------------------------------- */ + +/* a xino file */ +struct au_xino { + struct file **xi_file; + unsigned int xi_nfile; + + struct { + spinlock_t spin; + ino_t *array; + int total; + /* reserved for future use */ + /* unsigned long *bitmap; */ + wait_queue_head_t wqh; + } xi_nondir; + + struct mutex xi_mtx; /* protects xi_file array */ + struct hlist_bl_head xi_writing; + + atomic_t xi_truncating; + + struct kref xi_kref; +}; + +/* File-based Hierarchical Storage Management */ +struct au_br_fhsm { +#ifdef CONFIG_AUFS_FHSM + struct mutex bf_lock; + unsigned long bf_jiffy; + struct aufs_stfs bf_stfs; + int bf_readable; +#endif +}; + +/* members for writable branch only */ +enum {AuBrWh_BASE, AuBrWh_PLINK, AuBrWh_ORPH, AuBrWh_Last}; +struct au_wbr { + struct au_rwsem wbr_wh_rwsem; + struct dentry *wbr_wh[AuBrWh_Last]; + atomic_t wbr_wh_running; +#define wbr_whbase wbr_wh[AuBrWh_BASE] /* whiteout base */ +#define wbr_plink wbr_wh[AuBrWh_PLINK] /* pseudo-link dir */ +#define wbr_orph wbr_wh[AuBrWh_ORPH] /* dir for orphans */ + + /* mfs mode */ + unsigned long long wbr_bytes; +}; + +/* ext2 has 3 types of operations at least, ext3 has 4 */ +#define AuBrDynOp (AuDyLast * 4) + +#ifdef CONFIG_AUFS_HFSNOTIFY +/* support for asynchronous destruction */ +struct au_br_hfsnotify { + struct fsnotify_group *hfsn_group; +}; +#endif + +/* sysfs entries */ +struct au_brsysfs { + char name[16]; + struct attribute attr; +}; + +enum { + AuBrSysfs_BR, + AuBrSysfs_BRID, + AuBrSysfs_Last +}; + +/* protected by superblock rwsem */ +struct au_branch { + struct au_xino *br_xino; + + aufs_bindex_t br_id; + + int br_perm; + struct path br_path; + spinlock_t br_dykey_lock; + struct au_dykey *br_dykey[AuBrDynOp]; + au_lcnt_t br_nfiles; /* opened files */ + au_lcnt_t br_count; /* in-use for other */ + + struct au_wbr *br_wbr; + struct au_br_fhsm *br_fhsm; + +#ifdef CONFIG_AUFS_HFSNOTIFY + struct au_br_hfsnotify *br_hfsn; +#endif + +#ifdef CONFIG_SYSFS + /* entries under sysfs per mount-point */ + struct au_brsysfs br_sysfs[AuBrSysfs_Last]; +#endif + +#ifdef CONFIG_DEBUG_FS + struct dentry *br_dbgaufs; /* xino */ +#endif + + struct au_dr_br br_dirren; +}; + +/* ---------------------------------------------------------------------- */ + +static inline struct vfsmount *au_br_mnt(struct au_branch *br) +{ + return br->br_path.mnt; +} + +static inline struct dentry *au_br_dentry(struct au_branch *br) +{ + return br->br_path.dentry; +} + +static inline struct super_block *au_br_sb(struct au_branch *br) +{ + return au_br_mnt(br)->mnt_sb; +} + +static inline int au_br_rdonly(struct au_branch *br) +{ + return (sb_rdonly(au_br_sb(br)) + || !au_br_writable(br->br_perm)) + ? -EROFS : 0; +} + +static inline int au_br_hnotifyable(int brperm __maybe_unused) +{ +#ifdef CONFIG_AUFS_HNOTIFY + return !(brperm & AuBrPerm_RR); +#else + return 0; +#endif +} + +static inline int au_br_test_oflag(int oflag, struct au_branch *br) +{ + int err, exec_flag; + + err = 0; + exec_flag = oflag & __FMODE_EXEC; + if (unlikely(exec_flag && path_noexec(&br->br_path))) + err = -EACCES; + + return err; +} + +static inline void au_xino_get(struct au_branch *br) +{ + struct au_xino *xi; + + xi = br->br_xino; + if (xi) + kref_get(&xi->xi_kref); +} + +static inline int au_xino_count(struct au_branch *br) +{ + int v; + struct au_xino *xi; + + v = 0; + xi = br->br_xino; + if (xi) + v = kref_read(&xi->xi_kref); + + return v; +} + +/* ---------------------------------------------------------------------- */ + +/* branch.c */ +struct au_sbinfo; +void au_br_free(struct au_sbinfo *sinfo); +int au_br_index(struct super_block *sb, aufs_bindex_t br_id); +struct au_opt_add; +int au_br_add(struct super_block *sb, struct au_opt_add *add, int remount); +struct au_opt_del; +int au_br_del(struct super_block *sb, struct au_opt_del *del, int remount); +long au_ibusy_ioctl(struct file *file, unsigned long arg); +#ifdef CONFIG_COMPAT +long au_ibusy_compat_ioctl(struct file *file, unsigned long arg); +#endif +struct au_opt_mod; +int au_br_mod(struct super_block *sb, struct au_opt_mod *mod, int remount, + int *do_refresh); +struct aufs_stfs; +int au_br_stfs(struct au_branch *br, struct aufs_stfs *stfs); + +/* xino.c */ +static const loff_t au_loff_max = LLONG_MAX; + +aufs_bindex_t au_xi_root(struct super_block *sb, struct dentry *dentry); +struct file *au_xino_create(struct super_block *sb, char *fpath, int silent, + int wbrtop); +struct file *au_xino_create2(struct super_block *sb, struct path *base, + struct file *copy_src); +struct au_xi_new { + struct au_xino *xi; /* switch between xino and xigen */ + int idx; + struct path *base; + struct file *copy_src; +}; +struct file *au_xi_new(struct super_block *sb, struct au_xi_new *xinew); + +int au_xino_read(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + ino_t *ino); +int au_xino_write(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + ino_t ino); +ssize_t xino_fread(vfs_readf_t func, struct file *file, void *buf, size_t size, + loff_t *pos); +ssize_t xino_fwrite(vfs_writef_t func, struct file *file, void *buf, + size_t size, loff_t *pos); + +int au_xib_trunc(struct super_block *sb); +int au_xino_trunc(struct super_block *sb, aufs_bindex_t bindex, int idx_begin); + +struct au_xino *au_xino_alloc(unsigned int nfile); +int au_xino_put(struct au_branch *br); +struct file *au_xino_file1(struct au_xino *xi); + +struct au_opt_xino; +void au_xino_clr(struct super_block *sb); +int au_xino_set(struct super_block *sb, struct au_opt_xino *xiopt, int remount); +struct file *au_xino_def(struct super_block *sb); +int au_xino_init_br(struct super_block *sb, struct au_branch *br, ino_t hino, + struct path *base); + +ino_t au_xino_new_ino(struct super_block *sb); +void au_xino_delete_inode(struct inode *inode, const int unlinked); + +void au_xinondir_leave(struct super_block *sb, aufs_bindex_t bindex, + ino_t h_ino, int idx); +int au_xinondir_enter(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + int *idx); + +int au_xino_path(struct seq_file *seq, struct file *file); + +/* ---------------------------------------------------------------------- */ + +/* @idx is signed to accept -1 meaning the first file */ +static inline struct file *au_xino_file(struct au_xino *xi, int idx) +{ + struct file *file; + + file = NULL; + if (!xi) + goto out; + + if (idx >= 0) { + if (idx < xi->xi_nfile) + file = xi->xi_file[idx]; + } else + file = au_xino_file1(xi); + +out: + return file; +} + +/* ---------------------------------------------------------------------- */ + +/* Superblock to branch */ +static inline +aufs_bindex_t au_sbr_id(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_sbr(sb, bindex)->br_id; +} + +static inline +struct vfsmount *au_sbr_mnt(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_br_mnt(au_sbr(sb, bindex)); +} + +static inline +struct super_block *au_sbr_sb(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_br_sb(au_sbr(sb, bindex)); +} + +static inline int au_sbr_perm(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_sbr(sb, bindex)->br_perm; +} + +static inline int au_sbr_whable(struct super_block *sb, aufs_bindex_t bindex) +{ + return au_br_whable(au_sbr_perm(sb, bindex)); +} + +/* ---------------------------------------------------------------------- */ + +#define wbr_wh_read_lock(wbr) au_rw_read_lock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_write_lock(wbr) au_rw_write_lock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_read_trylock(wbr) au_rw_read_trylock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_write_trylock(wbr) au_rw_write_trylock(&(wbr)->wbr_wh_rwsem) +/* +#define wbr_wh_read_trylock_nested(wbr) \ + au_rw_read_trylock_nested(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_write_trylock_nested(wbr) \ + au_rw_write_trylock_nested(&(wbr)->wbr_wh_rwsem) +*/ + +#define wbr_wh_read_unlock(wbr) au_rw_read_unlock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_write_unlock(wbr) au_rw_write_unlock(&(wbr)->wbr_wh_rwsem) +#define wbr_wh_downgrade_lock(wbr) au_rw_dgrade_lock(&(wbr)->wbr_wh_rwsem) + +#define WbrWhMustNoWaiters(wbr) AuRwMustNoWaiters(&(wbr)->wbr_wh_rwsem) +#define WbrWhMustAnyLock(wbr) AuRwMustAnyLock(&(wbr)->wbr_wh_rwsem) +#define WbrWhMustWriteLock(wbr) AuRwMustWriteLock(&(wbr)->wbr_wh_rwsem) + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_FHSM +static inline void au_br_fhsm_init(struct au_br_fhsm *brfhsm) +{ + mutex_init(&brfhsm->bf_lock); + brfhsm->bf_jiffy = 0; + brfhsm->bf_readable = 0; +} + +static inline void au_br_fhsm_fin(struct au_br_fhsm *brfhsm) +{ + mutex_destroy(&brfhsm->bf_lock); +} +#else +AuStubVoid(au_br_fhsm_init, struct au_br_fhsm *brfhsm) +AuStubVoid(au_br_fhsm_fin, struct au_br_fhsm *brfhsm) +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_BRANCH_H__ */ diff --git a/fs/aufs/conf.mk b/fs/aufs/conf.mk new file mode 100644 index 0000000000000..12782f8e0f381 --- /dev/null +++ b/fs/aufs/conf.mk @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0 + +AuConfStr = CONFIG_AUFS_FS=${CONFIG_AUFS_FS} + +define AuConf +ifdef ${1} +AuConfStr += ${1}=${${1}} +endif +endef + +AuConfAll = BRANCH_MAX_127 BRANCH_MAX_511 BRANCH_MAX_1023 BRANCH_MAX_32767 \ + SBILIST \ + HNOTIFY HFSNOTIFY \ + EXPORT INO_T_64 \ + XATTR \ + FHSM \ + RDU \ + DIRREN \ + SHWH \ + BR_RAMFS \ + BR_FUSE POLL \ + BR_HFSPLUS \ + BDEV_LOOP \ + DEBUG MAGIC_SYSRQ +$(foreach i, ${AuConfAll}, \ + $(eval $(call AuConf,CONFIG_AUFS_${i}))) + +AuConfName = ${obj}/conf.str +${AuConfName}.tmp: FORCE + @echo ${AuConfStr} | tr ' ' '\n' | sed -e 's/^/"/' -e 's/$$/\\n"/' > $@ +${AuConfName}: ${AuConfName}.tmp + @diff -q $< $@ > /dev/null 2>&1 || { \ + echo ' GEN ' $@; \ + cp -p $< $@; \ + } +FORCE: +clean-files += ${AuConfName} ${AuConfName}.tmp +${obj}/sysfs.o: ${AuConfName} + +-include ${srctree}/${src}/conf_priv.mk diff --git a/fs/aufs/cpup.c b/fs/aufs/cpup.c new file mode 100644 index 0000000000000..ab69fcea78008 --- /dev/null +++ b/fs/aufs/cpup.c @@ -0,0 +1,1448 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * copy-up functions, see wbr_policy.c for copy-down + */ + +#include +#include +#include +#include "aufs.h" + +void au_cpup_attr_flags(struct inode *dst, unsigned int iflags) +{ + const unsigned int mask = S_DEAD | S_SWAPFILE | S_PRIVATE + | S_NOATIME | S_NOCMTIME | S_AUTOMOUNT; + + BUILD_BUG_ON(sizeof(iflags) != sizeof(dst->i_flags)); + + dst->i_flags |= iflags & ~mask; + if (au_test_fs_notime(dst->i_sb)) + dst->i_flags |= S_NOATIME | S_NOCMTIME; +} + +void au_cpup_attr_timesizes(struct inode *inode) +{ + struct inode *h_inode; + + h_inode = au_h_iptr(inode, au_ibtop(inode)); + fsstack_copy_attr_times(inode, h_inode); + fsstack_copy_inode_size(inode, h_inode); +} + +void au_cpup_attr_nlink(struct inode *inode, int force) +{ + struct inode *h_inode; + struct super_block *sb; + aufs_bindex_t bindex, bbot; + + sb = inode->i_sb; + bindex = au_ibtop(inode); + h_inode = au_h_iptr(inode, bindex); + if (!force + && !S_ISDIR(h_inode->i_mode) + && au_opt_test(au_mntflags(sb), PLINK) + && au_plink_test(inode)) + return; + + /* + * 0 can happen in revalidating. + * h_inode->i_mutex may not be held here, but it is harmless since once + * i_nlink reaches 0, it will never become positive except O_TMPFILE + * case. + * todo: O_TMPFILE+linkat(AT_SYMLINK_FOLLOW) bypassing aufs may cause + * the incorrect link count. + */ + set_nlink(inode, h_inode->i_nlink); + + /* + * fewer nlink makes find(1) noisy, but larger nlink doesn't. + * it may includes whplink directory. + */ + if (S_ISDIR(h_inode->i_mode)) { + bbot = au_ibbot(inode); + for (bindex++; bindex <= bbot; bindex++) { + h_inode = au_h_iptr(inode, bindex); + if (h_inode) + au_add_nlink(inode, h_inode); + } + } +} + +void au_cpup_attr_changeable(struct inode *inode) +{ + struct inode *h_inode; + + h_inode = au_h_iptr(inode, au_ibtop(inode)); + inode->i_mode = h_inode->i_mode; + inode->i_uid = h_inode->i_uid; + inode->i_gid = h_inode->i_gid; + au_cpup_attr_timesizes(inode); + au_cpup_attr_flags(inode, h_inode->i_flags); +} + +void au_cpup_igen(struct inode *inode, struct inode *h_inode) +{ + struct au_iinfo *iinfo = au_ii(inode); + + IiMustWriteLock(inode); + + iinfo->ii_higen = h_inode->i_generation; + iinfo->ii_hsb1 = h_inode->i_sb; +} + +void au_cpup_attr_all(struct inode *inode, int force) +{ + struct inode *h_inode; + + h_inode = au_h_iptr(inode, au_ibtop(inode)); + au_cpup_attr_changeable(inode); + if (inode->i_nlink > 0) + au_cpup_attr_nlink(inode, force); + inode->i_rdev = h_inode->i_rdev; + inode->i_blkbits = h_inode->i_blkbits; + au_cpup_igen(inode, h_inode); +} + +/* ---------------------------------------------------------------------- */ + +/* Note: dt_dentry and dt_h_dentry are not dget/dput-ed */ + +/* keep the timestamps of the parent dir when cpup */ +void au_dtime_store(struct au_dtime *dt, struct dentry *dentry, + struct path *h_path) +{ + struct inode *h_inode; + + dt->dt_dentry = dentry; + dt->dt_h_path = *h_path; + h_inode = d_inode(h_path->dentry); + dt->dt_atime = h_inode->i_atime; + dt->dt_mtime = h_inode->i_mtime; + /* smp_mb(); */ +} + +void au_dtime_revert(struct au_dtime *dt) +{ + struct iattr attr; + int err; + + attr.ia_atime = dt->dt_atime; + attr.ia_mtime = dt->dt_mtime; + attr.ia_valid = ATTR_FORCE | ATTR_MTIME | ATTR_MTIME_SET + | ATTR_ATIME | ATTR_ATIME_SET; + + /* no delegation since this is a directory */ + err = vfsub_notify_change(&dt->dt_h_path, &attr, /*delegated*/NULL); + if (unlikely(err)) + pr_warn("restoring timestamps failed(%d). ignored\n", err); +} + +/* ---------------------------------------------------------------------- */ + +/* internal use only */ +struct au_cpup_reg_attr { + int valid; + struct kstat st; + unsigned int iflags; /* inode->i_flags */ +}; + +static noinline_for_stack +int cpup_iattr(struct dentry *dst, aufs_bindex_t bindex, struct dentry *h_src, + struct au_cpup_reg_attr *h_src_attr) +{ + int err, sbits, icex; + unsigned int mnt_flags; + unsigned char verbose; + struct iattr ia; + struct path h_path; + struct inode *h_isrc, *h_idst; + struct kstat *h_st; + struct au_branch *br; + + h_path.dentry = au_h_dptr(dst, bindex); + h_idst = d_inode(h_path.dentry); + br = au_sbr(dst->d_sb, bindex); + h_path.mnt = au_br_mnt(br); + h_isrc = d_inode(h_src); + ia.ia_valid = ATTR_FORCE | ATTR_UID | ATTR_GID + | ATTR_ATIME | ATTR_MTIME + | ATTR_ATIME_SET | ATTR_MTIME_SET; + if (h_src_attr && h_src_attr->valid) { + h_st = &h_src_attr->st; + ia.ia_uid = h_st->uid; + ia.ia_gid = h_st->gid; + ia.ia_atime = h_st->atime; + ia.ia_mtime = h_st->mtime; + if (h_idst->i_mode != h_st->mode + && !S_ISLNK(h_idst->i_mode)) { + ia.ia_valid |= ATTR_MODE; + ia.ia_mode = h_st->mode; + } + sbits = !!(h_st->mode & (S_ISUID | S_ISGID)); + au_cpup_attr_flags(h_idst, h_src_attr->iflags); + } else { + ia.ia_uid = h_isrc->i_uid; + ia.ia_gid = h_isrc->i_gid; + ia.ia_atime = h_isrc->i_atime; + ia.ia_mtime = h_isrc->i_mtime; + if (h_idst->i_mode != h_isrc->i_mode + && !S_ISLNK(h_idst->i_mode)) { + ia.ia_valid |= ATTR_MODE; + ia.ia_mode = h_isrc->i_mode; + } + sbits = !!(h_isrc->i_mode & (S_ISUID | S_ISGID)); + au_cpup_attr_flags(h_idst, h_isrc->i_flags); + } + /* no delegation since it is just created */ + err = vfsub_notify_change(&h_path, &ia, /*delegated*/NULL); + + /* is this nfs only? */ + if (!err && sbits && au_test_nfs(h_path.dentry->d_sb)) { + ia.ia_valid = ATTR_FORCE | ATTR_MODE; + ia.ia_mode = h_isrc->i_mode; + err = vfsub_notify_change(&h_path, &ia, /*delegated*/NULL); + } + + icex = br->br_perm & AuBrAttr_ICEX; + if (!err) { + mnt_flags = au_mntflags(dst->d_sb); + verbose = !!au_opt_test(mnt_flags, VERBOSE); + err = au_cpup_xattr(h_path.dentry, h_src, icex, verbose); + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_copy_file(struct file *dst, struct file *src, loff_t len, + char *buf, unsigned long blksize) +{ + int err; + size_t sz, rbytes, wbytes; + unsigned char all_zero; + char *p, *zp; + struct inode *h_inode; + /* reduce stack usage */ + struct iattr *ia; + + zp = page_address(ZERO_PAGE(0)); + if (unlikely(!zp)) + return -ENOMEM; /* possible? */ + + err = 0; + all_zero = 0; + while (len) { + AuDbg("len %lld\n", len); + sz = blksize; + if (len < blksize) + sz = len; + + rbytes = 0; + /* todo: signal_pending? */ + while (!rbytes || err == -EAGAIN || err == -EINTR) { + rbytes = vfsub_read_k(src, buf, sz, &src->f_pos); + err = rbytes; + } + if (unlikely(err < 0)) + break; + + all_zero = 0; + if (len >= rbytes && rbytes == blksize) + all_zero = !memcmp(buf, zp, rbytes); + if (!all_zero) { + wbytes = rbytes; + p = buf; + while (wbytes) { + size_t b; + + b = vfsub_write_k(dst, p, wbytes, &dst->f_pos); + err = b; + /* todo: signal_pending? */ + if (unlikely(err == -EAGAIN || err == -EINTR)) + continue; + if (unlikely(err < 0)) + break; + wbytes -= b; + p += b; + } + if (unlikely(err < 0)) + break; + } else { + loff_t res; + + AuLabel(hole); + res = vfsub_llseek(dst, rbytes, SEEK_CUR); + err = res; + if (unlikely(res < 0)) + break; + } + len -= rbytes; + err = 0; + } + + /* the last block may be a hole */ + if (!err && all_zero) { + AuLabel(last hole); + + err = 1; + if (au_test_nfs(dst->f_path.dentry->d_sb)) { + /* nfs requires this step to make last hole */ + /* is this only nfs? */ + do { + /* todo: signal_pending? */ + err = vfsub_write_k(dst, "\0", 1, &dst->f_pos); + } while (err == -EAGAIN || err == -EINTR); + if (err == 1) + dst->f_pos--; + } + + if (err == 1) { + ia = (void *)buf; + ia->ia_size = dst->f_pos; + ia->ia_valid = ATTR_SIZE | ATTR_FILE; + ia->ia_file = dst; + h_inode = file_inode(dst); + inode_lock_nested(h_inode, AuLsc_I_CHILD2); + /* no delegation since it is just created */ + err = vfsub_notify_change(&dst->f_path, ia, + /*delegated*/NULL); + inode_unlock(h_inode); + } + } + + return err; +} + +int au_copy_file(struct file *dst, struct file *src, loff_t len) +{ + int err; + unsigned long blksize; + unsigned char do_kfree; + char *buf; + struct super_block *h_sb; + + err = -ENOMEM; + h_sb = file_inode(dst)->i_sb; + blksize = h_sb->s_blocksize; + if (!blksize || PAGE_SIZE < blksize) + blksize = PAGE_SIZE; + AuDbg("blksize %lu\n", blksize); + do_kfree = (blksize != PAGE_SIZE && blksize >= sizeof(struct iattr *)); + if (do_kfree) + buf = kmalloc(blksize, GFP_NOFS); + else + buf = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!buf)) + goto out; + + if (len > (1 << 22)) + AuDbg("copying a large file %lld\n", (long long)len); + + src->f_pos = 0; + dst->f_pos = 0; + err = au_do_copy_file(dst, src, len, buf, blksize); + if (do_kfree) { + AuDebugOn(!au_kfree_do_sz_test(blksize)); + au_kfree_do_rcu(buf); + } else + free_page((unsigned long)buf); + +out: + return err; +} + +static int au_do_copy(struct file *dst, struct file *src, loff_t len) +{ + int err; + struct super_block *h_src_sb; + struct inode *h_src_inode; + + h_src_inode = file_inode(src); + h_src_sb = h_src_inode->i_sb; + + /* XFS acquires inode_lock */ + if (!au_test_xfs(h_src_sb)) + err = au_copy_file(dst, src, len); + else { + inode_unlock_shared(h_src_inode); + err = au_copy_file(dst, src, len); + inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD); + } + + return err; +} + +static int au_clone_or_copy(struct file *dst, struct file *src, loff_t len) +{ + int err; + struct super_block *h_src_sb; + struct inode *h_src_inode; + + h_src_inode = file_inode(src); + h_src_sb = h_src_inode->i_sb; + if (h_src_sb != file_inode(dst)->i_sb + || !dst->f_op->clone_file_range) { + err = au_do_copy(dst, src, len); + goto out; + } + + if (!au_test_nfs(h_src_sb)) { + inode_unlock_shared(h_src_inode); + err = vfsub_clone_file_range(src, dst, len); + inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD); + } else + err = vfsub_clone_file_range(src, dst, len); + /* older XFS has a condition in cloning */ + if (unlikely(err != -EOPNOTSUPP)) + goto out; + + /* the backend fs on NFS may not support cloning */ + err = au_do_copy(dst, src, len); + +out: + AuTraceErr(err); + return err; +} + +/* + * to support a sparse file which is opened with O_APPEND, + * we need to close the file. + */ +static int au_cp_regular(struct au_cp_generic *cpg) +{ + int err, i; + enum { SRC, DST }; + struct { + aufs_bindex_t bindex; + unsigned int flags; + struct dentry *dentry; + int force_wr; + struct file *file; + } *f, file[] = { + { + .bindex = cpg->bsrc, + .flags = O_RDONLY | O_NOATIME | O_LARGEFILE, + }, + { + .bindex = cpg->bdst, + .flags = O_WRONLY | O_NOATIME | O_LARGEFILE, + .force_wr = !!au_ftest_cpup(cpg->flags, RWDST), + } + }; + struct au_branch *br; + struct super_block *sb, *h_src_sb; + struct inode *h_src_inode; + struct task_struct *tsk = current; + + /* bsrc branch can be ro/rw. */ + sb = cpg->dentry->d_sb; + f = file; + for (i = 0; i < 2; i++, f++) { + f->dentry = au_h_dptr(cpg->dentry, f->bindex); + f->file = au_h_open(cpg->dentry, f->bindex, f->flags, + /*file*/NULL, f->force_wr); + if (IS_ERR(f->file)) { + err = PTR_ERR(f->file); + if (i == SRC) + goto out; + else + goto out_src; + } + } + + /* try stopping to update while we copyup */ + h_src_inode = d_inode(file[SRC].dentry); + h_src_sb = h_src_inode->i_sb; + if (!au_test_nfs(h_src_sb)) + IMustLock(h_src_inode); + err = au_clone_or_copy(file[DST].file, file[SRC].file, cpg->len); + + /* i wonder if we had O_NO_DELAY_FPUT flag */ + if (tsk->flags & PF_KTHREAD) + __fput_sync(file[DST].file); + else { + /* it happened actually */ + fput(file[DST].file); + /* + * too bad. + * we have to call both since we don't know which place the file + * was added to. + */ + task_work_run(); + flush_delayed_fput(); + } + br = au_sbr(sb, file[DST].bindex); + au_lcnt_dec(&br->br_nfiles); + +out_src: + fput(file[SRC].file); + br = au_sbr(sb, file[SRC].bindex); + au_lcnt_dec(&br->br_nfiles); +out: + return err; +} + +static int au_do_cpup_regular(struct au_cp_generic *cpg, + struct au_cpup_reg_attr *h_src_attr) +{ + int err, rerr; + loff_t l; + struct path h_path; + struct inode *h_src_inode, *h_dst_inode; + + err = 0; + h_src_inode = au_h_iptr(d_inode(cpg->dentry), cpg->bsrc); + l = i_size_read(h_src_inode); + if (cpg->len == -1 || l < cpg->len) + cpg->len = l; + if (cpg->len) { + /* try stopping to update while we are referencing */ + inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD); + au_pin_hdir_unlock(cpg->pin); + + h_path.dentry = au_h_dptr(cpg->dentry, cpg->bsrc); + h_path.mnt = au_sbr_mnt(cpg->dentry->d_sb, cpg->bsrc); + h_src_attr->iflags = h_src_inode->i_flags; + if (!au_test_nfs(h_src_inode->i_sb)) + err = vfsub_getattr(&h_path, &h_src_attr->st); + else { + inode_unlock_shared(h_src_inode); + err = vfsub_getattr(&h_path, &h_src_attr->st); + inode_lock_shared_nested(h_src_inode, AuLsc_I_CHILD); + } + if (unlikely(err)) { + inode_unlock_shared(h_src_inode); + goto out; + } + h_src_attr->valid = 1; + if (!au_test_nfs(h_src_inode->i_sb)) { + err = au_cp_regular(cpg); + inode_unlock_shared(h_src_inode); + } else { + inode_unlock_shared(h_src_inode); + err = au_cp_regular(cpg); + } + rerr = au_pin_hdir_relock(cpg->pin); + if (!err && rerr) + err = rerr; + } + if (!err && (h_src_inode->i_state & I_LINKABLE)) { + h_path.dentry = au_h_dptr(cpg->dentry, cpg->bdst); + h_dst_inode = d_inode(h_path.dentry); + spin_lock(&h_dst_inode->i_lock); + h_dst_inode->i_state |= I_LINKABLE; + spin_unlock(&h_dst_inode->i_lock); + } + +out: + return err; +} + +static int au_do_cpup_symlink(struct path *h_path, struct dentry *h_src, + struct inode *h_dir) +{ + int err, symlen; + mm_segment_t old_fs; + union { + char *k; + char __user *u; + } sym; + + err = -ENOMEM; + sym.k = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!sym.k)) + goto out; + + /* unnecessary to support mmap_sem since symlink is not mmap-able */ + old_fs = get_fs(); + set_fs(KERNEL_DS); + symlen = vfs_readlink(h_src, sym.u, PATH_MAX); + err = symlen; + set_fs(old_fs); + + if (symlen > 0) { + sym.k[symlen] = 0; + err = vfsub_symlink(h_dir, h_path, sym.k); + } + free_page((unsigned long)sym.k); + +out: + return err; +} + +/* + * regardless 'acl' option, reset all ACL. + * All ACL will be copied up later from the original entry on the lower branch. + */ +static int au_reset_acl(struct inode *h_dir, struct path *h_path, umode_t mode) +{ + int err; + struct dentry *h_dentry; + struct inode *h_inode; + + h_dentry = h_path->dentry; + h_inode = d_inode(h_dentry); + /* forget_all_cached_acls(h_inode)); */ + err = vfsub_removexattr(h_dentry, XATTR_NAME_POSIX_ACL_ACCESS); + AuTraceErr(err); + if (err == -EOPNOTSUPP) + err = 0; + if (!err) + err = vfsub_acl_chmod(h_inode, mode); + + AuTraceErr(err); + return err; +} + +static int au_do_cpup_dir(struct au_cp_generic *cpg, struct dentry *dst_parent, + struct inode *h_dir, struct path *h_path) +{ + int err; + struct inode *dir, *inode; + + err = vfsub_removexattr(h_path->dentry, XATTR_NAME_POSIX_ACL_DEFAULT); + AuTraceErr(err); + if (err == -EOPNOTSUPP) + err = 0; + if (unlikely(err)) + goto out; + + /* + * strange behaviour from the users view, + * particularly setattr case + */ + dir = d_inode(dst_parent); + if (au_ibtop(dir) == cpg->bdst) + au_cpup_attr_nlink(dir, /*force*/1); + inode = d_inode(cpg->dentry); + au_cpup_attr_nlink(inode, /*force*/1); + +out: + return err; +} + +static noinline_for_stack +int cpup_entry(struct au_cp_generic *cpg, struct dentry *dst_parent, + struct au_cpup_reg_attr *h_src_attr) +{ + int err; + umode_t mode; + unsigned int mnt_flags; + unsigned char isdir, isreg, force; + const unsigned char do_dt = !!au_ftest_cpup(cpg->flags, DTIME); + struct au_dtime dt; + struct path h_path; + struct dentry *h_src, *h_dst, *h_parent; + struct inode *h_inode, *h_dir; + struct super_block *sb; + + /* bsrc branch can be ro/rw. */ + h_src = au_h_dptr(cpg->dentry, cpg->bsrc); + h_inode = d_inode(h_src); + AuDebugOn(h_inode != au_h_iptr(d_inode(cpg->dentry), cpg->bsrc)); + + /* try stopping to be referenced while we are creating */ + h_dst = au_h_dptr(cpg->dentry, cpg->bdst); + if (au_ftest_cpup(cpg->flags, RENAME)) + AuDebugOn(strncmp(h_dst->d_name.name, AUFS_WH_PFX, + AUFS_WH_PFX_LEN)); + h_parent = h_dst->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + AuDebugOn(h_parent != h_dst->d_parent); + + sb = cpg->dentry->d_sb; + h_path.mnt = au_sbr_mnt(sb, cpg->bdst); + if (do_dt) { + h_path.dentry = h_parent; + au_dtime_store(&dt, dst_parent, &h_path); + } + h_path.dentry = h_dst; + + isreg = 0; + isdir = 0; + mode = h_inode->i_mode; + switch (mode & S_IFMT) { + case S_IFREG: + isreg = 1; + err = vfsub_create(h_dir, &h_path, 0600, /*want_excl*/true); + if (!err) + err = au_do_cpup_regular(cpg, h_src_attr); + break; + case S_IFDIR: + isdir = 1; + err = vfsub_mkdir(h_dir, &h_path, mode); + if (!err) + err = au_do_cpup_dir(cpg, dst_parent, h_dir, &h_path); + break; + case S_IFLNK: + err = au_do_cpup_symlink(&h_path, h_src, h_dir); + break; + case S_IFCHR: + case S_IFBLK: + AuDebugOn(!capable(CAP_MKNOD)); + /*FALLTHROUGH*/ + case S_IFIFO: + case S_IFSOCK: + err = vfsub_mknod(h_dir, &h_path, mode, h_inode->i_rdev); + break; + default: + AuIOErr("Unknown inode type 0%o\n", mode); + err = -EIO; + } + if (!err) + err = au_reset_acl(h_dir, &h_path, mode); + + mnt_flags = au_mntflags(sb); + if (!au_opt_test(mnt_flags, UDBA_NONE) + && !isdir + && au_opt_test(mnt_flags, XINO) + && (h_inode->i_nlink == 1 + || (h_inode->i_state & I_LINKABLE)) + /* todo: unnecessary? */ + /* && d_inode(cpg->dentry)->i_nlink == 1 */ + && cpg->bdst < cpg->bsrc + && !au_ftest_cpup(cpg->flags, KEEPLINO)) + au_xino_write(sb, cpg->bsrc, h_inode->i_ino, /*ino*/0); + /* ignore this error */ + + if (!err) { + force = 0; + if (isreg) { + force = !!cpg->len; + if (cpg->len == -1) + force = !!i_size_read(h_inode); + } + au_fhsm_wrote(sb, cpg->bdst, force); + } + + if (do_dt) + au_dtime_revert(&dt); + return err; +} + +static int au_do_ren_after_cpup(struct au_cp_generic *cpg, struct path *h_path) +{ + int err; + struct dentry *dentry, *h_dentry, *h_parent, *parent; + struct inode *h_dir; + aufs_bindex_t bdst; + + dentry = cpg->dentry; + bdst = cpg->bdst; + h_dentry = au_h_dptr(dentry, bdst); + if (!au_ftest_cpup(cpg->flags, OVERWRITE)) { + dget(h_dentry); + au_set_h_dptr(dentry, bdst, NULL); + err = au_lkup_neg(dentry, bdst, /*wh*/0); + if (!err) + h_path->dentry = dget(au_h_dptr(dentry, bdst)); + au_set_h_dptr(dentry, bdst, h_dentry); + } else { + err = 0; + parent = dget_parent(dentry); + h_parent = au_h_dptr(parent, bdst); + dput(parent); + h_path->dentry = vfsub_lkup_one(&dentry->d_name, h_parent); + if (IS_ERR(h_path->dentry)) + err = PTR_ERR(h_path->dentry); + } + if (unlikely(err)) + goto out; + + h_parent = h_dentry->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + AuDbg("%pd %pd\n", h_dentry, h_path->dentry); + /* no delegation since it is just created */ + err = vfsub_rename(h_dir, h_dentry, h_dir, h_path, /*delegated*/NULL, + /*flags*/0); + dput(h_path->dentry); + +out: + return err; +} + +/* + * copyup the @dentry from @bsrc to @bdst. + * the caller must set the both of lower dentries. + * @len is for truncating when it is -1 copyup the entire file. + * in link/rename cases, @dst_parent may be different from the real one. + * basic->bsrc can be larger than basic->bdst. + * aufs doesn't touch the credential so + * security_inode_copy_up{,_xattr}() are unnecessary. + */ +static int au_cpup_single(struct au_cp_generic *cpg, struct dentry *dst_parent) +{ + int err, rerr; + aufs_bindex_t old_ibtop; + unsigned char isdir, plink; + struct dentry *h_src, *h_dst, *h_parent; + struct inode *dst_inode, *h_dir, *inode, *delegated, *src_inode; + struct super_block *sb; + struct au_branch *br; + /* to reduce stack size */ + struct { + struct au_dtime dt; + struct path h_path; + struct au_cpup_reg_attr h_src_attr; + } *a; + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + a->h_src_attr.valid = 0; + + sb = cpg->dentry->d_sb; + br = au_sbr(sb, cpg->bdst); + a->h_path.mnt = au_br_mnt(br); + h_dst = au_h_dptr(cpg->dentry, cpg->bdst); + h_parent = h_dst->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + + h_src = au_h_dptr(cpg->dentry, cpg->bsrc); + inode = d_inode(cpg->dentry); + + if (!dst_parent) + dst_parent = dget_parent(cpg->dentry); + else + dget(dst_parent); + + plink = !!au_opt_test(au_mntflags(sb), PLINK); + dst_inode = au_h_iptr(inode, cpg->bdst); + if (dst_inode) { + if (unlikely(!plink)) { + err = -EIO; + AuIOErr("hi%lu(i%lu) exists on b%d " + "but plink is disabled\n", + dst_inode->i_ino, inode->i_ino, cpg->bdst); + goto out_parent; + } + + if (dst_inode->i_nlink) { + const int do_dt = au_ftest_cpup(cpg->flags, DTIME); + + h_src = au_plink_lkup(inode, cpg->bdst); + err = PTR_ERR(h_src); + if (IS_ERR(h_src)) + goto out_parent; + if (unlikely(d_is_negative(h_src))) { + err = -EIO; + AuIOErr("i%lu exists on b%d " + "but not pseudo-linked\n", + inode->i_ino, cpg->bdst); + dput(h_src); + goto out_parent; + } + + if (do_dt) { + a->h_path.dentry = h_parent; + au_dtime_store(&a->dt, dst_parent, &a->h_path); + } + + a->h_path.dentry = h_dst; + delegated = NULL; + err = vfsub_link(h_src, h_dir, &a->h_path, &delegated); + if (!err && au_ftest_cpup(cpg->flags, RENAME)) + err = au_do_ren_after_cpup(cpg, &a->h_path); + if (do_dt) + au_dtime_revert(&a->dt); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + dput(h_src); + goto out_parent; + } else + /* todo: cpup_wh_file? */ + /* udba work */ + au_update_ibrange(inode, /*do_put_zero*/1); + } + + isdir = S_ISDIR(inode->i_mode); + old_ibtop = au_ibtop(inode); + err = cpup_entry(cpg, dst_parent, &a->h_src_attr); + if (unlikely(err)) + goto out_rev; + dst_inode = d_inode(h_dst); + inode_lock_nested(dst_inode, AuLsc_I_CHILD2); + /* todo: necessary? */ + /* au_pin_hdir_unlock(cpg->pin); */ + + err = cpup_iattr(cpg->dentry, cpg->bdst, h_src, &a->h_src_attr); + if (unlikely(err)) { + /* todo: necessary? */ + /* au_pin_hdir_relock(cpg->pin); */ /* ignore an error */ + inode_unlock(dst_inode); + goto out_rev; + } + + if (cpg->bdst < old_ibtop) { + if (S_ISREG(inode->i_mode)) { + err = au_dy_iaop(inode, cpg->bdst, dst_inode); + if (unlikely(err)) { + /* ignore an error */ + /* au_pin_hdir_relock(cpg->pin); */ + inode_unlock(dst_inode); + goto out_rev; + } + } + au_set_ibtop(inode, cpg->bdst); + } else + au_set_ibbot(inode, cpg->bdst); + au_set_h_iptr(inode, cpg->bdst, au_igrab(dst_inode), + au_hi_flags(inode, isdir)); + + /* todo: necessary? */ + /* err = au_pin_hdir_relock(cpg->pin); */ + inode_unlock(dst_inode); + if (unlikely(err)) + goto out_rev; + + src_inode = d_inode(h_src); + if (!isdir + && (src_inode->i_nlink > 1 + || src_inode->i_state & I_LINKABLE) + && plink) + au_plink_append(inode, cpg->bdst, h_dst); + + if (au_ftest_cpup(cpg->flags, RENAME)) { + a->h_path.dentry = h_dst; + err = au_do_ren_after_cpup(cpg, &a->h_path); + } + if (!err) + goto out_parent; /* success */ + + /* revert */ +out_rev: + a->h_path.dentry = h_parent; + au_dtime_store(&a->dt, dst_parent, &a->h_path); + a->h_path.dentry = h_dst; + rerr = 0; + if (d_is_positive(h_dst)) { + if (!isdir) { + /* no delegation since it is just created */ + rerr = vfsub_unlink(h_dir, &a->h_path, + /*delegated*/NULL, /*force*/0); + } else + rerr = vfsub_rmdir(h_dir, &a->h_path); + } + au_dtime_revert(&a->dt); + if (rerr) { + AuIOErr("failed removing broken entry(%d, %d)\n", err, rerr); + err = -EIO; + } +out_parent: + dput(dst_parent); + au_kfree_rcu(a); +out: + return err; +} + +#if 0 /* reserved */ +struct au_cpup_single_args { + int *errp; + struct au_cp_generic *cpg; + struct dentry *dst_parent; +}; + +static void au_call_cpup_single(void *args) +{ + struct au_cpup_single_args *a = args; + + au_pin_hdir_acquire_nest(a->cpg->pin); + *a->errp = au_cpup_single(a->cpg, a->dst_parent); + au_pin_hdir_release(a->cpg->pin); +} +#endif + +/* + * prevent SIGXFSZ in copy-up. + * testing CAP_MKNOD is for generic fs, + * but CAP_FSETID is for xfs only, currently. + */ +static int au_cpup_sio_test(struct au_pin *pin, umode_t mode) +{ + int do_sio; + struct super_block *sb; + struct inode *h_dir; + + do_sio = 0; + sb = au_pinned_parent(pin)->d_sb; + if (!au_wkq_test() + && (!au_sbi(sb)->si_plink_maint_pid + || au_plink_maint(sb, AuLock_NOPLM))) { + switch (mode & S_IFMT) { + case S_IFREG: + /* no condition about RLIMIT_FSIZE and the file size */ + do_sio = 1; + break; + case S_IFCHR: + case S_IFBLK: + do_sio = !capable(CAP_MKNOD); + break; + } + if (!do_sio) + do_sio = ((mode & (S_ISUID | S_ISGID)) + && !capable(CAP_FSETID)); + /* this workaround may be removed in the future */ + if (!do_sio) { + h_dir = au_pinned_h_dir(pin); + do_sio = h_dir->i_mode & S_ISVTX; + } + } + + return do_sio; +} + +#if 0 /* reserved */ +int au_sio_cpup_single(struct au_cp_generic *cpg, struct dentry *dst_parent) +{ + int err, wkq_err; + struct dentry *h_dentry; + + h_dentry = au_h_dptr(cpg->dentry, cpg->bsrc); + if (!au_cpup_sio_test(pin, d_inode(h_dentry)->i_mode)) + err = au_cpup_single(cpg, dst_parent); + else { + struct au_cpup_single_args args = { + .errp = &err, + .cpg = cpg, + .dst_parent = dst_parent + }; + wkq_err = au_wkq_wait(au_call_cpup_single, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + return err; +} +#endif + +/* + * copyup the @dentry from the first active lower branch to @bdst, + * using au_cpup_single(). + */ +static int au_cpup_simple(struct au_cp_generic *cpg) +{ + int err; + unsigned int flags_orig; + struct dentry *dentry; + + AuDebugOn(cpg->bsrc < 0); + + dentry = cpg->dentry; + DiMustWriteLock(dentry); + + err = au_lkup_neg(dentry, cpg->bdst, /*wh*/1); + if (!err) { + flags_orig = cpg->flags; + au_fset_cpup(cpg->flags, RENAME); + err = au_cpup_single(cpg, NULL); + cpg->flags = flags_orig; + if (!err) + return 0; /* success */ + + /* revert */ + au_set_h_dptr(dentry, cpg->bdst, NULL); + au_set_dbtop(dentry, cpg->bsrc); + } + + return err; +} + +struct au_cpup_simple_args { + int *errp; + struct au_cp_generic *cpg; +}; + +static void au_call_cpup_simple(void *args) +{ + struct au_cpup_simple_args *a = args; + + au_pin_hdir_acquire_nest(a->cpg->pin); + *a->errp = au_cpup_simple(a->cpg); + au_pin_hdir_release(a->cpg->pin); +} + +static int au_do_sio_cpup_simple(struct au_cp_generic *cpg) +{ + int err, wkq_err; + struct dentry *dentry, *parent; + struct file *h_file; + struct inode *h_dir; + + dentry = cpg->dentry; + h_file = NULL; + if (au_ftest_cpup(cpg->flags, HOPEN)) { + AuDebugOn(cpg->bsrc < 0); + h_file = au_h_open_pre(dentry, cpg->bsrc, /*force_wr*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + } + + parent = dget_parent(dentry); + h_dir = au_h_iptr(d_inode(parent), cpg->bdst); + if (!au_test_h_perm_sio(h_dir, MAY_EXEC | MAY_WRITE) + && !au_cpup_sio_test(cpg->pin, d_inode(dentry)->i_mode)) + err = au_cpup_simple(cpg); + else { + struct au_cpup_simple_args args = { + .errp = &err, + .cpg = cpg + }; + wkq_err = au_wkq_wait(au_call_cpup_simple, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + dput(parent); + if (h_file) + au_h_open_post(dentry, cpg->bsrc, h_file); + +out: + return err; +} + +int au_sio_cpup_simple(struct au_cp_generic *cpg) +{ + aufs_bindex_t bsrc, bbot; + struct dentry *dentry, *h_dentry; + + if (cpg->bsrc < 0) { + dentry = cpg->dentry; + bbot = au_dbbot(dentry); + for (bsrc = cpg->bdst + 1; bsrc <= bbot; bsrc++) { + h_dentry = au_h_dptr(dentry, bsrc); + if (h_dentry) { + AuDebugOn(d_is_negative(h_dentry)); + break; + } + } + AuDebugOn(bsrc > bbot); + cpg->bsrc = bsrc; + } + AuDebugOn(cpg->bsrc <= cpg->bdst); + return au_do_sio_cpup_simple(cpg); +} + +int au_sio_cpdown_simple(struct au_cp_generic *cpg) +{ + AuDebugOn(cpg->bdst <= cpg->bsrc); + return au_do_sio_cpup_simple(cpg); +} + +/* ---------------------------------------------------------------------- */ + +/* + * copyup the deleted file for writing. + */ +static int au_do_cpup_wh(struct au_cp_generic *cpg, struct dentry *wh_dentry, + struct file *file) +{ + int err; + unsigned int flags_orig; + aufs_bindex_t bsrc_orig; + struct au_dinfo *dinfo; + struct { + struct au_hdentry *hd; + struct dentry *h_dentry; + } hdst, hsrc; + + dinfo = au_di(cpg->dentry); + AuRwMustWriteLock(&dinfo->di_rwsem); + + bsrc_orig = cpg->bsrc; + cpg->bsrc = dinfo->di_btop; + hdst.hd = au_hdentry(dinfo, cpg->bdst); + hdst.h_dentry = hdst.hd->hd_dentry; + hdst.hd->hd_dentry = wh_dentry; + dinfo->di_btop = cpg->bdst; + + hsrc.h_dentry = NULL; + if (file) { + hsrc.hd = au_hdentry(dinfo, cpg->bsrc); + hsrc.h_dentry = hsrc.hd->hd_dentry; + hsrc.hd->hd_dentry = au_hf_top(file)->f_path.dentry; + } + flags_orig = cpg->flags; + cpg->flags = !AuCpup_DTIME; + err = au_cpup_single(cpg, /*h_parent*/NULL); + cpg->flags = flags_orig; + if (file) { + if (!err) + err = au_reopen_nondir(file); + hsrc.hd->hd_dentry = hsrc.h_dentry; + } + hdst.hd->hd_dentry = hdst.h_dentry; + dinfo->di_btop = cpg->bsrc; + cpg->bsrc = bsrc_orig; + + return err; +} + +static int au_cpup_wh(struct au_cp_generic *cpg, struct file *file) +{ + int err; + aufs_bindex_t bdst; + struct au_dtime dt; + struct dentry *dentry, *parent, *h_parent, *wh_dentry; + struct au_branch *br; + struct path h_path; + + dentry = cpg->dentry; + bdst = cpg->bdst; + br = au_sbr(dentry->d_sb, bdst); + parent = dget_parent(dentry); + h_parent = au_h_dptr(parent, bdst); + wh_dentry = au_whtmp_lkup(h_parent, br, &dentry->d_name); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out; + + h_path.dentry = h_parent; + h_path.mnt = au_br_mnt(br); + au_dtime_store(&dt, parent, &h_path); + err = au_do_cpup_wh(cpg, wh_dentry, file); + if (unlikely(err)) + goto out_wh; + + dget(wh_dentry); + h_path.dentry = wh_dentry; + if (!d_is_dir(wh_dentry)) { + /* no delegation since it is just created */ + err = vfsub_unlink(d_inode(h_parent), &h_path, + /*delegated*/NULL, /*force*/0); + } else + err = vfsub_rmdir(d_inode(h_parent), &h_path); + if (unlikely(err)) { + AuIOErr("failed remove copied-up tmp file %pd(%d)\n", + wh_dentry, err); + err = -EIO; + } + au_dtime_revert(&dt); + au_set_hi_wh(d_inode(dentry), bdst, wh_dentry); + +out_wh: + dput(wh_dentry); +out: + dput(parent); + return err; +} + +struct au_cpup_wh_args { + int *errp; + struct au_cp_generic *cpg; + struct file *file; +}; + +static void au_call_cpup_wh(void *args) +{ + struct au_cpup_wh_args *a = args; + + au_pin_hdir_acquire_nest(a->cpg->pin); + *a->errp = au_cpup_wh(a->cpg, a->file); + au_pin_hdir_release(a->cpg->pin); +} + +int au_sio_cpup_wh(struct au_cp_generic *cpg, struct file *file) +{ + int err, wkq_err; + aufs_bindex_t bdst; + struct dentry *dentry, *parent, *h_orph, *h_parent; + struct inode *dir, *h_dir, *h_tmpdir; + struct au_wbr *wbr; + struct au_pin wh_pin, *pin_orig; + + dentry = cpg->dentry; + bdst = cpg->bdst; + parent = dget_parent(dentry); + dir = d_inode(parent); + h_orph = NULL; + h_parent = NULL; + h_dir = au_igrab(au_h_iptr(dir, bdst)); + h_tmpdir = h_dir; + pin_orig = NULL; + if (!h_dir->i_nlink) { + wbr = au_sbr(dentry->d_sb, bdst)->br_wbr; + h_orph = wbr->wbr_orph; + + h_parent = dget(au_h_dptr(parent, bdst)); + au_set_h_dptr(parent, bdst, dget(h_orph)); + h_tmpdir = d_inode(h_orph); + au_set_h_iptr(dir, bdst, au_igrab(h_tmpdir), /*flags*/0); + + inode_lock_nested(h_tmpdir, AuLsc_I_PARENT3); + /* todo: au_h_open_pre()? */ + + pin_orig = cpg->pin; + au_pin_init(&wh_pin, dentry, bdst, AuLsc_DI_PARENT, + AuLsc_I_PARENT3, cpg->pin->udba, AuPin_DI_LOCKED); + cpg->pin = &wh_pin; + } + + if (!au_test_h_perm_sio(h_tmpdir, MAY_EXEC | MAY_WRITE) + && !au_cpup_sio_test(cpg->pin, d_inode(dentry)->i_mode)) + err = au_cpup_wh(cpg, file); + else { + struct au_cpup_wh_args args = { + .errp = &err, + .cpg = cpg, + .file = file + }; + wkq_err = au_wkq_wait(au_call_cpup_wh, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + if (h_orph) { + inode_unlock(h_tmpdir); + /* todo: au_h_open_post()? */ + au_set_h_iptr(dir, bdst, au_igrab(h_dir), /*flags*/0); + au_set_h_dptr(parent, bdst, h_parent); + AuDebugOn(!pin_orig); + cpg->pin = pin_orig; + } + iput(h_dir); + dput(parent); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * generic routine for both of copy-up and copy-down. + */ +/* cf. revalidate function in file.c */ +int au_cp_dirs(struct dentry *dentry, aufs_bindex_t bdst, + int (*cp)(struct dentry *dentry, aufs_bindex_t bdst, + struct au_pin *pin, + struct dentry *h_parent, void *arg), + void *arg) +{ + int err; + struct au_pin pin; + struct dentry *d, *parent, *h_parent, *real_parent, *h_dentry; + + err = 0; + parent = dget_parent(dentry); + if (IS_ROOT(parent)) + goto out; + + au_pin_init(&pin, dentry, bdst, AuLsc_DI_PARENT2, AuLsc_I_PARENT2, + au_opt_udba(dentry->d_sb), AuPin_MNT_WRITE); + + /* do not use au_dpage */ + real_parent = parent; + while (1) { + dput(parent); + parent = dget_parent(dentry); + h_parent = au_h_dptr(parent, bdst); + if (h_parent) + goto out; /* success */ + + /* find top dir which is necessary to cpup */ + do { + d = parent; + dput(parent); + parent = dget_parent(d); + di_read_lock_parent3(parent, !AuLock_IR); + h_parent = au_h_dptr(parent, bdst); + di_read_unlock(parent, !AuLock_IR); + } while (!h_parent); + + if (d != real_parent) + di_write_lock_child3(d); + + /* somebody else might create while we were sleeping */ + h_dentry = au_h_dptr(d, bdst); + if (!h_dentry || d_is_negative(h_dentry)) { + if (h_dentry) + au_update_dbtop(d); + + au_pin_set_dentry(&pin, d); + err = au_do_pin(&pin); + if (!err) { + err = cp(d, bdst, &pin, h_parent, arg); + au_unpin(&pin); + } + } + + if (d != real_parent) + di_write_unlock(d); + if (unlikely(err)) + break; + } + +out: + dput(parent); + return err; +} + +static int au_cpup_dir(struct dentry *dentry, aufs_bindex_t bdst, + struct au_pin *pin, + struct dentry *h_parent __maybe_unused, + void *arg __maybe_unused) +{ + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = bdst, + .bsrc = -1, + .len = 0, + .pin = pin, + .flags = AuCpup_DTIME + }; + return au_sio_cpup_simple(&cpg); +} + +int au_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst) +{ + return au_cp_dirs(dentry, bdst, au_cpup_dir, NULL); +} + +int au_test_and_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst) +{ + int err; + struct dentry *parent; + struct inode *dir; + + parent = dget_parent(dentry); + dir = d_inode(parent); + err = 0; + if (au_h_iptr(dir, bdst)) + goto out; + + di_read_unlock(parent, AuLock_IR); + di_write_lock_parent(parent); + /* someone else might change our inode while we were sleeping */ + if (!au_h_iptr(dir, bdst)) + err = au_cpup_dirs(dentry, bdst); + di_downgrade_lock(parent, AuLock_IR); + +out: + dput(parent); + return err; +} diff --git a/fs/aufs/cpup.h b/fs/aufs/cpup.h new file mode 100644 index 0000000000000..d02f8150fa05e --- /dev/null +++ b/fs/aufs/cpup.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * copy-up/down functions + */ + +#ifndef __AUFS_CPUP_H__ +#define __AUFS_CPUP_H__ + +#ifdef __KERNEL__ + +#include + +struct inode; +struct file; +struct au_pin; + +void au_cpup_attr_flags(struct inode *dst, unsigned int iflags); +void au_cpup_attr_timesizes(struct inode *inode); +void au_cpup_attr_nlink(struct inode *inode, int force); +void au_cpup_attr_changeable(struct inode *inode); +void au_cpup_igen(struct inode *inode, struct inode *h_inode); +void au_cpup_attr_all(struct inode *inode, int force); + +/* ---------------------------------------------------------------------- */ + +struct au_cp_generic { + struct dentry *dentry; + aufs_bindex_t bdst, bsrc; + loff_t len; + struct au_pin *pin; + unsigned int flags; +}; + +/* cpup flags */ +#define AuCpup_DTIME 1 /* do dtime_store/revert */ +#define AuCpup_KEEPLINO (1 << 1) /* do not clear the lower xino, + for link(2) */ +#define AuCpup_RENAME (1 << 2) /* rename after cpup */ +#define AuCpup_HOPEN (1 << 3) /* call h_open_pre/post() in + cpup */ +#define AuCpup_OVERWRITE (1 << 4) /* allow overwriting the + existing entry */ +#define AuCpup_RWDST (1 << 5) /* force write target even if + the branch is marked as RO */ + +#ifndef CONFIG_AUFS_BR_HFSPLUS +#undef AuCpup_HOPEN +#define AuCpup_HOPEN 0 +#endif + +#define au_ftest_cpup(flags, name) ((flags) & AuCpup_##name) +#define au_fset_cpup(flags, name) \ + do { (flags) |= AuCpup_##name; } while (0) +#define au_fclr_cpup(flags, name) \ + do { (flags) &= ~AuCpup_##name; } while (0) + +int au_copy_file(struct file *dst, struct file *src, loff_t len); +int au_sio_cpup_simple(struct au_cp_generic *cpg); +int au_sio_cpdown_simple(struct au_cp_generic *cpg); +int au_sio_cpup_wh(struct au_cp_generic *cpg, struct file *file); + +int au_cp_dirs(struct dentry *dentry, aufs_bindex_t bdst, + int (*cp)(struct dentry *dentry, aufs_bindex_t bdst, + struct au_pin *pin, + struct dentry *h_parent, void *arg), + void *arg); +int au_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst); +int au_test_and_cpup_dirs(struct dentry *dentry, aufs_bindex_t bdst); + +/* ---------------------------------------------------------------------- */ + +/* keep timestamps when copyup */ +struct au_dtime { + struct dentry *dt_dentry; + struct path dt_h_path; + struct timespec64 dt_atime, dt_mtime; +}; +void au_dtime_store(struct au_dtime *dt, struct dentry *dentry, + struct path *h_path); +void au_dtime_revert(struct au_dtime *dt); + +#endif /* __KERNEL__ */ +#endif /* __AUFS_CPUP_H__ */ diff --git a/fs/aufs/dbgaufs.c b/fs/aufs/dbgaufs.c new file mode 100644 index 0000000000000..80266f4fcf7d7 --- /dev/null +++ b/fs/aufs/dbgaufs.c @@ -0,0 +1,526 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * debugfs interface + */ + +#include +#include "aufs.h" + +#ifndef CONFIG_SYSFS +#error DEBUG_FS depends upon SYSFS +#endif + +static struct dentry *dbgaufs; +static const mode_t dbgaufs_mode = 0444; + +/* 20 is max digits length of ulong 64 */ +struct dbgaufs_arg { + int n; + char a[20 * 4]; +}; + +/* + * common function for all XINO files + */ +static int dbgaufs_xi_release(struct inode *inode __maybe_unused, + struct file *file) +{ + void *p; + + p = file->private_data; + if (p) { + /* this is struct dbgaufs_arg */ + AuDebugOn(!au_kfree_sz_test(p)); + au_kfree_do_rcu(p); + } + return 0; +} + +static int dbgaufs_xi_open(struct file *xf, struct file *file, int do_fcnt, + int cnt) +{ + int err; + struct kstat st; + struct dbgaufs_arg *p; + + err = -ENOMEM; + p = kmalloc(sizeof(*p), GFP_NOFS); + if (unlikely(!p)) + goto out; + + err = 0; + p->n = 0; + file->private_data = p; + if (!xf) + goto out; + + err = vfsub_getattr(&xf->f_path, &st); + if (!err) { + if (do_fcnt) + p->n = snprintf + (p->a, sizeof(p->a), "%d, %llux%u %lld\n", + cnt, st.blocks, st.blksize, + (long long)st.size); + else + p->n = snprintf(p->a, sizeof(p->a), "%llux%u %lld\n", + st.blocks, st.blksize, + (long long)st.size); + AuDebugOn(p->n >= sizeof(p->a)); + } else { + p->n = snprintf(p->a, sizeof(p->a), "err %d\n", err); + err = 0; + } + +out: + return err; +} + +static ssize_t dbgaufs_xi_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dbgaufs_arg *p; + + p = file->private_data; + return simple_read_from_buffer(buf, count, ppos, p->a, p->n); +} + +/* ---------------------------------------------------------------------- */ + +struct dbgaufs_plink_arg { + int n; + char a[]; +}; + +static int dbgaufs_plink_release(struct inode *inode __maybe_unused, + struct file *file) +{ + free_page((unsigned long)file->private_data); + return 0; +} + +static int dbgaufs_plink_open(struct inode *inode, struct file *file) +{ + int err, i, limit; + unsigned long n, sum; + struct dbgaufs_plink_arg *p; + struct au_sbinfo *sbinfo; + struct super_block *sb; + struct hlist_bl_head *hbl; + + err = -ENOMEM; + p = (void *)get_zeroed_page(GFP_NOFS); + if (unlikely(!p)) + goto out; + + err = -EFBIG; + sbinfo = inode->i_private; + sb = sbinfo->si_sb; + si_noflush_read_lock(sb); + if (au_opt_test(au_mntflags(sb), PLINK)) { + limit = PAGE_SIZE - sizeof(p->n); + + /* the number of buckets */ + n = snprintf(p->a + p->n, limit, "%d\n", AuPlink_NHASH); + p->n += n; + limit -= n; + + sum = 0; + for (i = 0, hbl = sbinfo->si_plink; i < AuPlink_NHASH; + i++, hbl++) { + n = au_hbl_count(hbl); + sum += n; + + n = snprintf(p->a + p->n, limit, "%lu ", n); + p->n += n; + limit -= n; + if (unlikely(limit <= 0)) + goto out_free; + } + p->a[p->n - 1] = '\n'; + + /* the sum of plinks */ + n = snprintf(p->a + p->n, limit, "%lu\n", sum); + p->n += n; + limit -= n; + if (unlikely(limit <= 0)) + goto out_free; + } else { +#define str "1\n0\n0\n" + p->n = sizeof(str) - 1; + strcpy(p->a, str); +#undef str + } + si_read_unlock(sb); + + err = 0; + file->private_data = p; + goto out; /* success */ + +out_free: + free_page((unsigned long)p); +out: + return err; +} + +static ssize_t dbgaufs_plink_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dbgaufs_plink_arg *p; + + p = file->private_data; + return simple_read_from_buffer(buf, count, ppos, p->a, p->n); +} + +static const struct file_operations dbgaufs_plink_fop = { + .owner = THIS_MODULE, + .open = dbgaufs_plink_open, + .release = dbgaufs_plink_release, + .read = dbgaufs_plink_read +}; + +/* ---------------------------------------------------------------------- */ + +static int dbgaufs_xib_open(struct inode *inode, struct file *file) +{ + int err; + struct au_sbinfo *sbinfo; + struct super_block *sb; + + sbinfo = inode->i_private; + sb = sbinfo->si_sb; + si_noflush_read_lock(sb); + err = dbgaufs_xi_open(sbinfo->si_xib, file, /*do_fcnt*/0, /*cnt*/0); + si_read_unlock(sb); + return err; +} + +static const struct file_operations dbgaufs_xib_fop = { + .owner = THIS_MODULE, + .open = dbgaufs_xib_open, + .release = dbgaufs_xi_release, + .read = dbgaufs_xi_read +}; + +/* ---------------------------------------------------------------------- */ + +#define DbgaufsXi_PREFIX "xi" + +static int dbgaufs_xino_open(struct inode *inode, struct file *file) +{ + int err, idx; + long l; + aufs_bindex_t bindex; + char *p, a[sizeof(DbgaufsXi_PREFIX) + 8]; + struct au_sbinfo *sbinfo; + struct super_block *sb; + struct au_xino *xi; + struct file *xf; + struct qstr *name; + struct au_branch *br; + + err = -ENOENT; + name = &file->f_path.dentry->d_name; + if (unlikely(name->len < sizeof(DbgaufsXi_PREFIX) + || memcmp(name->name, DbgaufsXi_PREFIX, + sizeof(DbgaufsXi_PREFIX) - 1))) + goto out; + + AuDebugOn(name->len >= sizeof(a)); + memcpy(a, name->name, name->len); + a[name->len] = '\0'; + p = strchr(a, '-'); + if (p) + *p = '\0'; + err = kstrtol(a + sizeof(DbgaufsXi_PREFIX) - 1, 10, &l); + if (unlikely(err)) + goto out; + bindex = l; + idx = 0; + if (p) { + err = kstrtol(p + 1, 10, &l); + if (unlikely(err)) + goto out; + idx = l; + } + + err = -ENOENT; + sbinfo = inode->i_private; + sb = sbinfo->si_sb; + si_noflush_read_lock(sb); + if (unlikely(bindex < 0 || bindex > au_sbbot(sb))) + goto out_si; + br = au_sbr(sb, bindex); + xi = br->br_xino; + if (unlikely(idx >= xi->xi_nfile)) + goto out_si; + xf = au_xino_file(xi, idx); + if (xf) + err = dbgaufs_xi_open(xf, file, /*do_fcnt*/1, + au_xino_count(br)); + +out_si: + si_read_unlock(sb); +out: + AuTraceErr(err); + return err; +} + +static const struct file_operations dbgaufs_xino_fop = { + .owner = THIS_MODULE, + .open = dbgaufs_xino_open, + .release = dbgaufs_xi_release, + .read = dbgaufs_xi_read +}; + +void dbgaufs_xino_del(struct au_branch *br) +{ + struct dentry *dbgaufs; + + dbgaufs = br->br_dbgaufs; + if (!dbgaufs) + return; + + br->br_dbgaufs = NULL; + /* debugfs acquires the parent i_mutex */ + lockdep_off(); + debugfs_remove(dbgaufs); + lockdep_on(); +} + +void dbgaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex) +{ + aufs_bindex_t bbot; + struct au_branch *br; + + if (!au_sbi(sb)->si_dbgaufs) + return; + + bbot = au_sbbot(sb); + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + dbgaufs_xino_del(br); + } +} + +static void dbgaufs_br_do_add(struct super_block *sb, aufs_bindex_t bindex, + unsigned int idx, struct dentry *parent, + struct au_sbinfo *sbinfo) +{ + struct au_branch *br; + struct dentry *d; + /* "xi" bindex(5) "-" idx(2) NULL */ + char name[sizeof(DbgaufsXi_PREFIX) + 8]; + + if (!idx) + snprintf(name, sizeof(name), DbgaufsXi_PREFIX "%d", bindex); + else + snprintf(name, sizeof(name), DbgaufsXi_PREFIX "%d-%u", + bindex, idx); + br = au_sbr(sb, bindex); + if (br->br_dbgaufs) { + struct qstr qstr = QSTR_INIT(name, strlen(name)); + + if (!au_qstreq(&br->br_dbgaufs->d_name, &qstr)) { + /* debugfs acquires the parent i_mutex */ + lockdep_off(); + d = debugfs_rename(parent, br->br_dbgaufs, parent, + name); + lockdep_on(); + if (unlikely(!d)) + pr_warn("failed renaming %pd/%s, ignored.\n", + parent, name); + } + } else { + lockdep_off(); + br->br_dbgaufs = debugfs_create_file(name, dbgaufs_mode, parent, + sbinfo, &dbgaufs_xino_fop); + lockdep_on(); + if (unlikely(!br->br_dbgaufs)) + pr_warn("failed creating %pd/%s, ignored.\n", + parent, name); + } +} + +static void dbgaufs_br_add(struct super_block *sb, aufs_bindex_t bindex, + struct dentry *parent, struct au_sbinfo *sbinfo) +{ + struct au_branch *br; + struct au_xino *xi; + unsigned int u; + + br = au_sbr(sb, bindex); + xi = br->br_xino; + for (u = 0; u < xi->xi_nfile; u++) + dbgaufs_br_do_add(sb, bindex, u, parent, sbinfo); +} + +void dbgaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex, int topdown) +{ + struct au_sbinfo *sbinfo; + struct dentry *parent; + aufs_bindex_t bbot; + + if (!au_opt_test(au_mntflags(sb), XINO)) + return; + + sbinfo = au_sbi(sb); + parent = sbinfo->si_dbgaufs; + if (!parent) + return; + + bbot = au_sbbot(sb); + if (topdown) + for (; bindex <= bbot; bindex++) + dbgaufs_br_add(sb, bindex, parent, sbinfo); + else + for (; bbot >= bindex; bbot--) + dbgaufs_br_add(sb, bbot, parent, sbinfo); +} + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_EXPORT +static int dbgaufs_xigen_open(struct inode *inode, struct file *file) +{ + int err; + struct au_sbinfo *sbinfo; + struct super_block *sb; + + sbinfo = inode->i_private; + sb = sbinfo->si_sb; + si_noflush_read_lock(sb); + err = dbgaufs_xi_open(sbinfo->si_xigen, file, /*do_fcnt*/0, /*cnt*/0); + si_read_unlock(sb); + return err; +} + +static const struct file_operations dbgaufs_xigen_fop = { + .owner = THIS_MODULE, + .open = dbgaufs_xigen_open, + .release = dbgaufs_xi_release, + .read = dbgaufs_xi_read +}; + +static int dbgaufs_xigen_init(struct au_sbinfo *sbinfo) +{ + int err; + + /* + * This function is a dynamic '__init' function actually, + * so the tiny check for si_rwsem is unnecessary. + */ + /* AuRwMustWriteLock(&sbinfo->si_rwsem); */ + + err = -EIO; + sbinfo->si_dbgaufs_xigen = debugfs_create_file + ("xigen", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo, + &dbgaufs_xigen_fop); + if (sbinfo->si_dbgaufs_xigen) + err = 0; + + return err; +} +#else +static int dbgaufs_xigen_init(struct au_sbinfo *sbinfo) +{ + return 0; +} +#endif /* CONFIG_AUFS_EXPORT */ + +/* ---------------------------------------------------------------------- */ + +void dbgaufs_si_fin(struct au_sbinfo *sbinfo) +{ + /* + * This function is a dynamic '__fin' function actually, + * so the tiny check for si_rwsem is unnecessary. + */ + /* AuRwMustWriteLock(&sbinfo->si_rwsem); */ + + debugfs_remove_recursive(sbinfo->si_dbgaufs); + sbinfo->si_dbgaufs = NULL; +} + +int dbgaufs_si_init(struct au_sbinfo *sbinfo) +{ + int err; + char name[SysaufsSiNameLen]; + + /* + * This function is a dynamic '__init' function actually, + * so the tiny check for si_rwsem is unnecessary. + */ + /* AuRwMustWriteLock(&sbinfo->si_rwsem); */ + + err = -ENOENT; + if (!dbgaufs) { + AuErr1("/debug/aufs is uninitialized\n"); + goto out; + } + + err = -EIO; + sysaufs_name(sbinfo, name); + sbinfo->si_dbgaufs = debugfs_create_dir(name, dbgaufs); + if (unlikely(!sbinfo->si_dbgaufs)) + goto out; + + /* regardless plink/noplink option */ + sbinfo->si_dbgaufs_plink = debugfs_create_file + ("plink", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo, + &dbgaufs_plink_fop); + if (unlikely(!sbinfo->si_dbgaufs_plink)) + goto out_dir; + + /* regardless xino/noxino option */ + sbinfo->si_dbgaufs_xib = debugfs_create_file + ("xib", dbgaufs_mode, sbinfo->si_dbgaufs, sbinfo, + &dbgaufs_xib_fop); + if (unlikely(!sbinfo->si_dbgaufs_xib)) + goto out_dir; + + err = dbgaufs_xigen_init(sbinfo); + if (!err) + goto out; /* success */ + +out_dir: + dbgaufs_si_fin(sbinfo); +out: + if (unlikely(err)) + pr_err("debugfs/aufs failed\n"); + return err; +} + +/* ---------------------------------------------------------------------- */ + +void dbgaufs_fin(void) +{ + debugfs_remove(dbgaufs); +} + +int __init dbgaufs_init(void) +{ + int err; + + err = -EIO; + dbgaufs = debugfs_create_dir(AUFS_NAME, NULL); + if (dbgaufs) + err = 0; + return err; +} diff --git a/fs/aufs/dbgaufs.h b/fs/aufs/dbgaufs.h new file mode 100644 index 0000000000000..7b4ccdebb678f --- /dev/null +++ b/fs/aufs/dbgaufs.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * debugfs interface + */ + +#ifndef __DBGAUFS_H__ +#define __DBGAUFS_H__ + +#ifdef __KERNEL__ + +struct super_block; +struct au_sbinfo; +struct au_branch; + +#ifdef CONFIG_DEBUG_FS +/* dbgaufs.c */ +void dbgaufs_xino_del(struct au_branch *br); +void dbgaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex); +void dbgaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex, int topdown); +void dbgaufs_si_fin(struct au_sbinfo *sbinfo); +int dbgaufs_si_init(struct au_sbinfo *sbinfo); +void dbgaufs_fin(void); +int __init dbgaufs_init(void); +#else +AuStubVoid(dbgaufs_xino_del, struct au_branch *br) +AuStubVoid(dbgaufs_brs_del, struct super_block *sb, aufs_bindex_t bindex) +AuStubVoid(dbgaufs_brs_add, struct super_block *sb, aufs_bindex_t bindex, + int topdown) +AuStubVoid(dbgaufs_si_fin, struct au_sbinfo *sbinfo) +AuStubInt0(dbgaufs_si_init, struct au_sbinfo *sbinfo) +AuStubVoid(dbgaufs_fin, void) +AuStubInt0(__init dbgaufs_init, void) +#endif /* CONFIG_DEBUG_FS */ + +#endif /* __KERNEL__ */ +#endif /* __DBGAUFS_H__ */ diff --git a/fs/aufs/dcsub.c b/fs/aufs/dcsub.c new file mode 100644 index 0000000000000..0b9b1862b563a --- /dev/null +++ b/fs/aufs/dcsub.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sub-routines for dentry cache + */ + +#include "aufs.h" + +static void au_dpage_free(struct au_dpage *dpage) +{ + int i; + struct dentry **p; + + p = dpage->dentries; + for (i = 0; i < dpage->ndentry; i++) + dput(*p++); + free_page((unsigned long)dpage->dentries); +} + +int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp) +{ + int err; + void *p; + + err = -ENOMEM; + dpages->dpages = kmalloc(sizeof(*dpages->dpages), gfp); + if (unlikely(!dpages->dpages)) + goto out; + + p = (void *)__get_free_page(gfp); + if (unlikely(!p)) + goto out_dpages; + + dpages->dpages[0].ndentry = 0; + dpages->dpages[0].dentries = p; + dpages->ndpage = 1; + return 0; /* success */ + +out_dpages: + au_kfree_try_rcu(dpages->dpages); +out: + return err; +} + +void au_dpages_free(struct au_dcsub_pages *dpages) +{ + int i; + struct au_dpage *p; + + p = dpages->dpages; + for (i = 0; i < dpages->ndpage; i++) + au_dpage_free(p++); + au_kfree_try_rcu(dpages->dpages); +} + +static int au_dpages_append(struct au_dcsub_pages *dpages, + struct dentry *dentry, gfp_t gfp) +{ + int err, sz; + struct au_dpage *dpage; + void *p; + + dpage = dpages->dpages + dpages->ndpage - 1; + sz = PAGE_SIZE / sizeof(dentry); + if (unlikely(dpage->ndentry >= sz)) { + AuLabel(new dpage); + err = -ENOMEM; + sz = dpages->ndpage * sizeof(*dpages->dpages); + p = au_kzrealloc(dpages->dpages, sz, + sz + sizeof(*dpages->dpages), gfp, + /*may_shrink*/0); + if (unlikely(!p)) + goto out; + + dpages->dpages = p; + dpage = dpages->dpages + dpages->ndpage; + p = (void *)__get_free_page(gfp); + if (unlikely(!p)) + goto out; + + dpage->ndentry = 0; + dpage->dentries = p; + dpages->ndpage++; + } + + AuDebugOn(au_dcount(dentry) <= 0); + dpage->dentries[dpage->ndentry++] = dget_dlock(dentry); + return 0; /* success */ + +out: + return err; +} + +/* todo: BAD approach */ +/* copied from linux/fs/dcache.c */ +enum d_walk_ret { + D_WALK_CONTINUE, + D_WALK_QUIT, + D_WALK_NORETRY, + D_WALK_SKIP, +}; + +extern void d_walk(struct dentry *parent, void *data, + enum d_walk_ret (*enter)(void *, struct dentry *)); + +struct ac_dpages_arg { + int err; + struct au_dcsub_pages *dpages; + struct super_block *sb; + au_dpages_test test; + void *arg; +}; + +static enum d_walk_ret au_call_dpages_append(void *_arg, struct dentry *dentry) +{ + enum d_walk_ret ret; + struct ac_dpages_arg *arg = _arg; + + ret = D_WALK_CONTINUE; + if (dentry->d_sb == arg->sb + && !IS_ROOT(dentry) + && au_dcount(dentry) > 0 + && au_di(dentry) + && (!arg->test || arg->test(dentry, arg->arg))) { + arg->err = au_dpages_append(arg->dpages, dentry, GFP_ATOMIC); + if (unlikely(arg->err)) + ret = D_WALK_QUIT; + } + + return ret; +} + +int au_dcsub_pages(struct au_dcsub_pages *dpages, struct dentry *root, + au_dpages_test test, void *arg) +{ + struct ac_dpages_arg args = { + .err = 0, + .dpages = dpages, + .sb = root->d_sb, + .test = test, + .arg = arg + }; + + d_walk(root, &args, au_call_dpages_append); + + return args.err; +} + +int au_dcsub_pages_rev(struct au_dcsub_pages *dpages, struct dentry *dentry, + int do_include, au_dpages_test test, void *arg) +{ + int err; + + err = 0; + write_seqlock(&rename_lock); + spin_lock(&dentry->d_lock); + if (do_include + && au_dcount(dentry) > 0 + && (!test || test(dentry, arg))) + err = au_dpages_append(dpages, dentry, GFP_ATOMIC); + spin_unlock(&dentry->d_lock); + if (unlikely(err)) + goto out; + + /* + * RCU for vfsmount is unnecessary since this is a traverse in a single + * mount + */ + while (!IS_ROOT(dentry)) { + dentry = dentry->d_parent; /* rename_lock is locked */ + spin_lock(&dentry->d_lock); + if (au_dcount(dentry) > 0 + && (!test || test(dentry, arg))) + err = au_dpages_append(dpages, dentry, GFP_ATOMIC); + spin_unlock(&dentry->d_lock); + if (unlikely(err)) + break; + } + +out: + write_sequnlock(&rename_lock); + return err; +} + +static inline int au_dcsub_dpages_aufs(struct dentry *dentry, void *arg) +{ + return au_di(dentry) && dentry->d_sb == arg; +} + +int au_dcsub_pages_rev_aufs(struct au_dcsub_pages *dpages, + struct dentry *dentry, int do_include) +{ + return au_dcsub_pages_rev(dpages, dentry, do_include, + au_dcsub_dpages_aufs, dentry->d_sb); +} + +int au_test_subdir(struct dentry *d1, struct dentry *d2) +{ + struct path path[2] = { + { + .dentry = d1 + }, + { + .dentry = d2 + } + }; + + return path_is_under(path + 0, path + 1); +} diff --git a/fs/aufs/dcsub.h b/fs/aufs/dcsub.h new file mode 100644 index 0000000000000..36f7fcdd7f233 --- /dev/null +++ b/fs/aufs/dcsub.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sub-routines for dentry cache + */ + +#ifndef __AUFS_DCSUB_H__ +#define __AUFS_DCSUB_H__ + +#ifdef __KERNEL__ + +#include +#include + +struct au_dpage { + int ndentry; + struct dentry **dentries; +}; + +struct au_dcsub_pages { + int ndpage; + struct au_dpage *dpages; +}; + +/* ---------------------------------------------------------------------- */ + +/* dcsub.c */ +int au_dpages_init(struct au_dcsub_pages *dpages, gfp_t gfp); +void au_dpages_free(struct au_dcsub_pages *dpages); +typedef int (*au_dpages_test)(struct dentry *dentry, void *arg); +int au_dcsub_pages(struct au_dcsub_pages *dpages, struct dentry *root, + au_dpages_test test, void *arg); +int au_dcsub_pages_rev(struct au_dcsub_pages *dpages, struct dentry *dentry, + int do_include, au_dpages_test test, void *arg); +int au_dcsub_pages_rev_aufs(struct au_dcsub_pages *dpages, + struct dentry *dentry, int do_include); +int au_test_subdir(struct dentry *d1, struct dentry *d2); + +/* ---------------------------------------------------------------------- */ + +/* + * todo: in linux-3.13, several similar (but faster) helpers are added to + * include/linux/dcache.h. Try them (in the future). + */ + +static inline int au_d_hashed_positive(struct dentry *d) +{ + int err; + struct inode *inode = d_inode(d); + + err = 0; + if (unlikely(d_unhashed(d) + || d_is_negative(d) + || !inode->i_nlink)) + err = -ENOENT; + return err; +} + +static inline int au_d_linkable(struct dentry *d) +{ + int err; + struct inode *inode = d_inode(d); + + err = au_d_hashed_positive(d); + if (err + && d_is_positive(d) + && (inode->i_state & I_LINKABLE)) + err = 0; + return err; +} + +static inline int au_d_alive(struct dentry *d) +{ + int err; + struct inode *inode; + + err = 0; + if (!IS_ROOT(d)) + err = au_d_hashed_positive(d); + else { + inode = d_inode(d); + if (unlikely(d_unlinked(d) + || d_is_negative(d) + || !inode->i_nlink)) + err = -ENOENT; + } + return err; +} + +static inline int au_alive_dir(struct dentry *d) +{ + int err; + + err = au_d_alive(d); + if (unlikely(err || IS_DEADDIR(d_inode(d)))) + err = -ENOENT; + return err; +} + +static inline int au_qstreq(struct qstr *a, struct qstr *b) +{ + return a->len == b->len + && !memcmp(a->name, b->name, a->len); +} + +/* + * by the commit + * 360f547 2015-01-25 dcache: let the dentry count go down to zero without + * taking d_lock + * the type of d_lockref.count became int, but the inlined function d_count() + * still returns unsigned int. + * I don't know why. Maybe it is for every d_count() users? + * Anyway au_dcount() lives on. + */ +static inline int au_dcount(struct dentry *d) +{ + return (int)d_count(d); +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DCSUB_H__ */ diff --git a/fs/aufs/debug.c b/fs/aufs/debug.c new file mode 100644 index 0000000000000..f0c076c61252b --- /dev/null +++ b/fs/aufs/debug.c @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * debug print functions + */ + +#include +#include "aufs.h" + +/* Returns 0, or -errno. arg is in kp->arg. */ +static int param_atomic_t_set(const char *val, const struct kernel_param *kp) +{ + int err, n; + + err = kstrtoint(val, 0, &n); + if (!err) { + if (n > 0) + au_debug_on(); + else + au_debug_off(); + } + return err; +} + +/* Returns length written or -errno. Buffer is 4k (ie. be short!) */ +static int param_atomic_t_get(char *buffer, const struct kernel_param *kp) +{ + atomic_t *a; + + a = kp->arg; + return sprintf(buffer, "%d", atomic_read(a)); +} + +static struct kernel_param_ops param_ops_atomic_t = { + .set = param_atomic_t_set, + .get = param_atomic_t_get + /* void (*free)(void *arg) */ +}; + +atomic_t aufs_debug = ATOMIC_INIT(0); +MODULE_PARM_DESC(debug, "debug print"); +module_param_named(debug, aufs_debug, atomic_t, 0664); + +DEFINE_MUTEX(au_dbg_mtx); /* just to serialize the dbg msgs */ +char *au_plevel = KERN_DEBUG; +#define dpri(fmt, ...) do { \ + if ((au_plevel \ + && strcmp(au_plevel, KERN_DEBUG)) \ + || au_debug_test()) \ + printk("%s" fmt, au_plevel, ##__VA_ARGS__); \ +} while (0) + +/* ---------------------------------------------------------------------- */ + +void au_dpri_whlist(struct au_nhash *whlist) +{ + unsigned long ul, n; + struct hlist_head *head; + struct au_vdir_wh *pos; + + n = whlist->nh_num; + head = whlist->nh_head; + for (ul = 0; ul < n; ul++) { + hlist_for_each_entry(pos, head, wh_hash) + dpri("b%d, %.*s, %d\n", + pos->wh_bindex, + pos->wh_str.len, pos->wh_str.name, + pos->wh_str.len); + head++; + } +} + +void au_dpri_vdir(struct au_vdir *vdir) +{ + unsigned long ul; + union au_vdir_deblk_p p; + unsigned char *o; + + if (!vdir || IS_ERR(vdir)) { + dpri("err %ld\n", PTR_ERR(vdir)); + return; + } + + dpri("deblk %u, nblk %lu, deblk %p, last{%lu, %p}, ver %llu\n", + vdir->vd_deblk_sz, vdir->vd_nblk, vdir->vd_deblk, + vdir->vd_last.ul, vdir->vd_last.p.deblk, vdir->vd_version); + for (ul = 0; ul < vdir->vd_nblk; ul++) { + p.deblk = vdir->vd_deblk[ul]; + o = p.deblk; + dpri("[%lu]: %p\n", ul, o); + } +} + +static int do_pri_inode(aufs_bindex_t bindex, struct inode *inode, int hn, + struct dentry *wh) +{ + char *n = NULL; + int l = 0; + + if (!inode || IS_ERR(inode)) { + dpri("i%d: err %ld\n", bindex, PTR_ERR(inode)); + return -1; + } + + /* the type of i_blocks depends upon CONFIG_LBDAF */ + BUILD_BUG_ON(sizeof(inode->i_blocks) != sizeof(unsigned long) + && sizeof(inode->i_blocks) != sizeof(u64)); + if (wh) { + n = (void *)wh->d_name.name; + l = wh->d_name.len; + } + + dpri("i%d: %p, i%lu, %s, cnt %d, nl %u, 0%o, sz %llu, blk %llu," + " hn %d, ct %lld, np %lu, st 0x%lx, f 0x%x, v %llu, g %x%s%.*s\n", + bindex, inode, + inode->i_ino, inode->i_sb ? au_sbtype(inode->i_sb) : "??", + atomic_read(&inode->i_count), inode->i_nlink, inode->i_mode, + i_size_read(inode), (unsigned long long)inode->i_blocks, + hn, (long long)timespec64_to_ns(&inode->i_ctime) & 0x0ffff, + inode->i_mapping ? inode->i_mapping->nrpages : 0, + inode->i_state, inode->i_flags, inode_peek_iversion(inode), + inode->i_generation, + l ? ", wh " : "", l, n); + return 0; +} + +void au_dpri_inode(struct inode *inode) +{ + struct au_iinfo *iinfo; + struct au_hinode *hi; + aufs_bindex_t bindex; + int err, hn; + + err = do_pri_inode(-1, inode, -1, NULL); + if (err || !au_test_aufs(inode->i_sb) || au_is_bad_inode(inode)) + return; + + iinfo = au_ii(inode); + dpri("i-1: btop %d, bbot %d, gen %d\n", + iinfo->ii_btop, iinfo->ii_bbot, au_iigen(inode, NULL)); + if (iinfo->ii_btop < 0) + return; + hn = 0; + for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot; bindex++) { + hi = au_hinode(iinfo, bindex); + hn = !!au_hn(hi); + do_pri_inode(bindex, hi->hi_inode, hn, hi->hi_whdentry); + } +} + +void au_dpri_dalias(struct inode *inode) +{ + struct dentry *d; + + spin_lock(&inode->i_lock); + hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias) + au_dpri_dentry(d); + spin_unlock(&inode->i_lock); +} + +static int do_pri_dentry(aufs_bindex_t bindex, struct dentry *dentry) +{ + struct dentry *wh = NULL; + int hn; + struct inode *inode; + struct au_iinfo *iinfo; + struct au_hinode *hi; + + if (!dentry || IS_ERR(dentry)) { + dpri("d%d: err %ld\n", bindex, PTR_ERR(dentry)); + return -1; + } + /* do not call dget_parent() here */ + /* note: access d_xxx without d_lock */ + dpri("d%d: %p, %pd2?, %s, cnt %d, flags 0x%x, %shashed\n", + bindex, dentry, dentry, + dentry->d_sb ? au_sbtype(dentry->d_sb) : "??", + au_dcount(dentry), dentry->d_flags, + d_unhashed(dentry) ? "un" : ""); + hn = -1; + inode = NULL; + if (d_is_positive(dentry)) + inode = d_inode(dentry); + if (inode + && au_test_aufs(dentry->d_sb) + && bindex >= 0 + && !au_is_bad_inode(inode)) { + iinfo = au_ii(inode); + hi = au_hinode(iinfo, bindex); + hn = !!au_hn(hi); + wh = hi->hi_whdentry; + } + do_pri_inode(bindex, inode, hn, wh); + return 0; +} + +void au_dpri_dentry(struct dentry *dentry) +{ + struct au_dinfo *dinfo; + aufs_bindex_t bindex; + int err; + + err = do_pri_dentry(-1, dentry); + if (err || !au_test_aufs(dentry->d_sb)) + return; + + dinfo = au_di(dentry); + if (!dinfo) + return; + dpri("d-1: btop %d, bbot %d, bwh %d, bdiropq %d, gen %d, tmp %d\n", + dinfo->di_btop, dinfo->di_bbot, + dinfo->di_bwh, dinfo->di_bdiropq, au_digen(dentry), + dinfo->di_tmpfile); + if (dinfo->di_btop < 0) + return; + for (bindex = dinfo->di_btop; bindex <= dinfo->di_bbot; bindex++) + do_pri_dentry(bindex, au_hdentry(dinfo, bindex)->hd_dentry); +} + +static int do_pri_file(aufs_bindex_t bindex, struct file *file) +{ + char a[32]; + + if (!file || IS_ERR(file)) { + dpri("f%d: err %ld\n", bindex, PTR_ERR(file)); + return -1; + } + a[0] = 0; + if (bindex < 0 + && !IS_ERR_OR_NULL(file->f_path.dentry) + && au_test_aufs(file->f_path.dentry->d_sb) + && au_fi(file)) + snprintf(a, sizeof(a), ", gen %d, mmapped %d", + au_figen(file), atomic_read(&au_fi(file)->fi_mmapped)); + dpri("f%d: mode 0x%x, flags 0%o, cnt %ld, v %llu, pos %llu%s\n", + bindex, file->f_mode, file->f_flags, (long)file_count(file), + file->f_version, file->f_pos, a); + if (!IS_ERR_OR_NULL(file->f_path.dentry)) + do_pri_dentry(bindex, file->f_path.dentry); + return 0; +} + +void au_dpri_file(struct file *file) +{ + struct au_finfo *finfo; + struct au_fidir *fidir; + struct au_hfile *hfile; + aufs_bindex_t bindex; + int err; + + err = do_pri_file(-1, file); + if (err + || IS_ERR_OR_NULL(file->f_path.dentry) + || !au_test_aufs(file->f_path.dentry->d_sb)) + return; + + finfo = au_fi(file); + if (!finfo) + return; + if (finfo->fi_btop < 0) + return; + fidir = finfo->fi_hdir; + if (!fidir) + do_pri_file(finfo->fi_btop, finfo->fi_htop.hf_file); + else + for (bindex = finfo->fi_btop; + bindex >= 0 && bindex <= fidir->fd_bbot; + bindex++) { + hfile = fidir->fd_hfile + bindex; + do_pri_file(bindex, hfile ? hfile->hf_file : NULL); + } +} + +static int do_pri_br(aufs_bindex_t bindex, struct au_branch *br) +{ + struct vfsmount *mnt; + struct super_block *sb; + + if (!br || IS_ERR(br)) + goto out; + mnt = au_br_mnt(br); + if (!mnt || IS_ERR(mnt)) + goto out; + sb = mnt->mnt_sb; + if (!sb || IS_ERR(sb)) + goto out; + + dpri("s%d: {perm 0x%x, id %d, wbr %p}, " + "%s, dev 0x%02x%02x, flags 0x%lx, cnt %d, active %d, " + "xino %d\n", + bindex, br->br_perm, br->br_id, br->br_wbr, + au_sbtype(sb), MAJOR(sb->s_dev), MINOR(sb->s_dev), + sb->s_flags, sb->s_count, + atomic_read(&sb->s_active), + !!au_xino_file(br->br_xino, /*idx*/-1)); + return 0; + +out: + dpri("s%d: err %ld\n", bindex, PTR_ERR(br)); + return -1; +} + +void au_dpri_sb(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + aufs_bindex_t bindex; + int err; + /* to reduce stack size */ + struct { + struct vfsmount mnt; + struct au_branch fake; + } *a; + + /* this function can be called from magic sysrq */ + a = kzalloc(sizeof(*a), GFP_ATOMIC); + if (unlikely(!a)) { + dpri("no memory\n"); + return; + } + + a->mnt.mnt_sb = sb; + a->fake.br_path.mnt = &a->mnt; + err = do_pri_br(-1, &a->fake); + au_kfree_rcu(a); + dpri("dev 0x%x\n", sb->s_dev); + if (err || !au_test_aufs(sb)) + return; + + sbinfo = au_sbi(sb); + if (!sbinfo) + return; + dpri("nw %d, gen %u, kobj %d\n", + atomic_read(&sbinfo->si_nowait.nw_len), sbinfo->si_generation, + kref_read(&sbinfo->si_kobj.kref)); + for (bindex = 0; bindex <= sbinfo->si_bbot; bindex++) + do_pri_br(bindex, sbinfo->si_branch[0 + bindex]); +} + +/* ---------------------------------------------------------------------- */ + +void __au_dbg_verify_dinode(struct dentry *dentry, const char *func, int line) +{ + struct inode *h_inode, *inode = d_inode(dentry); + struct dentry *h_dentry; + aufs_bindex_t bindex, bbot, bi; + + if (!inode /* || au_di(dentry)->di_lsc == AuLsc_DI_TMP */) + return; + + bbot = au_dbbot(dentry); + bi = au_ibbot(inode); + if (bi < bbot) + bbot = bi; + bindex = au_dbtop(dentry); + bi = au_ibtop(inode); + if (bi > bindex) + bindex = bi; + + for (; bindex <= bbot; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + h_inode = au_h_iptr(inode, bindex); + if (unlikely(h_inode != d_inode(h_dentry))) { + au_debug_on(); + AuDbg("b%d, %s:%d\n", bindex, func, line); + AuDbgDentry(dentry); + AuDbgInode(inode); + au_debug_off(); + BUG(); + } + } +} + +void au_dbg_verify_gen(struct dentry *parent, unsigned int sigen) +{ + int err, i, j; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry **dentries; + + err = au_dpages_init(&dpages, GFP_NOFS); + AuDebugOn(err); + err = au_dcsub_pages_rev_aufs(&dpages, parent, /*do_include*/1); + AuDebugOn(err); + for (i = dpages.ndpage - 1; !err && i >= 0; i--) { + dpage = dpages.dpages + i; + dentries = dpage->dentries; + for (j = dpage->ndentry - 1; !err && j >= 0; j--) + AuDebugOn(au_digen_test(dentries[j], sigen)); + } + au_dpages_free(&dpages); +} + +void au_dbg_verify_kthread(void) +{ + if (au_wkq_test()) { + au_dbg_blocked(); + /* + * It may be recursive, but udba=notify between two aufs mounts, + * where a single ro branch is shared, is not a problem. + */ + /* WARN_ON(1); */ + } +} + +/* ---------------------------------------------------------------------- */ + +int __init au_debug_init(void) +{ + aufs_bindex_t bindex; + struct au_vdir_destr destr; + + bindex = -1; + AuDebugOn(bindex >= 0); + + destr.len = -1; + AuDebugOn(destr.len < NAME_MAX); + +#ifdef CONFIG_4KSTACKS + pr_warn("CONFIG_4KSTACKS is defined.\n"); +#endif + + return 0; +} diff --git a/fs/aufs/debug.h b/fs/aufs/debug.h new file mode 100644 index 0000000000000..7e46953513f27 --- /dev/null +++ b/fs/aufs/debug.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * debug print functions + */ + +#ifndef __AUFS_DEBUG_H__ +#define __AUFS_DEBUG_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include + +#ifdef CONFIG_AUFS_DEBUG +#define AuDebugOn(a) BUG_ON(a) + +/* module parameter */ +extern atomic_t aufs_debug; +static inline void au_debug_on(void) +{ + atomic_inc(&aufs_debug); +} +static inline void au_debug_off(void) +{ + atomic_dec_if_positive(&aufs_debug); +} + +static inline int au_debug_test(void) +{ + return atomic_read(&aufs_debug) > 0; +} +#else +#define AuDebugOn(a) do {} while (0) +AuStubVoid(au_debug_on, void) +AuStubVoid(au_debug_off, void) +AuStubInt0(au_debug_test, void) +#endif /* CONFIG_AUFS_DEBUG */ + +#define param_check_atomic_t(name, p) __param_check(name, p, atomic_t) + +/* ---------------------------------------------------------------------- */ + +/* debug print */ + +#define AuDbg(fmt, ...) do { \ + if (au_debug_test()) \ + pr_debug("DEBUG: " fmt, ##__VA_ARGS__); \ +} while (0) +#define AuLabel(l) AuDbg(#l "\n") +#define AuIOErr(fmt, ...) pr_err("I/O Error, " fmt, ##__VA_ARGS__) +#define AuWarn1(fmt, ...) do { \ + static unsigned char _c; \ + if (!_c++) \ + pr_warn(fmt, ##__VA_ARGS__); \ +} while (0) + +#define AuErr1(fmt, ...) do { \ + static unsigned char _c; \ + if (!_c++) \ + pr_err(fmt, ##__VA_ARGS__); \ +} while (0) + +#define AuIOErr1(fmt, ...) do { \ + static unsigned char _c; \ + if (!_c++) \ + AuIOErr(fmt, ##__VA_ARGS__); \ +} while (0) + +#define AuUnsupportMsg "This operation is not supported." \ + " Please report this application to aufs-users ML." +#define AuUnsupport(fmt, ...) do { \ + pr_err(AuUnsupportMsg "\n" fmt, ##__VA_ARGS__); \ + dump_stack(); \ +} while (0) + +#define AuTraceErr(e) do { \ + if (unlikely((e) < 0)) \ + AuDbg("err %d\n", (int)(e)); \ +} while (0) + +#define AuTraceErrPtr(p) do { \ + if (IS_ERR(p)) \ + AuDbg("err %ld\n", PTR_ERR(p)); \ +} while (0) + +/* dirty macros for debug print, use with "%.*s" and caution */ +#define AuLNPair(qstr) (qstr)->len, (qstr)->name + +/* ---------------------------------------------------------------------- */ + +struct dentry; +#ifdef CONFIG_AUFS_DEBUG +extern struct mutex au_dbg_mtx; +extern char *au_plevel; +struct au_nhash; +void au_dpri_whlist(struct au_nhash *whlist); +struct au_vdir; +void au_dpri_vdir(struct au_vdir *vdir); +struct inode; +void au_dpri_inode(struct inode *inode); +void au_dpri_dalias(struct inode *inode); +void au_dpri_dentry(struct dentry *dentry); +struct file; +void au_dpri_file(struct file *filp); +struct super_block; +void au_dpri_sb(struct super_block *sb); + +#define au_dbg_verify_dinode(d) __au_dbg_verify_dinode(d, __func__, __LINE__) +void __au_dbg_verify_dinode(struct dentry *dentry, const char *func, int line); +void au_dbg_verify_gen(struct dentry *parent, unsigned int sigen); +void au_dbg_verify_kthread(void); + +int __init au_debug_init(void); + +#define AuDbgWhlist(w) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#w "\n"); \ + au_dpri_whlist(w); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgVdir(v) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#v "\n"); \ + au_dpri_vdir(v); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgInode(i) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#i "\n"); \ + au_dpri_inode(i); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgDAlias(i) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#i "\n"); \ + au_dpri_dalias(i); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgDentry(d) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#d "\n"); \ + au_dpri_dentry(d); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgFile(f) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#f "\n"); \ + au_dpri_file(f); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgSb(sb) do { \ + mutex_lock(&au_dbg_mtx); \ + AuDbg(#sb "\n"); \ + au_dpri_sb(sb); \ + mutex_unlock(&au_dbg_mtx); \ +} while (0) + +#define AuDbgSym(addr) do { \ + char sym[KSYM_SYMBOL_LEN]; \ + sprint_symbol(sym, (unsigned long)addr); \ + AuDbg("%s\n", sym); \ +} while (0) +#else +AuStubVoid(au_dbg_verify_dinode, struct dentry *dentry) +AuStubVoid(au_dbg_verify_gen, struct dentry *parent, unsigned int sigen) +AuStubVoid(au_dbg_verify_kthread, void) +AuStubInt0(__init au_debug_init, void) + +#define AuDbgWhlist(w) do {} while (0) +#define AuDbgVdir(v) do {} while (0) +#define AuDbgInode(i) do {} while (0) +#define AuDbgDAlias(i) do {} while (0) +#define AuDbgDentry(d) do {} while (0) +#define AuDbgFile(f) do {} while (0) +#define AuDbgSb(sb) do {} while (0) +#define AuDbgSym(addr) do {} while (0) +#endif /* CONFIG_AUFS_DEBUG */ + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_MAGIC_SYSRQ +int __init au_sysrq_init(void); +void au_sysrq_fin(void); + +#ifdef CONFIG_HW_CONSOLE +#define au_dbg_blocked() do { \ + WARN_ON(1); \ + handle_sysrq('w'); \ +} while (0) +#else +AuStubVoid(au_dbg_blocked, void) +#endif + +#else +AuStubInt0(__init au_sysrq_init, void) +AuStubVoid(au_sysrq_fin, void) +AuStubVoid(au_dbg_blocked, void) +#endif /* CONFIG_AUFS_MAGIC_SYSRQ */ + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DEBUG_H__ */ diff --git a/fs/aufs/dentry.c b/fs/aufs/dentry.c new file mode 100644 index 0000000000000..50aaaa21d0899 --- /dev/null +++ b/fs/aufs/dentry.c @@ -0,0 +1,1154 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * lookup and dentry operations + */ + +#include +#include +#include "aufs.h" + +/* + * returns positive/negative dentry, NULL or an error. + * NULL means whiteout-ed or not-found. + */ +static struct dentry* +au_do_lookup(struct dentry *h_parent, struct dentry *dentry, + aufs_bindex_t bindex, struct au_do_lookup_args *args) +{ + struct dentry *h_dentry; + struct inode *h_inode; + struct au_branch *br; + int wh_found, opq; + unsigned char wh_able; + const unsigned char allow_neg = !!au_ftest_lkup(args->flags, ALLOW_NEG); + const unsigned char ignore_perm = !!au_ftest_lkup(args->flags, + IGNORE_PERM); + + wh_found = 0; + br = au_sbr(dentry->d_sb, bindex); + wh_able = !!au_br_whable(br->br_perm); + if (wh_able) + wh_found = au_wh_test(h_parent, &args->whname, ignore_perm); + h_dentry = ERR_PTR(wh_found); + if (!wh_found) + goto real_lookup; + if (unlikely(wh_found < 0)) + goto out; + + /* We found a whiteout */ + /* au_set_dbbot(dentry, bindex); */ + au_set_dbwh(dentry, bindex); + if (!allow_neg) + return NULL; /* success */ + +real_lookup: + if (!ignore_perm) + h_dentry = vfsub_lkup_one(args->name, h_parent); + else + h_dentry = au_sio_lkup_one(args->name, h_parent); + if (IS_ERR(h_dentry)) { + if (PTR_ERR(h_dentry) == -ENAMETOOLONG + && !allow_neg) + h_dentry = NULL; + goto out; + } + + h_inode = d_inode(h_dentry); + if (d_is_negative(h_dentry)) { + if (!allow_neg) + goto out_neg; + } else if (wh_found + || (args->type && args->type != (h_inode->i_mode & S_IFMT))) + goto out_neg; + else if (au_ftest_lkup(args->flags, DIRREN) + /* && h_inode */ + && !au_dr_lkup_h_ino(args, bindex, h_inode->i_ino)) { + AuDbg("b%d %pd ignored hi%llu\n", bindex, h_dentry, + (unsigned long long)h_inode->i_ino); + goto out_neg; + } + + if (au_dbbot(dentry) <= bindex) + au_set_dbbot(dentry, bindex); + if (au_dbtop(dentry) < 0 || bindex < au_dbtop(dentry)) + au_set_dbtop(dentry, bindex); + au_set_h_dptr(dentry, bindex, h_dentry); + + if (!d_is_dir(h_dentry) + || !wh_able + || (d_really_is_positive(dentry) && !d_is_dir(dentry))) + goto out; /* success */ + + inode_lock_shared_nested(h_inode, AuLsc_I_CHILD); + opq = au_diropq_test(h_dentry); + inode_unlock_shared(h_inode); + if (opq > 0) + au_set_dbdiropq(dentry, bindex); + else if (unlikely(opq < 0)) { + au_set_h_dptr(dentry, bindex, NULL); + h_dentry = ERR_PTR(opq); + } + goto out; + +out_neg: + dput(h_dentry); + h_dentry = NULL; +out: + return h_dentry; +} + +static int au_test_shwh(struct super_block *sb, const struct qstr *name) +{ + if (unlikely(!au_opt_test(au_mntflags(sb), SHWH) + && !strncmp(name->name, AUFS_WH_PFX, AUFS_WH_PFX_LEN))) + return -EPERM; + return 0; +} + +/* + * returns the number of lower positive dentries, + * otherwise an error. + * can be called at unlinking with @type is zero. + */ +int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t btop, + unsigned int flags) +{ + int npositive, err; + aufs_bindex_t bindex, btail, bdiropq; + unsigned char isdir, dirperm1, dirren; + struct au_do_lookup_args args = { + .flags = flags, + .name = &dentry->d_name + }; + struct dentry *parent; + struct super_block *sb; + + sb = dentry->d_sb; + err = au_test_shwh(sb, args.name); + if (unlikely(err)) + goto out; + + err = au_wh_name_alloc(&args.whname, args.name); + if (unlikely(err)) + goto out; + + isdir = !!d_is_dir(dentry); + dirperm1 = !!au_opt_test(au_mntflags(sb), DIRPERM1); + dirren = !!au_opt_test(au_mntflags(sb), DIRREN); + if (dirren) + au_fset_lkup(args.flags, DIRREN); + + npositive = 0; + parent = dget_parent(dentry); + btail = au_dbtaildir(parent); + for (bindex = btop; bindex <= btail; bindex++) { + struct dentry *h_parent, *h_dentry; + struct inode *h_inode, *h_dir; + struct au_branch *br; + + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry) { + if (d_is_positive(h_dentry)) + npositive++; + break; + } + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || !d_is_dir(h_parent)) + continue; + + if (dirren) { + /* if the inum matches, then use the prepared name */ + err = au_dr_lkup_name(&args, bindex); + if (unlikely(err)) + goto out_parent; + } + + h_dir = d_inode(h_parent); + inode_lock_shared_nested(h_dir, AuLsc_I_PARENT); + h_dentry = au_do_lookup(h_parent, dentry, bindex, &args); + inode_unlock_shared(h_dir); + err = PTR_ERR(h_dentry); + if (IS_ERR(h_dentry)) + goto out_parent; + if (h_dentry) + au_fclr_lkup(args.flags, ALLOW_NEG); + if (dirperm1) + au_fset_lkup(args.flags, IGNORE_PERM); + + if (au_dbwh(dentry) == bindex) + break; + if (!h_dentry) + continue; + if (d_is_negative(h_dentry)) + continue; + h_inode = d_inode(h_dentry); + npositive++; + if (!args.type) + args.type = h_inode->i_mode & S_IFMT; + if (args.type != S_IFDIR) + break; + else if (isdir) { + /* the type of lower may be different */ + bdiropq = au_dbdiropq(dentry); + if (bdiropq >= 0 && bdiropq <= bindex) + break; + } + br = au_sbr(sb, bindex); + if (dirren + && au_dr_hino_test_add(&br->br_dirren, h_inode->i_ino, + /*add_ent*/NULL)) { + /* prepare next name to lookup */ + err = au_dr_lkup(&args, dentry, bindex); + if (unlikely(err)) + goto out_parent; + } + } + + if (npositive) { + AuLabel(positive); + au_update_dbtop(dentry); + } + err = npositive; + if (unlikely(!au_opt_test(au_mntflags(sb), UDBA_NONE) + && au_dbtop(dentry) < 0)) { + err = -EIO; + AuIOErr("both of real entry and whiteout found, %pd, err %d\n", + dentry, err); + } + +out_parent: + dput(parent); + au_kfree_try_rcu(args.whname.name); + if (dirren) + au_dr_lkup_fin(&args); +out: + return err; +} + +struct dentry *au_sio_lkup_one(struct qstr *name, struct dentry *parent) +{ + struct dentry *dentry; + int wkq_err; + + if (!au_test_h_perm_sio(d_inode(parent), MAY_EXEC)) + dentry = vfsub_lkup_one(name, parent); + else { + struct vfsub_lkup_one_args args = { + .errp = &dentry, + .name = name, + .parent = parent + }; + + wkq_err = au_wkq_wait(vfsub_call_lkup_one, &args); + if (unlikely(wkq_err)) + dentry = ERR_PTR(wkq_err); + } + + return dentry; +} + +/* + * lookup @dentry on @bindex which should be negative. + */ +int au_lkup_neg(struct dentry *dentry, aufs_bindex_t bindex, int wh) +{ + int err; + struct dentry *parent, *h_parent, *h_dentry; + struct au_branch *br; + + parent = dget_parent(dentry); + h_parent = au_h_dptr(parent, bindex); + br = au_sbr(dentry->d_sb, bindex); + if (wh) + h_dentry = au_whtmp_lkup(h_parent, br, &dentry->d_name); + else + h_dentry = au_sio_lkup_one(&dentry->d_name, h_parent); + err = PTR_ERR(h_dentry); + if (IS_ERR(h_dentry)) + goto out; + if (unlikely(d_is_positive(h_dentry))) { + err = -EIO; + AuIOErr("%pd should be negative on b%d.\n", h_dentry, bindex); + dput(h_dentry); + goto out; + } + + err = 0; + if (bindex < au_dbtop(dentry)) + au_set_dbtop(dentry, bindex); + if (au_dbbot(dentry) < bindex) + au_set_dbbot(dentry, bindex); + au_set_h_dptr(dentry, bindex, h_dentry); + +out: + dput(parent); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* subset of struct inode */ +struct au_iattr { + unsigned long i_ino; + /* unsigned int i_nlink; */ + kuid_t i_uid; + kgid_t i_gid; + u64 i_version; +/* + loff_t i_size; + blkcnt_t i_blocks; +*/ + umode_t i_mode; +}; + +static void au_iattr_save(struct au_iattr *ia, struct inode *h_inode) +{ + ia->i_ino = h_inode->i_ino; + /* ia->i_nlink = h_inode->i_nlink; */ + ia->i_uid = h_inode->i_uid; + ia->i_gid = h_inode->i_gid; + ia->i_version = inode_query_iversion(h_inode); +/* + ia->i_size = h_inode->i_size; + ia->i_blocks = h_inode->i_blocks; +*/ + ia->i_mode = (h_inode->i_mode & S_IFMT); +} + +static int au_iattr_test(struct au_iattr *ia, struct inode *h_inode) +{ + return ia->i_ino != h_inode->i_ino + /* || ia->i_nlink != h_inode->i_nlink */ + || !uid_eq(ia->i_uid, h_inode->i_uid) + || !gid_eq(ia->i_gid, h_inode->i_gid) + || !inode_eq_iversion(h_inode, ia->i_version) +/* + || ia->i_size != h_inode->i_size + || ia->i_blocks != h_inode->i_blocks +*/ + || ia->i_mode != (h_inode->i_mode & S_IFMT); +} + +static int au_h_verify_dentry(struct dentry *h_dentry, struct dentry *h_parent, + struct au_branch *br) +{ + int err; + struct au_iattr ia; + struct inode *h_inode; + struct dentry *h_d; + struct super_block *h_sb; + + err = 0; + memset(&ia, -1, sizeof(ia)); + h_sb = h_dentry->d_sb; + h_inode = NULL; + if (d_is_positive(h_dentry)) { + h_inode = d_inode(h_dentry); + au_iattr_save(&ia, h_inode); + } else if (au_test_nfs(h_sb) || au_test_fuse(h_sb)) + /* nfs d_revalidate may return 0 for negative dentry */ + /* fuse d_revalidate always return 0 for negative dentry */ + goto out; + + /* main purpose is namei.c:cached_lookup() and d_revalidate */ + h_d = vfsub_lkup_one(&h_dentry->d_name, h_parent); + err = PTR_ERR(h_d); + if (IS_ERR(h_d)) + goto out; + + err = 0; + if (unlikely(h_d != h_dentry + || d_inode(h_d) != h_inode + || (h_inode && au_iattr_test(&ia, h_inode)))) + err = au_busy_or_stale(); + dput(h_d); + +out: + AuTraceErr(err); + return err; +} + +int au_h_verify(struct dentry *h_dentry, unsigned int udba, struct inode *h_dir, + struct dentry *h_parent, struct au_branch *br) +{ + int err; + + err = 0; + if (udba == AuOpt_UDBA_REVAL + && !au_test_fs_remote(h_dentry->d_sb)) { + IMustLock(h_dir); + err = (d_inode(h_dentry->d_parent) != h_dir); + } else if (udba != AuOpt_UDBA_NONE) + err = au_h_verify_dentry(h_dentry, h_parent, br); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_refresh_hdentry(struct dentry *dentry, struct dentry *parent) +{ + int err; + aufs_bindex_t new_bindex, bindex, bbot, bwh, bdiropq; + struct au_hdentry tmp, *p, *q; + struct au_dinfo *dinfo; + struct super_block *sb; + + DiMustWriteLock(dentry); + + sb = dentry->d_sb; + dinfo = au_di(dentry); + bbot = dinfo->di_bbot; + bwh = dinfo->di_bwh; + bdiropq = dinfo->di_bdiropq; + bindex = dinfo->di_btop; + p = au_hdentry(dinfo, bindex); + for (; bindex <= bbot; bindex++, p++) { + if (!p->hd_dentry) + continue; + + new_bindex = au_br_index(sb, p->hd_id); + if (new_bindex == bindex) + continue; + + if (dinfo->di_bwh == bindex) + bwh = new_bindex; + if (dinfo->di_bdiropq == bindex) + bdiropq = new_bindex; + if (new_bindex < 0) { + au_hdput(p); + p->hd_dentry = NULL; + continue; + } + + /* swap two lower dentries, and loop again */ + q = au_hdentry(dinfo, new_bindex); + tmp = *q; + *q = *p; + *p = tmp; + if (tmp.hd_dentry) { + bindex--; + p--; + } + } + + dinfo->di_bwh = -1; + if (bwh >= 0 && bwh <= au_sbbot(sb) && au_sbr_whable(sb, bwh)) + dinfo->di_bwh = bwh; + + dinfo->di_bdiropq = -1; + if (bdiropq >= 0 + && bdiropq <= au_sbbot(sb) + && au_sbr_whable(sb, bdiropq)) + dinfo->di_bdiropq = bdiropq; + + err = -EIO; + dinfo->di_btop = -1; + dinfo->di_bbot = -1; + bbot = au_dbbot(parent); + bindex = 0; + p = au_hdentry(dinfo, bindex); + for (; bindex <= bbot; bindex++, p++) + if (p->hd_dentry) { + dinfo->di_btop = bindex; + break; + } + + if (dinfo->di_btop >= 0) { + bindex = bbot; + p = au_hdentry(dinfo, bindex); + for (; bindex >= 0; bindex--, p--) + if (p->hd_dentry) { + dinfo->di_bbot = bindex; + err = 0; + break; + } + } + + return err; +} + +static void au_do_hide(struct dentry *dentry) +{ + struct inode *inode; + + if (d_really_is_positive(dentry)) { + inode = d_inode(dentry); + if (!d_is_dir(dentry)) { + if (inode->i_nlink && !d_unhashed(dentry)) + drop_nlink(inode); + } else { + clear_nlink(inode); + /* stop next lookup */ + inode->i_flags |= S_DEAD; + } + smp_mb(); /* necessary? */ + } + d_drop(dentry); +} + +static int au_hide_children(struct dentry *parent) +{ + int err, i, j, ndentry; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry *dentry; + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_dcsub_pages(&dpages, parent, NULL, NULL); + if (unlikely(err)) + goto out_dpages; + + /* in reverse order */ + for (i = dpages.ndpage - 1; i >= 0; i--) { + dpage = dpages.dpages + i; + ndentry = dpage->ndentry; + for (j = ndentry - 1; j >= 0; j--) { + dentry = dpage->dentries[j]; + if (dentry != parent) + au_do_hide(dentry); + } + } + +out_dpages: + au_dpages_free(&dpages); +out: + return err; +} + +static void au_hide(struct dentry *dentry) +{ + int err; + + AuDbgDentry(dentry); + if (d_is_dir(dentry)) { + /* shrink_dcache_parent(dentry); */ + err = au_hide_children(dentry); + if (unlikely(err)) + AuIOErr("%pd, failed hiding children, ignored %d\n", + dentry, err); + } + au_do_hide(dentry); +} + +/* + * By adding a dirty branch, a cached dentry may be affected in various ways. + * + * a dirty branch is added + * - on the top of layers + * - in the middle of layers + * - to the bottom of layers + * + * on the added branch there exists + * - a whiteout + * - a diropq + * - a same named entry + * + exist + * * negative --> positive + * * positive --> positive + * - type is unchanged + * - type is changed + * + doesn't exist + * * negative --> negative + * * positive --> negative (rejected by au_br_del() for non-dir case) + * - none + */ +static int au_refresh_by_dinfo(struct dentry *dentry, struct au_dinfo *dinfo, + struct au_dinfo *tmp) +{ + int err; + aufs_bindex_t bindex, bbot; + struct { + struct dentry *dentry; + struct inode *inode; + mode_t mode; + } orig_h, tmp_h = { + .dentry = NULL + }; + struct au_hdentry *hd; + struct inode *inode, *h_inode; + struct dentry *h_dentry; + + err = 0; + AuDebugOn(dinfo->di_btop < 0); + orig_h.mode = 0; + orig_h.dentry = au_hdentry(dinfo, dinfo->di_btop)->hd_dentry; + orig_h.inode = NULL; + if (d_is_positive(orig_h.dentry)) { + orig_h.inode = d_inode(orig_h.dentry); + orig_h.mode = orig_h.inode->i_mode & S_IFMT; + } + if (tmp->di_btop >= 0) { + tmp_h.dentry = au_hdentry(tmp, tmp->di_btop)->hd_dentry; + if (d_is_positive(tmp_h.dentry)) { + tmp_h.inode = d_inode(tmp_h.dentry); + tmp_h.mode = tmp_h.inode->i_mode & S_IFMT; + } + } + + inode = NULL; + if (d_really_is_positive(dentry)) + inode = d_inode(dentry); + if (!orig_h.inode) { + AuDbg("negative originally\n"); + if (inode) { + au_hide(dentry); + goto out; + } + AuDebugOn(inode); + AuDebugOn(dinfo->di_btop != dinfo->di_bbot); + AuDebugOn(dinfo->di_bdiropq != -1); + + if (!tmp_h.inode) { + AuDbg("negative --> negative\n"); + /* should have only one negative lower */ + if (tmp->di_btop >= 0 + && tmp->di_btop < dinfo->di_btop) { + AuDebugOn(tmp->di_btop != tmp->di_bbot); + AuDebugOn(dinfo->di_btop != dinfo->di_bbot); + au_set_h_dptr(dentry, dinfo->di_btop, NULL); + au_di_cp(dinfo, tmp); + hd = au_hdentry(tmp, tmp->di_btop); + au_set_h_dptr(dentry, tmp->di_btop, + dget(hd->hd_dentry)); + } + au_dbg_verify_dinode(dentry); + } else { + AuDbg("negative --> positive\n"); + /* + * similar to the behaviour of creating with bypassing + * aufs. + * unhash it in order to force an error in the + * succeeding create operation. + * we should not set S_DEAD here. + */ + d_drop(dentry); + /* au_di_swap(tmp, dinfo); */ + au_dbg_verify_dinode(dentry); + } + } else { + AuDbg("positive originally\n"); + /* inode may be NULL */ + AuDebugOn(inode && (inode->i_mode & S_IFMT) != orig_h.mode); + if (!tmp_h.inode) { + AuDbg("positive --> negative\n"); + /* or bypassing aufs */ + au_hide(dentry); + if (tmp->di_bwh >= 0 && tmp->di_bwh <= dinfo->di_btop) + dinfo->di_bwh = tmp->di_bwh; + if (inode) + err = au_refresh_hinode_self(inode); + au_dbg_verify_dinode(dentry); + } else if (orig_h.mode == tmp_h.mode) { + AuDbg("positive --> positive, same type\n"); + if (!S_ISDIR(orig_h.mode) + && dinfo->di_btop > tmp->di_btop) { + /* + * similar to the behaviour of removing and + * creating. + */ + au_hide(dentry); + if (inode) + err = au_refresh_hinode_self(inode); + au_dbg_verify_dinode(dentry); + } else { + /* fill empty slots */ + if (dinfo->di_btop > tmp->di_btop) + dinfo->di_btop = tmp->di_btop; + if (dinfo->di_bbot < tmp->di_bbot) + dinfo->di_bbot = tmp->di_bbot; + dinfo->di_bwh = tmp->di_bwh; + dinfo->di_bdiropq = tmp->di_bdiropq; + bbot = dinfo->di_bbot; + bindex = tmp->di_btop; + hd = au_hdentry(tmp, bindex); + for (; bindex <= bbot; bindex++, hd++) { + if (au_h_dptr(dentry, bindex)) + continue; + h_dentry = hd->hd_dentry; + if (!h_dentry) + continue; + AuDebugOn(d_is_negative(h_dentry)); + h_inode = d_inode(h_dentry); + AuDebugOn(orig_h.mode + != (h_inode->i_mode + & S_IFMT)); + au_set_h_dptr(dentry, bindex, + dget(h_dentry)); + } + if (inode) + err = au_refresh_hinode(inode, dentry); + au_dbg_verify_dinode(dentry); + } + } else { + AuDbg("positive --> positive, different type\n"); + /* similar to the behaviour of removing and creating */ + au_hide(dentry); + if (inode) + err = au_refresh_hinode_self(inode); + au_dbg_verify_dinode(dentry); + } + } + +out: + return err; +} + +void au_refresh_dop(struct dentry *dentry, int force_reval) +{ + const struct dentry_operations *dop + = force_reval ? &aufs_dop : dentry->d_sb->s_d_op; + static const unsigned int mask + = DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE; + + BUILD_BUG_ON(sizeof(mask) != sizeof(dentry->d_flags)); + + if (dentry->d_op == dop) + return; + + AuDbg("%pd\n", dentry); + spin_lock(&dentry->d_lock); + if (dop == &aufs_dop) + dentry->d_flags |= mask; + else + dentry->d_flags &= ~mask; + dentry->d_op = dop; + spin_unlock(&dentry->d_lock); +} + +int au_refresh_dentry(struct dentry *dentry, struct dentry *parent) +{ + int err, ebrange, nbr; + unsigned int sigen; + struct au_dinfo *dinfo, *tmp; + struct super_block *sb; + struct inode *inode; + + DiMustWriteLock(dentry); + AuDebugOn(IS_ROOT(dentry)); + AuDebugOn(d_really_is_negative(parent)); + + sb = dentry->d_sb; + sigen = au_sigen(sb); + err = au_digen_test(parent, sigen); + if (unlikely(err)) + goto out; + + nbr = au_sbbot(sb) + 1; + dinfo = au_di(dentry); + err = au_di_realloc(dinfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + ebrange = au_dbrange_test(dentry); + if (!ebrange) + ebrange = au_do_refresh_hdentry(dentry, parent); + + if (d_unhashed(dentry) || ebrange /* || dinfo->di_tmpfile */) { + AuDebugOn(au_dbtop(dentry) < 0 && au_dbbot(dentry) >= 0); + if (d_really_is_positive(dentry)) { + inode = d_inode(dentry); + err = au_refresh_hinode_self(inode); + } + au_dbg_verify_dinode(dentry); + if (!err) + goto out_dgen; /* success */ + goto out; + } + + /* temporary dinfo */ + AuDbgDentry(dentry); + err = -ENOMEM; + tmp = au_di_alloc(sb, AuLsc_DI_TMP); + if (unlikely(!tmp)) + goto out; + au_di_swap(tmp, dinfo); + /* returns the number of positive dentries */ + /* + * if current working dir is removed, it returns an error. + * but the dentry is legal. + */ + err = au_lkup_dentry(dentry, /*btop*/0, AuLkup_ALLOW_NEG); + AuDbgDentry(dentry); + au_di_swap(tmp, dinfo); + if (err == -ENOENT) + err = 0; + if (err >= 0) { + /* compare/refresh by dinfo */ + AuDbgDentry(dentry); + err = au_refresh_by_dinfo(dentry, dinfo, tmp); + au_dbg_verify_dinode(dentry); + AuTraceErr(err); + } + au_di_realloc(dinfo, nbr, /*may_shrink*/1); /* harmless if err */ + au_rw_write_unlock(&tmp->di_rwsem); + au_di_free(tmp); + if (unlikely(err)) + goto out; + +out_dgen: + au_update_digen(dentry); +out: + if (unlikely(err && !(dentry->d_flags & DCACHE_NFSFS_RENAMED))) { + AuIOErr("failed refreshing %pd, %d\n", dentry, err); + AuDbgDentry(dentry); + } + AuTraceErr(err); + return err; +} + +static int au_do_h_d_reval(struct dentry *h_dentry, unsigned int flags, + struct dentry *dentry, aufs_bindex_t bindex) +{ + int err, valid; + + err = 0; + if (!(h_dentry->d_flags & DCACHE_OP_REVALIDATE)) + goto out; + + AuDbg("b%d\n", bindex); + /* + * gave up supporting LOOKUP_CREATE/OPEN for lower fs, + * due to whiteout and branch permission. + */ + flags &= ~(/*LOOKUP_PARENT |*/ LOOKUP_OPEN | LOOKUP_CREATE + | LOOKUP_FOLLOW | LOOKUP_EXCL); + /* it may return tri-state */ + valid = h_dentry->d_op->d_revalidate(h_dentry, flags); + + if (unlikely(valid < 0)) + err = valid; + else if (!valid) + err = -EINVAL; + +out: + AuTraceErr(err); + return err; +} + +/* todo: remove this */ +static int h_d_revalidate(struct dentry *dentry, struct inode *inode, + unsigned int flags, int do_udba, int dirren) +{ + int err; + umode_t mode, h_mode; + aufs_bindex_t bindex, btail, btop, ibs, ibe; + unsigned char plus, unhashed, is_root, h_plus, h_nfs, tmpfile; + struct inode *h_inode, *h_cached_inode; + struct dentry *h_dentry; + struct qstr *name, *h_name; + + err = 0; + plus = 0; + mode = 0; + ibs = -1; + ibe = -1; + unhashed = !!d_unhashed(dentry); + is_root = !!IS_ROOT(dentry); + name = &dentry->d_name; + tmpfile = au_di(dentry)->di_tmpfile; + + /* + * Theoretically, REVAL test should be unnecessary in case of + * {FS,I}NOTIFY. + * But {fs,i}notify doesn't fire some necessary events, + * IN_ATTRIB for atime/nlink/pageio + * Let's do REVAL test too. + */ + if (do_udba && inode) { + mode = (inode->i_mode & S_IFMT); + plus = (inode->i_nlink > 0); + ibs = au_ibtop(inode); + ibe = au_ibbot(inode); + } + + btop = au_dbtop(dentry); + btail = btop; + if (inode && S_ISDIR(inode->i_mode)) + btail = au_dbtaildir(dentry); + for (bindex = btop; bindex <= btail; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + + AuDbg("b%d, %pd\n", bindex, h_dentry); + h_nfs = !!au_test_nfs(h_dentry->d_sb); + spin_lock(&h_dentry->d_lock); + h_name = &h_dentry->d_name; + if (unlikely(do_udba + && !is_root + && ((!h_nfs + && (unhashed != !!d_unhashed(h_dentry) + || (!tmpfile && !dirren + && !au_qstreq(name, h_name)) + )) + || (h_nfs + && !(flags & LOOKUP_OPEN) + && (h_dentry->d_flags + & DCACHE_NFSFS_RENAMED))) + )) { + int h_unhashed; + + h_unhashed = d_unhashed(h_dentry); + spin_unlock(&h_dentry->d_lock); + AuDbg("unhash 0x%x 0x%x, %pd %pd\n", + unhashed, h_unhashed, dentry, h_dentry); + goto err; + } + spin_unlock(&h_dentry->d_lock); + + err = au_do_h_d_reval(h_dentry, flags, dentry, bindex); + if (unlikely(err)) + /* do not goto err, to keep the errno */ + break; + + /* todo: plink too? */ + if (!do_udba) + continue; + + /* UDBA tests */ + if (unlikely(!!inode != d_is_positive(h_dentry))) + goto err; + + h_inode = NULL; + if (d_is_positive(h_dentry)) + h_inode = d_inode(h_dentry); + h_plus = plus; + h_mode = mode; + h_cached_inode = h_inode; + if (h_inode) { + h_mode = (h_inode->i_mode & S_IFMT); + h_plus = (h_inode->i_nlink > 0); + } + if (inode && ibs <= bindex && bindex <= ibe) + h_cached_inode = au_h_iptr(inode, bindex); + + if (!h_nfs) { + if (unlikely(plus != h_plus && !tmpfile)) + goto err; + } else { + if (unlikely(!(h_dentry->d_flags & DCACHE_NFSFS_RENAMED) + && !is_root + && !IS_ROOT(h_dentry) + && unhashed != d_unhashed(h_dentry))) + goto err; + } + if (unlikely(mode != h_mode + || h_cached_inode != h_inode)) + goto err; + continue; + +err: + err = -EINVAL; + break; + } + + AuTraceErr(err); + return err; +} + +/* todo: consolidate with do_refresh() and au_reval_for_attr() */ +static int simple_reval_dpath(struct dentry *dentry, unsigned int sigen) +{ + int err; + struct dentry *parent; + + if (!au_digen_test(dentry, sigen)) + return 0; + + parent = dget_parent(dentry); + di_read_lock_parent(parent, AuLock_IR); + AuDebugOn(au_digen_test(parent, sigen)); + au_dbg_verify_gen(parent, sigen); + err = au_refresh_dentry(dentry, parent); + di_read_unlock(parent, AuLock_IR); + dput(parent); + AuTraceErr(err); + return err; +} + +int au_reval_dpath(struct dentry *dentry, unsigned int sigen) +{ + int err; + struct dentry *d, *parent; + + if (!au_ftest_si(au_sbi(dentry->d_sb), FAILED_REFRESH_DIR)) + return simple_reval_dpath(dentry, sigen); + + /* slow loop, keep it simple and stupid */ + /* cf: au_cpup_dirs() */ + err = 0; + parent = NULL; + while (au_digen_test(dentry, sigen)) { + d = dentry; + while (1) { + dput(parent); + parent = dget_parent(d); + if (!au_digen_test(parent, sigen)) + break; + d = parent; + } + + if (d != dentry) + di_write_lock_child2(d); + + /* someone might update our dentry while we were sleeping */ + if (au_digen_test(d, sigen)) { + /* + * todo: consolidate with simple_reval_dpath(), + * do_refresh() and au_reval_for_attr(). + */ + di_read_lock_parent(parent, AuLock_IR); + err = au_refresh_dentry(d, parent); + di_read_unlock(parent, AuLock_IR); + } + + if (d != dentry) + di_write_unlock(d); + dput(parent); + if (unlikely(err)) + break; + } + + return err; +} + +/* + * if valid returns 1, otherwise 0. + */ +static int aufs_d_revalidate(struct dentry *dentry, unsigned int flags) +{ + int valid, err; + unsigned int sigen; + unsigned char do_udba, dirren; + struct super_block *sb; + struct inode *inode; + + /* todo: support rcu-walk? */ + if (flags & LOOKUP_RCU) + return -ECHILD; + + valid = 0; + if (unlikely(!au_di(dentry))) + goto out; + + valid = 1; + sb = dentry->d_sb; + /* + * todo: very ugly + * i_mutex of parent dir may be held, + * but we should not return 'invalid' due to busy. + */ + err = aufs_read_lock(dentry, AuLock_FLUSH | AuLock_DW | AuLock_NOPLM); + if (unlikely(err)) { + valid = err; + AuTraceErr(err); + goto out; + } + inode = NULL; + if (d_really_is_positive(dentry)) + inode = d_inode(dentry); + if (unlikely(inode && au_is_bad_inode(inode))) { + err = -EINVAL; + AuTraceErr(err); + goto out_dgrade; + } + if (unlikely(au_dbrange_test(dentry))) { + err = -EINVAL; + AuTraceErr(err); + goto out_dgrade; + } + + sigen = au_sigen(sb); + if (au_digen_test(dentry, sigen)) { + AuDebugOn(IS_ROOT(dentry)); + err = au_reval_dpath(dentry, sigen); + if (unlikely(err)) { + AuTraceErr(err); + goto out_dgrade; + } + } + di_downgrade_lock(dentry, AuLock_IR); + + err = -EINVAL; + if (!(flags & (LOOKUP_OPEN | LOOKUP_EMPTY)) + && inode + && !(inode->i_state && I_LINKABLE) + && (IS_DEADDIR(inode) || !inode->i_nlink)) { + AuTraceErr(err); + goto out_inval; + } + + do_udba = !au_opt_test(au_mntflags(sb), UDBA_NONE); + if (do_udba && inode) { + aufs_bindex_t btop = au_ibtop(inode); + struct inode *h_inode; + + if (btop >= 0) { + h_inode = au_h_iptr(inode, btop); + if (h_inode && au_test_higen(inode, h_inode)) { + AuTraceErr(err); + goto out_inval; + } + } + } + + dirren = !!au_opt_test(au_mntflags(sb), DIRREN); + err = h_d_revalidate(dentry, inode, flags, do_udba, dirren); + if (unlikely(!err && do_udba && au_dbtop(dentry) < 0)) { + err = -EIO; + AuDbg("both of real entry and whiteout found, %p, err %d\n", + dentry, err); + } + goto out_inval; + +out_dgrade: + di_downgrade_lock(dentry, AuLock_IR); +out_inval: + aufs_read_unlock(dentry, AuLock_IR); + AuTraceErr(err); + valid = !err; +out: + if (!valid) { + AuDbg("%pd invalid, %d\n", dentry, valid); + d_drop(dentry); + } + return valid; +} + +static void aufs_d_release(struct dentry *dentry) +{ + if (au_di(dentry)) { + au_di_fin(dentry); + au_hn_di_reinit(dentry); + } +} + +const struct dentry_operations aufs_dop = { + .d_revalidate = aufs_d_revalidate, + .d_weak_revalidate = aufs_d_revalidate, + .d_release = aufs_d_release +}; + +/* aufs_dop without d_revalidate */ +const struct dentry_operations aufs_dop_noreval = { + .d_release = aufs_d_release +}; diff --git a/fs/aufs/dentry.h b/fs/aufs/dentry.h new file mode 100644 index 0000000000000..c9f9c704da0a8 --- /dev/null +++ b/fs/aufs/dentry.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * lookup and dentry operations + */ + +#ifndef __AUFS_DENTRY_H__ +#define __AUFS_DENTRY_H__ + +#ifdef __KERNEL__ + +#include +#include "dirren.h" +#include "rwsem.h" + +struct au_hdentry { + struct dentry *hd_dentry; + aufs_bindex_t hd_id; +}; + +struct au_dinfo { + atomic_t di_generation; + + struct au_rwsem di_rwsem; + aufs_bindex_t di_btop, di_bbot, di_bwh, di_bdiropq; + unsigned char di_tmpfile; /* to allow the different name */ + struct au_hdentry *di_hdentry; + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ + +/* flags for au_lkup_dentry() */ +#define AuLkup_ALLOW_NEG 1 +#define AuLkup_IGNORE_PERM (1 << 1) +#define AuLkup_DIRREN (1 << 2) +#define au_ftest_lkup(flags, name) ((flags) & AuLkup_##name) +#define au_fset_lkup(flags, name) \ + do { (flags) |= AuLkup_##name; } while (0) +#define au_fclr_lkup(flags, name) \ + do { (flags) &= ~AuLkup_##name; } while (0) + +#ifndef CONFIG_AUFS_DIRREN +#undef AuLkup_DIRREN +#define AuLkup_DIRREN 0 +#endif + +struct au_do_lookup_args { + unsigned int flags; + mode_t type; + struct qstr whname, *name; + struct au_dr_lookup dirren; +}; + +/* ---------------------------------------------------------------------- */ + +/* dentry.c */ +extern const struct dentry_operations aufs_dop, aufs_dop_noreval; +struct au_branch; +struct dentry *au_sio_lkup_one(struct qstr *name, struct dentry *parent); +int au_h_verify(struct dentry *h_dentry, unsigned int udba, struct inode *h_dir, + struct dentry *h_parent, struct au_branch *br); + +int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t btop, + unsigned int flags); +int au_lkup_neg(struct dentry *dentry, aufs_bindex_t bindex, int wh); +int au_refresh_dentry(struct dentry *dentry, struct dentry *parent); +int au_reval_dpath(struct dentry *dentry, unsigned int sigen); +void au_refresh_dop(struct dentry *dentry, int force_reval); + +/* dinfo.c */ +void au_di_init_once(void *_di); +struct au_dinfo *au_di_alloc(struct super_block *sb, unsigned int lsc); +void au_di_free(struct au_dinfo *dinfo); +void au_di_swap(struct au_dinfo *a, struct au_dinfo *b); +void au_di_cp(struct au_dinfo *dst, struct au_dinfo *src); +int au_di_init(struct dentry *dentry); +void au_di_fin(struct dentry *dentry); +int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink); + +void di_read_lock(struct dentry *d, int flags, unsigned int lsc); +void di_read_unlock(struct dentry *d, int flags); +void di_downgrade_lock(struct dentry *d, int flags); +void di_write_lock(struct dentry *d, unsigned int lsc); +void di_write_unlock(struct dentry *d); +void di_write_lock2_child(struct dentry *d1, struct dentry *d2, int isdir); +void di_write_lock2_parent(struct dentry *d1, struct dentry *d2, int isdir); +void di_write_unlock2(struct dentry *d1, struct dentry *d2); + +struct dentry *au_h_dptr(struct dentry *dentry, aufs_bindex_t bindex); +struct dentry *au_h_d_alias(struct dentry *dentry, aufs_bindex_t bindex); +aufs_bindex_t au_dbtail(struct dentry *dentry); +aufs_bindex_t au_dbtaildir(struct dentry *dentry); + +void au_set_h_dptr(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_dentry); +int au_digen_test(struct dentry *dentry, unsigned int sigen); +int au_dbrange_test(struct dentry *dentry); +void au_update_digen(struct dentry *dentry); +void au_update_dbrange(struct dentry *dentry, int do_put_zero); +void au_update_dbtop(struct dentry *dentry); +void au_update_dbbot(struct dentry *dentry); +int au_find_dbindex(struct dentry *dentry, struct dentry *h_dentry); + +/* ---------------------------------------------------------------------- */ + +static inline struct au_dinfo *au_di(struct dentry *dentry) +{ + return dentry->d_fsdata; +} + +/* ---------------------------------------------------------------------- */ + +/* lock subclass for dinfo */ +enum { + AuLsc_DI_CHILD, /* child first */ + AuLsc_DI_CHILD2, /* rename(2), link(2), and cpup at hnotify */ + AuLsc_DI_CHILD3, /* copyup dirs */ + AuLsc_DI_PARENT, + AuLsc_DI_PARENT2, + AuLsc_DI_PARENT3, + AuLsc_DI_TMP /* temp for replacing dinfo */ +}; + +/* + * di_read_lock_child, di_write_lock_child, + * di_read_lock_child2, di_write_lock_child2, + * di_read_lock_child3, di_write_lock_child3, + * di_read_lock_parent, di_write_lock_parent, + * di_read_lock_parent2, di_write_lock_parent2, + * di_read_lock_parent3, di_write_lock_parent3, + */ +#define AuReadLockFunc(name, lsc) \ +static inline void di_read_lock_##name(struct dentry *d, int flags) \ +{ di_read_lock(d, flags, AuLsc_DI_##lsc); } + +#define AuWriteLockFunc(name, lsc) \ +static inline void di_write_lock_##name(struct dentry *d) \ +{ di_write_lock(d, AuLsc_DI_##lsc); } + +#define AuRWLockFuncs(name, lsc) \ + AuReadLockFunc(name, lsc) \ + AuWriteLockFunc(name, lsc) + +AuRWLockFuncs(child, CHILD); +AuRWLockFuncs(child2, CHILD2); +AuRWLockFuncs(child3, CHILD3); +AuRWLockFuncs(parent, PARENT); +AuRWLockFuncs(parent2, PARENT2); +AuRWLockFuncs(parent3, PARENT3); + +#undef AuReadLockFunc +#undef AuWriteLockFunc +#undef AuRWLockFuncs + +#define DiMustNoWaiters(d) AuRwMustNoWaiters(&au_di(d)->di_rwsem) +#define DiMustAnyLock(d) AuRwMustAnyLock(&au_di(d)->di_rwsem) +#define DiMustWriteLock(d) AuRwMustWriteLock(&au_di(d)->di_rwsem) + +/* ---------------------------------------------------------------------- */ + +/* todo: memory barrier? */ +static inline unsigned int au_digen(struct dentry *d) +{ + return atomic_read(&au_di(d)->di_generation); +} + +static inline void au_h_dentry_init(struct au_hdentry *hdentry) +{ + hdentry->hd_dentry = NULL; +} + +static inline struct au_hdentry *au_hdentry(struct au_dinfo *di, + aufs_bindex_t bindex) +{ + return di->di_hdentry + bindex; +} + +static inline void au_hdput(struct au_hdentry *hd) +{ + if (hd) + dput(hd->hd_dentry); +} + +static inline aufs_bindex_t au_dbtop(struct dentry *dentry) +{ + DiMustAnyLock(dentry); + return au_di(dentry)->di_btop; +} + +static inline aufs_bindex_t au_dbbot(struct dentry *dentry) +{ + DiMustAnyLock(dentry); + return au_di(dentry)->di_bbot; +} + +static inline aufs_bindex_t au_dbwh(struct dentry *dentry) +{ + DiMustAnyLock(dentry); + return au_di(dentry)->di_bwh; +} + +static inline aufs_bindex_t au_dbdiropq(struct dentry *dentry) +{ + DiMustAnyLock(dentry); + return au_di(dentry)->di_bdiropq; +} + +/* todo: hard/soft set? */ +static inline void au_set_dbtop(struct dentry *dentry, aufs_bindex_t bindex) +{ + DiMustWriteLock(dentry); + au_di(dentry)->di_btop = bindex; +} + +static inline void au_set_dbbot(struct dentry *dentry, aufs_bindex_t bindex) +{ + DiMustWriteLock(dentry); + au_di(dentry)->di_bbot = bindex; +} + +static inline void au_set_dbwh(struct dentry *dentry, aufs_bindex_t bindex) +{ + DiMustWriteLock(dentry); + /* dbwh can be outside of btop - bbot range */ + au_di(dentry)->di_bwh = bindex; +} + +static inline void au_set_dbdiropq(struct dentry *dentry, aufs_bindex_t bindex) +{ + DiMustWriteLock(dentry); + au_di(dentry)->di_bdiropq = bindex; +} + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_HNOTIFY +static inline void au_digen_dec(struct dentry *d) +{ + atomic_dec(&au_di(d)->di_generation); +} + +static inline void au_hn_di_reinit(struct dentry *dentry) +{ + dentry->d_fsdata = NULL; +} +#else +AuStubVoid(au_hn_di_reinit, struct dentry *dentry __maybe_unused) +#endif /* CONFIG_AUFS_HNOTIFY */ + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DENTRY_H__ */ diff --git a/fs/aufs/dinfo.c b/fs/aufs/dinfo.c new file mode 100644 index 0000000000000..be959106d9808 --- /dev/null +++ b/fs/aufs/dinfo.c @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * dentry private data + */ + +#include "aufs.h" + +void au_di_init_once(void *_dinfo) +{ + struct au_dinfo *dinfo = _dinfo; + + au_rw_init(&dinfo->di_rwsem); +} + +struct au_dinfo *au_di_alloc(struct super_block *sb, unsigned int lsc) +{ + struct au_dinfo *dinfo; + int nbr, i; + + dinfo = au_cache_alloc_dinfo(); + if (unlikely(!dinfo)) + goto out; + + nbr = au_sbbot(sb) + 1; + if (nbr <= 0) + nbr = 1; + dinfo->di_hdentry = kcalloc(nbr, sizeof(*dinfo->di_hdentry), GFP_NOFS); + if (dinfo->di_hdentry) { + au_rw_write_lock_nested(&dinfo->di_rwsem, lsc); + dinfo->di_btop = -1; + dinfo->di_bbot = -1; + dinfo->di_bwh = -1; + dinfo->di_bdiropq = -1; + dinfo->di_tmpfile = 0; + for (i = 0; i < nbr; i++) + dinfo->di_hdentry[i].hd_id = -1; + goto out; + } + + au_cache_free_dinfo(dinfo); + dinfo = NULL; + +out: + return dinfo; +} + +void au_di_free(struct au_dinfo *dinfo) +{ + struct au_hdentry *p; + aufs_bindex_t bbot, bindex; + + /* dentry may not be revalidated */ + bindex = dinfo->di_btop; + if (bindex >= 0) { + bbot = dinfo->di_bbot; + p = au_hdentry(dinfo, bindex); + while (bindex++ <= bbot) + au_hdput(p++); + } + au_kfree_try_rcu(dinfo->di_hdentry); + au_cache_free_dinfo(dinfo); +} + +void au_di_swap(struct au_dinfo *a, struct au_dinfo *b) +{ + struct au_hdentry *p; + aufs_bindex_t bi; + + AuRwMustWriteLock(&a->di_rwsem); + AuRwMustWriteLock(&b->di_rwsem); + +#define DiSwap(v, name) \ + do { \ + v = a->di_##name; \ + a->di_##name = b->di_##name; \ + b->di_##name = v; \ + } while (0) + + DiSwap(p, hdentry); + DiSwap(bi, btop); + DiSwap(bi, bbot); + DiSwap(bi, bwh); + DiSwap(bi, bdiropq); + /* smp_mb(); */ + +#undef DiSwap +} + +void au_di_cp(struct au_dinfo *dst, struct au_dinfo *src) +{ + AuRwMustWriteLock(&dst->di_rwsem); + AuRwMustWriteLock(&src->di_rwsem); + + dst->di_btop = src->di_btop; + dst->di_bbot = src->di_bbot; + dst->di_bwh = src->di_bwh; + dst->di_bdiropq = src->di_bdiropq; + /* smp_mb(); */ +} + +int au_di_init(struct dentry *dentry) +{ + int err; + struct super_block *sb; + struct au_dinfo *dinfo; + + err = 0; + sb = dentry->d_sb; + dinfo = au_di_alloc(sb, AuLsc_DI_CHILD); + if (dinfo) { + atomic_set(&dinfo->di_generation, au_sigen(sb)); + /* smp_mb(); */ /* atomic_set */ + dentry->d_fsdata = dinfo; + } else + err = -ENOMEM; + + return err; +} + +void au_di_fin(struct dentry *dentry) +{ + struct au_dinfo *dinfo; + + dinfo = au_di(dentry); + AuRwDestroy(&dinfo->di_rwsem); + au_di_free(dinfo); +} + +int au_di_realloc(struct au_dinfo *dinfo, int nbr, int may_shrink) +{ + int err, sz; + struct au_hdentry *hdp; + + AuRwMustWriteLock(&dinfo->di_rwsem); + + err = -ENOMEM; + sz = sizeof(*hdp) * (dinfo->di_bbot + 1); + if (!sz) + sz = sizeof(*hdp); + hdp = au_kzrealloc(dinfo->di_hdentry, sz, sizeof(*hdp) * nbr, GFP_NOFS, + may_shrink); + if (hdp) { + dinfo->di_hdentry = hdp; + err = 0; + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static void do_ii_write_lock(struct inode *inode, unsigned int lsc) +{ + switch (lsc) { + case AuLsc_DI_CHILD: + ii_write_lock_child(inode); + break; + case AuLsc_DI_CHILD2: + ii_write_lock_child2(inode); + break; + case AuLsc_DI_CHILD3: + ii_write_lock_child3(inode); + break; + case AuLsc_DI_PARENT: + ii_write_lock_parent(inode); + break; + case AuLsc_DI_PARENT2: + ii_write_lock_parent2(inode); + break; + case AuLsc_DI_PARENT3: + ii_write_lock_parent3(inode); + break; + default: + BUG(); + } +} + +static void do_ii_read_lock(struct inode *inode, unsigned int lsc) +{ + switch (lsc) { + case AuLsc_DI_CHILD: + ii_read_lock_child(inode); + break; + case AuLsc_DI_CHILD2: + ii_read_lock_child2(inode); + break; + case AuLsc_DI_CHILD3: + ii_read_lock_child3(inode); + break; + case AuLsc_DI_PARENT: + ii_read_lock_parent(inode); + break; + case AuLsc_DI_PARENT2: + ii_read_lock_parent2(inode); + break; + case AuLsc_DI_PARENT3: + ii_read_lock_parent3(inode); + break; + default: + BUG(); + } +} + +void di_read_lock(struct dentry *d, int flags, unsigned int lsc) +{ + struct inode *inode; + + au_rw_read_lock_nested(&au_di(d)->di_rwsem, lsc); + if (d_really_is_positive(d)) { + inode = d_inode(d); + if (au_ftest_lock(flags, IW)) + do_ii_write_lock(inode, lsc); + else if (au_ftest_lock(flags, IR)) + do_ii_read_lock(inode, lsc); + } +} + +void di_read_unlock(struct dentry *d, int flags) +{ + struct inode *inode; + + if (d_really_is_positive(d)) { + inode = d_inode(d); + if (au_ftest_lock(flags, IW)) { + au_dbg_verify_dinode(d); + ii_write_unlock(inode); + } else if (au_ftest_lock(flags, IR)) { + au_dbg_verify_dinode(d); + ii_read_unlock(inode); + } + } + au_rw_read_unlock(&au_di(d)->di_rwsem); +} + +void di_downgrade_lock(struct dentry *d, int flags) +{ + if (d_really_is_positive(d) && au_ftest_lock(flags, IR)) + ii_downgrade_lock(d_inode(d)); + au_rw_dgrade_lock(&au_di(d)->di_rwsem); +} + +void di_write_lock(struct dentry *d, unsigned int lsc) +{ + au_rw_write_lock_nested(&au_di(d)->di_rwsem, lsc); + if (d_really_is_positive(d)) + do_ii_write_lock(d_inode(d), lsc); +} + +void di_write_unlock(struct dentry *d) +{ + au_dbg_verify_dinode(d); + if (d_really_is_positive(d)) + ii_write_unlock(d_inode(d)); + au_rw_write_unlock(&au_di(d)->di_rwsem); +} + +void di_write_lock2_child(struct dentry *d1, struct dentry *d2, int isdir) +{ + AuDebugOn(d1 == d2 + || d_inode(d1) == d_inode(d2) + || d1->d_sb != d2->d_sb); + + if ((isdir && au_test_subdir(d1, d2)) + || d1 < d2) { + di_write_lock_child(d1); + di_write_lock_child2(d2); + } else { + di_write_lock_child(d2); + di_write_lock_child2(d1); + } +} + +void di_write_lock2_parent(struct dentry *d1, struct dentry *d2, int isdir) +{ + AuDebugOn(d1 == d2 + || d_inode(d1) == d_inode(d2) + || d1->d_sb != d2->d_sb); + + if ((isdir && au_test_subdir(d1, d2)) + || d1 < d2) { + di_write_lock_parent(d1); + di_write_lock_parent2(d2); + } else { + di_write_lock_parent(d2); + di_write_lock_parent2(d1); + } +} + +void di_write_unlock2(struct dentry *d1, struct dentry *d2) +{ + di_write_unlock(d1); + if (d_inode(d1) == d_inode(d2)) + au_rw_write_unlock(&au_di(d2)->di_rwsem); + else + di_write_unlock(d2); +} + +/* ---------------------------------------------------------------------- */ + +struct dentry *au_h_dptr(struct dentry *dentry, aufs_bindex_t bindex) +{ + struct dentry *d; + + DiMustAnyLock(dentry); + + if (au_dbtop(dentry) < 0 || bindex < au_dbtop(dentry)) + return NULL; + AuDebugOn(bindex < 0); + d = au_hdentry(au_di(dentry), bindex)->hd_dentry; + AuDebugOn(d && au_dcount(d) <= 0); + return d; +} + +/* + * extended version of au_h_dptr(). + * returns a hashed and positive (or linkable) h_dentry in bindex, NULL, or + * error. + */ +struct dentry *au_h_d_alias(struct dentry *dentry, aufs_bindex_t bindex) +{ + struct dentry *h_dentry; + struct inode *inode, *h_inode; + + AuDebugOn(d_really_is_negative(dentry)); + + h_dentry = NULL; + if (au_dbtop(dentry) <= bindex + && bindex <= au_dbbot(dentry)) + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry && !au_d_linkable(h_dentry)) { + dget(h_dentry); + goto out; /* success */ + } + + inode = d_inode(dentry); + AuDebugOn(bindex < au_ibtop(inode)); + AuDebugOn(au_ibbot(inode) < bindex); + h_inode = au_h_iptr(inode, bindex); + h_dentry = d_find_alias(h_inode); + if (h_dentry) { + if (!IS_ERR(h_dentry)) { + if (!au_d_linkable(h_dentry)) + goto out; /* success */ + dput(h_dentry); + } else + goto out; + } + + if (au_opt_test(au_mntflags(dentry->d_sb), PLINK)) { + h_dentry = au_plink_lkup(inode, bindex); + AuDebugOn(!h_dentry); + if (!IS_ERR(h_dentry)) { + if (!au_d_hashed_positive(h_dentry)) + goto out; /* success */ + dput(h_dentry); + h_dentry = NULL; + } + } + +out: + AuDbgDentry(h_dentry); + return h_dentry; +} + +aufs_bindex_t au_dbtail(struct dentry *dentry) +{ + aufs_bindex_t bbot, bwh; + + bbot = au_dbbot(dentry); + if (0 <= bbot) { + bwh = au_dbwh(dentry); + if (!bwh) + return bwh; + if (0 < bwh && bwh < bbot) + return bwh - 1; + } + return bbot; +} + +aufs_bindex_t au_dbtaildir(struct dentry *dentry) +{ + aufs_bindex_t bbot, bopq; + + bbot = au_dbtail(dentry); + if (0 <= bbot) { + bopq = au_dbdiropq(dentry); + if (0 <= bopq && bopq < bbot) + bbot = bopq; + } + return bbot; +} + +/* ---------------------------------------------------------------------- */ + +void au_set_h_dptr(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_dentry) +{ + struct au_dinfo *dinfo; + struct au_hdentry *hd; + struct au_branch *br; + + DiMustWriteLock(dentry); + + dinfo = au_di(dentry); + hd = au_hdentry(dinfo, bindex); + au_hdput(hd); + hd->hd_dentry = h_dentry; + if (h_dentry) { + br = au_sbr(dentry->d_sb, bindex); + hd->hd_id = br->br_id; + } +} + +int au_dbrange_test(struct dentry *dentry) +{ + int err; + aufs_bindex_t btop, bbot; + + err = 0; + btop = au_dbtop(dentry); + bbot = au_dbbot(dentry); + if (btop >= 0) + AuDebugOn(bbot < 0 && btop > bbot); + else { + err = -EIO; + AuDebugOn(bbot >= 0); + } + + return err; +} + +int au_digen_test(struct dentry *dentry, unsigned int sigen) +{ + int err; + + err = 0; + if (unlikely(au_digen(dentry) != sigen + || au_iigen_test(d_inode(dentry), sigen))) + err = -EIO; + + return err; +} + +void au_update_digen(struct dentry *dentry) +{ + atomic_set(&au_di(dentry)->di_generation, au_sigen(dentry->d_sb)); + /* smp_mb(); */ /* atomic_set */ +} + +void au_update_dbrange(struct dentry *dentry, int do_put_zero) +{ + struct au_dinfo *dinfo; + struct dentry *h_d; + struct au_hdentry *hdp; + aufs_bindex_t bindex, bbot; + + DiMustWriteLock(dentry); + + dinfo = au_di(dentry); + if (!dinfo || dinfo->di_btop < 0) + return; + + if (do_put_zero) { + bbot = dinfo->di_bbot; + bindex = dinfo->di_btop; + hdp = au_hdentry(dinfo, bindex); + for (; bindex <= bbot; bindex++, hdp++) { + h_d = hdp->hd_dentry; + if (h_d && d_is_negative(h_d)) + au_set_h_dptr(dentry, bindex, NULL); + } + } + + dinfo->di_btop = 0; + hdp = au_hdentry(dinfo, dinfo->di_btop); + for (; dinfo->di_btop <= dinfo->di_bbot; dinfo->di_btop++, hdp++) + if (hdp->hd_dentry) + break; + if (dinfo->di_btop > dinfo->di_bbot) { + dinfo->di_btop = -1; + dinfo->di_bbot = -1; + return; + } + + hdp = au_hdentry(dinfo, dinfo->di_bbot); + for (; dinfo->di_bbot >= 0; dinfo->di_bbot--, hdp--) + if (hdp->hd_dentry) + break; + AuDebugOn(dinfo->di_btop > dinfo->di_bbot || dinfo->di_bbot < 0); +} + +void au_update_dbtop(struct dentry *dentry) +{ + aufs_bindex_t bindex, bbot; + struct dentry *h_dentry; + + bbot = au_dbbot(dentry); + for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + if (d_is_positive(h_dentry)) { + au_set_dbtop(dentry, bindex); + return; + } + au_set_h_dptr(dentry, bindex, NULL); + } +} + +void au_update_dbbot(struct dentry *dentry) +{ + aufs_bindex_t bindex, btop; + struct dentry *h_dentry; + + btop = au_dbtop(dentry); + for (bindex = au_dbbot(dentry); bindex >= btop; bindex--) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + if (d_is_positive(h_dentry)) { + au_set_dbbot(dentry, bindex); + return; + } + au_set_h_dptr(dentry, bindex, NULL); + } +} + +int au_find_dbindex(struct dentry *dentry, struct dentry *h_dentry) +{ + aufs_bindex_t bindex, bbot; + + bbot = au_dbbot(dentry); + for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++) + if (au_h_dptr(dentry, bindex) == h_dentry) + return bindex; + return -1; +} diff --git a/fs/aufs/dir.c b/fs/aufs/dir.c new file mode 100644 index 0000000000000..0bcb39ee72558 --- /dev/null +++ b/fs/aufs/dir.c @@ -0,0 +1,763 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * directory operations + */ + +#include +#include +#include "aufs.h" + +void au_add_nlink(struct inode *dir, struct inode *h_dir) +{ + unsigned int nlink; + + AuDebugOn(!S_ISDIR(dir->i_mode) || !S_ISDIR(h_dir->i_mode)); + + nlink = dir->i_nlink; + nlink += h_dir->i_nlink - 2; + if (h_dir->i_nlink < 2) + nlink += 2; + smp_mb(); /* for i_nlink */ + /* 0 can happen in revaliding */ + set_nlink(dir, nlink); +} + +void au_sub_nlink(struct inode *dir, struct inode *h_dir) +{ + unsigned int nlink; + + AuDebugOn(!S_ISDIR(dir->i_mode) || !S_ISDIR(h_dir->i_mode)); + + nlink = dir->i_nlink; + nlink -= h_dir->i_nlink - 2; + if (h_dir->i_nlink < 2) + nlink -= 2; + smp_mb(); /* for i_nlink */ + /* nlink == 0 means the branch-fs is broken */ + set_nlink(dir, nlink); +} + +loff_t au_dir_size(struct file *file, struct dentry *dentry) +{ + loff_t sz; + aufs_bindex_t bindex, bbot; + struct file *h_file; + struct dentry *h_dentry; + + sz = 0; + if (file) { + AuDebugOn(!d_is_dir(file->f_path.dentry)); + + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); + bindex <= bbot && sz < KMALLOC_MAX_SIZE; + bindex++) { + h_file = au_hf_dir(file, bindex); + if (h_file && file_inode(h_file)) + sz += vfsub_f_size_read(h_file); + } + } else { + AuDebugOn(!dentry); + AuDebugOn(!d_is_dir(dentry)); + + bbot = au_dbtaildir(dentry); + for (bindex = au_dbtop(dentry); + bindex <= bbot && sz < KMALLOC_MAX_SIZE; + bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry && d_is_positive(h_dentry)) + sz += i_size_read(d_inode(h_dentry)); + } + } + if (sz < KMALLOC_MAX_SIZE) + sz = roundup_pow_of_two(sz); + if (sz > KMALLOC_MAX_SIZE) + sz = KMALLOC_MAX_SIZE; + else if (sz < NAME_MAX) { + BUILD_BUG_ON(AUFS_RDBLK_DEF < NAME_MAX); + sz = AUFS_RDBLK_DEF; + } + return sz; +} + +struct au_dir_ts_arg { + struct dentry *dentry; + aufs_bindex_t brid; +}; + +static void au_do_dir_ts(void *arg) +{ + struct au_dir_ts_arg *a = arg; + struct au_dtime dt; + struct path h_path; + struct inode *dir, *h_dir; + struct super_block *sb; + struct au_branch *br; + struct au_hinode *hdir; + int err; + aufs_bindex_t btop, bindex; + + sb = a->dentry->d_sb; + if (d_really_is_negative(a->dentry)) + goto out; + /* no dir->i_mutex lock */ + aufs_read_lock(a->dentry, AuLock_DW); /* noflush */ + + dir = d_inode(a->dentry); + btop = au_ibtop(dir); + bindex = au_br_index(sb, a->brid); + if (bindex < btop) + goto out_unlock; + + br = au_sbr(sb, bindex); + h_path.dentry = au_h_dptr(a->dentry, bindex); + if (!h_path.dentry) + goto out_unlock; + h_path.mnt = au_br_mnt(br); + au_dtime_store(&dt, a->dentry, &h_path); + + br = au_sbr(sb, btop); + if (!au_br_writable(br->br_perm)) + goto out_unlock; + h_path.dentry = au_h_dptr(a->dentry, btop); + h_path.mnt = au_br_mnt(br); + err = vfsub_mnt_want_write(h_path.mnt); + if (err) + goto out_unlock; + hdir = au_hi(dir, btop); + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + h_dir = au_h_iptr(dir, btop); + if (h_dir->i_nlink + && timespec64_compare(&h_dir->i_mtime, &dt.dt_mtime) < 0) { + dt.dt_h_path = h_path; + au_dtime_revert(&dt); + } + au_hn_inode_unlock(hdir); + vfsub_mnt_drop_write(h_path.mnt); + au_cpup_attr_timesizes(dir); + +out_unlock: + aufs_read_unlock(a->dentry, AuLock_DW); +out: + dput(a->dentry); + au_nwt_done(&au_sbi(sb)->si_nowait); + au_kfree_try_rcu(arg); +} + +void au_dir_ts(struct inode *dir, aufs_bindex_t bindex) +{ + int perm, wkq_err; + aufs_bindex_t btop; + struct au_dir_ts_arg *arg; + struct dentry *dentry; + struct super_block *sb; + + IMustLock(dir); + + dentry = d_find_any_alias(dir); + AuDebugOn(!dentry); + sb = dentry->d_sb; + btop = au_ibtop(dir); + if (btop == bindex) { + au_cpup_attr_timesizes(dir); + goto out; + } + + perm = au_sbr_perm(sb, btop); + if (!au_br_writable(perm)) + goto out; + + arg = kmalloc(sizeof(*arg), GFP_NOFS); + if (!arg) + goto out; + + arg->dentry = dget(dentry); /* will be dput-ted by au_do_dir_ts() */ + arg->brid = au_sbr_id(sb, bindex); + wkq_err = au_wkq_nowait(au_do_dir_ts, arg, sb, /*flags*/0); + if (unlikely(wkq_err)) { + pr_err("wkq %d\n", wkq_err); + dput(dentry); + au_kfree_try_rcu(arg); + } + +out: + dput(dentry); +} + +/* ---------------------------------------------------------------------- */ + +static int reopen_dir(struct file *file) +{ + int err; + unsigned int flags; + aufs_bindex_t bindex, btail, btop; + struct dentry *dentry, *h_dentry; + struct file *h_file; + + /* open all lower dirs */ + dentry = file->f_path.dentry; + btop = au_dbtop(dentry); + for (bindex = au_fbtop(file); bindex < btop; bindex++) + au_set_h_fptr(file, bindex, NULL); + au_set_fbtop(file, btop); + + btail = au_dbtaildir(dentry); + for (bindex = au_fbbot_dir(file); btail < bindex; bindex--) + au_set_h_fptr(file, bindex, NULL); + au_set_fbbot_dir(file, btail); + + flags = vfsub_file_flags(file); + for (bindex = btop; bindex <= btail; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + h_file = au_hf_dir(file, bindex); + if (h_file) + continue; + + h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; /* close all? */ + au_set_h_fptr(file, bindex, h_file); + } + au_update_figen(file); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + err = 0; + +out: + return err; +} + +static int do_open_dir(struct file *file, int flags, struct file *h_file) +{ + int err; + aufs_bindex_t bindex, btail; + struct dentry *dentry, *h_dentry; + struct vfsmount *mnt; + + FiMustWriteLock(file); + AuDebugOn(h_file); + + err = 0; + mnt = file->f_path.mnt; + dentry = file->f_path.dentry; + file->f_version = inode_query_iversion(d_inode(dentry)); + bindex = au_dbtop(dentry); + au_set_fbtop(file, bindex); + btail = au_dbtaildir(dentry); + au_set_fbbot_dir(file, btail); + for (; !err && bindex <= btail; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (!h_dentry) + continue; + + err = vfsub_test_mntns(mnt, h_dentry->d_sb); + if (unlikely(err)) + break; + h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0); + if (IS_ERR(h_file)) { + err = PTR_ERR(h_file); + break; + } + au_set_h_fptr(file, bindex, h_file); + } + au_update_figen(file); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + if (!err) + return 0; /* success */ + + /* close all */ + for (bindex = au_fbtop(file); bindex <= btail; bindex++) + au_set_h_fptr(file, bindex, NULL); + au_set_fbtop(file, -1); + au_set_fbbot_dir(file, -1); + + return err; +} + +static int aufs_open_dir(struct inode *inode __maybe_unused, + struct file *file) +{ + int err; + struct super_block *sb; + struct au_fidir *fidir; + + err = -ENOMEM; + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH); + fidir = au_fidir_alloc(sb); + if (fidir) { + struct au_do_open_args args = { + .open = do_open_dir, + .fidir = fidir + }; + err = au_do_open(file, &args); + if (unlikely(err)) + au_kfree_rcu(fidir); + } + si_read_unlock(sb); + return err; +} + +static int aufs_release_dir(struct inode *inode __maybe_unused, + struct file *file) +{ + struct au_vdir *vdir_cache; + struct au_finfo *finfo; + struct au_fidir *fidir; + struct au_hfile *hf; + aufs_bindex_t bindex, bbot; + + finfo = au_fi(file); + fidir = finfo->fi_hdir; + if (fidir) { + au_hbl_del(&finfo->fi_hlist, + &au_sbi(file->f_path.dentry->d_sb)->si_files); + vdir_cache = fidir->fd_vdir_cache; /* lock-free */ + if (vdir_cache) + au_vdir_free(vdir_cache); + + bindex = finfo->fi_btop; + if (bindex >= 0) { + hf = fidir->fd_hfile + bindex; + /* + * calls fput() instead of filp_close(), + * since no dnotify or lock for the lower file. + */ + bbot = fidir->fd_bbot; + for (; bindex <= bbot; bindex++, hf++) + if (hf->hf_file) + au_hfput(hf, /*execed*/0); + } + au_kfree_rcu(fidir); + finfo->fi_hdir = NULL; + } + au_finfo_fin(file); + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_flush_dir(struct file *file, fl_owner_t id) +{ + int err; + aufs_bindex_t bindex, bbot; + struct file *h_file; + + err = 0; + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); !err && bindex <= bbot; bindex++) { + h_file = au_hf_dir(file, bindex); + if (h_file) + err = vfsub_flush(h_file, id); + } + return err; +} + +static int aufs_flush_dir(struct file *file, fl_owner_t id) +{ + return au_do_flush(file, id, au_do_flush_dir); +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_fsync_dir_no_file(struct dentry *dentry, int datasync) +{ + int err; + aufs_bindex_t bbot, bindex; + struct inode *inode; + struct super_block *sb; + + err = 0; + sb = dentry->d_sb; + inode = d_inode(dentry); + IMustLock(inode); + bbot = au_dbbot(dentry); + for (bindex = au_dbtop(dentry); !err && bindex <= bbot; bindex++) { + struct path h_path; + + if (au_test_ro(sb, bindex, inode)) + continue; + h_path.dentry = au_h_dptr(dentry, bindex); + if (!h_path.dentry) + continue; + + h_path.mnt = au_sbr_mnt(sb, bindex); + err = vfsub_fsync(NULL, &h_path, datasync); + } + + return err; +} + +static int au_do_fsync_dir(struct file *file, int datasync) +{ + int err; + aufs_bindex_t bbot, bindex; + struct file *h_file; + struct super_block *sb; + struct inode *inode; + + err = au_reval_and_lock_fdi(file, reopen_dir, /*wlock*/1, /*fi_lsc*/0); + if (unlikely(err)) + goto out; + + inode = file_inode(file); + sb = inode->i_sb; + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); !err && bindex <= bbot; bindex++) { + h_file = au_hf_dir(file, bindex); + if (!h_file || au_test_ro(sb, bindex, inode)) + continue; + + err = vfsub_fsync(h_file, &h_file->f_path, datasync); + } + +out: + return err; +} + +/* + * @file may be NULL + */ +static int aufs_fsync_dir(struct file *file, loff_t start, loff_t end, + int datasync) +{ + int err; + struct dentry *dentry; + struct inode *inode; + struct super_block *sb; + + err = 0; + dentry = file->f_path.dentry; + inode = d_inode(dentry); + inode_lock(inode); + sb = dentry->d_sb; + si_noflush_read_lock(sb); + if (file) + err = au_do_fsync_dir(file, datasync); + else { + di_write_lock_child(dentry); + err = au_do_fsync_dir_no_file(dentry, datasync); + } + au_cpup_attr_timesizes(inode); + di_write_unlock(dentry); + if (file) + fi_write_unlock(file); + + si_read_unlock(sb); + inode_unlock(inode); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int aufs_iterate_shared(struct file *file, struct dir_context *ctx) +{ + int err; + struct dentry *dentry; + struct inode *inode, *h_inode; + struct super_block *sb; + + AuDbg("%pD, ctx{%ps, %llu}\n", file, ctx->actor, ctx->pos); + + dentry = file->f_path.dentry; + inode = d_inode(dentry); + IMustLock(inode); + + sb = dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH); + err = au_reval_and_lock_fdi(file, reopen_dir, /*wlock*/1, /*fi_lsc*/0); + if (unlikely(err)) + goto out; + err = au_alive_dir(dentry); + if (!err) + err = au_vdir_init(file); + di_downgrade_lock(dentry, AuLock_IR); + if (unlikely(err)) + goto out_unlock; + + h_inode = au_h_iptr(inode, au_ibtop(inode)); + if (!au_test_nfsd()) { + err = au_vdir_fill_de(file, ctx); + fsstack_copy_attr_atime(inode, h_inode); + } else { + /* + * nfsd filldir may call lookup_one_len(), vfs_getattr(), + * encode_fh() and others. + */ + atomic_inc(&h_inode->i_count); + di_read_unlock(dentry, AuLock_IR); + si_read_unlock(sb); + err = au_vdir_fill_de(file, ctx); + fsstack_copy_attr_atime(inode, h_inode); + fi_write_unlock(file); + iput(h_inode); + + AuTraceErr(err); + return err; + } + +out_unlock: + di_read_unlock(dentry, AuLock_IR); + fi_write_unlock(file); +out: + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +#define AuTestEmpty_WHONLY 1 +#define AuTestEmpty_CALLED (1 << 1) +#define AuTestEmpty_SHWH (1 << 2) +#define au_ftest_testempty(flags, name) ((flags) & AuTestEmpty_##name) +#define au_fset_testempty(flags, name) \ + do { (flags) |= AuTestEmpty_##name; } while (0) +#define au_fclr_testempty(flags, name) \ + do { (flags) &= ~AuTestEmpty_##name; } while (0) + +#ifndef CONFIG_AUFS_SHWH +#undef AuTestEmpty_SHWH +#define AuTestEmpty_SHWH 0 +#endif + +struct test_empty_arg { + struct dir_context ctx; + struct au_nhash *whlist; + unsigned int flags; + int err; + aufs_bindex_t bindex; +}; + +static int test_empty_cb(struct dir_context *ctx, const char *__name, + int namelen, loff_t offset __maybe_unused, u64 ino, + unsigned int d_type) +{ + struct test_empty_arg *arg = container_of(ctx, struct test_empty_arg, + ctx); + char *name = (void *)__name; + + arg->err = 0; + au_fset_testempty(arg->flags, CALLED); + /* smp_mb(); */ + if (name[0] == '.' + && (namelen == 1 || (name[1] == '.' && namelen == 2))) + goto out; /* success */ + + if (namelen <= AUFS_WH_PFX_LEN + || memcmp(name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) { + if (au_ftest_testempty(arg->flags, WHONLY) + && !au_nhash_test_known_wh(arg->whlist, name, namelen)) + arg->err = -ENOTEMPTY; + goto out; + } + + name += AUFS_WH_PFX_LEN; + namelen -= AUFS_WH_PFX_LEN; + if (!au_nhash_test_known_wh(arg->whlist, name, namelen)) + arg->err = au_nhash_append_wh + (arg->whlist, name, namelen, ino, d_type, arg->bindex, + au_ftest_testempty(arg->flags, SHWH)); + +out: + /* smp_mb(); */ + AuTraceErr(arg->err); + return arg->err; +} + +static int do_test_empty(struct dentry *dentry, struct test_empty_arg *arg) +{ + int err; + struct file *h_file; + struct au_branch *br; + + h_file = au_h_open(dentry, arg->bindex, + O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_LARGEFILE, + /*file*/NULL, /*force_wr*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = 0; + if (!au_opt_test(au_mntflags(dentry->d_sb), UDBA_NONE) + && !file_inode(h_file)->i_nlink) + goto out_put; + + do { + arg->err = 0; + au_fclr_testempty(arg->flags, CALLED); + /* smp_mb(); */ + err = vfsub_iterate_dir(h_file, &arg->ctx); + if (err >= 0) + err = arg->err; + } while (!err && au_ftest_testempty(arg->flags, CALLED)); + +out_put: + fput(h_file); + br = au_sbr(dentry->d_sb, arg->bindex); + au_lcnt_dec(&br->br_nfiles); +out: + return err; +} + +struct do_test_empty_args { + int *errp; + struct dentry *dentry; + struct test_empty_arg *arg; +}; + +static void call_do_test_empty(void *args) +{ + struct do_test_empty_args *a = args; + *a->errp = do_test_empty(a->dentry, a->arg); +} + +static int sio_test_empty(struct dentry *dentry, struct test_empty_arg *arg) +{ + int err, wkq_err; + struct dentry *h_dentry; + struct inode *h_inode; + + h_dentry = au_h_dptr(dentry, arg->bindex); + h_inode = d_inode(h_dentry); + /* todo: i_mode changes anytime? */ + inode_lock_shared_nested(h_inode, AuLsc_I_CHILD); + err = au_test_h_perm_sio(h_inode, MAY_EXEC | MAY_READ); + inode_unlock_shared(h_inode); + if (!err) + err = do_test_empty(dentry, arg); + else { + struct do_test_empty_args args = { + .errp = &err, + .dentry = dentry, + .arg = arg + }; + unsigned int flags = arg->flags; + + wkq_err = au_wkq_wait(call_do_test_empty, &args); + if (unlikely(wkq_err)) + err = wkq_err; + arg->flags = flags; + } + + return err; +} + +int au_test_empty_lower(struct dentry *dentry) +{ + int err; + unsigned int rdhash; + aufs_bindex_t bindex, btop, btail; + struct au_nhash whlist; + struct test_empty_arg arg = { + .ctx = { + .actor = test_empty_cb + } + }; + int (*test_empty)(struct dentry *dentry, struct test_empty_arg *arg); + + SiMustAnyLock(dentry->d_sb); + + rdhash = au_sbi(dentry->d_sb)->si_rdhash; + if (!rdhash) + rdhash = au_rdhash_est(au_dir_size(/*file*/NULL, dentry)); + err = au_nhash_alloc(&whlist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out; + + arg.flags = 0; + arg.whlist = &whlist; + btop = au_dbtop(dentry); + if (au_opt_test(au_mntflags(dentry->d_sb), SHWH)) + au_fset_testempty(arg.flags, SHWH); + test_empty = do_test_empty; + if (au_opt_test(au_mntflags(dentry->d_sb), DIRPERM1)) + test_empty = sio_test_empty; + arg.bindex = btop; + err = test_empty(dentry, &arg); + if (unlikely(err)) + goto out_whlist; + + au_fset_testempty(arg.flags, WHONLY); + btail = au_dbtaildir(dentry); + for (bindex = btop + 1; !err && bindex <= btail; bindex++) { + struct dentry *h_dentry; + + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry && d_is_positive(h_dentry)) { + arg.bindex = bindex; + err = test_empty(dentry, &arg); + } + } + +out_whlist: + au_nhash_wh_free(&whlist); +out: + return err; +} + +int au_test_empty(struct dentry *dentry, struct au_nhash *whlist) +{ + int err; + struct test_empty_arg arg = { + .ctx = { + .actor = test_empty_cb + } + }; + aufs_bindex_t bindex, btail; + + err = 0; + arg.whlist = whlist; + arg.flags = AuTestEmpty_WHONLY; + if (au_opt_test(au_mntflags(dentry->d_sb), SHWH)) + au_fset_testempty(arg.flags, SHWH); + btail = au_dbtaildir(dentry); + for (bindex = au_dbtop(dentry); !err && bindex <= btail; bindex++) { + struct dentry *h_dentry; + + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry && d_is_positive(h_dentry)) { + arg.bindex = bindex; + err = sio_test_empty(dentry, &arg); + } + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +const struct file_operations aufs_dir_fop = { + .owner = THIS_MODULE, + .llseek = default_llseek, + .read = generic_read_dir, + .iterate_shared = aufs_iterate_shared, + .unlocked_ioctl = aufs_ioctl_dir, +#ifdef CONFIG_COMPAT + .compat_ioctl = aufs_compat_ioctl_dir, +#endif + .open = aufs_open_dir, + .release = aufs_release_dir, + .flush = aufs_flush_dir, + .fsync = aufs_fsync_dir +}; diff --git a/fs/aufs/dir.h b/fs/aufs/dir.h new file mode 100644 index 0000000000000..e44c2a1a849b0 --- /dev/null +++ b/fs/aufs/dir.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * directory operations + */ + +#ifndef __AUFS_DIR_H__ +#define __AUFS_DIR_H__ + +#ifdef __KERNEL__ + +#include + +/* ---------------------------------------------------------------------- */ + +/* need to be faster and smaller */ + +struct au_nhash { + unsigned int nh_num; + struct hlist_head *nh_head; +}; + +struct au_vdir_destr { + unsigned char len; + unsigned char name[]; +} __packed; + +struct au_vdir_dehstr { + struct hlist_node hash; + struct au_vdir_destr *str; + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +struct au_vdir_de { + ino_t de_ino; + unsigned char de_type; + /* caution: packed */ + struct au_vdir_destr de_str; +} __packed; + +struct au_vdir_wh { + struct hlist_node wh_hash; +#ifdef CONFIG_AUFS_SHWH + ino_t wh_ino; + aufs_bindex_t wh_bindex; + unsigned char wh_type; +#else + aufs_bindex_t wh_bindex; +#endif + /* caution: packed */ + struct au_vdir_destr wh_str; +} __packed; + +union au_vdir_deblk_p { + unsigned char *deblk; + struct au_vdir_de *de; +}; + +struct au_vdir { + unsigned char **vd_deblk; + unsigned long vd_nblk; + struct { + unsigned long ul; + union au_vdir_deblk_p p; + } vd_last; + + u64 vd_version; + unsigned int vd_deblk_sz; + unsigned long vd_jiffy; + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ + +/* dir.c */ +extern const struct file_operations aufs_dir_fop; +void au_add_nlink(struct inode *dir, struct inode *h_dir); +void au_sub_nlink(struct inode *dir, struct inode *h_dir); +loff_t au_dir_size(struct file *file, struct dentry *dentry); +void au_dir_ts(struct inode *dir, aufs_bindex_t bsrc); +int au_test_empty_lower(struct dentry *dentry); +int au_test_empty(struct dentry *dentry, struct au_nhash *whlist); + +/* vdir.c */ +unsigned int au_rdhash_est(loff_t sz); +int au_nhash_alloc(struct au_nhash *nhash, unsigned int num_hash, gfp_t gfp); +void au_nhash_wh_free(struct au_nhash *whlist); +int au_nhash_test_longer_wh(struct au_nhash *whlist, aufs_bindex_t btgt, + int limit); +int au_nhash_test_known_wh(struct au_nhash *whlist, char *name, int nlen); +int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino, + unsigned int d_type, aufs_bindex_t bindex, + unsigned char shwh); +void au_vdir_free(struct au_vdir *vdir); +int au_vdir_init(struct file *file); +int au_vdir_fill_de(struct file *file, struct dir_context *ctx); + +/* ioctl.c */ +long aufs_ioctl_dir(struct file *file, unsigned int cmd, unsigned long arg); + +#ifdef CONFIG_AUFS_RDU +/* rdu.c */ +long au_rdu_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +#ifdef CONFIG_COMPAT +long au_rdu_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg); +#endif +#else +AuStub(long, au_rdu_ioctl, return -EINVAL, struct file *file, + unsigned int cmd, unsigned long arg) +#ifdef CONFIG_COMPAT +AuStub(long, au_rdu_compat_ioctl, return -EINVAL, struct file *file, + unsigned int cmd, unsigned long arg) +#endif +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DIR_H__ */ diff --git a/fs/aufs/dirren.c b/fs/aufs/dirren.c new file mode 100644 index 0000000000000..ba4bf56cfb874 --- /dev/null +++ b/fs/aufs/dirren.c @@ -0,0 +1,1316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * special handling in renaming a directory + * in order to support looking-up the before-renamed name on the lower readonly + * branches + */ + +#include +#include "aufs.h" + +static void au_dr_hino_del(struct au_dr_br *dr, struct au_dr_hino *ent) +{ + int idx; + + idx = au_dr_ihash(ent->dr_h_ino); + au_hbl_del(&ent->dr_hnode, dr->dr_h_ino + idx); +} + +static int au_dr_hino_test_empty(struct au_dr_br *dr) +{ + int ret, i; + struct hlist_bl_head *hbl; + + ret = 1; + for (i = 0; ret && i < AuDirren_NHASH; i++) { + hbl = dr->dr_h_ino + i; + hlist_bl_lock(hbl); + ret &= hlist_bl_empty(hbl); + hlist_bl_unlock(hbl); + } + + return ret; +} + +static struct au_dr_hino *au_dr_hino_find(struct au_dr_br *dr, ino_t ino) +{ + struct au_dr_hino *found, *ent; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + int idx; + + found = NULL; + idx = au_dr_ihash(ino); + hbl = dr->dr_h_ino + idx; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(ent, pos, hbl, dr_hnode) + if (ent->dr_h_ino == ino) { + found = ent; + break; + } + hlist_bl_unlock(hbl); + + return found; +} + +int au_dr_hino_test_add(struct au_dr_br *dr, ino_t ino, + struct au_dr_hino *add_ent) +{ + int found, idx; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_dr_hino *ent; + + found = 0; + idx = au_dr_ihash(ino); + hbl = dr->dr_h_ino + idx; +#if 0 /* debug print */ + { + struct hlist_bl_node *tmp; + + hlist_bl_for_each_entry_safe(ent, pos, tmp, hbl, dr_hnode) + AuDbg("hi%llu\n", (unsigned long long)ent->dr_h_ino); + } +#endif + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(ent, pos, hbl, dr_hnode) + if (ent->dr_h_ino == ino) { + found = 1; + break; + } + if (!found && add_ent) + hlist_bl_add_head(&add_ent->dr_hnode, hbl); + hlist_bl_unlock(hbl); + + if (!found && add_ent) + AuDbg("i%llu added\n", (unsigned long long)add_ent->dr_h_ino); + + return found; +} + +void au_dr_hino_free(struct au_dr_br *dr) +{ + int i; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos, *tmp; + struct au_dr_hino *ent; + + /* SiMustWriteLock(sb); */ + + for (i = 0; i < AuDirren_NHASH; i++) { + hbl = dr->dr_h_ino + i; + /* no spinlock since sbinfo must be write-locked */ + hlist_bl_for_each_entry_safe(ent, pos, tmp, hbl, dr_hnode) + au_kfree_rcu(ent); + INIT_HLIST_BL_HEAD(hbl); + } +} + +/* returns the number of inodes or an error */ +static int au_dr_hino_store(struct super_block *sb, struct au_branch *br, + struct file *hinofile) +{ + int err, i; + ssize_t ssz; + loff_t pos, oldsize; + __be64 u64; + struct inode *hinoinode; + struct hlist_bl_head *hbl; + struct hlist_bl_node *n1, *n2; + struct au_dr_hino *ent; + + SiMustWriteLock(sb); + AuDebugOn(!au_br_writable(br->br_perm)); + + hinoinode = file_inode(hinofile); + oldsize = i_size_read(hinoinode); + + err = 0; + pos = 0; + hbl = br->br_dirren.dr_h_ino; + for (i = 0; !err && i < AuDirren_NHASH; i++, hbl++) { + /* no bit-lock since sbinfo must be write-locked */ + hlist_bl_for_each_entry_safe(ent, n1, n2, hbl, dr_hnode) { + AuDbg("hi%llu, %pD2\n", + (unsigned long long)ent->dr_h_ino, hinofile); + u64 = cpu_to_be64(ent->dr_h_ino); + ssz = vfsub_write_k(hinofile, &u64, sizeof(u64), &pos); + if (ssz == sizeof(u64)) + continue; + + /* write error */ + pr_err("ssz %zd, %pD2\n", ssz, hinofile); + err = -ENOSPC; + if (ssz < 0) + err = ssz; + break; + } + } + /* regardless the error */ + if (pos < oldsize) { + err = vfsub_trunc(&hinofile->f_path, pos, /*attr*/0, hinofile); + AuTraceErr(err); + } + + AuTraceErr(err); + return err; +} + +static int au_dr_hino_load(struct au_dr_br *dr, struct file *hinofile) +{ + int err, hidx; + ssize_t ssz; + size_t sz, n; + loff_t pos; + uint64_t u64; + struct au_dr_hino *ent; + struct inode *hinoinode; + struct hlist_bl_head *hbl; + + err = 0; + pos = 0; + hbl = dr->dr_h_ino; + hinoinode = file_inode(hinofile); + sz = i_size_read(hinoinode); + AuDebugOn(sz % sizeof(u64)); + n = sz / sizeof(u64); + while (n--) { + ssz = vfsub_read_k(hinofile, &u64, sizeof(u64), &pos); + if (unlikely(ssz != sizeof(u64))) { + pr_err("ssz %zd, %pD2\n", ssz, hinofile); + err = -EINVAL; + if (ssz < 0) + err = ssz; + goto out_free; + } + + ent = kmalloc(sizeof(*ent), GFP_NOFS); + if (!ent) { + err = -ENOMEM; + AuTraceErr(err); + goto out_free; + } + ent->dr_h_ino = be64_to_cpu((__force __be64)u64); + AuDbg("hi%llu, %pD2\n", + (unsigned long long)ent->dr_h_ino, hinofile); + hidx = au_dr_ihash(ent->dr_h_ino); + au_hbl_add(&ent->dr_hnode, hbl + hidx); + } + goto out; /* success */ + +out_free: + au_dr_hino_free(dr); +out: + AuTraceErr(err); + return err; +} + +/* + * @bindex/@br is a switch to distinguish whether suspending hnotify or not. + * @path is a switch to distinguish load and store. + */ +static int au_dr_hino(struct super_block *sb, aufs_bindex_t bindex, + struct au_branch *br, const struct path *path) +{ + int err, flags; + unsigned char load, suspend; + struct file *hinofile; + struct au_hinode *hdir; + struct inode *dir, *delegated; + struct path hinopath; + struct qstr hinoname = QSTR_INIT(AUFS_WH_DR_BRHINO, + sizeof(AUFS_WH_DR_BRHINO) - 1); + + AuDebugOn(bindex < 0 && !br); + AuDebugOn(bindex >= 0 && br); + + err = -EINVAL; + suspend = !br; + if (suspend) + br = au_sbr(sb, bindex); + load = !!path; + if (!load) { + path = &br->br_path; + AuDebugOn(!au_br_writable(br->br_perm)); + if (unlikely(!au_br_writable(br->br_perm))) + goto out; + } + + hdir = NULL; + if (suspend) { + dir = d_inode(sb->s_root); + hdir = au_hinode(au_ii(dir), bindex); + dir = hdir->hi_inode; + au_hn_inode_lock_nested(hdir, AuLsc_I_CHILD); + } else { + dir = d_inode(path->dentry); + inode_lock_nested(dir, AuLsc_I_CHILD); + } + hinopath.dentry = vfsub_lkup_one(&hinoname, path->dentry); + err = PTR_ERR(hinopath.dentry); + if (IS_ERR(hinopath.dentry)) + goto out_unlock; + + err = 0; + flags = O_RDONLY; + if (load) { + if (d_is_negative(hinopath.dentry)) + goto out_dput; /* success */ + } else { + if (au_dr_hino_test_empty(&br->br_dirren)) { + if (d_is_positive(hinopath.dentry)) { + delegated = NULL; + err = vfsub_unlink(dir, &hinopath, &delegated, + /*force*/0); + AuTraceErr(err); + if (unlikely(err)) + pr_err("ignored err %d, %pd2\n", + err, hinopath.dentry); + if (unlikely(err == -EWOULDBLOCK)) + iput(delegated); + err = 0; + } + goto out_dput; + } else if (!d_is_positive(hinopath.dentry)) { + err = vfsub_create(dir, &hinopath, 0600, + /*want_excl*/false); + AuTraceErr(err); + if (unlikely(err)) + goto out_dput; + } + flags = O_WRONLY; + } + hinopath.mnt = path->mnt; + hinofile = vfsub_dentry_open(&hinopath, flags); + if (suspend) + au_hn_inode_unlock(hdir); + else + inode_unlock(dir); + dput(hinopath.dentry); + AuTraceErrPtr(hinofile); + if (IS_ERR(hinofile)) { + err = PTR_ERR(hinofile); + goto out; + } + + if (load) + err = au_dr_hino_load(&br->br_dirren, hinofile); + else + err = au_dr_hino_store(sb, br, hinofile); + fput(hinofile); + goto out; + +out_dput: + dput(hinopath.dentry); +out_unlock: + if (suspend) + au_hn_inode_unlock(hdir); + else + inode_unlock(dir); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_dr_brid_init(struct au_dr_brid *brid, const struct path *path) +{ + int err; + struct kstatfs kstfs; + dev_t dev; + struct dentry *dentry; + struct super_block *sb; + + err = vfs_statfs((void *)path, &kstfs); + AuTraceErr(err); + if (unlikely(err)) + goto out; + + /* todo: support for UUID */ + + if (kstfs.f_fsid.val[0] || kstfs.f_fsid.val[1]) { + brid->type = AuBrid_FSID; + brid->fsid = kstfs.f_fsid; + } else { + dentry = path->dentry; + sb = dentry->d_sb; + dev = sb->s_dev; + if (dev) { + brid->type = AuBrid_DEV; + brid->dev = dev; + } + } + +out: + return err; +} + +int au_dr_br_init(struct super_block *sb, struct au_branch *br, + const struct path *path) +{ + int err, i; + struct au_dr_br *dr; + struct hlist_bl_head *hbl; + + dr = &br->br_dirren; + hbl = dr->dr_h_ino; + for (i = 0; i < AuDirren_NHASH; i++, hbl++) + INIT_HLIST_BL_HEAD(hbl); + + err = au_dr_brid_init(&dr->dr_brid, path); + if (unlikely(err)) + goto out; + + if (au_opt_test(au_mntflags(sb), DIRREN)) + err = au_dr_hino(sb, /*bindex*/-1, br, path); + +out: + AuTraceErr(err); + return err; +} + +int au_dr_br_fin(struct super_block *sb, struct au_branch *br) +{ + int err; + + err = 0; + if (au_br_writable(br->br_perm)) + err = au_dr_hino(sb, /*bindex*/-1, br, /*path*/NULL); + if (!err) + au_dr_hino_free(&br->br_dirren); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_brid_str(struct au_dr_brid *brid, struct inode *h_inode, + char *buf, size_t sz) +{ + int err; + unsigned int major, minor; + char *p; + + p = buf; + err = snprintf(p, sz, "%d_", brid->type); + AuDebugOn(err > sz); + p += err; + sz -= err; + switch (brid->type) { + case AuBrid_Unset: + return -EINVAL; + case AuBrid_UUID: + err = snprintf(p, sz, "%pU", brid->uuid.b); + break; + case AuBrid_FSID: + err = snprintf(p, sz, "%08x-%08x", + brid->fsid.val[0], brid->fsid.val[1]); + break; + case AuBrid_DEV: + major = MAJOR(brid->dev); + minor = MINOR(brid->dev); + if (major <= 0xff && minor <= 0xff) + err = snprintf(p, sz, "%02x%02x", major, minor); + else + err = snprintf(p, sz, "%03x:%05x", major, minor); + break; + } + AuDebugOn(err > sz); + p += err; + sz -= err; + err = snprintf(p, sz, "_%llu", (unsigned long long)h_inode->i_ino); + AuDebugOn(err > sz); + p += err; + sz -= err; + + return p - buf; +} + +static int au_drinfo_name(struct au_branch *br, char *name, int len) +{ + int rlen; + struct dentry *br_dentry; + struct inode *br_inode; + + br_dentry = au_br_dentry(br); + br_inode = d_inode(br_dentry); + rlen = au_brid_str(&br->br_dirren.dr_brid, br_inode, name, len); + AuDebugOn(rlen >= AUFS_DIRREN_ENV_VAL_SZ); + AuDebugOn(rlen > len); + + return rlen; +} + +/* ---------------------------------------------------------------------- */ + +/* + * from the given @h_dentry, construct drinfo at @*fdata. + * when the size of @*fdata is not enough, reallocate and return new @fdata and + * @allocated. + */ +static int au_drinfo_construct(struct au_drinfo_fdata **fdata, + struct dentry *h_dentry, + unsigned char *allocated) +{ + int err, v; + struct au_drinfo_fdata *f, *p; + struct au_drinfo *drinfo; + struct inode *h_inode; + struct qstr *qname; + + err = 0; + f = *fdata; + h_inode = d_inode(h_dentry); + qname = &h_dentry->d_name; + drinfo = &f->drinfo; + drinfo->ino = (__force uint64_t)cpu_to_be64(h_inode->i_ino); + drinfo->oldnamelen = qname->len; + if (*allocated < sizeof(*f) + qname->len) { + v = roundup_pow_of_two(*allocated + qname->len); + p = au_krealloc(f, v, GFP_NOFS, /*may_shrink*/0); + if (unlikely(!p)) { + err = -ENOMEM; + AuTraceErr(err); + goto out; + } + f = p; + *fdata = f; + *allocated = v; + drinfo = &f->drinfo; + } + memcpy(drinfo->oldname, qname->name, qname->len); + AuDbg("i%llu, %.*s\n", + be64_to_cpu((__force __be64)drinfo->ino), drinfo->oldnamelen, + drinfo->oldname); + +out: + AuTraceErr(err); + return err; +} + +/* callers have to free the return value */ +static struct au_drinfo *au_drinfo_read_k(struct file *file, ino_t h_ino) +{ + struct au_drinfo *ret, *drinfo; + struct au_drinfo_fdata fdata; + int len; + loff_t pos; + ssize_t ssz; + + ret = ERR_PTR(-EIO); + pos = 0; + ssz = vfsub_read_k(file, &fdata, sizeof(fdata), &pos); + if (unlikely(ssz != sizeof(fdata))) { + AuIOErr("ssz %zd, %u, %pD2\n", + ssz, (unsigned int)sizeof(fdata), file); + goto out; + } + + fdata.magic = ntohl((__force __be32)fdata.magic); + switch (fdata.magic) { + case AUFS_DRINFO_MAGIC_V1: + break; + default: + AuIOErr("magic-num 0x%x, 0x%x, %pD2\n", + fdata.magic, AUFS_DRINFO_MAGIC_V1, file); + goto out; + } + + drinfo = &fdata.drinfo; + len = drinfo->oldnamelen; + if (!len) { + AuIOErr("broken drinfo %pD2\n", file); + goto out; + } + + ret = NULL; + drinfo->ino = be64_to_cpu((__force __be64)drinfo->ino); + if (unlikely(h_ino && drinfo->ino != h_ino)) { + AuDbg("ignored i%llu, i%llu, %pD2\n", + (unsigned long long)drinfo->ino, + (unsigned long long)h_ino, file); + goto out; /* success */ + } + + ret = kmalloc(sizeof(*ret) + len, GFP_NOFS); + if (unlikely(!ret)) { + ret = ERR_PTR(-ENOMEM); + AuTraceErrPtr(ret); + goto out; + } + + *ret = *drinfo; + ssz = vfsub_read_k(file, (void *)ret->oldname, len, &pos); + if (unlikely(ssz != len)) { + au_kfree_rcu(ret); + ret = ERR_PTR(-EIO); + AuIOErr("ssz %zd, %u, %pD2\n", ssz, len, file); + goto out; + } + + AuDbg("oldname %.*s\n", ret->oldnamelen, ret->oldname); + +out: + return ret; +} + +/* ---------------------------------------------------------------------- */ + +/* in order to be revertible */ +struct au_drinfo_rev_elm { + int created; + struct dentry *info_dentry; + struct au_drinfo *info_last; +}; + +struct au_drinfo_rev { + unsigned char already; + aufs_bindex_t nelm; + struct au_drinfo_rev_elm elm[]; +}; + +/* todo: isn't it too large? */ +struct au_drinfo_store { + struct path h_ppath; + struct dentry *h_dentry; + struct au_drinfo_fdata *fdata; + char *infoname; /* inside of whname, just after PFX */ + char whname[sizeof(AUFS_WH_DR_INFO_PFX) + AUFS_DIRREN_ENV_VAL_SZ]; + aufs_bindex_t btgt, btail; + unsigned char no_sio, + allocated, /* current size of *fdata */ + infonamelen, /* room size for p */ + whnamelen, /* length of the generated name */ + renameback; /* renamed back */ +}; + +/* on rename(2) error, the caller should revert it using @elm */ +static int au_drinfo_do_store(struct au_drinfo_store *w, + struct au_drinfo_rev_elm *elm) +{ + int err, len; + ssize_t ssz; + loff_t pos; + struct path infopath = { + .mnt = w->h_ppath.mnt + }; + struct inode *h_dir, *h_inode, *delegated; + struct file *infofile; + struct qstr *qname; + + AuDebugOn(elm + && memcmp(elm, page_address(ZERO_PAGE(0)), sizeof(*elm))); + + infopath.dentry = vfsub_lookup_one_len(w->whname, w->h_ppath.dentry, + w->whnamelen); + AuTraceErrPtr(infopath.dentry); + if (IS_ERR(infopath.dentry)) { + err = PTR_ERR(infopath.dentry); + goto out; + } + + err = 0; + h_dir = d_inode(w->h_ppath.dentry); + if (elm && d_is_negative(infopath.dentry)) { + err = vfsub_create(h_dir, &infopath, 0600, /*want_excl*/true); + AuTraceErr(err); + if (unlikely(err)) + goto out_dput; + elm->created = 1; + elm->info_dentry = dget(infopath.dentry); + } + + infofile = vfsub_dentry_open(&infopath, O_RDWR); + AuTraceErrPtr(infofile); + if (IS_ERR(infofile)) { + err = PTR_ERR(infofile); + goto out_dput; + } + + h_inode = d_inode(infopath.dentry); + if (elm && i_size_read(h_inode)) { + h_inode = d_inode(w->h_dentry); + elm->info_last = au_drinfo_read_k(infofile, h_inode->i_ino); + AuTraceErrPtr(elm->info_last); + if (IS_ERR(elm->info_last)) { + err = PTR_ERR(elm->info_last); + elm->info_last = NULL; + AuDebugOn(elm->info_dentry); + goto out_fput; + } + } + + if (elm && w->renameback) { + delegated = NULL; + err = vfsub_unlink(h_dir, &infopath, &delegated, /*force*/0); + AuTraceErr(err); + if (unlikely(err == -EWOULDBLOCK)) + iput(delegated); + goto out_fput; + } + + pos = 0; + qname = &w->h_dentry->d_name; + len = sizeof(*w->fdata) + qname->len; + if (!elm) + len = sizeof(*w->fdata) + w->fdata->drinfo.oldnamelen; + ssz = vfsub_write_k(infofile, w->fdata, len, &pos); + if (ssz == len) { + AuDbg("hi%llu, %.*s\n", w->fdata->drinfo.ino, + w->fdata->drinfo.oldnamelen, w->fdata->drinfo.oldname); + goto out_fput; /* success */ + } else { + err = -EIO; + if (ssz < 0) + err = ssz; + /* the caller should revert it using @elm */ + } + +out_fput: + fput(infofile); +out_dput: + dput(infopath.dentry); +out: + AuTraceErr(err); + return err; +} + +struct au_call_drinfo_do_store_args { + int *errp; + struct au_drinfo_store *w; + struct au_drinfo_rev_elm *elm; +}; + +static void au_call_drinfo_do_store(void *args) +{ + struct au_call_drinfo_do_store_args *a = args; + + *a->errp = au_drinfo_do_store(a->w, a->elm); +} + +static int au_drinfo_store_sio(struct au_drinfo_store *w, + struct au_drinfo_rev_elm *elm) +{ + int err, wkq_err; + + if (w->no_sio) + err = au_drinfo_do_store(w, elm); + else { + struct au_call_drinfo_do_store_args a = { + .errp = &err, + .w = w, + .elm = elm + }; + wkq_err = au_wkq_wait(au_call_drinfo_do_store, &a); + if (unlikely(wkq_err)) + err = wkq_err; + } + AuTraceErr(err); + + return err; +} + +static int au_drinfo_store_work_init(struct au_drinfo_store *w, + aufs_bindex_t btgt) +{ + int err; + + memset(w, 0, sizeof(*w)); + w->allocated = roundup_pow_of_two(sizeof(*w->fdata) + 40); + strcpy(w->whname, AUFS_WH_DR_INFO_PFX); + w->infoname = w->whname + sizeof(AUFS_WH_DR_INFO_PFX) - 1; + w->infonamelen = sizeof(w->whname) - sizeof(AUFS_WH_DR_INFO_PFX); + w->btgt = btgt; + w->no_sio = !!uid_eq(current_fsuid(), GLOBAL_ROOT_UID); + + err = -ENOMEM; + w->fdata = kcalloc(1, w->allocated, GFP_NOFS); + if (unlikely(!w->fdata)) { + AuTraceErr(err); + goto out; + } + w->fdata->magic = (__force uint32_t)htonl(AUFS_DRINFO_MAGIC_V1); + err = 0; + +out: + return err; +} + +static void au_drinfo_store_work_fin(struct au_drinfo_store *w) +{ + au_kfree_rcu(w->fdata); +} + +static void au_drinfo_store_rev(struct au_drinfo_rev *rev, + struct au_drinfo_store *w) +{ + struct au_drinfo_rev_elm *elm; + struct inode *h_dir, *delegated; + int err, nelm; + struct path infopath = { + .mnt = w->h_ppath.mnt + }; + + h_dir = d_inode(w->h_ppath.dentry); + IMustLock(h_dir); + + err = 0; + elm = rev->elm; + for (nelm = rev->nelm; nelm > 0; nelm--, elm++) { + AuDebugOn(elm->created && elm->info_last); + if (elm->created) { + AuDbg("here\n"); + delegated = NULL; + infopath.dentry = elm->info_dentry; + err = vfsub_unlink(h_dir, &infopath, &delegated, + !w->no_sio); + AuTraceErr(err); + if (unlikely(err == -EWOULDBLOCK)) + iput(delegated); + dput(elm->info_dentry); + } else if (elm->info_last) { + AuDbg("here\n"); + w->fdata->drinfo = *elm->info_last; + memcpy(w->fdata->drinfo.oldname, + elm->info_last->oldname, + elm->info_last->oldnamelen); + err = au_drinfo_store_sio(w, /*elm*/NULL); + au_kfree_rcu(elm->info_last); + } + if (unlikely(err)) + AuIOErr("%d, %s\n", err, w->whname); + /* go on even if err */ + } +} + +/* caller has to call au_dr_rename_fin() later */ +static int au_drinfo_store(struct dentry *dentry, aufs_bindex_t btgt, + struct qstr *dst_name, void *_rev) +{ + int err, sz, nelm; + aufs_bindex_t bindex, btail; + struct au_drinfo_store work; + struct au_drinfo_rev *rev, **p; + struct au_drinfo_rev_elm *elm; + struct super_block *sb; + struct au_branch *br; + struct au_hinode *hdir; + + err = au_drinfo_store_work_init(&work, btgt); + AuTraceErr(err); + if (unlikely(err)) + goto out; + + err = -ENOMEM; + btail = au_dbtaildir(dentry); + nelm = btail - btgt; + sz = sizeof(*rev) + sizeof(*elm) * nelm; + rev = kcalloc(1, sz, GFP_NOFS); + if (unlikely(!rev)) { + AuTraceErr(err); + goto out_args; + } + rev->nelm = nelm; + elm = rev->elm; + p = _rev; + *p = rev; + + err = 0; + sb = dentry->d_sb; + work.h_ppath.dentry = au_h_dptr(dentry, btgt); + work.h_ppath.mnt = au_sbr_mnt(sb, btgt); + hdir = au_hi(d_inode(dentry), btgt); + au_hn_inode_lock_nested(hdir, AuLsc_I_CHILD); + for (bindex = btgt + 1; bindex <= btail; bindex++, elm++) { + work.h_dentry = au_h_dptr(dentry, bindex); + if (!work.h_dentry) + continue; + + err = au_drinfo_construct(&work.fdata, work.h_dentry, + &work.allocated); + AuTraceErr(err); + if (unlikely(err)) + break; + + work.renameback = au_qstreq(&work.h_dentry->d_name, dst_name); + br = au_sbr(sb, bindex); + work.whnamelen = sizeof(AUFS_WH_DR_INFO_PFX) - 1; + work.whnamelen += au_drinfo_name(br, work.infoname, + work.infonamelen); + AuDbg("whname %.*s, i%llu, %.*s\n", + work.whnamelen, work.whname, + be64_to_cpu((__force __be64)work.fdata->drinfo.ino), + work.fdata->drinfo.oldnamelen, + work.fdata->drinfo.oldname); + + err = au_drinfo_store_sio(&work, elm); + AuTraceErr(err); + if (unlikely(err)) + break; + } + if (unlikely(err)) { + /* revert all drinfo */ + au_drinfo_store_rev(rev, &work); + au_kfree_try_rcu(rev); + *p = NULL; + } + au_hn_inode_unlock(hdir); + +out_args: + au_drinfo_store_work_fin(&work); +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_dr_rename(struct dentry *src, aufs_bindex_t bindex, + struct qstr *dst_name, void *_rev) +{ + int err, already; + ino_t ino; + struct super_block *sb; + struct au_branch *br; + struct au_dr_br *dr; + struct dentry *h_dentry; + struct inode *h_inode; + struct au_dr_hino *ent; + struct au_drinfo_rev *rev, **p; + + AuDbg("bindex %d\n", bindex); + + err = -ENOMEM; + ent = kmalloc(sizeof(*ent), GFP_NOFS); + if (unlikely(!ent)) + goto out; + + sb = src->d_sb; + br = au_sbr(sb, bindex); + dr = &br->br_dirren; + h_dentry = au_h_dptr(src, bindex); + h_inode = d_inode(h_dentry); + ino = h_inode->i_ino; + ent->dr_h_ino = ino; + already = au_dr_hino_test_add(dr, ino, ent); + AuDbg("b%d, hi%llu, already %d\n", + bindex, (unsigned long long)ino, already); + + err = au_drinfo_store(src, bindex, dst_name, _rev); + AuTraceErr(err); + if (!err) { + p = _rev; + rev = *p; + rev->already = already; + goto out; /* success */ + } + + /* revert */ + if (!already) + au_dr_hino_del(dr, ent); + au_kfree_rcu(ent); + +out: + AuTraceErr(err); + return err; +} + +void au_dr_rename_fin(struct dentry *src, aufs_bindex_t btgt, void *_rev) +{ + struct au_drinfo_rev *rev; + struct au_drinfo_rev_elm *elm; + int nelm; + + rev = _rev; + elm = rev->elm; + for (nelm = rev->nelm; nelm > 0; nelm--, elm++) { + dput(elm->info_dentry); + au_kfree_rcu(elm->info_last); + } + au_kfree_try_rcu(rev); +} + +void au_dr_rename_rev(struct dentry *src, aufs_bindex_t btgt, void *_rev) +{ + int err; + struct au_drinfo_store work; + struct au_drinfo_rev *rev = _rev; + struct super_block *sb; + struct au_branch *br; + struct inode *h_inode; + struct au_dr_br *dr; + struct au_dr_hino *ent; + + err = au_drinfo_store_work_init(&work, btgt); + if (unlikely(err)) + goto out; + + sb = src->d_sb; + br = au_sbr(sb, btgt); + work.h_ppath.dentry = au_h_dptr(src, btgt); + work.h_ppath.mnt = au_br_mnt(br); + au_drinfo_store_rev(rev, &work); + au_drinfo_store_work_fin(&work); + if (rev->already) + goto out; + + dr = &br->br_dirren; + h_inode = d_inode(work.h_ppath.dentry); + ent = au_dr_hino_find(dr, h_inode->i_ino); + BUG_ON(!ent); + au_dr_hino_del(dr, ent); + au_kfree_rcu(ent); + +out: + au_kfree_try_rcu(rev); + if (unlikely(err)) + pr_err("failed to remove dirren info\n"); +} + +/* ---------------------------------------------------------------------- */ + +static struct au_drinfo *au_drinfo_do_load(struct path *h_ppath, + char *whname, int whnamelen, + struct dentry **info_dentry) +{ + struct au_drinfo *drinfo; + struct file *f; + struct inode *h_dir; + struct path infopath; + int unlocked; + + AuDbg("%pd/%.*s\n", h_ppath->dentry, whnamelen, whname); + + *info_dentry = NULL; + drinfo = NULL; + unlocked = 0; + h_dir = d_inode(h_ppath->dentry); + inode_lock_shared_nested(h_dir, AuLsc_I_PARENT); + infopath.dentry = vfsub_lookup_one_len(whname, h_ppath->dentry, + whnamelen); + if (IS_ERR(infopath.dentry)) { + drinfo = (void *)infopath.dentry; + goto out; + } + + if (d_is_negative(infopath.dentry)) + goto out_dput; /* success */ + + infopath.mnt = h_ppath->mnt; + f = vfsub_dentry_open(&infopath, O_RDONLY); + inode_unlock_shared(h_dir); + unlocked = 1; + if (IS_ERR(f)) { + drinfo = (void *)f; + goto out_dput; + } + + drinfo = au_drinfo_read_k(f, /*h_ino*/0); + if (IS_ERR_OR_NULL(drinfo)) + goto out_fput; + + AuDbg("oldname %.*s\n", drinfo->oldnamelen, drinfo->oldname); + *info_dentry = dget(infopath.dentry); /* keep it alive */ + +out_fput: + fput(f); +out_dput: + dput(infopath.dentry); +out: + if (!unlocked) + inode_unlock_shared(h_dir); + AuTraceErrPtr(drinfo); + return drinfo; +} + +struct au_drinfo_do_load_args { + struct au_drinfo **drinfop; + struct path *h_ppath; + char *whname; + int whnamelen; + struct dentry **info_dentry; +}; + +static void au_call_drinfo_do_load(void *args) +{ + struct au_drinfo_do_load_args *a = args; + + *a->drinfop = au_drinfo_do_load(a->h_ppath, a->whname, a->whnamelen, + a->info_dentry); +} + +struct au_drinfo_load { + struct path h_ppath; + struct qstr *qname; + unsigned char no_sio; + + aufs_bindex_t ninfo; + struct au_drinfo **drinfo; +}; + +static int au_drinfo_load(struct au_drinfo_load *w, aufs_bindex_t bindex, + struct au_branch *br) +{ + int err, wkq_err, whnamelen, e; + char whname[sizeof(AUFS_WH_DR_INFO_PFX) + AUFS_DIRREN_ENV_VAL_SZ] + = AUFS_WH_DR_INFO_PFX; + struct au_drinfo *drinfo; + struct qstr oldname; + struct inode *h_dir, *delegated; + struct dentry *info_dentry; + struct path infopath; + + whnamelen = sizeof(AUFS_WH_DR_INFO_PFX) - 1; + whnamelen += au_drinfo_name(br, whname + whnamelen, + sizeof(whname) - whnamelen); + if (w->no_sio) + drinfo = au_drinfo_do_load(&w->h_ppath, whname, whnamelen, + &info_dentry); + else { + struct au_drinfo_do_load_args args = { + .drinfop = &drinfo, + .h_ppath = &w->h_ppath, + .whname = whname, + .whnamelen = whnamelen, + .info_dentry = &info_dentry + }; + wkq_err = au_wkq_wait(au_call_drinfo_do_load, &args); + if (unlikely(wkq_err)) + drinfo = ERR_PTR(wkq_err); + } + err = PTR_ERR(drinfo); + if (IS_ERR_OR_NULL(drinfo)) + goto out; + + err = 0; + oldname.len = drinfo->oldnamelen; + oldname.name = drinfo->oldname; + if (au_qstreq(w->qname, &oldname)) { + /* the name is renamed back */ + au_kfree_rcu(drinfo); + drinfo = NULL; + + infopath.dentry = info_dentry; + infopath.mnt = w->h_ppath.mnt; + h_dir = d_inode(w->h_ppath.dentry); + delegated = NULL; + inode_lock_nested(h_dir, AuLsc_I_PARENT); + e = vfsub_unlink(h_dir, &infopath, &delegated, !w->no_sio); + inode_unlock(h_dir); + if (unlikely(e)) + AuIOErr("ignored %d, %pd2\n", e, &infopath.dentry); + if (unlikely(e == -EWOULDBLOCK)) + iput(delegated); + } + au_kfree_rcu(w->drinfo[bindex]); + w->drinfo[bindex] = drinfo; + dput(info_dentry); + +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static void au_dr_lkup_free(struct au_drinfo **drinfo, int n) +{ + struct au_drinfo **p = drinfo; + + while (n-- > 0) + au_kfree_rcu(*drinfo++); + au_kfree_try_rcu(p); +} + +int au_dr_lkup(struct au_do_lookup_args *lkup, struct dentry *dentry, + aufs_bindex_t btgt) +{ + int err, ninfo; + struct au_drinfo_load w; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + struct inode *h_dir; + struct au_dr_hino *ent; + struct super_block *sb; + + AuDbg("%.*s, name %.*s, whname %.*s, b%d\n", + AuLNPair(&dentry->d_name), AuLNPair(&lkup->dirren.dr_name), + AuLNPair(&lkup->whname), btgt); + + sb = dentry->d_sb; + bbot = au_sbbot(sb); + w.ninfo = bbot + 1; + if (!lkup->dirren.drinfo) { + lkup->dirren.drinfo = kcalloc(w.ninfo, + sizeof(*lkup->dirren.drinfo), + GFP_NOFS); + if (unlikely(!lkup->dirren.drinfo)) { + err = -ENOMEM; + goto out; + } + lkup->dirren.ninfo = w.ninfo; + } + w.drinfo = lkup->dirren.drinfo; + w.no_sio = !!uid_eq(current_fsuid(), GLOBAL_ROOT_UID); + w.h_ppath.dentry = au_h_dptr(dentry, btgt); + AuDebugOn(!w.h_ppath.dentry); + w.h_ppath.mnt = au_sbr_mnt(sb, btgt); + w.qname = &dentry->d_name; + + ninfo = 0; + for (bindex = btgt + 1; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + err = au_drinfo_load(&w, bindex, br); + if (unlikely(err)) + goto out_free; + if (w.drinfo[bindex]) + ninfo++; + } + if (!ninfo) { + br = au_sbr(sb, btgt); + h_dir = d_inode(w.h_ppath.dentry); + ent = au_dr_hino_find(&br->br_dirren, h_dir->i_ino); + AuDebugOn(!ent); + au_dr_hino_del(&br->br_dirren, ent); + au_kfree_rcu(ent); + } + goto out; /* success */ + +out_free: + au_dr_lkup_free(lkup->dirren.drinfo, lkup->dirren.ninfo); + lkup->dirren.ninfo = 0; + lkup->dirren.drinfo = NULL; +out: + AuTraceErr(err); + return err; +} + +void au_dr_lkup_fin(struct au_do_lookup_args *lkup) +{ + au_dr_lkup_free(lkup->dirren.drinfo, lkup->dirren.ninfo); +} + +int au_dr_lkup_name(struct au_do_lookup_args *lkup, aufs_bindex_t btgt) +{ + int err; + struct au_drinfo *drinfo; + + err = 0; + if (!lkup->dirren.drinfo) + goto out; + AuDebugOn(lkup->dirren.ninfo <= btgt); + drinfo = lkup->dirren.drinfo[btgt]; + if (!drinfo) + goto out; + + au_kfree_try_rcu(lkup->whname.name); + lkup->whname.name = NULL; + lkup->dirren.dr_name.len = drinfo->oldnamelen; + lkup->dirren.dr_name.name = drinfo->oldname; + lkup->name = &lkup->dirren.dr_name; + err = au_wh_name_alloc(&lkup->whname, lkup->name); + if (!err) + AuDbg("name %.*s, whname %.*s, b%d\n", + AuLNPair(lkup->name), AuLNPair(&lkup->whname), + btgt); + +out: + AuTraceErr(err); + return err; +} + +int au_dr_lkup_h_ino(struct au_do_lookup_args *lkup, aufs_bindex_t bindex, + ino_t h_ino) +{ + int match; + struct au_drinfo *drinfo; + + match = 1; + if (!lkup->dirren.drinfo) + goto out; + AuDebugOn(lkup->dirren.ninfo <= bindex); + drinfo = lkup->dirren.drinfo[bindex]; + if (!drinfo) + goto out; + + match = (drinfo->ino == h_ino); + AuDbg("match %d\n", match); + +out: + return match; +} + +/* ---------------------------------------------------------------------- */ + +int au_dr_opt_set(struct super_block *sb) +{ + int err; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + err = 0; + bbot = au_sbbot(sb); + for (bindex = 0; !err && bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + err = au_dr_hino(sb, bindex, /*br*/NULL, &br->br_path); + } + + return err; +} + +int au_dr_opt_flush(struct super_block *sb) +{ + int err; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + err = 0; + bbot = au_sbbot(sb); + for (bindex = 0; !err && bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_writable(br->br_perm)) + err = au_dr_hino(sb, bindex, /*br*/NULL, /*path*/NULL); + } + + return err; +} + +int au_dr_opt_clr(struct super_block *sb, int no_flush) +{ + int err; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + err = 0; + if (!no_flush) { + err = au_dr_opt_flush(sb); + if (unlikely(err)) + goto out; + } + + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + au_dr_hino_free(&br->br_dirren); + } + +out: + return err; +} diff --git a/fs/aufs/dirren.h b/fs/aufs/dirren.h new file mode 100644 index 0000000000000..1fbc8fb20defb --- /dev/null +++ b/fs/aufs/dirren.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2017-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * renamed dir info + */ + +#ifndef __AUFS_DIRREN_H__ +#define __AUFS_DIRREN_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include "hbl.h" + +#define AuDirren_NHASH 100 + +#ifdef CONFIG_AUFS_DIRREN +enum au_brid_type { + AuBrid_Unset, + AuBrid_UUID, + AuBrid_FSID, + AuBrid_DEV +}; + +struct au_dr_brid { + enum au_brid_type type; + union { + uuid_t uuid; /* unimplemented yet */ + fsid_t fsid; + dev_t dev; + }; +}; + +/* 20 is the max digits length of ulong 64 */ +/* brid-type "_" uuid "_" inum */ +#define AUFS_DIRREN_FNAME_SZ (1 + 1 + UUID_STRING_LEN + 20) +#define AUFS_DIRREN_ENV_VAL_SZ (AUFS_DIRREN_FNAME_SZ + 1 + 20) + +struct au_dr_hino { + struct hlist_bl_node dr_hnode; + ino_t dr_h_ino; +}; + +struct au_dr_br { + struct hlist_bl_head dr_h_ino[AuDirren_NHASH]; + struct au_dr_brid dr_brid; +}; + +struct au_dr_lookup { + /* dr_name is pointed by struct au_do_lookup_args.name */ + struct qstr dr_name; /* subset of dr_info */ + aufs_bindex_t ninfo; + struct au_drinfo **drinfo; +}; +#else +struct au_dr_hino; +/* empty */ +struct au_dr_br { }; +struct au_dr_lookup { }; +#endif + +/* ---------------------------------------------------------------------- */ + +struct au_branch; +struct au_do_lookup_args; +struct au_hinode; +#ifdef CONFIG_AUFS_DIRREN +int au_dr_hino_test_add(struct au_dr_br *dr, ino_t h_ino, + struct au_dr_hino *add_ent); +void au_dr_hino_free(struct au_dr_br *dr); +int au_dr_br_init(struct super_block *sb, struct au_branch *br, + const struct path *path); +int au_dr_br_fin(struct super_block *sb, struct au_branch *br); +int au_dr_rename(struct dentry *src, aufs_bindex_t bindex, + struct qstr *dst_name, void *_rev); +void au_dr_rename_fin(struct dentry *src, aufs_bindex_t btgt, void *rev); +void au_dr_rename_rev(struct dentry *src, aufs_bindex_t bindex, void *rev); +int au_dr_lkup(struct au_do_lookup_args *lkup, struct dentry *dentry, + aufs_bindex_t bindex); +int au_dr_lkup_name(struct au_do_lookup_args *lkup, aufs_bindex_t btgt); +int au_dr_lkup_h_ino(struct au_do_lookup_args *lkup, aufs_bindex_t bindex, + ino_t h_ino); +void au_dr_lkup_fin(struct au_do_lookup_args *lkup); +int au_dr_opt_set(struct super_block *sb); +int au_dr_opt_flush(struct super_block *sb); +int au_dr_opt_clr(struct super_block *sb, int no_flush); +#else +AuStubInt0(au_dr_hino_test_add, struct au_dr_br *dr, ino_t h_ino, + struct au_dr_hino *add_ent); +AuStubVoid(au_dr_hino_free, struct au_dr_br *dr); +AuStubInt0(au_dr_br_init, struct super_block *sb, struct au_branch *br, + const struct path *path); +AuStubInt0(au_dr_br_fin, struct super_block *sb, struct au_branch *br); +AuStubInt0(au_dr_rename, struct dentry *src, aufs_bindex_t bindex, + struct qstr *dst_name, void *_rev); +AuStubVoid(au_dr_rename_fin, struct dentry *src, aufs_bindex_t btgt, void *rev); +AuStubVoid(au_dr_rename_rev, struct dentry *src, aufs_bindex_t bindex, + void *rev); +AuStubInt0(au_dr_lkup, struct au_do_lookup_args *lkup, struct dentry *dentry, + aufs_bindex_t bindex); +AuStubInt0(au_dr_lkup_name, struct au_do_lookup_args *lkup, aufs_bindex_t btgt); +AuStubInt0(au_dr_lkup_h_ino, struct au_do_lookup_args *lkup, + aufs_bindex_t bindex, ino_t h_ino); +AuStubVoid(au_dr_lkup_fin, struct au_do_lookup_args *lkup); +AuStubInt0(au_dr_opt_set, struct super_block *sb); +AuStubInt0(au_dr_opt_flush, struct super_block *sb); +AuStubInt0(au_dr_opt_clr, struct super_block *sb, int no_flush); +#endif + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_DIRREN +static inline int au_dr_ihash(ino_t h_ino) +{ + return h_ino % AuDirren_NHASH; +} +#else +AuStubInt0(au_dr_ihash, ino_t h_ino); +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DIRREN_H__ */ diff --git a/fs/aufs/dynop.c b/fs/aufs/dynop.c new file mode 100644 index 0000000000000..41f38fb9a1442 --- /dev/null +++ b/fs/aufs/dynop.c @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * dynamically customizable operations for regular files + */ + +#include "aufs.h" + +#define DyPrSym(key) AuDbgSym(key->dk_op.dy_hop) + +/* + * How large will these lists be? + * Usually just a few elements, 20-30 at most for each, I guess. + */ +static struct hlist_bl_head dynop[AuDyLast]; + +static struct au_dykey *dy_gfind_get(struct hlist_bl_head *hbl, + const void *h_op) +{ + struct au_dykey *key, *tmp; + struct hlist_bl_node *pos; + + key = NULL; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(tmp, pos, hbl, dk_hnode) + if (tmp->dk_op.dy_hop == h_op) { + if (kref_get_unless_zero(&tmp->dk_kref)) + key = tmp; + break; + } + hlist_bl_unlock(hbl); + + return key; +} + +static struct au_dykey *dy_bradd(struct au_branch *br, struct au_dykey *key) +{ + struct au_dykey **k, *found; + const void *h_op = key->dk_op.dy_hop; + int i; + + found = NULL; + k = br->br_dykey; + for (i = 0; i < AuBrDynOp; i++) + if (k[i]) { + if (k[i]->dk_op.dy_hop == h_op) { + found = k[i]; + break; + } + } else + break; + if (!found) { + spin_lock(&br->br_dykey_lock); + for (; i < AuBrDynOp; i++) + if (k[i]) { + if (k[i]->dk_op.dy_hop == h_op) { + found = k[i]; + break; + } + } else { + k[i] = key; + break; + } + spin_unlock(&br->br_dykey_lock); + BUG_ON(i == AuBrDynOp); /* expand the array */ + } + + return found; +} + +/* kref_get() if @key is already added */ +static struct au_dykey *dy_gadd(struct hlist_bl_head *hbl, struct au_dykey *key) +{ + struct au_dykey *tmp, *found; + struct hlist_bl_node *pos; + const void *h_op = key->dk_op.dy_hop; + + found = NULL; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(tmp, pos, hbl, dk_hnode) + if (tmp->dk_op.dy_hop == h_op) { + if (kref_get_unless_zero(&tmp->dk_kref)) + found = tmp; + break; + } + if (!found) + hlist_bl_add_head(&key->dk_hnode, hbl); + hlist_bl_unlock(hbl); + + if (!found) + DyPrSym(key); + return found; +} + +static void dy_free_rcu(struct rcu_head *rcu) +{ + struct au_dykey *key; + + key = container_of(rcu, struct au_dykey, dk_rcu); + DyPrSym(key); + kfree(key); +} + +static void dy_free(struct kref *kref) +{ + struct au_dykey *key; + struct hlist_bl_head *hbl; + + key = container_of(kref, struct au_dykey, dk_kref); + hbl = dynop + key->dk_op.dy_type; + au_hbl_del(&key->dk_hnode, hbl); + call_rcu(&key->dk_rcu, dy_free_rcu); +} + +void au_dy_put(struct au_dykey *key) +{ + kref_put(&key->dk_kref, dy_free); +} + +/* ---------------------------------------------------------------------- */ + +#define DyDbgSize(cnt, op) AuDebugOn(cnt != sizeof(op)/sizeof(void *)) + +#ifdef CONFIG_AUFS_DEBUG +#define DyDbgDeclare(cnt) unsigned int cnt = 0 +#define DyDbgInc(cnt) do { cnt++; } while (0) +#else +#define DyDbgDeclare(cnt) do {} while (0) +#define DyDbgInc(cnt) do {} while (0) +#endif + +#define DySet(func, dst, src, h_op, h_sb) do { \ + DyDbgInc(cnt); \ + if (h_op->func) { \ + if (src.func) \ + dst.func = src.func; \ + else \ + AuDbg("%s %s\n", au_sbtype(h_sb), #func); \ + } \ +} while (0) + +#define DySetForce(func, dst, src) do { \ + AuDebugOn(!src.func); \ + DyDbgInc(cnt); \ + dst.func = src.func; \ +} while (0) + +#define DySetAop(func) \ + DySet(func, dyaop->da_op, aufs_aop, h_aop, h_sb) +#define DySetAopForce(func) \ + DySetForce(func, dyaop->da_op, aufs_aop) + +static void dy_aop(struct au_dykey *key, const void *h_op, + struct super_block *h_sb __maybe_unused) +{ + struct au_dyaop *dyaop = (void *)key; + const struct address_space_operations *h_aop = h_op; + DyDbgDeclare(cnt); + + AuDbg("%s\n", au_sbtype(h_sb)); + + DySetAop(writepage); + DySetAopForce(readpage); /* force */ + DySetAop(writepages); + DySetAop(set_page_dirty); + DySetAop(readpages); + DySetAop(write_begin); + DySetAop(write_end); + DySetAop(bmap); + DySetAop(invalidatepage); + DySetAop(releasepage); + DySetAop(freepage); + /* this one will be changed according to an aufs mount option */ + DySetAop(direct_IO); + DySetAop(migratepage); + DySetAop(isolate_page); + DySetAop(putback_page); + DySetAop(launder_page); + DySetAop(is_partially_uptodate); + DySetAop(is_dirty_writeback); + DySetAop(error_remove_page); + DySetAop(swap_activate); + DySetAop(swap_deactivate); + + DyDbgSize(cnt, *h_aop); +} + +/* ---------------------------------------------------------------------- */ + +static void dy_bug(struct kref *kref) +{ + BUG(); +} + +static struct au_dykey *dy_get(struct au_dynop *op, struct au_branch *br) +{ + struct au_dykey *key, *old; + struct hlist_bl_head *hbl; + struct op { + unsigned int sz; + void (*set)(struct au_dykey *key, const void *h_op, + struct super_block *h_sb __maybe_unused); + }; + static const struct op a[] = { + [AuDy_AOP] = { + .sz = sizeof(struct au_dyaop), + .set = dy_aop + } + }; + const struct op *p; + + hbl = dynop + op->dy_type; + key = dy_gfind_get(hbl, op->dy_hop); + if (key) + goto out_add; /* success */ + + p = a + op->dy_type; + key = kzalloc(p->sz, GFP_NOFS); + if (unlikely(!key)) { + key = ERR_PTR(-ENOMEM); + goto out; + } + + key->dk_op.dy_hop = op->dy_hop; + kref_init(&key->dk_kref); + p->set(key, op->dy_hop, au_br_sb(br)); + old = dy_gadd(hbl, key); + if (old) { + au_kfree_rcu(key); + key = old; + } + +out_add: + old = dy_bradd(br, key); + if (old) + /* its ref-count should never be zero here */ + kref_put(&key->dk_kref, dy_bug); +out: + return key; +} + +/* ---------------------------------------------------------------------- */ +/* + * Aufs prohibits O_DIRECT by default even if the branch supports it. + * This behaviour is necessary to return an error from open(O_DIRECT) instead + * of the succeeding I/O. The dio mount option enables O_DIRECT and makes + * open(O_DIRECT) always succeed, but the succeeding I/O may return an error. + * See the aufs manual in detail. + */ +static void dy_adx(struct au_dyaop *dyaop, int do_dx) +{ + if (!do_dx) + dyaop->da_op.direct_IO = NULL; + else + dyaop->da_op.direct_IO = aufs_aop.direct_IO; +} + +static struct au_dyaop *dy_aget(struct au_branch *br, + const struct address_space_operations *h_aop, + int do_dx) +{ + struct au_dyaop *dyaop; + struct au_dynop op; + + op.dy_type = AuDy_AOP; + op.dy_haop = h_aop; + dyaop = (void *)dy_get(&op, br); + if (IS_ERR(dyaop)) + goto out; + dy_adx(dyaop, do_dx); + +out: + return dyaop; +} + +int au_dy_iaop(struct inode *inode, aufs_bindex_t bindex, + struct inode *h_inode) +{ + int err, do_dx; + struct super_block *sb; + struct au_branch *br; + struct au_dyaop *dyaop; + + AuDebugOn(!S_ISREG(h_inode->i_mode)); + IiMustWriteLock(inode); + + sb = inode->i_sb; + br = au_sbr(sb, bindex); + do_dx = !!au_opt_test(au_mntflags(sb), DIO); + dyaop = dy_aget(br, h_inode->i_mapping->a_ops, do_dx); + err = PTR_ERR(dyaop); + if (IS_ERR(dyaop)) + /* unnecessary to call dy_fput() */ + goto out; + + err = 0; + inode->i_mapping->a_ops = &dyaop->da_op; + +out: + return err; +} + +/* + * Is it safe to replace a_ops during the inode/file is in operation? + * Yes, I hope so. + */ +int au_dy_irefresh(struct inode *inode) +{ + int err; + aufs_bindex_t btop; + struct inode *h_inode; + + err = 0; + if (S_ISREG(inode->i_mode)) { + btop = au_ibtop(inode); + h_inode = au_h_iptr(inode, btop); + err = au_dy_iaop(inode, btop, h_inode); + } + return err; +} + +void au_dy_arefresh(int do_dx) +{ + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_dykey *key; + + hbl = dynop + AuDy_AOP; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(key, pos, hbl, dk_hnode) + dy_adx((void *)key, do_dx); + hlist_bl_unlock(hbl); +} + +/* ---------------------------------------------------------------------- */ + +void __init au_dy_init(void) +{ + int i; + + /* make sure that 'struct au_dykey *' can be any type */ + BUILD_BUG_ON(offsetof(struct au_dyaop, da_key)); + + for (i = 0; i < AuDyLast; i++) + INIT_HLIST_BL_HEAD(dynop + i); +} + +void au_dy_fin(void) +{ + int i; + + for (i = 0; i < AuDyLast; i++) + WARN_ON(!hlist_bl_empty(dynop + i)); +} diff --git a/fs/aufs/dynop.h b/fs/aufs/dynop.h new file mode 100644 index 0000000000000..8b35ab5813e7d --- /dev/null +++ b/fs/aufs/dynop.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2010-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * dynamically customizable operations (for regular files only) + */ + +#ifndef __AUFS_DYNOP_H__ +#define __AUFS_DYNOP_H__ + +#ifdef __KERNEL__ + +#include +#include + +enum {AuDy_AOP, AuDyLast}; + +struct au_dynop { + int dy_type; + union { + const void *dy_hop; + const struct address_space_operations *dy_haop; + }; +}; + +struct au_dykey { + union { + struct hlist_bl_node dk_hnode; + struct rcu_head dk_rcu; + }; + struct au_dynop dk_op; + + /* + * during I am in the branch local array, kref is gotten. when the + * branch is removed, kref is put. + */ + struct kref dk_kref; +}; + +/* stop unioning since their sizes are very different from each other */ +struct au_dyaop { + struct au_dykey da_key; + struct address_space_operations da_op; /* not const */ +}; + +/* ---------------------------------------------------------------------- */ + +/* dynop.c */ +struct au_branch; +void au_dy_put(struct au_dykey *key); +int au_dy_iaop(struct inode *inode, aufs_bindex_t bindex, + struct inode *h_inode); +int au_dy_irefresh(struct inode *inode); +void au_dy_arefresh(int do_dio); + +void __init au_dy_init(void); +void au_dy_fin(void); + +#endif /* __KERNEL__ */ +#endif /* __AUFS_DYNOP_H__ */ diff --git a/fs/aufs/export.c b/fs/aufs/export.c new file mode 100644 index 0000000000000..842df6f05517c --- /dev/null +++ b/fs/aufs/export.c @@ -0,0 +1,838 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * export via nfs + */ + +#include +#include +#include +#include +#include +#include +#include "aufs.h" + +union conv { +#ifdef CONFIG_AUFS_INO_T_64 + __u32 a[2]; +#else + __u32 a[1]; +#endif + ino_t ino; +}; + +static ino_t decode_ino(__u32 *a) +{ + union conv u; + + BUILD_BUG_ON(sizeof(u.ino) != sizeof(u.a)); + u.a[0] = a[0]; +#ifdef CONFIG_AUFS_INO_T_64 + u.a[1] = a[1]; +#endif + return u.ino; +} + +static void encode_ino(__u32 *a, ino_t ino) +{ + union conv u; + + u.ino = ino; + a[0] = u.a[0]; +#ifdef CONFIG_AUFS_INO_T_64 + a[1] = u.a[1]; +#endif +} + +/* NFS file handle */ +enum { + Fh_br_id, + Fh_sigen, +#ifdef CONFIG_AUFS_INO_T_64 + /* support 64bit inode number */ + Fh_ino1, + Fh_ino2, + Fh_dir_ino1, + Fh_dir_ino2, +#else + Fh_ino1, + Fh_dir_ino1, +#endif + Fh_igen, + Fh_h_type, + Fh_tail, + + Fh_ino = Fh_ino1, + Fh_dir_ino = Fh_dir_ino1 +}; + +static int au_test_anon(struct dentry *dentry) +{ + /* note: read d_flags without d_lock */ + return !!(dentry->d_flags & DCACHE_DISCONNECTED); +} + +int au_test_nfsd(void) +{ + int ret; + struct task_struct *tsk = current; + char comm[sizeof(tsk->comm)]; + + ret = 0; + if (tsk->flags & PF_KTHREAD) { + get_task_comm(comm, tsk); + ret = !strcmp(comm, "nfsd"); + } + + return ret; +} + +/* ---------------------------------------------------------------------- */ +/* inode generation external table */ + +void au_xigen_inc(struct inode *inode) +{ + loff_t pos; + ssize_t sz; + __u32 igen; + struct super_block *sb; + struct au_sbinfo *sbinfo; + + sb = inode->i_sb; + AuDebugOn(!au_opt_test(au_mntflags(sb), XINO)); + + sbinfo = au_sbi(sb); + pos = inode->i_ino; + pos *= sizeof(igen); + igen = inode->i_generation + 1; + sz = xino_fwrite(sbinfo->si_xwrite, sbinfo->si_xigen, &igen, + sizeof(igen), &pos); + if (sz == sizeof(igen)) + return; /* success */ + + if (unlikely(sz >= 0)) + AuIOErr("xigen error (%zd)\n", sz); +} + +int au_xigen_new(struct inode *inode) +{ + int err; + loff_t pos; + ssize_t sz; + struct super_block *sb; + struct au_sbinfo *sbinfo; + struct file *file; + + err = 0; + /* todo: dirty, at mount time */ + if (inode->i_ino == AUFS_ROOT_INO) + goto out; + sb = inode->i_sb; + SiMustAnyLock(sb); + if (unlikely(!au_opt_test(au_mntflags(sb), XINO))) + goto out; + + err = -EFBIG; + pos = inode->i_ino; + if (unlikely(au_loff_max / sizeof(inode->i_generation) - 1 < pos)) { + AuIOErr1("too large i%lld\n", pos); + goto out; + } + pos *= sizeof(inode->i_generation); + + err = 0; + sbinfo = au_sbi(sb); + file = sbinfo->si_xigen; + BUG_ON(!file); + + if (vfsub_f_size_read(file) + < pos + sizeof(inode->i_generation)) { + inode->i_generation = atomic_inc_return(&sbinfo->si_xigen_next); + sz = xino_fwrite(sbinfo->si_xwrite, file, &inode->i_generation, + sizeof(inode->i_generation), &pos); + } else + sz = xino_fread(sbinfo->si_xread, file, &inode->i_generation, + sizeof(inode->i_generation), &pos); + if (sz == sizeof(inode->i_generation)) + goto out; /* success */ + + err = sz; + if (unlikely(sz >= 0)) { + err = -EIO; + AuIOErr("xigen error (%zd)\n", sz); + } + +out: + return err; +} + +int au_xigen_set(struct super_block *sb, struct path *path) +{ + int err; + struct au_sbinfo *sbinfo; + struct file *file; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + file = au_xino_create2(sb, path, sbinfo->si_xigen); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + err = 0; + if (sbinfo->si_xigen) + fput(sbinfo->si_xigen); + sbinfo->si_xigen = file; + +out: + AuTraceErr(err); + return err; +} + +void au_xigen_clr(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + if (sbinfo->si_xigen) { + fput(sbinfo->si_xigen); + sbinfo->si_xigen = NULL; + } +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry *decode_by_ino(struct super_block *sb, ino_t ino, + ino_t dir_ino) +{ + struct dentry *dentry, *d; + struct inode *inode; + unsigned int sigen; + + dentry = NULL; + inode = ilookup(sb, ino); + if (!inode) + goto out; + + dentry = ERR_PTR(-ESTALE); + sigen = au_sigen(sb); + if (unlikely(au_is_bad_inode(inode) + || IS_DEADDIR(inode) + || sigen != au_iigen(inode, NULL))) + goto out_iput; + + dentry = NULL; + if (!dir_ino || S_ISDIR(inode->i_mode)) + dentry = d_find_alias(inode); + else { + spin_lock(&inode->i_lock); + hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias) { + spin_lock(&d->d_lock); + if (!au_test_anon(d) + && d_inode(d->d_parent)->i_ino == dir_ino) { + dentry = dget_dlock(d); + spin_unlock(&d->d_lock); + break; + } + spin_unlock(&d->d_lock); + } + spin_unlock(&inode->i_lock); + } + if (unlikely(dentry && au_digen_test(dentry, sigen))) { + /* need to refresh */ + dput(dentry); + dentry = NULL; + } + +out_iput: + iput(inode); +out: + AuTraceErrPtr(dentry); + return dentry; +} + +/* ---------------------------------------------------------------------- */ + +/* todo: dirty? */ +/* if exportfs_decode_fh() passed vfsmount*, we could be happy */ + +struct au_compare_mnt_args { + /* input */ + struct super_block *sb; + + /* output */ + struct vfsmount *mnt; +}; + +static int au_compare_mnt(struct vfsmount *mnt, void *arg) +{ + struct au_compare_mnt_args *a = arg; + + if (mnt->mnt_sb != a->sb) + return 0; + a->mnt = mntget(mnt); + return 1; +} + +static struct vfsmount *au_mnt_get(struct super_block *sb) +{ + int err; + struct path root; + struct au_compare_mnt_args args = { + .sb = sb + }; + + get_fs_root(current->fs, &root); + rcu_read_lock(); + err = iterate_mounts(au_compare_mnt, &args, root.mnt); + rcu_read_unlock(); + path_put(&root); + AuDebugOn(!err); + AuDebugOn(!args.mnt); + return args.mnt; +} + +struct au_nfsd_si_lock { + unsigned int sigen; + aufs_bindex_t bindex, br_id; + unsigned char force_lock; +}; + +static int si_nfsd_read_lock(struct super_block *sb, + struct au_nfsd_si_lock *nsi_lock) +{ + int err; + aufs_bindex_t bindex; + + si_read_lock(sb, AuLock_FLUSH); + + /* branch id may be wrapped around */ + err = 0; + bindex = au_br_index(sb, nsi_lock->br_id); + if (bindex >= 0 && nsi_lock->sigen + AUFS_BRANCH_MAX > au_sigen(sb)) + goto out; /* success */ + + err = -ESTALE; + bindex = -1; + if (!nsi_lock->force_lock) + si_read_unlock(sb); + +out: + nsi_lock->bindex = bindex; + return err; +} + +struct find_name_by_ino { + struct dir_context ctx; + int called, found; + ino_t ino; + char *name; + int namelen; +}; + +static int +find_name_by_ino(struct dir_context *ctx, const char *name, int namelen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct find_name_by_ino *a = container_of(ctx, struct find_name_by_ino, + ctx); + + a->called++; + if (a->ino != ino) + return 0; + + memcpy(a->name, name, namelen); + a->namelen = namelen; + a->found = 1; + return 1; +} + +static struct dentry *au_lkup_by_ino(struct path *path, ino_t ino, + struct au_nfsd_si_lock *nsi_lock) +{ + struct dentry *dentry, *parent; + struct file *file; + struct inode *dir; + struct find_name_by_ino arg = { + .ctx = { + .actor = find_name_by_ino + } + }; + int err; + + parent = path->dentry; + if (nsi_lock) + si_read_unlock(parent->d_sb); + file = vfsub_dentry_open(path, au_dir_roflags); + dentry = (void *)file; + if (IS_ERR(file)) + goto out; + + dentry = ERR_PTR(-ENOMEM); + arg.name = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!arg.name)) + goto out_file; + arg.ino = ino; + arg.found = 0; + do { + arg.called = 0; + /* smp_mb(); */ + err = vfsub_iterate_dir(file, &arg.ctx); + } while (!err && !arg.found && arg.called); + dentry = ERR_PTR(err); + if (unlikely(err)) + goto out_name; + /* instead of ENOENT */ + dentry = ERR_PTR(-ESTALE); + if (!arg.found) + goto out_name; + + /* do not call vfsub_lkup_one() */ + dir = d_inode(parent); + dentry = vfsub_lookup_one_len_unlocked(arg.name, parent, arg.namelen); + AuTraceErrPtr(dentry); + if (IS_ERR(dentry)) + goto out_name; + AuDebugOn(au_test_anon(dentry)); + if (unlikely(d_really_is_negative(dentry))) { + dput(dentry); + dentry = ERR_PTR(-ENOENT); + } + +out_name: + free_page((unsigned long)arg.name); +out_file: + fput(file); +out: + if (unlikely(nsi_lock + && si_nfsd_read_lock(parent->d_sb, nsi_lock) < 0)) + if (!IS_ERR(dentry)) { + dput(dentry); + dentry = ERR_PTR(-ESTALE); + } + AuTraceErrPtr(dentry); + return dentry; +} + +static struct dentry *decode_by_dir_ino(struct super_block *sb, ino_t ino, + ino_t dir_ino, + struct au_nfsd_si_lock *nsi_lock) +{ + struct dentry *dentry; + struct path path; + + if (dir_ino != AUFS_ROOT_INO) { + path.dentry = decode_by_ino(sb, dir_ino, 0); + dentry = path.dentry; + if (!path.dentry || IS_ERR(path.dentry)) + goto out; + AuDebugOn(au_test_anon(path.dentry)); + } else + path.dentry = dget(sb->s_root); + + path.mnt = au_mnt_get(sb); + dentry = au_lkup_by_ino(&path, ino, nsi_lock); + path_put(&path); + +out: + AuTraceErrPtr(dentry); + return dentry; +} + +/* ---------------------------------------------------------------------- */ + +static int h_acceptable(void *expv, struct dentry *dentry) +{ + return 1; +} + +static char *au_build_path(struct dentry *h_parent, struct path *h_rootpath, + char *buf, int len, struct super_block *sb) +{ + char *p; + int n; + struct path path; + + p = d_path(h_rootpath, buf, len); + if (IS_ERR(p)) + goto out; + n = strlen(p); + + path.mnt = h_rootpath->mnt; + path.dentry = h_parent; + p = d_path(&path, buf, len); + if (IS_ERR(p)) + goto out; + if (n != 1) + p += n; + + path.mnt = au_mnt_get(sb); + path.dentry = sb->s_root; + p = d_path(&path, buf, len - strlen(p)); + mntput(path.mnt); + if (IS_ERR(p)) + goto out; + if (n != 1) + p[strlen(p)] = '/'; + +out: + AuTraceErrPtr(p); + return p; +} + +static +struct dentry *decode_by_path(struct super_block *sb, ino_t ino, __u32 *fh, + int fh_len, struct au_nfsd_si_lock *nsi_lock) +{ + struct dentry *dentry, *h_parent, *root; + struct super_block *h_sb; + char *pathname, *p; + struct vfsmount *h_mnt; + struct au_branch *br; + int err; + struct path path; + + br = au_sbr(sb, nsi_lock->bindex); + h_mnt = au_br_mnt(br); + h_sb = h_mnt->mnt_sb; + /* todo: call lower fh_to_dentry()? fh_to_parent()? */ + lockdep_off(); + h_parent = exportfs_decode_fh(h_mnt, (void *)(fh + Fh_tail), + fh_len - Fh_tail, fh[Fh_h_type], + h_acceptable, /*context*/NULL); + lockdep_on(); + dentry = h_parent; + if (unlikely(!h_parent || IS_ERR(h_parent))) { + AuWarn1("%s decode_fh failed, %ld\n", + au_sbtype(h_sb), PTR_ERR(h_parent)); + goto out; + } + dentry = NULL; + if (unlikely(au_test_anon(h_parent))) { + AuWarn1("%s decode_fh returned a disconnected dentry\n", + au_sbtype(h_sb)); + goto out_h_parent; + } + + dentry = ERR_PTR(-ENOMEM); + pathname = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!pathname)) + goto out_h_parent; + + root = sb->s_root; + path.mnt = h_mnt; + di_read_lock_parent(root, !AuLock_IR); + path.dentry = au_h_dptr(root, nsi_lock->bindex); + di_read_unlock(root, !AuLock_IR); + p = au_build_path(h_parent, &path, pathname, PAGE_SIZE, sb); + dentry = (void *)p; + if (IS_ERR(p)) + goto out_pathname; + + si_read_unlock(sb); + err = vfsub_kern_path(p, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + dentry = ERR_PTR(err); + if (unlikely(err)) + goto out_relock; + + dentry = ERR_PTR(-ENOENT); + AuDebugOn(au_test_anon(path.dentry)); + if (unlikely(d_really_is_negative(path.dentry))) + goto out_path; + + if (ino != d_inode(path.dentry)->i_ino) + dentry = au_lkup_by_ino(&path, ino, /*nsi_lock*/NULL); + else + dentry = dget(path.dentry); + +out_path: + path_put(&path); +out_relock: + if (unlikely(si_nfsd_read_lock(sb, nsi_lock) < 0)) + if (!IS_ERR(dentry)) { + dput(dentry); + dentry = ERR_PTR(-ESTALE); + } +out_pathname: + free_page((unsigned long)pathname); +out_h_parent: + dput(h_parent); +out: + AuTraceErrPtr(dentry); + return dentry; +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry * +aufs_fh_to_dentry(struct super_block *sb, struct fid *fid, int fh_len, + int fh_type) +{ + struct dentry *dentry; + __u32 *fh = fid->raw; + struct au_branch *br; + ino_t ino, dir_ino; + struct au_nfsd_si_lock nsi_lock = { + .force_lock = 0 + }; + + dentry = ERR_PTR(-ESTALE); + /* it should never happen, but the file handle is unreliable */ + if (unlikely(fh_len < Fh_tail)) + goto out; + nsi_lock.sigen = fh[Fh_sigen]; + nsi_lock.br_id = fh[Fh_br_id]; + + /* branch id may be wrapped around */ + br = NULL; + if (unlikely(si_nfsd_read_lock(sb, &nsi_lock))) + goto out; + nsi_lock.force_lock = 1; + + /* is this inode still cached? */ + ino = decode_ino(fh + Fh_ino); + /* it should never happen */ + if (unlikely(ino == AUFS_ROOT_INO)) + goto out_unlock; + + dir_ino = decode_ino(fh + Fh_dir_ino); + dentry = decode_by_ino(sb, ino, dir_ino); + if (IS_ERR(dentry)) + goto out_unlock; + if (dentry) + goto accept; + + /* is the parent dir cached? */ + br = au_sbr(sb, nsi_lock.bindex); + au_lcnt_inc(&br->br_nfiles); + dentry = decode_by_dir_ino(sb, ino, dir_ino, &nsi_lock); + if (IS_ERR(dentry)) + goto out_unlock; + if (dentry) + goto accept; + + /* lookup path */ + dentry = decode_by_path(sb, ino, fh, fh_len, &nsi_lock); + if (IS_ERR(dentry)) + goto out_unlock; + if (unlikely(!dentry)) + /* todo?: make it ESTALE */ + goto out_unlock; + +accept: + if (!au_digen_test(dentry, au_sigen(sb)) + && d_inode(dentry)->i_generation == fh[Fh_igen]) + goto out_unlock; /* success */ + + dput(dentry); + dentry = ERR_PTR(-ESTALE); +out_unlock: + if (br) + au_lcnt_dec(&br->br_nfiles); + si_read_unlock(sb); +out: + AuTraceErrPtr(dentry); + return dentry; +} + +#if 0 /* reserved for future use */ +/* support subtreecheck option */ +static struct dentry *aufs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + struct dentry *parent; + __u32 *fh = fid->raw; + ino_t dir_ino; + + dir_ino = decode_ino(fh + Fh_dir_ino); + parent = decode_by_ino(sb, dir_ino, 0); + if (IS_ERR(parent)) + goto out; + if (!parent) + parent = decode_by_path(sb, au_br_index(sb, fh[Fh_br_id]), + dir_ino, fh, fh_len); + +out: + AuTraceErrPtr(parent); + return parent; +} +#endif + +/* ---------------------------------------------------------------------- */ + +static int aufs_encode_fh(struct inode *inode, __u32 *fh, int *max_len, + struct inode *dir) +{ + int err; + aufs_bindex_t bindex; + struct super_block *sb, *h_sb; + struct dentry *dentry, *parent, *h_parent; + struct inode *h_dir; + struct au_branch *br; + + err = -ENOSPC; + if (unlikely(*max_len <= Fh_tail)) { + AuWarn1("NFSv2 client (max_len %d)?\n", *max_len); + goto out; + } + + err = FILEID_ROOT; + if (inode->i_ino == AUFS_ROOT_INO) { + AuDebugOn(inode->i_ino != AUFS_ROOT_INO); + goto out; + } + + h_parent = NULL; + sb = inode->i_sb; + err = si_read_lock(sb, AuLock_FLUSH); + if (unlikely(err)) + goto out; + +#ifdef CONFIG_AUFS_DEBUG + if (unlikely(!au_opt_test(au_mntflags(sb), XINO))) + AuWarn1("NFS-exporting requires xino\n"); +#endif + err = -EIO; + parent = NULL; + ii_read_lock_child(inode); + bindex = au_ibtop(inode); + if (!dir) { + dentry = d_find_any_alias(inode); + if (unlikely(!dentry)) + goto out_unlock; + AuDebugOn(au_test_anon(dentry)); + parent = dget_parent(dentry); + dput(dentry); + if (unlikely(!parent)) + goto out_unlock; + if (d_really_is_positive(parent)) + dir = d_inode(parent); + } + + ii_read_lock_parent(dir); + h_dir = au_h_iptr(dir, bindex); + ii_read_unlock(dir); + if (unlikely(!h_dir)) + goto out_parent; + h_parent = d_find_any_alias(h_dir); + if (unlikely(!h_parent)) + goto out_hparent; + + err = -EPERM; + br = au_sbr(sb, bindex); + h_sb = au_br_sb(br); + if (unlikely(!h_sb->s_export_op)) { + AuErr1("%s branch is not exportable\n", au_sbtype(h_sb)); + goto out_hparent; + } + + fh[Fh_br_id] = br->br_id; + fh[Fh_sigen] = au_sigen(sb); + encode_ino(fh + Fh_ino, inode->i_ino); + encode_ino(fh + Fh_dir_ino, dir->i_ino); + fh[Fh_igen] = inode->i_generation; + + *max_len -= Fh_tail; + fh[Fh_h_type] = exportfs_encode_fh(h_parent, (void *)(fh + Fh_tail), + max_len, + /*connectable or subtreecheck*/0); + err = fh[Fh_h_type]; + *max_len += Fh_tail; + /* todo: macros? */ + if (err != FILEID_INVALID) + err = 99; + else + AuWarn1("%s encode_fh failed\n", au_sbtype(h_sb)); + +out_hparent: + dput(h_parent); +out_parent: + dput(parent); +out_unlock: + ii_read_unlock(inode); + si_read_unlock(sb); +out: + if (unlikely(err < 0)) + err = FILEID_INVALID; + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int aufs_commit_metadata(struct inode *inode) +{ + int err; + aufs_bindex_t bindex; + struct super_block *sb; + struct inode *h_inode; + int (*f)(struct inode *inode); + + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + ii_write_lock_child(inode); + bindex = au_ibtop(inode); + AuDebugOn(bindex < 0); + h_inode = au_h_iptr(inode, bindex); + + f = h_inode->i_sb->s_export_op->commit_metadata; + if (f) + err = f(h_inode); + else { + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 0 /* metadata only */ + }; + + err = sync_inode(h_inode, &wbc); + } + + au_cpup_attr_timesizes(inode); + ii_write_unlock(inode); + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static struct export_operations aufs_export_op = { + .fh_to_dentry = aufs_fh_to_dentry, + /* .fh_to_parent = aufs_fh_to_parent, */ + .encode_fh = aufs_encode_fh, + .commit_metadata = aufs_commit_metadata +}; + +void au_export_init(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + __u32 u; + + BUILD_BUG_ON_MSG(IS_BUILTIN(CONFIG_AUFS_FS) + && IS_MODULE(CONFIG_EXPORTFS), + AUFS_NAME ": unsupported configuration " + "CONFIG_EXPORTFS=m and CONFIG_AUFS_FS=y"); + + sb->s_export_op = &aufs_export_op; + sbinfo = au_sbi(sb); + sbinfo->si_xigen = NULL; + get_random_bytes(&u, sizeof(u)); + BUILD_BUG_ON(sizeof(u) != sizeof(int)); + atomic_set(&sbinfo->si_xigen_next, u); +} diff --git a/fs/aufs/f_op.c b/fs/aufs/f_op.c new file mode 100644 index 0000000000000..9894e2124bd55 --- /dev/null +++ b/fs/aufs/f_op.c @@ -0,0 +1,819 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * file and vm operations + */ + +#include +#include +#include +#include +#include "aufs.h" + +int au_do_open_nondir(struct file *file, int flags, struct file *h_file) +{ + int err; + aufs_bindex_t bindex; + struct dentry *dentry, *h_dentry; + struct au_finfo *finfo; + struct inode *h_inode; + + FiMustWriteLock(file); + + err = 0; + dentry = file->f_path.dentry; + AuDebugOn(IS_ERR_OR_NULL(dentry)); + finfo = au_fi(file); + memset(&finfo->fi_htop, 0, sizeof(finfo->fi_htop)); + atomic_set(&finfo->fi_mmapped, 0); + bindex = au_dbtop(dentry); + if (!h_file) { + h_dentry = au_h_dptr(dentry, bindex); + err = vfsub_test_mntns(file->f_path.mnt, h_dentry->d_sb); + if (unlikely(err)) + goto out; + h_file = au_h_open(dentry, bindex, flags, file, /*force_wr*/0); + if (IS_ERR(h_file)) { + err = PTR_ERR(h_file); + goto out; + } + } else { + h_dentry = h_file->f_path.dentry; + err = vfsub_test_mntns(file->f_path.mnt, h_dentry->d_sb); + if (unlikely(err)) + goto out; + /* br ref is already inc-ed */ + } + + if ((flags & __O_TMPFILE) + && !(flags & O_EXCL)) { + h_inode = file_inode(h_file); + spin_lock(&h_inode->i_lock); + h_inode->i_state |= I_LINKABLE; + spin_unlock(&h_inode->i_lock); + } + au_set_fbtop(file, bindex); + au_set_h_fptr(file, bindex, h_file); + au_update_figen(file); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + +out: + return err; +} + +static int aufs_open_nondir(struct inode *inode __maybe_unused, + struct file *file) +{ + int err; + struct super_block *sb; + struct au_do_open_args args = { + .open = au_do_open_nondir + }; + + AuDbg("%pD, f_flags 0x%x, f_mode 0x%x\n", + file, vfsub_file_flags(file), file->f_mode); + + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH); + err = au_do_open(file, &args); + si_read_unlock(sb); + return err; +} + +int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file) +{ + struct au_finfo *finfo; + aufs_bindex_t bindex; + + finfo = au_fi(file); + au_hbl_del(&finfo->fi_hlist, + &au_sbi(file->f_path.dentry->d_sb)->si_files); + bindex = finfo->fi_btop; + if (bindex >= 0) + au_set_h_fptr(file, bindex, NULL); + + au_finfo_fin(file); + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static int au_do_flush_nondir(struct file *file, fl_owner_t id) +{ + int err; + struct file *h_file; + + err = 0; + h_file = au_hf_top(file); + if (h_file) + err = vfsub_flush(h_file, id); + return err; +} + +static int aufs_flush_nondir(struct file *file, fl_owner_t id) +{ + return au_do_flush(file, id, au_do_flush_nondir); +} + +/* ---------------------------------------------------------------------- */ +/* + * read and write functions acquire [fdi]_rwsem once, but release before + * mmap_sem. This is because to stop a race condition between mmap(2). + * Releasing these aufs-rwsem should be safe, no branch-management (by keeping + * si_rwsem), no harmful copy-up should happen. Actually copy-up may happen in + * read functions after [fdi]_rwsem are released, but it should be harmless. + */ + +/* Callers should call au_read_post() or fput() in the end */ +struct file *au_read_pre(struct file *file, int keep_fi, unsigned int lsc) +{ + struct file *h_file; + int err; + + err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/0, lsc); + if (!err) { + di_read_unlock(file->f_path.dentry, AuLock_IR); + h_file = au_hf_top(file); + get_file(h_file); + if (!keep_fi) + fi_read_unlock(file); + } else + h_file = ERR_PTR(err); + + return h_file; +} + +static void au_read_post(struct inode *inode, struct file *h_file) +{ + /* update without lock, I don't think it a problem */ + fsstack_copy_attr_atime(inode, file_inode(h_file)); + fput(h_file); +} + +struct au_write_pre { + /* input */ + unsigned int lsc; + + /* output */ + blkcnt_t blks; + aufs_bindex_t btop; +}; + +/* + * return with iinfo is write-locked + * callers should call au_write_post() or iinfo_write_unlock() + fput() in the + * end + */ +static struct file *au_write_pre(struct file *file, int do_ready, + struct au_write_pre *wpre) +{ + struct file *h_file; + struct dentry *dentry; + int err; + unsigned int lsc; + struct au_pin pin; + + lsc = 0; + if (wpre) + lsc = wpre->lsc; + err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/1, lsc); + h_file = ERR_PTR(err); + if (unlikely(err)) + goto out; + + dentry = file->f_path.dentry; + if (do_ready) { + err = au_ready_to_write(file, -1, &pin); + if (unlikely(err)) { + h_file = ERR_PTR(err); + di_write_unlock(dentry); + goto out_fi; + } + } + + di_downgrade_lock(dentry, /*flags*/0); + if (wpre) + wpre->btop = au_fbtop(file); + h_file = au_hf_top(file); + get_file(h_file); + if (wpre) + wpre->blks = file_inode(h_file)->i_blocks; + if (do_ready) + au_unpin(&pin); + di_read_unlock(dentry, /*flags*/0); + +out_fi: + fi_write_unlock(file); +out: + return h_file; +} + +static void au_write_post(struct inode *inode, struct file *h_file, + struct au_write_pre *wpre, ssize_t written) +{ + struct inode *h_inode; + + au_cpup_attr_timesizes(inode); + AuDebugOn(au_ibtop(inode) != wpre->btop); + h_inode = file_inode(h_file); + inode->i_mode = h_inode->i_mode; + ii_write_unlock(inode); + /* AuDbg("blks %llu, %llu\n", (u64)blks, (u64)h_inode->i_blocks); */ + if (written > 0) + au_fhsm_wrote(inode->i_sb, wpre->btop, + /*force*/h_inode->i_blocks > wpre->blks); + fput(h_file); +} + +static ssize_t aufs_read(struct file *file, char __user *buf, size_t count, + loff_t *ppos) +{ + ssize_t err; + struct inode *inode; + struct file *h_file; + struct super_block *sb; + + inode = file_inode(file); + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + /* filedata may be obsoleted by concurrent copyup, but no problem */ + err = vfsub_read_u(h_file, buf, count, ppos); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + au_read_post(inode, h_file); + +out: + si_read_unlock(sb); + return err; +} + +/* + * todo: very ugly + * it locks both of i_mutex and si_rwsem for read in safe. + * if the plink maintenance mode continues forever (that is the problem), + * may loop forever. + */ +static void au_mtx_and_read_lock(struct inode *inode) +{ + int err; + struct super_block *sb = inode->i_sb; + + while (1) { + inode_lock(inode); + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (!err) + break; + inode_unlock(inode); + si_read_lock(sb, AuLock_NOPLMW); + si_read_unlock(sb); + } +} + +static ssize_t aufs_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + ssize_t err; + struct au_write_pre wpre; + struct inode *inode; + struct file *h_file; + char __user *buf = (char __user *)ubuf; + + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = vfsub_write_u(h_file, buf, count, ppos); + au_write_post(inode, h_file, &wpre, err); + +out: + si_read_unlock(inode->i_sb); + inode_unlock(inode); + return err; +} + +static ssize_t au_do_iter(struct file *h_file, int rw, struct kiocb *kio, + struct iov_iter *iov_iter) +{ + ssize_t err; + struct file *file; + ssize_t (*iter)(struct kiocb *, struct iov_iter *); + + err = security_file_permission(h_file, rw); + if (unlikely(err)) + goto out; + + err = -ENOSYS; /* the branch doesn't have its ->(read|write)_iter() */ + iter = NULL; + if (rw == MAY_READ) + iter = h_file->f_op->read_iter; + else if (rw == MAY_WRITE) + iter = h_file->f_op->write_iter; + + file = kio->ki_filp; + kio->ki_filp = h_file; + if (iter) { + lockdep_off(); + err = iter(kio, iov_iter); + lockdep_on(); + } else + /* currently there is no such fs */ + WARN_ON_ONCE(1); + kio->ki_filp = file; + +out: + return err; +} + +static ssize_t aufs_read_iter(struct kiocb *kio, struct iov_iter *iov_iter) +{ + ssize_t err; + struct file *file, *h_file; + struct inode *inode; + struct super_block *sb; + + file = kio->ki_filp; + inode = file_inode(file); + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/1, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + if (au_test_loopback_kthread()) { + au_warn_loopback(h_file->f_path.dentry->d_sb); + if (file->f_mapping != h_file->f_mapping) { + file->f_mapping = h_file->f_mapping; + smp_mb(); /* unnecessary? */ + } + } + fi_read_unlock(file); + + err = au_do_iter(h_file, MAY_READ, kio, iov_iter); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + au_read_post(inode, h_file); + +out: + si_read_unlock(sb); + return err; +} + +static ssize_t aufs_write_iter(struct kiocb *kio, struct iov_iter *iov_iter) +{ + ssize_t err; + struct au_write_pre wpre; + struct inode *inode; + struct file *file, *h_file; + + file = kio->ki_filp; + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = au_do_iter(h_file, MAY_WRITE, kio, iov_iter); + au_write_post(inode, h_file, &wpre, err); + +out: + si_read_unlock(inode->i_sb); + inode_unlock(inode); + return err; +} + +static ssize_t aufs_splice_read(struct file *file, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + ssize_t err; + struct file *h_file; + struct inode *inode; + struct super_block *sb; + + inode = file_inode(file); + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = vfsub_splice_to(h_file, ppos, pipe, len, flags); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + au_read_post(inode, h_file); + +out: + si_read_unlock(sb); + return err; +} + +static ssize_t +aufs_splice_write(struct pipe_inode_info *pipe, struct file *file, loff_t *ppos, + size_t len, unsigned int flags) +{ + ssize_t err; + struct au_write_pre wpre; + struct inode *inode; + struct file *h_file; + + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = vfsub_splice_from(pipe, h_file, ppos, len, flags); + au_write_post(inode, h_file, &wpre, err); + +out: + si_read_unlock(inode->i_sb); + inode_unlock(inode); + return err; +} + +static long aufs_fallocate(struct file *file, int mode, loff_t offset, + loff_t len) +{ + long err; + struct au_write_pre wpre; + struct inode *inode; + struct file *h_file; + + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + lockdep_off(); + err = vfs_fallocate(h_file, mode, offset, len); + lockdep_on(); + au_write_post(inode, h_file, &wpre, /*written*/1); + +out: + si_read_unlock(inode->i_sb); + inode_unlock(inode); + return err; +} + +static ssize_t aufs_copy_file_range(struct file *src, loff_t src_pos, + struct file *dst, loff_t dst_pos, + size_t len, unsigned int flags) +{ + ssize_t err; + struct au_write_pre wpre; + enum { SRC, DST }; + struct { + struct inode *inode; + struct file *h_file; + struct super_block *h_sb; + } a[2]; +#define a_src a[SRC] +#define a_dst a[DST] + + err = -EINVAL; + a_src.inode = file_inode(src); + if (unlikely(!S_ISREG(a_src.inode->i_mode))) + goto out; + a_dst.inode = file_inode(dst); + if (unlikely(!S_ISREG(a_dst.inode->i_mode))) + goto out; + + au_mtx_and_read_lock(a_dst.inode); + /* + * in order to match the order in di_write_lock2_{child,parent}(), + * use f_path.dentry for this comparison. + */ + if (src->f_path.dentry < dst->f_path.dentry) { + a_src.h_file = au_read_pre(src, /*keep_fi*/1, AuLsc_FI_1); + err = PTR_ERR(a_src.h_file); + if (IS_ERR(a_src.h_file)) + goto out_si; + + wpre.lsc = AuLsc_FI_2; + a_dst.h_file = au_write_pre(dst, /*do_ready*/1, &wpre); + err = PTR_ERR(a_dst.h_file); + if (IS_ERR(a_dst.h_file)) { + au_read_post(a_src.inode, a_src.h_file); + goto out_si; + } + } else { + wpre.lsc = AuLsc_FI_1; + a_dst.h_file = au_write_pre(dst, /*do_ready*/1, &wpre); + err = PTR_ERR(a_dst.h_file); + if (IS_ERR(a_dst.h_file)) + goto out_si; + + a_src.h_file = au_read_pre(src, /*keep_fi*/1, AuLsc_FI_2); + err = PTR_ERR(a_src.h_file); + if (IS_ERR(a_src.h_file)) { + au_write_post(a_dst.inode, a_dst.h_file, &wpre, + /*written*/0); + goto out_si; + } + } + + err = -EXDEV; + a_src.h_sb = file_inode(a_src.h_file)->i_sb; + a_dst.h_sb = file_inode(a_dst.h_file)->i_sb; + if (unlikely(a_src.h_sb != a_dst.h_sb)) { + AuDbgFile(src); + AuDbgFile(dst); + goto out_file; + } + + err = vfsub_copy_file_range(a_src.h_file, src_pos, a_dst.h_file, + dst_pos, len, flags); + +out_file: + au_write_post(a_dst.inode, a_dst.h_file, &wpre, err); + fi_read_unlock(src); + au_read_post(a_src.inode, a_src.h_file); +out_si: + si_read_unlock(a_dst.inode->i_sb); + inode_unlock(a_dst.inode); +out: + return err; +#undef a_src +#undef a_dst +} + +/* ---------------------------------------------------------------------- */ + +/* + * The locking order around current->mmap_sem. + * - in most and regular cases + * file I/O syscall -- aufs_read() or something + * -- si_rwsem for read -- mmap_sem + * (Note that [fdi]i_rwsem are released before mmap_sem). + * - in mmap case + * mmap(2) -- mmap_sem -- aufs_mmap() -- si_rwsem for read -- [fdi]i_rwsem + * This AB-BA order is definitely bad, but is not a problem since "si_rwsem for + * read" allows multiple processes to acquire it and [fdi]i_rwsem are not held + * in file I/O. Aufs needs to stop lockdep in aufs_mmap() though. + * It means that when aufs acquires si_rwsem for write, the process should never + * acquire mmap_sem. + * + * Actually aufs_iterate() holds [fdi]i_rwsem before mmap_sem, but this is not a + * problem either since any directory is not able to be mmap-ed. + * The similar scenario is applied to aufs_readlink() too. + */ + +#if 0 /* stop calling security_file_mmap() */ +/* cf. linux/include/linux/mman.h: calc_vm_prot_bits() */ +#define AuConv_VM_PROT(f, b) _calc_vm_trans(f, VM_##b, PROT_##b) + +static unsigned long au_arch_prot_conv(unsigned long flags) +{ + /* currently ppc64 only */ +#ifdef CONFIG_PPC64 + /* cf. linux/arch/powerpc/include/asm/mman.h */ + AuDebugOn(arch_calc_vm_prot_bits(-1) != VM_SAO); + return AuConv_VM_PROT(flags, SAO); +#else + AuDebugOn(arch_calc_vm_prot_bits(-1)); + return 0; +#endif +} + +static unsigned long au_prot_conv(unsigned long flags) +{ + return AuConv_VM_PROT(flags, READ) + | AuConv_VM_PROT(flags, WRITE) + | AuConv_VM_PROT(flags, EXEC) + | au_arch_prot_conv(flags); +} + +/* cf. linux/include/linux/mman.h: calc_vm_flag_bits() */ +#define AuConv_VM_MAP(f, b) _calc_vm_trans(f, VM_##b, MAP_##b) + +static unsigned long au_flag_conv(unsigned long flags) +{ + return AuConv_VM_MAP(flags, GROWSDOWN) + | AuConv_VM_MAP(flags, DENYWRITE) + | AuConv_VM_MAP(flags, LOCKED); +} +#endif + +static int aufs_mmap(struct file *file, struct vm_area_struct *vma) +{ + int err; + const unsigned char wlock + = (file->f_mode & FMODE_WRITE) && (vma->vm_flags & VM_SHARED); + struct super_block *sb; + struct file *h_file; + struct inode *inode; + + AuDbgVmRegion(file, vma); + + inode = file_inode(file); + sb = inode->i_sb; + lockdep_off(); + si_read_lock(sb, AuLock_NOPLMW); + + h_file = au_write_pre(file, wlock, /*wpre*/NULL); + lockdep_on(); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + err = 0; + au_set_mmapped(file); + au_vm_file_reset(vma, h_file); + /* + * we cannot call security_mmap_file() here since it may acquire + * mmap_sem or i_mutex. + * + * err = security_mmap_file(h_file, au_prot_conv(vma->vm_flags), + * au_flag_conv(vma->vm_flags)); + */ + if (!err) + err = call_mmap(h_file, vma); + if (!err) { + au_vm_prfile_set(vma, file); + fsstack_copy_attr_atime(inode, file_inode(h_file)); + goto out_fput; /* success */ + } + au_unset_mmapped(file); + au_vm_file_reset(vma, file); + +out_fput: + lockdep_off(); + ii_write_unlock(inode); + lockdep_on(); + fput(h_file); +out: + lockdep_off(); + si_read_unlock(sb); + lockdep_on(); + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int aufs_fsync_nondir(struct file *file, loff_t start, loff_t end, + int datasync) +{ + int err; + struct au_write_pre wpre; + struct inode *inode; + struct file *h_file; + + err = 0; /* -EBADF; */ /* posix? */ + if (unlikely(!(file->f_mode & FMODE_WRITE))) + goto out; + + inode = file_inode(file); + au_mtx_and_read_lock(inode); + + wpre.lsc = 0; + h_file = au_write_pre(file, /*do_ready*/1, &wpre); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out_unlock; + + err = vfsub_fsync(h_file, &h_file->f_path, datasync); + au_write_post(inode, h_file, &wpre, /*written*/0); + +out_unlock: + si_read_unlock(inode->i_sb); + inode_unlock(inode); +out: + return err; +} + +static int aufs_fasync(int fd, struct file *file, int flag) +{ + int err; + struct file *h_file; + struct super_block *sb; + + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + if (h_file->f_op->fasync) + err = h_file->f_op->fasync(fd, h_file, flag); + fput(h_file); /* instead of au_read_post() */ + +out: + si_read_unlock(sb); + return err; +} + +static int aufs_setfl(struct file *file, unsigned long arg) +{ + int err; + struct file *h_file; + struct super_block *sb; + + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out; + + /* stop calling h_file->fasync */ + arg |= vfsub_file_flags(file) & FASYNC; + err = setfl(/*unused fd*/-1, h_file, arg); + fput(h_file); /* instead of au_read_post() */ + +out: + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* no one supports this operation, currently */ +#if 0 /* reserved for future use */ +static ssize_t aufs_sendpage(struct file *file, struct page *page, int offset, + size_t len, loff_t *pos, int more) +{ +} +#endif + +/* ---------------------------------------------------------------------- */ + +const struct file_operations aufs_file_fop = { + .owner = THIS_MODULE, + + .llseek = default_llseek, + + .read = aufs_read, + .write = aufs_write, + .read_iter = aufs_read_iter, + .write_iter = aufs_write_iter, + +#ifdef CONFIG_AUFS_POLL + .poll = aufs_poll, +#endif + .unlocked_ioctl = aufs_ioctl_nondir, +#ifdef CONFIG_COMPAT + .compat_ioctl = aufs_compat_ioctl_nondir, +#endif + .mmap = aufs_mmap, + .open = aufs_open_nondir, + .flush = aufs_flush_nondir, + .release = aufs_release_nondir, + .fsync = aufs_fsync_nondir, + .fasync = aufs_fasync, + /* .sendpage = aufs_sendpage, */ + .setfl = aufs_setfl, + .splice_write = aufs_splice_write, + .splice_read = aufs_splice_read, +#if 0 /* reserved for future use */ + .aio_splice_write = aufs_aio_splice_write, + .aio_splice_read = aufs_aio_splice_read, +#endif + .fallocate = aufs_fallocate, + .copy_file_range = aufs_copy_file_range +}; diff --git a/fs/aufs/fhsm.c b/fs/aufs/fhsm.c new file mode 100644 index 0000000000000..9cef93b429933 --- /dev/null +++ b/fs/aufs/fhsm.c @@ -0,0 +1,427 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* + * File-based Hierarchy Storage Management + */ + +#include +#include +#include +#include +#include "aufs.h" + +static aufs_bindex_t au_fhsm_bottom(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + AuDebugOn(!fhsm); + return fhsm->fhsm_bottom; +} + +void au_fhsm_set_bottom(struct super_block *sb, aufs_bindex_t bindex) +{ + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + AuDebugOn(!fhsm); + fhsm->fhsm_bottom = bindex; +} + +/* ---------------------------------------------------------------------- */ + +static int au_fhsm_test_jiffy(struct au_sbinfo *sbinfo, struct au_branch *br) +{ + struct au_br_fhsm *bf; + + bf = br->br_fhsm; + MtxMustLock(&bf->bf_lock); + + return !bf->bf_readable + || time_after(jiffies, + bf->bf_jiffy + sbinfo->si_fhsm.fhsm_expire); +} + +/* ---------------------------------------------------------------------- */ + +static void au_fhsm_notify(struct super_block *sb, int val) +{ + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + if (au_fhsm_pid(fhsm) + && atomic_read(&fhsm->fhsm_readable) != -1) { + atomic_set(&fhsm->fhsm_readable, val); + if (val) + wake_up(&fhsm->fhsm_wqh); + } +} + +static int au_fhsm_stfs(struct super_block *sb, aufs_bindex_t bindex, + struct aufs_stfs *rstfs, int do_lock, int do_notify) +{ + int err; + struct au_branch *br; + struct au_br_fhsm *bf; + + br = au_sbr(sb, bindex); + AuDebugOn(au_br_rdonly(br)); + bf = br->br_fhsm; + AuDebugOn(!bf); + + if (do_lock) + mutex_lock(&bf->bf_lock); + else + MtxMustLock(&bf->bf_lock); + + /* sb->s_root for NFS is unreliable */ + err = au_br_stfs(br, &bf->bf_stfs); + if (unlikely(err)) { + AuErr1("FHSM failed (%d), b%d, ignored.\n", bindex, err); + goto out; + } + + bf->bf_jiffy = jiffies; + bf->bf_readable = 1; + if (do_notify) + au_fhsm_notify(sb, /*val*/1); + if (rstfs) + *rstfs = bf->bf_stfs; + +out: + if (do_lock) + mutex_unlock(&bf->bf_lock); + au_fhsm_notify(sb, /*val*/1); + + return err; +} + +void au_fhsm_wrote(struct super_block *sb, aufs_bindex_t bindex, int force) +{ + int err; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + struct au_branch *br; + struct au_br_fhsm *bf; + + AuDbg("b%d, force %d\n", bindex, force); + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + if (!au_ftest_si(sbinfo, FHSM) + || fhsm->fhsm_bottom == bindex) + return; + + br = au_sbr(sb, bindex); + bf = br->br_fhsm; + AuDebugOn(!bf); + mutex_lock(&bf->bf_lock); + if (force + || au_fhsm_pid(fhsm) + || au_fhsm_test_jiffy(sbinfo, br)) + err = au_fhsm_stfs(sb, bindex, /*rstfs*/NULL, /*do_lock*/0, + /*do_notify*/1); + mutex_unlock(&bf->bf_lock); +} + +void au_fhsm_wrote_all(struct super_block *sb, int force) +{ + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + /* exclude the bottom */ + bbot = au_fhsm_bottom(sb); + for (bindex = 0; bindex < bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_fhsm(br->br_perm)) + au_fhsm_wrote(sb, bindex, force); + } +} + +/* ---------------------------------------------------------------------- */ + +static __poll_t au_fhsm_poll(struct file *file, struct poll_table_struct *wait) +{ + __poll_t mask; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + mask = 0; + sbinfo = file->private_data; + fhsm = &sbinfo->si_fhsm; + poll_wait(file, &fhsm->fhsm_wqh, wait); + if (atomic_read(&fhsm->fhsm_readable)) + mask = EPOLLIN /* | EPOLLRDNORM */; + + if (!mask) + AuDbg("mask 0x%x\n", mask); + return mask; +} + +static int au_fhsm_do_read_one(struct aufs_stbr __user *stbr, + struct aufs_stfs *stfs, __s16 brid) +{ + int err; + + err = copy_to_user(&stbr->stfs, stfs, sizeof(*stfs)); + if (!err) + err = __put_user(brid, &stbr->brid); + if (unlikely(err)) + err = -EFAULT; + + return err; +} + +static ssize_t au_fhsm_do_read(struct super_block *sb, + struct aufs_stbr __user *stbr, size_t count) +{ + ssize_t err; + int nstbr; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + struct au_br_fhsm *bf; + + /* except the bottom branch */ + err = 0; + nstbr = 0; + bbot = au_fhsm_bottom(sb); + for (bindex = 0; !err && bindex < bbot; bindex++) { + br = au_sbr(sb, bindex); + if (!au_br_fhsm(br->br_perm)) + continue; + + bf = br->br_fhsm; + mutex_lock(&bf->bf_lock); + if (bf->bf_readable) { + err = -EFAULT; + if (count >= sizeof(*stbr)) + err = au_fhsm_do_read_one(stbr++, &bf->bf_stfs, + br->br_id); + if (!err) { + bf->bf_readable = 0; + count -= sizeof(*stbr); + nstbr++; + } + } + mutex_unlock(&bf->bf_lock); + } + if (!err) + err = sizeof(*stbr) * nstbr; + + return err; +} + +static ssize_t au_fhsm_read(struct file *file, char __user *buf, size_t count, + loff_t *pos) +{ + ssize_t err; + int readable; + aufs_bindex_t nfhsm, bindex, bbot; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + struct au_branch *br; + struct super_block *sb; + + err = 0; + sbinfo = file->private_data; + fhsm = &sbinfo->si_fhsm; +need_data: + spin_lock_irq(&fhsm->fhsm_wqh.lock); + if (!atomic_read(&fhsm->fhsm_readable)) { + if (vfsub_file_flags(file) & O_NONBLOCK) + err = -EAGAIN; + else + err = wait_event_interruptible_locked_irq + (fhsm->fhsm_wqh, + atomic_read(&fhsm->fhsm_readable)); + } + spin_unlock_irq(&fhsm->fhsm_wqh.lock); + if (unlikely(err)) + goto out; + + /* sb may already be dead */ + au_rw_read_lock(&sbinfo->si_rwsem); + readable = atomic_read(&fhsm->fhsm_readable); + if (readable > 0) { + sb = sbinfo->si_sb; + AuDebugOn(!sb); + /* exclude the bottom branch */ + nfhsm = 0; + bbot = au_fhsm_bottom(sb); + for (bindex = 0; bindex < bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_fhsm(br->br_perm)) + nfhsm++; + } + err = -EMSGSIZE; + if (nfhsm * sizeof(struct aufs_stbr) <= count) { + atomic_set(&fhsm->fhsm_readable, 0); + err = au_fhsm_do_read(sbinfo->si_sb, (void __user *)buf, + count); + } + } + au_rw_read_unlock(&sbinfo->si_rwsem); + if (!readable) + goto need_data; + +out: + return err; +} + +static int au_fhsm_release(struct inode *inode, struct file *file) +{ + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + /* sb may already be dead */ + sbinfo = file->private_data; + fhsm = &sbinfo->si_fhsm; + spin_lock(&fhsm->fhsm_spin); + fhsm->fhsm_pid = 0; + spin_unlock(&fhsm->fhsm_spin); + kobject_put(&sbinfo->si_kobj); + + return 0; +} + +static const struct file_operations au_fhsm_fops = { + .owner = THIS_MODULE, + .llseek = noop_llseek, + .read = au_fhsm_read, + .poll = au_fhsm_poll, + .release = au_fhsm_release +}; + +int au_fhsm_fd(struct super_block *sb, int oflags) +{ + int err, fd; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + + err = -EPERM; + if (unlikely(!capable(CAP_SYS_ADMIN))) + goto out; + + err = -EINVAL; + if (unlikely(oflags & ~(O_CLOEXEC | O_NONBLOCK))) + goto out; + + err = 0; + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + spin_lock(&fhsm->fhsm_spin); + if (!fhsm->fhsm_pid) + fhsm->fhsm_pid = current->pid; + else + err = -EBUSY; + spin_unlock(&fhsm->fhsm_spin); + if (unlikely(err)) + goto out; + + oflags |= O_RDONLY; + /* oflags |= FMODE_NONOTIFY; */ + fd = anon_inode_getfd("[aufs_fhsm]", &au_fhsm_fops, sbinfo, oflags); + err = fd; + if (unlikely(fd < 0)) + goto out_pid; + + /* succeed regardless 'fhsm' status */ + kobject_get(&sbinfo->si_kobj); + si_noflush_read_lock(sb); + if (au_ftest_si(sbinfo, FHSM)) + au_fhsm_wrote_all(sb, /*force*/0); + si_read_unlock(sb); + goto out; /* success */ + +out_pid: + spin_lock(&fhsm->fhsm_spin); + fhsm->fhsm_pid = 0; + spin_unlock(&fhsm->fhsm_spin); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_fhsm_br_alloc(struct au_branch *br) +{ + int err; + + err = 0; + br->br_fhsm = kmalloc(sizeof(*br->br_fhsm), GFP_NOFS); + if (br->br_fhsm) + au_br_fhsm_init(br->br_fhsm); + else + err = -ENOMEM; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_fhsm_fin(struct super_block *sb) +{ + au_fhsm_notify(sb, /*val*/-1); +} + +void au_fhsm_init(struct au_sbinfo *sbinfo) +{ + struct au_fhsm *fhsm; + + fhsm = &sbinfo->si_fhsm; + spin_lock_init(&fhsm->fhsm_spin); + init_waitqueue_head(&fhsm->fhsm_wqh); + atomic_set(&fhsm->fhsm_readable, 0); + fhsm->fhsm_expire + = msecs_to_jiffies(AUFS_FHSM_CACHE_DEF_SEC * MSEC_PER_SEC); + fhsm->fhsm_bottom = -1; +} + +void au_fhsm_set(struct au_sbinfo *sbinfo, unsigned int sec) +{ + sbinfo->si_fhsm.fhsm_expire + = msecs_to_jiffies(sec * MSEC_PER_SEC); +} + +void au_fhsm_show(struct seq_file *seq, struct au_sbinfo *sbinfo) +{ + unsigned int u; + + if (!au_ftest_si(sbinfo, FHSM)) + return; + + u = jiffies_to_msecs(sbinfo->si_fhsm.fhsm_expire) / MSEC_PER_SEC; + if (u != AUFS_FHSM_CACHE_DEF_SEC) + seq_printf(seq, ",fhsm_sec=%u", u); +} diff --git a/fs/aufs/file.c b/fs/aufs/file.c new file mode 100644 index 0000000000000..b0075b57d8bc5 --- /dev/null +++ b/fs/aufs/file.c @@ -0,0 +1,863 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * handling file/dir, and address_space operation + */ + +#ifdef CONFIG_AUFS_DEBUG +#include +#endif +#include +#include "aufs.h" + +/* drop flags for writing */ +unsigned int au_file_roflags(unsigned int flags) +{ + flags &= ~(O_WRONLY | O_RDWR | O_APPEND | O_CREAT | O_TRUNC); + flags |= O_RDONLY | O_NOATIME; + return flags; +} + +/* common functions to regular file and dir */ +struct file *au_h_open(struct dentry *dentry, aufs_bindex_t bindex, int flags, + struct file *file, int force_wr) +{ + struct file *h_file; + struct dentry *h_dentry; + struct inode *h_inode; + struct super_block *sb; + struct au_branch *br; + struct path h_path; + int err; + + /* a race condition can happen between open and unlink/rmdir */ + h_file = ERR_PTR(-ENOENT); + h_dentry = au_h_dptr(dentry, bindex); + if (au_test_nfsd() && (!h_dentry || d_is_negative(h_dentry))) + goto out; + h_inode = d_inode(h_dentry); + spin_lock(&h_dentry->d_lock); + err = (!d_unhashed(dentry) && d_unlinked(h_dentry)) + /* || !d_inode(dentry)->i_nlink */ + ; + spin_unlock(&h_dentry->d_lock); + if (unlikely(err)) + goto out; + + sb = dentry->d_sb; + br = au_sbr(sb, bindex); + err = au_br_test_oflag(flags, br); + h_file = ERR_PTR(err); + if (unlikely(err)) + goto out; + + /* drop flags for writing */ + if (au_test_ro(sb, bindex, d_inode(dentry))) { + if (force_wr && !(flags & O_WRONLY)) + force_wr = 0; + flags = au_file_roflags(flags); + if (force_wr) { + h_file = ERR_PTR(-EROFS); + flags = au_file_roflags(flags); + if (unlikely(vfsub_native_ro(h_inode) + || IS_APPEND(h_inode))) + goto out; + flags &= ~O_ACCMODE; + flags |= O_WRONLY; + } + } + flags &= ~O_CREAT; + au_lcnt_inc(&br->br_nfiles); + h_path.dentry = h_dentry; + h_path.mnt = au_br_mnt(br); + h_file = vfsub_dentry_open(&h_path, flags); + if (IS_ERR(h_file)) + goto out_br; + + if (flags & __FMODE_EXEC) { + err = deny_write_access(h_file); + if (unlikely(err)) { + fput(h_file); + h_file = ERR_PTR(err); + goto out_br; + } + } + fsnotify_open(h_file); + goto out; /* success */ + +out_br: + au_lcnt_dec(&br->br_nfiles); +out: + return h_file; +} + +static int au_cmoo(struct dentry *dentry) +{ + int err, cmoo, matched; + unsigned int udba; + struct path h_path; + struct au_pin pin; + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = -1, + .bsrc = -1, + .len = -1, + .pin = &pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + struct inode *delegated; + struct super_block *sb; + struct au_sbinfo *sbinfo; + struct au_fhsm *fhsm; + pid_t pid; + struct au_branch *br; + struct dentry *parent; + struct au_hinode *hdir; + + DiMustWriteLock(dentry); + IiMustWriteLock(d_inode(dentry)); + + err = 0; + if (IS_ROOT(dentry)) + goto out; + cpg.bsrc = au_dbtop(dentry); + if (!cpg.bsrc) + goto out; + + sb = dentry->d_sb; + sbinfo = au_sbi(sb); + fhsm = &sbinfo->si_fhsm; + pid = au_fhsm_pid(fhsm); + rcu_read_lock(); + matched = (pid + && (current->pid == pid + || rcu_dereference(current->real_parent)->pid == pid)); + rcu_read_unlock(); + if (matched) + goto out; + + br = au_sbr(sb, cpg.bsrc); + cmoo = au_br_cmoo(br->br_perm); + if (!cmoo) + goto out; + if (!d_is_reg(dentry)) + cmoo &= AuBrAttr_COO_ALL; + if (!cmoo) + goto out; + + parent = dget_parent(dentry); + di_write_lock_parent(parent); + err = au_wbr_do_copyup_bu(dentry, cpg.bsrc - 1); + cpg.bdst = err; + if (unlikely(err < 0)) { + err = 0; /* there is no upper writable branch */ + goto out_dgrade; + } + AuDbg("bsrc %d, bdst %d\n", cpg.bsrc, cpg.bdst); + + /* do not respect the coo attrib for the target branch */ + err = au_cpup_dirs(dentry, cpg.bdst); + if (unlikely(err)) + goto out_dgrade; + + di_downgrade_lock(parent, AuLock_IR); + udba = au_opt_udba(sb); + err = au_pin(&pin, dentry, cpg.bdst, udba, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out_parent; + + err = au_sio_cpup_simple(&cpg); + au_unpin(&pin); + if (unlikely(err)) + goto out_parent; + if (!(cmoo & AuBrWAttr_MOO)) + goto out_parent; /* success */ + + err = au_pin(&pin, dentry, cpg.bsrc, udba, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out_parent; + + h_path.mnt = au_br_mnt(br); + h_path.dentry = au_h_dptr(dentry, cpg.bsrc); + hdir = au_hi(d_inode(parent), cpg.bsrc); + delegated = NULL; + err = vfsub_unlink(hdir->hi_inode, &h_path, &delegated, /*force*/1); + au_unpin(&pin); + /* todo: keep h_dentry or not? */ + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + if (unlikely(err)) { + pr_err("unlink %pd after coo failed (%d), ignored\n", + dentry, err); + err = 0; + } + goto out_parent; /* success */ + +out_dgrade: + di_downgrade_lock(parent, AuLock_IR); +out_parent: + di_read_unlock(parent, AuLock_IR); + dput(parent); +out: + AuTraceErr(err); + return err; +} + +int au_do_open(struct file *file, struct au_do_open_args *args) +{ + int err, aopen = args->aopen; + struct dentry *dentry; + struct au_finfo *finfo; + + if (!aopen) + err = au_finfo_init(file, args->fidir); + else { + lockdep_off(); + err = au_finfo_init(file, args->fidir); + lockdep_on(); + } + if (unlikely(err)) + goto out; + + dentry = file->f_path.dentry; + AuDebugOn(IS_ERR_OR_NULL(dentry)); + di_write_lock_child(dentry); + err = au_cmoo(dentry); + di_downgrade_lock(dentry, AuLock_IR); + if (!err) { + if (!aopen) + err = args->open(file, vfsub_file_flags(file), NULL); + else { + lockdep_off(); + err = args->open(file, vfsub_file_flags(file), + args->h_file); + lockdep_on(); + } + } + di_read_unlock(dentry, AuLock_IR); + + finfo = au_fi(file); + if (!err) { + finfo->fi_file = file; + au_hbl_add(&finfo->fi_hlist, + &au_sbi(file->f_path.dentry->d_sb)->si_files); + } + if (!aopen) + fi_write_unlock(file); + else { + lockdep_off(); + fi_write_unlock(file); + lockdep_on(); + } + if (unlikely(err)) { + finfo->fi_hdir = NULL; + au_finfo_fin(file); + } + +out: + AuTraceErr(err); + return err; +} + +int au_reopen_nondir(struct file *file) +{ + int err; + aufs_bindex_t btop; + struct dentry *dentry; + struct au_branch *br; + struct file *h_file, *h_file_tmp; + + dentry = file->f_path.dentry; + btop = au_dbtop(dentry); + br = au_sbr(dentry->d_sb, btop); + h_file_tmp = NULL; + if (au_fbtop(file) == btop) { + h_file = au_hf_top(file); + if (file->f_mode == h_file->f_mode) + return 0; /* success */ + h_file_tmp = h_file; + get_file(h_file_tmp); + au_lcnt_inc(&br->br_nfiles); + au_set_h_fptr(file, btop, NULL); + } + AuDebugOn(au_fi(file)->fi_hdir); + /* + * it can happen + * file exists on both of rw and ro + * open --> dbtop and fbtop are both 0 + * prepend a branch as rw, "rw" become ro + * remove rw/file + * delete the top branch, "rw" becomes rw again + * --> dbtop is 1, fbtop is still 0 + * write --> fbtop is 0 but dbtop is 1 + */ + /* AuDebugOn(au_fbtop(file) < btop); */ + + h_file = au_h_open(dentry, btop, vfsub_file_flags(file) & ~O_TRUNC, + file, /*force_wr*/0); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) { + if (h_file_tmp) { + /* revert */ + au_set_h_fptr(file, btop, h_file_tmp); + h_file_tmp = NULL; + } + goto out; /* todo: close all? */ + } + + err = 0; + au_set_fbtop(file, btop); + au_set_h_fptr(file, btop, h_file); + au_update_figen(file); + /* todo: necessary? */ + /* file->f_ra = h_file->f_ra; */ + +out: + if (h_file_tmp) { + fput(h_file_tmp); + au_lcnt_dec(&br->br_nfiles); + } + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_reopen_wh(struct file *file, aufs_bindex_t btgt, + struct dentry *hi_wh) +{ + int err; + aufs_bindex_t btop; + struct au_dinfo *dinfo; + struct dentry *h_dentry; + struct au_hdentry *hdp; + + dinfo = au_di(file->f_path.dentry); + AuRwMustWriteLock(&dinfo->di_rwsem); + + btop = dinfo->di_btop; + dinfo->di_btop = btgt; + hdp = au_hdentry(dinfo, btgt); + h_dentry = hdp->hd_dentry; + hdp->hd_dentry = hi_wh; + err = au_reopen_nondir(file); + hdp->hd_dentry = h_dentry; + dinfo->di_btop = btop; + + return err; +} + +static int au_ready_to_write_wh(struct file *file, loff_t len, + aufs_bindex_t bcpup, struct au_pin *pin) +{ + int err; + struct inode *inode, *h_inode; + struct dentry *h_dentry, *hi_wh; + struct au_cp_generic cpg = { + .dentry = file->f_path.dentry, + .bdst = bcpup, + .bsrc = -1, + .len = len, + .pin = pin + }; + + au_update_dbtop(cpg.dentry); + inode = d_inode(cpg.dentry); + h_inode = NULL; + if (au_dbtop(cpg.dentry) <= bcpup + && au_dbbot(cpg.dentry) >= bcpup) { + h_dentry = au_h_dptr(cpg.dentry, bcpup); + if (h_dentry && d_is_positive(h_dentry)) + h_inode = d_inode(h_dentry); + } + hi_wh = au_hi_wh(inode, bcpup); + if (!hi_wh && !h_inode) + err = au_sio_cpup_wh(&cpg, file); + else + /* already copied-up after unlink */ + err = au_reopen_wh(file, bcpup, hi_wh); + + if (!err + && (inode->i_nlink > 1 + || (inode->i_state & I_LINKABLE)) + && au_opt_test(au_mntflags(cpg.dentry->d_sb), PLINK)) + au_plink_append(inode, bcpup, au_h_dptr(cpg.dentry, bcpup)); + + return err; +} + +/* + * prepare the @file for writing. + */ +int au_ready_to_write(struct file *file, loff_t len, struct au_pin *pin) +{ + int err; + aufs_bindex_t dbtop; + struct dentry *parent; + struct inode *inode; + struct super_block *sb; + struct file *h_file; + struct au_cp_generic cpg = { + .dentry = file->f_path.dentry, + .bdst = -1, + .bsrc = -1, + .len = len, + .pin = pin, + .flags = AuCpup_DTIME + }; + + sb = cpg.dentry->d_sb; + inode = d_inode(cpg.dentry); + cpg.bsrc = au_fbtop(file); + err = au_test_ro(sb, cpg.bsrc, inode); + if (!err && (au_hf_top(file)->f_mode & FMODE_WRITE)) { + err = au_pin(pin, cpg.dentry, cpg.bsrc, AuOpt_UDBA_NONE, + /*flags*/0); + goto out; + } + + /* need to cpup or reopen */ + parent = dget_parent(cpg.dentry); + di_write_lock_parent(parent); + err = AuWbrCopyup(au_sbi(sb), cpg.dentry); + cpg.bdst = err; + if (unlikely(err < 0)) + goto out_dgrade; + err = 0; + + if (!d_unhashed(cpg.dentry) && !au_h_dptr(parent, cpg.bdst)) { + err = au_cpup_dirs(cpg.dentry, cpg.bdst); + if (unlikely(err)) + goto out_dgrade; + } + + err = au_pin(pin, cpg.dentry, cpg.bdst, AuOpt_UDBA_NONE, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out_dgrade; + + dbtop = au_dbtop(cpg.dentry); + if (dbtop <= cpg.bdst) + cpg.bsrc = cpg.bdst; + + if (dbtop <= cpg.bdst /* just reopen */ + || !d_unhashed(cpg.dentry) /* copyup and reopen */ + ) { + h_file = au_h_open_pre(cpg.dentry, cpg.bsrc, /*force_wr*/0); + if (IS_ERR(h_file)) + err = PTR_ERR(h_file); + else { + di_downgrade_lock(parent, AuLock_IR); + if (dbtop > cpg.bdst) + err = au_sio_cpup_simple(&cpg); + if (!err) + err = au_reopen_nondir(file); + au_h_open_post(cpg.dentry, cpg.bsrc, h_file); + } + } else { /* copyup as wh and reopen */ + /* + * since writable hfsplus branch is not supported, + * h_open_pre/post() are unnecessary. + */ + err = au_ready_to_write_wh(file, len, cpg.bdst, pin); + di_downgrade_lock(parent, AuLock_IR); + } + + if (!err) { + au_pin_set_parent_lflag(pin, /*lflag*/0); + goto out_dput; /* success */ + } + au_unpin(pin); + goto out_unlock; + +out_dgrade: + di_downgrade_lock(parent, AuLock_IR); +out_unlock: + di_read_unlock(parent, AuLock_IR); +out_dput: + dput(parent); +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_do_flush(struct file *file, fl_owner_t id, + int (*flush)(struct file *file, fl_owner_t id)) +{ + int err; + struct super_block *sb; + struct inode *inode; + + inode = file_inode(file); + sb = inode->i_sb; + si_noflush_read_lock(sb); + fi_read_lock(file); + ii_read_lock_child(inode); + + err = flush(file, id); + au_cpup_attr_timesizes(inode); + + ii_read_unlock(inode); + fi_read_unlock(file); + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_file_refresh_by_inode(struct file *file, int *need_reopen) +{ + int err; + struct au_pin pin; + struct au_finfo *finfo; + struct dentry *parent, *hi_wh; + struct inode *inode; + struct super_block *sb; + struct au_cp_generic cpg = { + .dentry = file->f_path.dentry, + .bdst = -1, + .bsrc = -1, + .len = -1, + .pin = &pin, + .flags = AuCpup_DTIME + }; + + FiMustWriteLock(file); + + err = 0; + finfo = au_fi(file); + sb = cpg.dentry->d_sb; + inode = d_inode(cpg.dentry); + cpg.bdst = au_ibtop(inode); + if (cpg.bdst == finfo->fi_btop || IS_ROOT(cpg.dentry)) + goto out; + + parent = dget_parent(cpg.dentry); + if (au_test_ro(sb, cpg.bdst, inode)) { + di_read_lock_parent(parent, !AuLock_IR); + err = AuWbrCopyup(au_sbi(sb), cpg.dentry); + cpg.bdst = err; + di_read_unlock(parent, !AuLock_IR); + if (unlikely(err < 0)) + goto out_parent; + err = 0; + } + + di_read_lock_parent(parent, AuLock_IR); + hi_wh = au_hi_wh(inode, cpg.bdst); + if (!S_ISDIR(inode->i_mode) + && au_opt_test(au_mntflags(sb), PLINK) + && au_plink_test(inode) + && !d_unhashed(cpg.dentry) + && cpg.bdst < au_dbtop(cpg.dentry)) { + err = au_test_and_cpup_dirs(cpg.dentry, cpg.bdst); + if (unlikely(err)) + goto out_unlock; + + /* always superio. */ + err = au_pin(&pin, cpg.dentry, cpg.bdst, AuOpt_UDBA_NONE, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (!err) { + err = au_sio_cpup_simple(&cpg); + au_unpin(&pin); + } + } else if (hi_wh) { + /* already copied-up after unlink */ + err = au_reopen_wh(file, cpg.bdst, hi_wh); + *need_reopen = 0; + } + +out_unlock: + di_read_unlock(parent, AuLock_IR); +out_parent: + dput(parent); +out: + return err; +} + +static void au_do_refresh_dir(struct file *file) +{ + aufs_bindex_t bindex, bbot, new_bindex, brid; + struct au_hfile *p, tmp, *q; + struct au_finfo *finfo; + struct super_block *sb; + struct au_fidir *fidir; + + FiMustWriteLock(file); + + sb = file->f_path.dentry->d_sb; + finfo = au_fi(file); + fidir = finfo->fi_hdir; + AuDebugOn(!fidir); + p = fidir->fd_hfile + finfo->fi_btop; + brid = p->hf_br->br_id; + bbot = fidir->fd_bbot; + for (bindex = finfo->fi_btop; bindex <= bbot; bindex++, p++) { + if (!p->hf_file) + continue; + + new_bindex = au_br_index(sb, p->hf_br->br_id); + if (new_bindex == bindex) + continue; + if (new_bindex < 0) { + au_set_h_fptr(file, bindex, NULL); + continue; + } + + /* swap two lower inode, and loop again */ + q = fidir->fd_hfile + new_bindex; + tmp = *q; + *q = *p; + *p = tmp; + if (tmp.hf_file) { + bindex--; + p--; + } + } + + p = fidir->fd_hfile; + if (!au_test_mmapped(file) && !d_unlinked(file->f_path.dentry)) { + bbot = au_sbbot(sb); + for (finfo->fi_btop = 0; finfo->fi_btop <= bbot; + finfo->fi_btop++, p++) + if (p->hf_file) { + if (file_inode(p->hf_file)) + break; + au_hfput(p, /*execed*/0); + } + } else { + bbot = au_br_index(sb, brid); + for (finfo->fi_btop = 0; finfo->fi_btop < bbot; + finfo->fi_btop++, p++) + if (p->hf_file) + au_hfput(p, /*execed*/0); + bbot = au_sbbot(sb); + } + + p = fidir->fd_hfile + bbot; + for (fidir->fd_bbot = bbot; fidir->fd_bbot >= finfo->fi_btop; + fidir->fd_bbot--, p--) + if (p->hf_file) { + if (file_inode(p->hf_file)) + break; + au_hfput(p, /*execed*/0); + } + AuDebugOn(fidir->fd_bbot < finfo->fi_btop); +} + +/* + * after branch manipulating, refresh the file. + */ +static int refresh_file(struct file *file, int (*reopen)(struct file *file)) +{ + int err, need_reopen, nbr; + aufs_bindex_t bbot, bindex; + struct dentry *dentry; + struct super_block *sb; + struct au_finfo *finfo; + struct au_hfile *hfile; + + dentry = file->f_path.dentry; + sb = dentry->d_sb; + nbr = au_sbbot(sb) + 1; + finfo = au_fi(file); + if (!finfo->fi_hdir) { + hfile = &finfo->fi_htop; + AuDebugOn(!hfile->hf_file); + bindex = au_br_index(sb, hfile->hf_br->br_id); + AuDebugOn(bindex < 0); + if (bindex != finfo->fi_btop) + au_set_fbtop(file, bindex); + } else { + err = au_fidir_realloc(finfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + au_do_refresh_dir(file); + } + + err = 0; + need_reopen = 1; + if (!au_test_mmapped(file)) + err = au_file_refresh_by_inode(file, &need_reopen); + if (finfo->fi_hdir) + /* harmless if err */ + au_fidir_realloc(finfo, nbr, /*may_shrink*/1); + if (!err && need_reopen && !d_unlinked(dentry)) + err = reopen(file); + if (!err) { + au_update_figen(file); + goto out; /* success */ + } + + /* error, close all lower files */ + if (finfo->fi_hdir) { + bbot = au_fbbot_dir(file); + for (bindex = au_fbtop(file); bindex <= bbot; bindex++) + au_set_h_fptr(file, bindex, NULL); + } + +out: + return err; +} + +/* common function to regular file and dir */ +int au_reval_and_lock_fdi(struct file *file, int (*reopen)(struct file *file), + int wlock, unsigned int fi_lsc) +{ + int err; + unsigned int sigen, figen; + aufs_bindex_t btop; + unsigned char pseudo_link; + struct dentry *dentry; + struct inode *inode; + + err = 0; + dentry = file->f_path.dentry; + inode = d_inode(dentry); + sigen = au_sigen(dentry->d_sb); + fi_write_lock_nested(file, fi_lsc); + figen = au_figen(file); + if (!fi_lsc) + di_write_lock_child(dentry); + else + di_write_lock_child2(dentry); + btop = au_dbtop(dentry); + pseudo_link = (btop != au_ibtop(inode)); + if (sigen == figen && !pseudo_link && au_fbtop(file) == btop) { + if (!wlock) { + di_downgrade_lock(dentry, AuLock_IR); + fi_downgrade_lock(file); + } + goto out; /* success */ + } + + AuDbg("sigen %d, figen %d\n", sigen, figen); + if (au_digen_test(dentry, sigen)) { + err = au_reval_dpath(dentry, sigen); + AuDebugOn(!err && au_digen_test(dentry, sigen)); + } + + if (!err) + err = refresh_file(file, reopen); + if (!err) { + if (!wlock) { + di_downgrade_lock(dentry, AuLock_IR); + fi_downgrade_lock(file); + } + } else { + di_write_unlock(dentry); + fi_write_unlock(file); + } + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* cf. aufs_nopage() */ +/* for madvise(2) */ +static int aufs_readpage(struct file *file __maybe_unused, struct page *page) +{ + unlock_page(page); + return 0; +} + +/* it will never be called, but necessary to support O_DIRECT */ +static ssize_t aufs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ BUG(); return 0; } + +/* they will never be called. */ +#ifdef CONFIG_AUFS_DEBUG +static int aufs_write_begin(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned flags, + struct page **pagep, void **fsdata) +{ AuUnsupport(); return 0; } +static int aufs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct page *page, void *fsdata) +{ AuUnsupport(); return 0; } +static int aufs_writepage(struct page *page, struct writeback_control *wbc) +{ AuUnsupport(); return 0; } + +static int aufs_set_page_dirty(struct page *page) +{ AuUnsupport(); return 0; } +static void aufs_invalidatepage(struct page *page, unsigned int offset, + unsigned int length) +{ AuUnsupport(); } +static int aufs_releasepage(struct page *page, gfp_t gfp) +{ AuUnsupport(); return 0; } +#if 0 /* called by memory compaction regardless file */ +static int aufs_migratepage(struct address_space *mapping, struct page *newpage, + struct page *page, enum migrate_mode mode) +{ AuUnsupport(); return 0; } +#endif +static bool aufs_isolate_page(struct page *page, isolate_mode_t mode) +{ AuUnsupport(); return true; } +static void aufs_putback_page(struct page *page) +{ AuUnsupport(); } +static int aufs_launder_page(struct page *page) +{ AuUnsupport(); return 0; } +static int aufs_is_partially_uptodate(struct page *page, + unsigned long from, + unsigned long count) +{ AuUnsupport(); return 0; } +static void aufs_is_dirty_writeback(struct page *page, bool *dirty, + bool *writeback) +{ AuUnsupport(); } +static int aufs_error_remove_page(struct address_space *mapping, + struct page *page) +{ AuUnsupport(); return 0; } +static int aufs_swap_activate(struct swap_info_struct *sis, struct file *file, + sector_t *span) +{ AuUnsupport(); return 0; } +static void aufs_swap_deactivate(struct file *file) +{ AuUnsupport(); } +#endif /* CONFIG_AUFS_DEBUG */ + +const struct address_space_operations aufs_aop = { + .readpage = aufs_readpage, + .direct_IO = aufs_direct_IO, +#ifdef CONFIG_AUFS_DEBUG + .writepage = aufs_writepage, + /* no writepages, because of writepage */ + .set_page_dirty = aufs_set_page_dirty, + /* no readpages, because of readpage */ + .write_begin = aufs_write_begin, + .write_end = aufs_write_end, + /* no bmap, no block device */ + .invalidatepage = aufs_invalidatepage, + .releasepage = aufs_releasepage, + /* is fallback_migrate_page ok? */ + /* .migratepage = aufs_migratepage, */ + .isolate_page = aufs_isolate_page, + .putback_page = aufs_putback_page, + .launder_page = aufs_launder_page, + .is_partially_uptodate = aufs_is_partially_uptodate, + .is_dirty_writeback = aufs_is_dirty_writeback, + .error_remove_page = aufs_error_remove_page, + .swap_activate = aufs_swap_activate, + .swap_deactivate = aufs_swap_deactivate +#endif /* CONFIG_AUFS_DEBUG */ +}; diff --git a/fs/aufs/file.h b/fs/aufs/file.h new file mode 100644 index 0000000000000..d124d9c8216d9 --- /dev/null +++ b/fs/aufs/file.h @@ -0,0 +1,342 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * file operations + */ + +#ifndef __AUFS_FILE_H__ +#define __AUFS_FILE_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include "rwsem.h" + +struct au_branch; +struct au_hfile { + struct file *hf_file; + struct au_branch *hf_br; +}; + +struct au_vdir; +struct au_fidir { + aufs_bindex_t fd_bbot; + aufs_bindex_t fd_nent; + struct au_vdir *fd_vdir_cache; + struct au_hfile fd_hfile[]; +}; + +static inline int au_fidir_sz(int nent) +{ + AuDebugOn(nent < 0); + return sizeof(struct au_fidir) + sizeof(struct au_hfile) * nent; +} + +struct au_finfo { + atomic_t fi_generation; + + struct au_rwsem fi_rwsem; + aufs_bindex_t fi_btop; + + /* do not union them */ + struct { /* for non-dir */ + struct au_hfile fi_htop; + atomic_t fi_mmapped; + }; + struct au_fidir *fi_hdir; /* for dir only */ + + struct hlist_bl_node fi_hlist; + struct file *fi_file; /* very ugly */ + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +/* ---------------------------------------------------------------------- */ + +/* file.c */ +extern const struct address_space_operations aufs_aop; +unsigned int au_file_roflags(unsigned int flags); +struct file *au_h_open(struct dentry *dentry, aufs_bindex_t bindex, int flags, + struct file *file, int force_wr); +struct au_do_open_args { + int aopen; + int (*open)(struct file *file, int flags, + struct file *h_file); + struct au_fidir *fidir; + struct file *h_file; +}; +int au_do_open(struct file *file, struct au_do_open_args *args); +int au_reopen_nondir(struct file *file); +struct au_pin; +int au_ready_to_write(struct file *file, loff_t len, struct au_pin *pin); +int au_reval_and_lock_fdi(struct file *file, int (*reopen)(struct file *file), + int wlock, unsigned int fi_lsc); +int au_do_flush(struct file *file, fl_owner_t id, + int (*flush)(struct file *file, fl_owner_t id)); + +/* poll.c */ +#ifdef CONFIG_AUFS_POLL +__poll_t aufs_poll(struct file *file, struct poll_table_struct *pt); +#endif + +#ifdef CONFIG_AUFS_BR_HFSPLUS +/* hfsplus.c */ +struct file *au_h_open_pre(struct dentry *dentry, aufs_bindex_t bindex, + int force_wr); +void au_h_open_post(struct dentry *dentry, aufs_bindex_t bindex, + struct file *h_file); +#else +AuStub(struct file *, au_h_open_pre, return NULL, struct dentry *dentry, + aufs_bindex_t bindex, int force_wr) +AuStubVoid(au_h_open_post, struct dentry *dentry, aufs_bindex_t bindex, + struct file *h_file); +#endif + +/* f_op.c */ +extern const struct file_operations aufs_file_fop; +int au_do_open_nondir(struct file *file, int flags, struct file *h_file); +int aufs_release_nondir(struct inode *inode __maybe_unused, struct file *file); +struct file *au_read_pre(struct file *file, int keep_fi, unsigned int lsc); + +/* finfo.c */ +void au_hfput(struct au_hfile *hf, int execed); +void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, + struct file *h_file); + +void au_update_figen(struct file *file); +struct au_fidir *au_fidir_alloc(struct super_block *sb); +int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink); + +void au_fi_init_once(void *_fi); +void au_finfo_fin(struct file *file); +int au_finfo_init(struct file *file, struct au_fidir *fidir); + +/* ioctl.c */ +long aufs_ioctl_nondir(struct file *file, unsigned int cmd, unsigned long arg); +#ifdef CONFIG_COMPAT +long aufs_compat_ioctl_dir(struct file *file, unsigned int cmd, + unsigned long arg); +long aufs_compat_ioctl_nondir(struct file *file, unsigned int cmd, + unsigned long arg); +#endif + +/* ---------------------------------------------------------------------- */ + +static inline struct au_finfo *au_fi(struct file *file) +{ + return file->private_data; +} + +/* ---------------------------------------------------------------------- */ + +#define fi_read_lock(f) au_rw_read_lock(&au_fi(f)->fi_rwsem) +#define fi_write_lock(f) au_rw_write_lock(&au_fi(f)->fi_rwsem) +#define fi_read_trylock(f) au_rw_read_trylock(&au_fi(f)->fi_rwsem) +#define fi_write_trylock(f) au_rw_write_trylock(&au_fi(f)->fi_rwsem) +/* +#define fi_read_trylock_nested(f) \ + au_rw_read_trylock_nested(&au_fi(f)->fi_rwsem) +#define fi_write_trylock_nested(f) \ + au_rw_write_trylock_nested(&au_fi(f)->fi_rwsem) +*/ + +#define fi_read_unlock(f) au_rw_read_unlock(&au_fi(f)->fi_rwsem) +#define fi_write_unlock(f) au_rw_write_unlock(&au_fi(f)->fi_rwsem) +#define fi_downgrade_lock(f) au_rw_dgrade_lock(&au_fi(f)->fi_rwsem) + +/* lock subclass for finfo */ +enum { + AuLsc_FI_1, + AuLsc_FI_2 +}; + +static inline void fi_read_lock_nested(struct file *f, unsigned int lsc) +{ + au_rw_read_lock_nested(&au_fi(f)->fi_rwsem, lsc); +} + +static inline void fi_write_lock_nested(struct file *f, unsigned int lsc) +{ + au_rw_write_lock_nested(&au_fi(f)->fi_rwsem, lsc); +} + +/* + * fi_read_lock_1, fi_write_lock_1, + * fi_read_lock_2, fi_write_lock_2 + */ +#define AuReadLockFunc(name) \ +static inline void fi_read_lock_##name(struct file *f) \ +{ fi_read_lock_nested(f, AuLsc_FI_##name); } + +#define AuWriteLockFunc(name) \ +static inline void fi_write_lock_##name(struct file *f) \ +{ fi_write_lock_nested(f, AuLsc_FI_##name); } + +#define AuRWLockFuncs(name) \ + AuReadLockFunc(name) \ + AuWriteLockFunc(name) + +AuRWLockFuncs(1); +AuRWLockFuncs(2); + +#undef AuReadLockFunc +#undef AuWriteLockFunc +#undef AuRWLockFuncs + +#define FiMustNoWaiters(f) AuRwMustNoWaiters(&au_fi(f)->fi_rwsem) +#define FiMustAnyLock(f) AuRwMustAnyLock(&au_fi(f)->fi_rwsem) +#define FiMustWriteLock(f) AuRwMustWriteLock(&au_fi(f)->fi_rwsem) + +/* ---------------------------------------------------------------------- */ + +/* todo: hard/soft set? */ +static inline aufs_bindex_t au_fbtop(struct file *file) +{ + FiMustAnyLock(file); + return au_fi(file)->fi_btop; +} + +static inline aufs_bindex_t au_fbbot_dir(struct file *file) +{ + FiMustAnyLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + return au_fi(file)->fi_hdir->fd_bbot; +} + +static inline struct au_vdir *au_fvdir_cache(struct file *file) +{ + FiMustAnyLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + return au_fi(file)->fi_hdir->fd_vdir_cache; +} + +static inline void au_set_fbtop(struct file *file, aufs_bindex_t bindex) +{ + FiMustWriteLock(file); + au_fi(file)->fi_btop = bindex; +} + +static inline void au_set_fbbot_dir(struct file *file, aufs_bindex_t bindex) +{ + FiMustWriteLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + au_fi(file)->fi_hdir->fd_bbot = bindex; +} + +static inline void au_set_fvdir_cache(struct file *file, + struct au_vdir *vdir_cache) +{ + FiMustWriteLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + au_fi(file)->fi_hdir->fd_vdir_cache = vdir_cache; +} + +static inline struct file *au_hf_top(struct file *file) +{ + FiMustAnyLock(file); + AuDebugOn(au_fi(file)->fi_hdir); + return au_fi(file)->fi_htop.hf_file; +} + +static inline struct file *au_hf_dir(struct file *file, aufs_bindex_t bindex) +{ + FiMustAnyLock(file); + AuDebugOn(!au_fi(file)->fi_hdir); + return au_fi(file)->fi_hdir->fd_hfile[0 + bindex].hf_file; +} + +/* todo: memory barrier? */ +static inline unsigned int au_figen(struct file *f) +{ + return atomic_read(&au_fi(f)->fi_generation); +} + +static inline void au_set_mmapped(struct file *f) +{ + if (atomic_inc_return(&au_fi(f)->fi_mmapped)) + return; + pr_warn("fi_mmapped wrapped around\n"); + while (!atomic_inc_return(&au_fi(f)->fi_mmapped)) + ; +} + +static inline void au_unset_mmapped(struct file *f) +{ + atomic_dec(&au_fi(f)->fi_mmapped); +} + +static inline int au_test_mmapped(struct file *f) +{ + return atomic_read(&au_fi(f)->fi_mmapped); +} + +/* customize vma->vm_file */ + +static inline void au_do_vm_file_reset(struct vm_area_struct *vma, + struct file *file) +{ + struct file *f; + + f = vma->vm_file; + get_file(file); + vma->vm_file = file; + fput(f); +} + +#ifdef CONFIG_MMU +#define AuDbgVmRegion(file, vma) do {} while (0) + +static inline void au_vm_file_reset(struct vm_area_struct *vma, + struct file *file) +{ + au_do_vm_file_reset(vma, file); +} +#else +#define AuDbgVmRegion(file, vma) \ + AuDebugOn((vma)->vm_region && (vma)->vm_region->vm_file != (file)) + +static inline void au_vm_file_reset(struct vm_area_struct *vma, + struct file *file) +{ + struct file *f; + + au_do_vm_file_reset(vma, file); + f = vma->vm_region->vm_file; + get_file(file); + vma->vm_region->vm_file = file; + fput(f); +} +#endif /* CONFIG_MMU */ + +/* handle vma->vm_prfile */ +static inline void au_vm_prfile_set(struct vm_area_struct *vma, + struct file *file) +{ + get_file(file); + vma->vm_prfile = file; +#ifndef CONFIG_MMU + get_file(file); + vma->vm_region->vm_prfile = file; +#endif +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_FILE_H__ */ diff --git a/fs/aufs/finfo.c b/fs/aufs/finfo.c new file mode 100644 index 0000000000000..25077670a5078 --- /dev/null +++ b/fs/aufs/finfo.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * file private data + */ + +#include "aufs.h" + +void au_hfput(struct au_hfile *hf, int execed) +{ + if (execed) + allow_write_access(hf->hf_file); + fput(hf->hf_file); + hf->hf_file = NULL; + au_lcnt_dec(&hf->hf_br->br_nfiles); + hf->hf_br = NULL; +} + +void au_set_h_fptr(struct file *file, aufs_bindex_t bindex, struct file *val) +{ + struct au_finfo *finfo = au_fi(file); + struct au_hfile *hf; + struct au_fidir *fidir; + + fidir = finfo->fi_hdir; + if (!fidir) { + AuDebugOn(finfo->fi_btop != bindex); + hf = &finfo->fi_htop; + } else + hf = fidir->fd_hfile + bindex; + + if (hf && hf->hf_file) + au_hfput(hf, vfsub_file_execed(file)); + if (val) { + FiMustWriteLock(file); + AuDebugOn(IS_ERR_OR_NULL(file->f_path.dentry)); + hf->hf_file = val; + hf->hf_br = au_sbr(file->f_path.dentry->d_sb, bindex); + } +} + +void au_update_figen(struct file *file) +{ + atomic_set(&au_fi(file)->fi_generation, au_digen(file->f_path.dentry)); + /* smp_mb(); */ /* atomic_set */ +} + +/* ---------------------------------------------------------------------- */ + +struct au_fidir *au_fidir_alloc(struct super_block *sb) +{ + struct au_fidir *fidir; + int nbr; + + nbr = au_sbbot(sb) + 1; + if (nbr < 2) + nbr = 2; /* initial allocate for 2 branches */ + fidir = kzalloc(au_fidir_sz(nbr), GFP_NOFS); + if (fidir) { + fidir->fd_bbot = -1; + fidir->fd_nent = nbr; + } + + return fidir; +} + +int au_fidir_realloc(struct au_finfo *finfo, int nbr, int may_shrink) +{ + int err; + struct au_fidir *fidir, *p; + + AuRwMustWriteLock(&finfo->fi_rwsem); + fidir = finfo->fi_hdir; + AuDebugOn(!fidir); + + err = -ENOMEM; + p = au_kzrealloc(fidir, au_fidir_sz(fidir->fd_nent), au_fidir_sz(nbr), + GFP_NOFS, may_shrink); + if (p) { + p->fd_nent = nbr; + finfo->fi_hdir = p; + err = 0; + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_finfo_fin(struct file *file) +{ + struct au_finfo *finfo; + + au_lcnt_dec(&au_sbi(file->f_path.dentry->d_sb)->si_nfiles); + + finfo = au_fi(file); + AuDebugOn(finfo->fi_hdir); + AuRwDestroy(&finfo->fi_rwsem); + au_cache_free_finfo(finfo); +} + +void au_fi_init_once(void *_finfo) +{ + struct au_finfo *finfo = _finfo; + + au_rw_init(&finfo->fi_rwsem); +} + +int au_finfo_init(struct file *file, struct au_fidir *fidir) +{ + int err; + struct au_finfo *finfo; + struct dentry *dentry; + + err = -ENOMEM; + dentry = file->f_path.dentry; + finfo = au_cache_alloc_finfo(); + if (unlikely(!finfo)) + goto out; + + err = 0; + au_lcnt_inc(&au_sbi(dentry->d_sb)->si_nfiles); + au_rw_write_lock(&finfo->fi_rwsem); + finfo->fi_btop = -1; + finfo->fi_hdir = fidir; + atomic_set(&finfo->fi_generation, au_digen(dentry)); + /* smp_mb(); */ /* atomic_set */ + + file->private_data = finfo; + +out: + return err; +} diff --git a/fs/aufs/fstype.h b/fs/aufs/fstype.h new file mode 100644 index 0000000000000..af4bc6c0dd427 --- /dev/null +++ b/fs/aufs/fstype.h @@ -0,0 +1,401 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * judging filesystem type + */ + +#ifndef __AUFS_FSTYPE_H__ +#define __AUFS_FSTYPE_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include + +static inline int au_test_aufs(struct super_block *sb) +{ + return sb->s_magic == AUFS_SUPER_MAGIC; +} + +static inline const char *au_sbtype(struct super_block *sb) +{ + return sb->s_type->name; +} + +static inline int au_test_iso9660(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_ISO9660_FS) + return sb->s_magic == ISOFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_romfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_ROMFS_FS) + return sb->s_magic == ROMFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_cramfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_CRAMFS) + return sb->s_magic == CRAMFS_MAGIC; +#endif + return 0; +} + +static inline int au_test_nfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_NFS_FS) + return sb->s_magic == NFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_fuse(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_FUSE_FS) + return sb->s_magic == FUSE_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_xfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_XFS_FS) + return sb->s_magic == XFS_SB_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_tmpfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_TMPFS + return sb->s_magic == TMPFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_ecryptfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_ECRYPT_FS) + return !strcmp(au_sbtype(sb), "ecryptfs"); +#else + return 0; +#endif +} + +static inline int au_test_ramfs(struct super_block *sb) +{ + return sb->s_magic == RAMFS_MAGIC; +} + +static inline int au_test_ubifs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_UBIFS_FS) + return sb->s_magic == UBIFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_procfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_PROC_FS + return sb->s_magic == PROC_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_sysfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_SYSFS + return sb->s_magic == SYSFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_configfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_CONFIGFS_FS) + return sb->s_magic == CONFIGFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_minix(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_MINIX_FS) + return sb->s_magic == MINIX3_SUPER_MAGIC + || sb->s_magic == MINIX2_SUPER_MAGIC + || sb->s_magic == MINIX2_SUPER_MAGIC2 + || sb->s_magic == MINIX_SUPER_MAGIC + || sb->s_magic == MINIX_SUPER_MAGIC2; +#else + return 0; +#endif +} + +static inline int au_test_fat(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_FAT_FS) + return sb->s_magic == MSDOS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_msdos(struct super_block *sb) +{ + return au_test_fat(sb); +} + +static inline int au_test_vfat(struct super_block *sb) +{ + return au_test_fat(sb); +} + +static inline int au_test_securityfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_SECURITYFS + return sb->s_magic == SECURITYFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_squashfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_SQUASHFS) + return sb->s_magic == SQUASHFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_btrfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_BTRFS_FS) + return sb->s_magic == BTRFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_xenfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_XENFS) + return sb->s_magic == XENFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_debugfs(struct super_block *sb __maybe_unused) +{ +#ifdef CONFIG_DEBUG_FS + return sb->s_magic == DEBUGFS_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_nilfs(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_NILFS) + return sb->s_magic == NILFS_SUPER_MAGIC; +#else + return 0; +#endif +} + +static inline int au_test_hfsplus(struct super_block *sb __maybe_unused) +{ +#if IS_ENABLED(CONFIG_HFSPLUS_FS) + return sb->s_magic == HFSPLUS_SUPER_MAGIC; +#else + return 0; +#endif +} + +/* ---------------------------------------------------------------------- */ +/* + * they can't be an aufs branch. + */ +static inline int au_test_fs_unsuppoted(struct super_block *sb) +{ + return +#ifndef CONFIG_AUFS_BR_RAMFS + au_test_ramfs(sb) || +#endif + au_test_procfs(sb) + || au_test_sysfs(sb) + || au_test_configfs(sb) + || au_test_debugfs(sb) + || au_test_securityfs(sb) + || au_test_xenfs(sb) + || au_test_ecryptfs(sb) + /* || !strcmp(au_sbtype(sb), "unionfs") */ + || au_test_aufs(sb); /* will be supported in next version */ +} + +static inline int au_test_fs_remote(struct super_block *sb) +{ + return !au_test_tmpfs(sb) +#ifdef CONFIG_AUFS_BR_RAMFS + && !au_test_ramfs(sb) +#endif + && !(sb->s_type->fs_flags & FS_REQUIRES_DEV); +} + +/* ---------------------------------------------------------------------- */ + +/* + * Note: these functions (below) are created after reading ->getattr() in all + * filesystems under linux/fs. it means we have to do so in every update... + */ + +/* + * some filesystems require getattr to refresh the inode attributes before + * referencing. + * in most cases, we can rely on the inode attribute in NFS (or every remote fs) + * and leave the work for d_revalidate() + */ +static inline int au_test_fs_refresh_iattr(struct super_block *sb) +{ + return au_test_nfs(sb) + || au_test_fuse(sb) + /* || au_test_btrfs(sb) */ /* untested */ + ; +} + +/* + * filesystems which don't maintain i_size or i_blocks. + */ +static inline int au_test_fs_bad_iattr_size(struct super_block *sb) +{ + return au_test_xfs(sb) + || au_test_btrfs(sb) + || au_test_ubifs(sb) + || au_test_hfsplus(sb) /* maintained, but incorrect */ + /* || au_test_minix(sb) */ /* untested */ + ; +} + +/* + * filesystems which don't store the correct value in some of their inode + * attributes. + */ +static inline int au_test_fs_bad_iattr(struct super_block *sb) +{ + return au_test_fs_bad_iattr_size(sb) + || au_test_fat(sb) + || au_test_msdos(sb) + || au_test_vfat(sb); +} + +/* they don't check i_nlink in link(2) */ +static inline int au_test_fs_no_limit_nlink(struct super_block *sb) +{ + return au_test_tmpfs(sb) +#ifdef CONFIG_AUFS_BR_RAMFS + || au_test_ramfs(sb) +#endif + || au_test_ubifs(sb) + || au_test_hfsplus(sb); +} + +/* + * filesystems which sets S_NOATIME and S_NOCMTIME. + */ +static inline int au_test_fs_notime(struct super_block *sb) +{ + return au_test_nfs(sb) + || au_test_fuse(sb) + || au_test_ubifs(sb) + ; +} + +/* temporary support for i#1 in cramfs */ +static inline int au_test_fs_unique_ino(struct inode *inode) +{ + if (au_test_cramfs(inode->i_sb)) + return inode->i_ino != 1; + return 1; +} + +/* ---------------------------------------------------------------------- */ + +/* + * the filesystem where the xino files placed must support i/o after unlink and + * maintain i_size and i_blocks. + */ +static inline int au_test_fs_bad_xino(struct super_block *sb) +{ + return au_test_fs_remote(sb) + || au_test_fs_bad_iattr_size(sb) + /* don't want unnecessary work for xino */ + || au_test_aufs(sb) + || au_test_ecryptfs(sb) + || au_test_nilfs(sb); +} + +static inline int au_test_fs_trunc_xino(struct super_block *sb) +{ + return au_test_tmpfs(sb) + || au_test_ramfs(sb); +} + +/* + * test if the @sb is real-readonly. + */ +static inline int au_test_fs_rr(struct super_block *sb) +{ + return au_test_squashfs(sb) + || au_test_iso9660(sb) + || au_test_cramfs(sb) + || au_test_romfs(sb); +} + +/* + * test if the @inode is nfs with 'noacl' option + * NFS always sets SB_POSIXACL regardless its mount option 'noacl.' + */ +static inline int au_test_nfs_noacl(struct inode *inode) +{ + return au_test_nfs(inode->i_sb) + /* && IS_POSIXACL(inode) */ + && !nfs_server_capable(inode, NFS_CAP_ACLS); +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_FSTYPE_H__ */ diff --git a/fs/aufs/hbl.h b/fs/aufs/hbl.h new file mode 100644 index 0000000000000..4d5016bccc51b --- /dev/null +++ b/fs/aufs/hbl.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2017-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * helpers for hlist_bl.h + */ + +#ifndef __AUFS_HBL_H__ +#define __AUFS_HBL_H__ + +#ifdef __KERNEL__ + +#include + +static inline void au_hbl_add(struct hlist_bl_node *node, + struct hlist_bl_head *hbl) +{ + hlist_bl_lock(hbl); + hlist_bl_add_head(node, hbl); + hlist_bl_unlock(hbl); +} + +static inline void au_hbl_del(struct hlist_bl_node *node, + struct hlist_bl_head *hbl) +{ + hlist_bl_lock(hbl); + hlist_bl_del(node); + hlist_bl_unlock(hbl); +} + +#define au_hbl_for_each(pos, head) \ + for (pos = hlist_bl_first(head); \ + pos; \ + pos = pos->next) + +static inline unsigned long au_hbl_count(struct hlist_bl_head *hbl) +{ + unsigned long cnt; + struct hlist_bl_node *pos; + + cnt = 0; + hlist_bl_lock(hbl); + au_hbl_for_each(pos, hbl) + cnt++; + hlist_bl_unlock(hbl); + return cnt; +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_HBL_H__ */ diff --git a/fs/aufs/hfsnotify.c b/fs/aufs/hfsnotify.c new file mode 100644 index 0000000000000..2e0543f7cc9f3 --- /dev/null +++ b/fs/aufs/hfsnotify.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * fsnotify for the lower directories + */ + +#include "aufs.h" + +/* FS_IN_IGNORED is unnecessary */ +static const __u32 AuHfsnMask = (FS_MOVED_TO | FS_MOVED_FROM | FS_DELETE + | FS_CREATE | FS_EVENT_ON_CHILD); +static DECLARE_WAIT_QUEUE_HEAD(au_hfsn_wq); +static __cacheline_aligned_in_smp atomic64_t au_hfsn_ifree = ATOMIC64_INIT(0); + +static void au_hfsn_free_mark(struct fsnotify_mark *mark) +{ + struct au_hnotify *hn = container_of(mark, struct au_hnotify, + hn_mark); + /* AuDbg("here\n"); */ + au_cache_free_hnotify(hn); + smp_mb__before_atomic(); /* for atomic64_dec */ + if (atomic64_dec_and_test(&au_hfsn_ifree)) + wake_up(&au_hfsn_wq); +} + +static int au_hfsn_alloc(struct au_hinode *hinode) +{ + int err; + struct au_hnotify *hn; + struct super_block *sb; + struct au_branch *br; + struct fsnotify_mark *mark; + aufs_bindex_t bindex; + + hn = hinode->hi_notify; + sb = hn->hn_aufs_inode->i_sb; + bindex = au_br_index(sb, hinode->hi_id); + br = au_sbr(sb, bindex); + AuDebugOn(!br->br_hfsn); + + mark = &hn->hn_mark; + fsnotify_init_mark(mark, br->br_hfsn->hfsn_group); + mark->mask = AuHfsnMask; + /* + * by udba rename or rmdir, aufs assign a new inode to the known + * h_inode, so specify 1 to allow dups. + */ + lockdep_off(); + err = fsnotify_add_inode_mark(mark, hinode->hi_inode, /*allow_dups*/1); + lockdep_on(); + + return err; +} + +static int au_hfsn_free(struct au_hinode *hinode, struct au_hnotify *hn) +{ + struct fsnotify_mark *mark; + unsigned long long ull; + struct fsnotify_group *group; + + ull = atomic64_inc_return(&au_hfsn_ifree); + BUG_ON(!ull); + + mark = &hn->hn_mark; + spin_lock(&mark->lock); + group = mark->group; + fsnotify_get_group(group); + spin_unlock(&mark->lock); + lockdep_off(); + fsnotify_destroy_mark(mark, group); + fsnotify_put_mark(mark); + fsnotify_put_group(group); + lockdep_on(); + + /* free hn by myself */ + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static void au_hfsn_ctl(struct au_hinode *hinode, int do_set) +{ + struct fsnotify_mark *mark; + + mark = &hinode->hi_notify->hn_mark; + spin_lock(&mark->lock); + if (do_set) { + AuDebugOn(mark->mask & AuHfsnMask); + mark->mask |= AuHfsnMask; + } else { + AuDebugOn(!(mark->mask & AuHfsnMask)); + mark->mask &= ~AuHfsnMask; + } + spin_unlock(&mark->lock); + /* fsnotify_recalc_inode_mask(hinode->hi_inode); */ +} + +/* ---------------------------------------------------------------------- */ + +/* #define AuDbgHnotify */ +#ifdef AuDbgHnotify +static char *au_hfsn_name(u32 mask) +{ +#ifdef CONFIG_AUFS_DEBUG +#define test_ret(flag) \ + do { \ + if (mask & flag) \ + return #flag; \ + } while (0) + test_ret(FS_ACCESS); + test_ret(FS_MODIFY); + test_ret(FS_ATTRIB); + test_ret(FS_CLOSE_WRITE); + test_ret(FS_CLOSE_NOWRITE); + test_ret(FS_OPEN); + test_ret(FS_MOVED_FROM); + test_ret(FS_MOVED_TO); + test_ret(FS_CREATE); + test_ret(FS_DELETE); + test_ret(FS_DELETE_SELF); + test_ret(FS_MOVE_SELF); + test_ret(FS_UNMOUNT); + test_ret(FS_Q_OVERFLOW); + test_ret(FS_IN_IGNORED); + test_ret(FS_ISDIR); + test_ret(FS_IN_ONESHOT); + test_ret(FS_EVENT_ON_CHILD); + return ""; +#undef test_ret +#else + return "??"; +#endif +} +#endif + +/* ---------------------------------------------------------------------- */ + +static void au_hfsn_free_group(struct fsnotify_group *group) +{ + struct au_br_hfsnotify *hfsn = group->private; + + /* AuDbg("here\n"); */ + au_kfree_try_rcu(hfsn); +} + +static int au_hfsn_handle_event(struct fsnotify_group *group, + struct inode *inode, + u32 mask, const void *data, int data_type, + const unsigned char *file_name, u32 cookie, + struct fsnotify_iter_info *iter_info) +{ + int err; + struct au_hnotify *hnotify; + struct inode *h_dir, *h_inode; + struct qstr h_child_qstr = QSTR_INIT(file_name, strlen(file_name)); + struct fsnotify_mark *inode_mark; + + AuDebugOn(data_type != FSNOTIFY_EVENT_INODE); + + err = 0; + /* if FS_UNMOUNT happens, there must be another bug */ + AuDebugOn(mask & FS_UNMOUNT); + if (mask & (FS_IN_IGNORED | FS_UNMOUNT)) + goto out; + + h_dir = inode; + h_inode = NULL; +#ifdef AuDbgHnotify + au_debug_on(); + if (1 || h_child_qstr.len != sizeof(AUFS_XINO_FNAME) - 1 + || strncmp(h_child_qstr.name, AUFS_XINO_FNAME, h_child_qstr.len)) { + AuDbg("i%lu, mask 0x%x %s, hcname %.*s, hi%lu\n", + h_dir->i_ino, mask, au_hfsn_name(mask), + AuLNPair(&h_child_qstr), h_inode ? h_inode->i_ino : 0); + /* WARN_ON(1); */ + } + au_debug_off(); +#endif + + inode_mark = fsnotify_iter_inode_mark(iter_info); + AuDebugOn(!inode_mark); + hnotify = container_of(inode_mark, struct au_hnotify, hn_mark); + err = au_hnotify(h_dir, hnotify, mask, &h_child_qstr, h_inode); + +out: + return err; +} + +static struct fsnotify_ops au_hfsn_ops = { + .handle_event = au_hfsn_handle_event, + .free_group_priv = au_hfsn_free_group, + .free_mark = au_hfsn_free_mark +}; + +/* ---------------------------------------------------------------------- */ + +static void au_hfsn_fin_br(struct au_branch *br) +{ + struct au_br_hfsnotify *hfsn; + + hfsn = br->br_hfsn; + if (hfsn) { + lockdep_off(); + fsnotify_put_group(hfsn->hfsn_group); + lockdep_on(); + } +} + +static int au_hfsn_init_br(struct au_branch *br, int perm) +{ + int err; + struct fsnotify_group *group; + struct au_br_hfsnotify *hfsn; + + err = 0; + br->br_hfsn = NULL; + if (!au_br_hnotifyable(perm)) + goto out; + + err = -ENOMEM; + hfsn = kmalloc(sizeof(*hfsn), GFP_NOFS); + if (unlikely(!hfsn)) + goto out; + + err = 0; + group = fsnotify_alloc_group(&au_hfsn_ops); + if (IS_ERR(group)) { + err = PTR_ERR(group); + pr_err("fsnotify_alloc_group() failed, %d\n", err); + goto out_hfsn; + } + + group->private = hfsn; + hfsn->hfsn_group = group; + br->br_hfsn = hfsn; + goto out; /* success */ + +out_hfsn: + au_kfree_try_rcu(hfsn); +out: + return err; +} + +static int au_hfsn_reset_br(unsigned int udba, struct au_branch *br, int perm) +{ + int err; + + err = 0; + if (!br->br_hfsn) + err = au_hfsn_init_br(br, perm); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static void au_hfsn_fin(void) +{ + AuDbg("au_hfsn_ifree %lld\n", (long long)atomic64_read(&au_hfsn_ifree)); + wait_event(au_hfsn_wq, !atomic64_read(&au_hfsn_ifree)); +} + +const struct au_hnotify_op au_hnotify_op = { + .ctl = au_hfsn_ctl, + .alloc = au_hfsn_alloc, + .free = au_hfsn_free, + + .fin = au_hfsn_fin, + + .reset_br = au_hfsn_reset_br, + .fin_br = au_hfsn_fin_br, + .init_br = au_hfsn_init_br +}; diff --git a/fs/aufs/hfsplus.c b/fs/aufs/hfsplus.c new file mode 100644 index 0000000000000..d250f019bedfe --- /dev/null +++ b/fs/aufs/hfsplus.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * special support for filesystems which acquires an inode mutex + * at final closing a file, eg, hfsplus. + * + * This trick is very simple and stupid, just to open the file before really + * necessary open to tell hfsplus that this is not the final closing. + * The caller should call au_h_open_pre() after acquiring the inode mutex, + * and au_h_open_post() after releasing it. + */ + +#include "aufs.h" + +struct file *au_h_open_pre(struct dentry *dentry, aufs_bindex_t bindex, + int force_wr) +{ + struct file *h_file; + struct dentry *h_dentry; + + h_dentry = au_h_dptr(dentry, bindex); + AuDebugOn(!h_dentry); + AuDebugOn(d_is_negative(h_dentry)); + + h_file = NULL; + if (au_test_hfsplus(h_dentry->d_sb) + && d_is_reg(h_dentry)) + h_file = au_h_open(dentry, bindex, + O_RDONLY | O_NOATIME | O_LARGEFILE, + /*file*/NULL, force_wr); + return h_file; +} + +void au_h_open_post(struct dentry *dentry, aufs_bindex_t bindex, + struct file *h_file) +{ + struct au_branch *br; + + if (h_file) { + fput(h_file); + br = au_sbr(dentry->d_sb, bindex); + au_lcnt_dec(&br->br_nfiles); + } +} diff --git a/fs/aufs/hnotify.c b/fs/aufs/hnotify.c new file mode 100644 index 0000000000000..e42e2a201d050 --- /dev/null +++ b/fs/aufs/hnotify.c @@ -0,0 +1,715 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * abstraction to notify the direct changes on lower directories + */ + +/* #include */ +#include "aufs.h" + +int au_hn_alloc(struct au_hinode *hinode, struct inode *inode) +{ + int err; + struct au_hnotify *hn; + + err = -ENOMEM; + hn = au_cache_alloc_hnotify(); + if (hn) { + hn->hn_aufs_inode = inode; + hinode->hi_notify = hn; + err = au_hnotify_op.alloc(hinode); + AuTraceErr(err); + if (unlikely(err)) { + hinode->hi_notify = NULL; + au_cache_free_hnotify(hn); + /* + * The upper dir was removed by udba, but the same named + * dir left. In this case, aufs assigns a new inode + * number and set the monitor again. + * For the lower dir, the old monitor is still left. + */ + if (err == -EEXIST) + err = 0; + } + } + + AuTraceErr(err); + return err; +} + +void au_hn_free(struct au_hinode *hinode) +{ + struct au_hnotify *hn; + + hn = hinode->hi_notify; + if (hn) { + hinode->hi_notify = NULL; + if (au_hnotify_op.free(hinode, hn)) + au_cache_free_hnotify(hn); + } +} + +/* ---------------------------------------------------------------------- */ + +void au_hn_ctl(struct au_hinode *hinode, int do_set) +{ + if (hinode->hi_notify) + au_hnotify_op.ctl(hinode, do_set); +} + +void au_hn_reset(struct inode *inode, unsigned int flags) +{ + aufs_bindex_t bindex, bbot; + struct inode *hi; + struct dentry *iwhdentry; + + bbot = au_ibbot(inode); + for (bindex = au_ibtop(inode); bindex <= bbot; bindex++) { + hi = au_h_iptr(inode, bindex); + if (!hi) + continue; + + /* inode_lock_nested(hi, AuLsc_I_CHILD); */ + iwhdentry = au_hi_wh(inode, bindex); + if (iwhdentry) + dget(iwhdentry); + au_igrab(hi); + au_set_h_iptr(inode, bindex, NULL, 0); + au_set_h_iptr(inode, bindex, au_igrab(hi), + flags & ~AuHi_XINO); + iput(hi); + dput(iwhdentry); + /* inode_unlock(hi); */ + } +} + +/* ---------------------------------------------------------------------- */ + +static int hn_xino(struct inode *inode, struct inode *h_inode) +{ + int err; + aufs_bindex_t bindex, bbot, bfound, btop; + struct inode *h_i; + + err = 0; + if (unlikely(inode->i_ino == AUFS_ROOT_INO)) { + pr_warn("branch root dir was changed\n"); + goto out; + } + + bfound = -1; + bbot = au_ibbot(inode); + btop = au_ibtop(inode); +#if 0 /* reserved for future use */ + if (bindex == bbot) { + /* keep this ino in rename case */ + goto out; + } +#endif + for (bindex = btop; bindex <= bbot; bindex++) + if (au_h_iptr(inode, bindex) == h_inode) { + bfound = bindex; + break; + } + if (bfound < 0) + goto out; + + for (bindex = btop; bindex <= bbot; bindex++) { + h_i = au_h_iptr(inode, bindex); + if (!h_i) + continue; + + err = au_xino_write(inode->i_sb, bindex, h_i->i_ino, /*ino*/0); + /* ignore this error */ + /* bad action? */ + } + + /* children inode number will be broken */ + +out: + AuTraceErr(err); + return err; +} + +static int hn_gen_tree(struct dentry *dentry) +{ + int err, i, j, ndentry; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry **dentries; + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_dcsub_pages(&dpages, dentry, NULL, NULL); + if (unlikely(err)) + goto out_dpages; + + for (i = 0; i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + dentries = dpage->dentries; + ndentry = dpage->ndentry; + for (j = 0; j < ndentry; j++) { + struct dentry *d; + + d = dentries[j]; + if (IS_ROOT(d)) + continue; + + au_digen_dec(d); + if (d_really_is_positive(d)) + /* todo: reset children xino? + cached children only? */ + au_iigen_dec(d_inode(d)); + } + } + +out_dpages: + au_dpages_free(&dpages); +out: + return err; +} + +/* + * return 0 if processed. + */ +static int hn_gen_by_inode(char *name, unsigned int nlen, struct inode *inode, + const unsigned int isdir) +{ + int err; + struct dentry *d; + struct qstr *dname; + + err = 1; + if (unlikely(inode->i_ino == AUFS_ROOT_INO)) { + pr_warn("branch root dir was changed\n"); + err = 0; + goto out; + } + + if (!isdir) { + AuDebugOn(!name); + au_iigen_dec(inode); + spin_lock(&inode->i_lock); + hlist_for_each_entry(d, &inode->i_dentry, d_u.d_alias) { + spin_lock(&d->d_lock); + dname = &d->d_name; + if (dname->len != nlen + && memcmp(dname->name, name, nlen)) { + spin_unlock(&d->d_lock); + continue; + } + err = 0; + au_digen_dec(d); + spin_unlock(&d->d_lock); + break; + } + spin_unlock(&inode->i_lock); + } else { + au_fset_si(au_sbi(inode->i_sb), FAILED_REFRESH_DIR); + d = d_find_any_alias(inode); + if (!d) { + au_iigen_dec(inode); + goto out; + } + + spin_lock(&d->d_lock); + dname = &d->d_name; + if (dname->len == nlen && !memcmp(dname->name, name, nlen)) { + spin_unlock(&d->d_lock); + err = hn_gen_tree(d); + spin_lock(&d->d_lock); + } + spin_unlock(&d->d_lock); + dput(d); + } + +out: + AuTraceErr(err); + return err; +} + +static int hn_gen_by_name(struct dentry *dentry, const unsigned int isdir) +{ + int err; + + if (IS_ROOT(dentry)) { + pr_warn("branch root dir was changed\n"); + return 0; + } + + err = 0; + if (!isdir) { + au_digen_dec(dentry); + if (d_really_is_positive(dentry)) + au_iigen_dec(d_inode(dentry)); + } else { + au_fset_si(au_sbi(dentry->d_sb), FAILED_REFRESH_DIR); + if (d_really_is_positive(dentry)) + err = hn_gen_tree(dentry); + } + + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* hnotify job flags */ +#define AuHnJob_XINO0 1 +#define AuHnJob_GEN (1 << 1) +#define AuHnJob_DIRENT (1 << 2) +#define AuHnJob_ISDIR (1 << 3) +#define AuHnJob_TRYXINO0 (1 << 4) +#define AuHnJob_MNTPNT (1 << 5) +#define au_ftest_hnjob(flags, name) ((flags) & AuHnJob_##name) +#define au_fset_hnjob(flags, name) \ + do { (flags) |= AuHnJob_##name; } while (0) +#define au_fclr_hnjob(flags, name) \ + do { (flags) &= ~AuHnJob_##name; } while (0) + +enum { + AuHn_CHILD, + AuHn_PARENT, + AuHnLast +}; + +struct au_hnotify_args { + struct inode *h_dir, *dir, *h_child_inode; + u32 mask; + unsigned int flags[AuHnLast]; + unsigned int h_child_nlen; + char h_child_name[]; +}; + +struct hn_job_args { + unsigned int flags; + struct inode *inode, *h_inode, *dir, *h_dir; + struct dentry *dentry; + char *h_name; + int h_nlen; +}; + +static int hn_job(struct hn_job_args *a) +{ + const unsigned int isdir = au_ftest_hnjob(a->flags, ISDIR); + int e; + + /* reset xino */ + if (au_ftest_hnjob(a->flags, XINO0) && a->inode) + hn_xino(a->inode, a->h_inode); /* ignore this error */ + + if (au_ftest_hnjob(a->flags, TRYXINO0) + && a->inode + && a->h_inode) { + inode_lock_shared_nested(a->h_inode, AuLsc_I_CHILD); + if (!a->h_inode->i_nlink + && !(a->h_inode->i_state & I_LINKABLE)) + hn_xino(a->inode, a->h_inode); /* ignore this error */ + inode_unlock_shared(a->h_inode); + } + + /* make the generation obsolete */ + if (au_ftest_hnjob(a->flags, GEN)) { + e = -1; + if (a->inode) + e = hn_gen_by_inode(a->h_name, a->h_nlen, a->inode, + isdir); + if (e && a->dentry) + hn_gen_by_name(a->dentry, isdir); + /* ignore this error */ + } + + /* make dir entries obsolete */ + if (au_ftest_hnjob(a->flags, DIRENT) && a->inode) { + struct au_vdir *vdir; + + vdir = au_ivdir(a->inode); + if (vdir) + vdir->vd_jiffy = 0; + /* IMustLock(a->inode); */ + /* inode_inc_iversion(a->inode); */ + } + + /* can do nothing but warn */ + if (au_ftest_hnjob(a->flags, MNTPNT) + && a->dentry + && d_mountpoint(a->dentry)) + pr_warn("mount-point %pd is removed or renamed\n", a->dentry); + + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry *lookup_wlock_by_name(char *name, unsigned int nlen, + struct inode *dir) +{ + struct dentry *dentry, *d, *parent; + struct qstr *dname; + + parent = d_find_any_alias(dir); + if (!parent) + return NULL; + + dentry = NULL; + spin_lock(&parent->d_lock); + list_for_each_entry(d, &parent->d_subdirs, d_child) { + /* AuDbg("%pd\n", d); */ + spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED); + dname = &d->d_name; + if (dname->len != nlen || memcmp(dname->name, name, nlen)) + goto cont_unlock; + if (au_di(d)) + au_digen_dec(d); + else + goto cont_unlock; + if (au_dcount(d) > 0) { + dentry = dget_dlock(d); + spin_unlock(&d->d_lock); + break; + } + +cont_unlock: + spin_unlock(&d->d_lock); + } + spin_unlock(&parent->d_lock); + dput(parent); + + if (dentry) + di_write_lock_child(dentry); + + return dentry; +} + +static struct inode *lookup_wlock_by_ino(struct super_block *sb, + aufs_bindex_t bindex, ino_t h_ino) +{ + struct inode *inode; + ino_t ino; + int err; + + inode = NULL; + err = au_xino_read(sb, bindex, h_ino, &ino); + if (!err && ino) + inode = ilookup(sb, ino); + if (!inode) + goto out; + + if (unlikely(inode->i_ino == AUFS_ROOT_INO)) { + pr_warn("wrong root branch\n"); + iput(inode); + inode = NULL; + goto out; + } + + ii_write_lock_child(inode); + +out: + return inode; +} + +static void au_hn_bh(void *_args) +{ + struct au_hnotify_args *a = _args; + struct super_block *sb; + aufs_bindex_t bindex, bbot, bfound; + unsigned char xino, try_iput; + int err; + struct inode *inode; + ino_t h_ino; + struct hn_job_args args; + struct dentry *dentry; + struct au_sbinfo *sbinfo; + + AuDebugOn(!_args); + AuDebugOn(!a->h_dir); + AuDebugOn(!a->dir); + AuDebugOn(!a->mask); + AuDbg("mask 0x%x, i%lu, hi%lu, hci%lu\n", + a->mask, a->dir->i_ino, a->h_dir->i_ino, + a->h_child_inode ? a->h_child_inode->i_ino : 0); + + inode = NULL; + dentry = NULL; + /* + * do not lock a->dir->i_mutex here + * because of d_revalidate() may cause a deadlock. + */ + sb = a->dir->i_sb; + AuDebugOn(!sb); + sbinfo = au_sbi(sb); + AuDebugOn(!sbinfo); + si_write_lock(sb, AuLock_NOPLMW); + + if (au_opt_test(sbinfo->si_mntflags, DIRREN)) + switch (a->mask & FS_EVENTS_POSS_ON_CHILD) { + case FS_MOVED_FROM: + case FS_MOVED_TO: + AuWarn1("DIRREN with UDBA may not work correctly " + "for the direct rename(2)\n"); + } + + ii_read_lock_parent(a->dir); + bfound = -1; + bbot = au_ibbot(a->dir); + for (bindex = au_ibtop(a->dir); bindex <= bbot; bindex++) + if (au_h_iptr(a->dir, bindex) == a->h_dir) { + bfound = bindex; + break; + } + ii_read_unlock(a->dir); + if (unlikely(bfound < 0)) + goto out; + + xino = !!au_opt_test(au_mntflags(sb), XINO); + h_ino = 0; + if (a->h_child_inode) + h_ino = a->h_child_inode->i_ino; + + if (a->h_child_nlen + && (au_ftest_hnjob(a->flags[AuHn_CHILD], GEN) + || au_ftest_hnjob(a->flags[AuHn_CHILD], MNTPNT))) + dentry = lookup_wlock_by_name(a->h_child_name, a->h_child_nlen, + a->dir); + try_iput = 0; + if (dentry && d_really_is_positive(dentry)) + inode = d_inode(dentry); + if (xino && !inode && h_ino + && (au_ftest_hnjob(a->flags[AuHn_CHILD], XINO0) + || au_ftest_hnjob(a->flags[AuHn_CHILD], TRYXINO0) + || au_ftest_hnjob(a->flags[AuHn_CHILD], GEN))) { + inode = lookup_wlock_by_ino(sb, bfound, h_ino); + try_iput = 1; + } + + args.flags = a->flags[AuHn_CHILD]; + args.dentry = dentry; + args.inode = inode; + args.h_inode = a->h_child_inode; + args.dir = a->dir; + args.h_dir = a->h_dir; + args.h_name = a->h_child_name; + args.h_nlen = a->h_child_nlen; + err = hn_job(&args); + if (dentry) { + if (au_di(dentry)) + di_write_unlock(dentry); + dput(dentry); + } + if (inode && try_iput) { + ii_write_unlock(inode); + iput(inode); + } + + ii_write_lock_parent(a->dir); + args.flags = a->flags[AuHn_PARENT]; + args.dentry = NULL; + args.inode = a->dir; + args.h_inode = a->h_dir; + args.dir = NULL; + args.h_dir = NULL; + args.h_name = NULL; + args.h_nlen = 0; + err = hn_job(&args); + ii_write_unlock(a->dir); + +out: + iput(a->h_child_inode); + iput(a->h_dir); + iput(a->dir); + si_write_unlock(sb); + au_nwt_done(&sbinfo->si_nowait); + au_kfree_rcu(a); +} + +/* ---------------------------------------------------------------------- */ + +int au_hnotify(struct inode *h_dir, struct au_hnotify *hnotify, u32 mask, + struct qstr *h_child_qstr, struct inode *h_child_inode) +{ + int err, len; + unsigned int flags[AuHnLast], f; + unsigned char isdir, isroot, wh; + struct inode *dir; + struct au_hnotify_args *args; + char *p, *h_child_name; + + err = 0; + AuDebugOn(!hnotify || !hnotify->hn_aufs_inode); + dir = igrab(hnotify->hn_aufs_inode); + if (!dir) + goto out; + + isroot = (dir->i_ino == AUFS_ROOT_INO); + wh = 0; + h_child_name = (void *)h_child_qstr->name; + len = h_child_qstr->len; + if (h_child_name) { + if (len > AUFS_WH_PFX_LEN + && !memcmp(h_child_name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) { + h_child_name += AUFS_WH_PFX_LEN; + len -= AUFS_WH_PFX_LEN; + wh = 1; + } + } + + isdir = 0; + if (h_child_inode) + isdir = !!S_ISDIR(h_child_inode->i_mode); + flags[AuHn_PARENT] = AuHnJob_ISDIR; + flags[AuHn_CHILD] = 0; + if (isdir) + flags[AuHn_CHILD] = AuHnJob_ISDIR; + au_fset_hnjob(flags[AuHn_PARENT], DIRENT); + au_fset_hnjob(flags[AuHn_CHILD], GEN); + switch (mask & FS_EVENTS_POSS_ON_CHILD) { + case FS_MOVED_FROM: + case FS_MOVED_TO: + au_fset_hnjob(flags[AuHn_CHILD], XINO0); + au_fset_hnjob(flags[AuHn_CHILD], MNTPNT); + /*FALLTHROUGH*/ + case FS_CREATE: + AuDebugOn(!h_child_name); + break; + + case FS_DELETE: + /* + * aufs never be able to get this child inode. + * revalidation should be in d_revalidate() + * by checking i_nlink, i_generation or d_unhashed(). + */ + AuDebugOn(!h_child_name); + au_fset_hnjob(flags[AuHn_CHILD], TRYXINO0); + au_fset_hnjob(flags[AuHn_CHILD], MNTPNT); + break; + + default: + AuDebugOn(1); + } + + if (wh) + h_child_inode = NULL; + + err = -ENOMEM; + /* iput() and kfree() will be called in au_hnotify() */ + args = kmalloc(sizeof(*args) + len + 1, GFP_NOFS); + if (unlikely(!args)) { + AuErr1("no memory\n"); + iput(dir); + goto out; + } + args->flags[AuHn_PARENT] = flags[AuHn_PARENT]; + args->flags[AuHn_CHILD] = flags[AuHn_CHILD]; + args->mask = mask; + args->dir = dir; + args->h_dir = igrab(h_dir); + if (h_child_inode) + h_child_inode = igrab(h_child_inode); /* can be NULL */ + args->h_child_inode = h_child_inode; + args->h_child_nlen = len; + if (len) { + p = (void *)args; + p += sizeof(*args); + memcpy(p, h_child_name, len); + p[len] = 0; + } + + /* NFS fires the event for silly-renamed one from kworker */ + f = 0; + if (!dir->i_nlink + || (au_test_nfs(h_dir->i_sb) && (mask & FS_DELETE))) + f = AuWkq_NEST; + err = au_wkq_nowait(au_hn_bh, args, dir->i_sb, f); + if (unlikely(err)) { + pr_err("wkq %d\n", err); + iput(args->h_child_inode); + iput(args->h_dir); + iput(args->dir); + au_kfree_rcu(args); + } + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_hnotify_reset_br(unsigned int udba, struct au_branch *br, int perm) +{ + int err; + + AuDebugOn(!(udba & AuOptMask_UDBA)); + + err = 0; + if (au_hnotify_op.reset_br) + err = au_hnotify_op.reset_br(udba, br, perm); + + return err; +} + +int au_hnotify_init_br(struct au_branch *br, int perm) +{ + int err; + + err = 0; + if (au_hnotify_op.init_br) + err = au_hnotify_op.init_br(br, perm); + + return err; +} + +void au_hnotify_fin_br(struct au_branch *br) +{ + if (au_hnotify_op.fin_br) + au_hnotify_op.fin_br(br); +} + +static void au_hn_destroy_cache(void) +{ + kmem_cache_destroy(au_cache[AuCache_HNOTIFY]); + au_cache[AuCache_HNOTIFY] = NULL; +} + +int __init au_hnotify_init(void) +{ + int err; + + err = -ENOMEM; + au_cache[AuCache_HNOTIFY] = AuCache(au_hnotify); + if (au_cache[AuCache_HNOTIFY]) { + err = 0; + if (au_hnotify_op.init) + err = au_hnotify_op.init(); + if (unlikely(err)) + au_hn_destroy_cache(); + } + AuTraceErr(err); + return err; +} + +void au_hnotify_fin(void) +{ + if (au_hnotify_op.fin) + au_hnotify_op.fin(); + + /* cf. au_cache_fin() */ + if (au_cache[AuCache_HNOTIFY]) + au_hn_destroy_cache(); +} diff --git a/fs/aufs/i_op.c b/fs/aufs/i_op.c new file mode 100644 index 0000000000000..fd9953e620801 --- /dev/null +++ b/fs/aufs/i_op.c @@ -0,0 +1,1505 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operations (except add/del/rename) + */ + +#include +#include +#include +#include +#include +#include "aufs.h" + +static int h_permission(struct inode *h_inode, int mask, + struct path *h_path, int brperm) +{ + int err; + const unsigned char write_mask = !!(mask & (MAY_WRITE | MAY_APPEND)); + + err = -EPERM; + if (write_mask && IS_IMMUTABLE(h_inode)) + goto out; + + err = -EACCES; + if (((mask & MAY_EXEC) + && S_ISREG(h_inode->i_mode) + && (path_noexec(h_path) + || !(h_inode->i_mode & 0111)))) + goto out; + + /* + * - skip the lower fs test in the case of write to ro branch. + * - nfs dir permission write check is optimized, but a policy for + * link/rename requires a real check. + * - nfs always sets SB_POSIXACL regardless its mount option 'noacl.' + * in this case, generic_permission() returns -EOPNOTSUPP. + */ + if ((write_mask && !au_br_writable(brperm)) + || (au_test_nfs(h_inode->i_sb) && S_ISDIR(h_inode->i_mode) + && write_mask && !(mask & MAY_READ)) + || !h_inode->i_op->permission) { + /* AuLabel(generic_permission); */ + /* AuDbg("get_acl %ps\n", h_inode->i_op->get_acl); */ + err = generic_permission(h_inode, mask); + if (err == -EOPNOTSUPP && au_test_nfs_noacl(h_inode)) + err = h_inode->i_op->permission(h_inode, mask); + AuTraceErr(err); + } else { + /* AuLabel(h_inode->permission); */ + err = h_inode->i_op->permission(h_inode, mask); + AuTraceErr(err); + } + + if (!err) + err = devcgroup_inode_permission(h_inode, mask); + if (!err) + err = security_inode_permission(h_inode, mask); + +out: + return err; +} + +static int aufs_permission(struct inode *inode, int mask) +{ + int err; + aufs_bindex_t bindex, bbot; + const unsigned char isdir = !!S_ISDIR(inode->i_mode), + write_mask = !!(mask & (MAY_WRITE | MAY_APPEND)); + struct inode *h_inode; + struct super_block *sb; + struct au_branch *br; + + /* todo: support rcu-walk? */ + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH); + ii_read_lock_child(inode); +#if 0 /* reserved for future use */ + /* + * This test may be rather 'too much' since the test is essentially done + * in the aufs_lookup(). Theoretically it is possible that the inode + * generation doesn't match to the superblock's here. But it isn't a + * big deal I suppose. + */ + err = au_iigen_test(inode, au_sigen(sb)); + if (unlikely(err)) + goto out; +#endif + + if (!isdir + || write_mask + || au_opt_test(au_mntflags(sb), DIRPERM1)) { + err = au_busy_or_stale(); + h_inode = au_h_iptr(inode, au_ibtop(inode)); + if (unlikely(!h_inode + || (h_inode->i_mode & S_IFMT) + != (inode->i_mode & S_IFMT))) + goto out; + + err = 0; + bindex = au_ibtop(inode); + br = au_sbr(sb, bindex); + err = h_permission(h_inode, mask, &br->br_path, br->br_perm); + if (write_mask + && !err + && !special_file(h_inode->i_mode)) { + /* test whether the upper writable branch exists */ + err = -EROFS; + for (; bindex >= 0; bindex--) + if (!au_br_rdonly(au_sbr(sb, bindex))) { + err = 0; + break; + } + } + goto out; + } + + /* non-write to dir */ + err = 0; + bbot = au_ibbot(inode); + for (bindex = au_ibtop(inode); !err && bindex <= bbot; bindex++) { + h_inode = au_h_iptr(inode, bindex); + if (h_inode) { + err = au_busy_or_stale(); + if (unlikely(!S_ISDIR(h_inode->i_mode))) + break; + + br = au_sbr(sb, bindex); + err = h_permission(h_inode, mask, &br->br_path, + br->br_perm); + } + } + +out: + ii_read_unlock(inode); + si_read_unlock(sb); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry *aufs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct dentry *ret, *parent; + struct inode *inode; + struct super_block *sb; + int err, npositive; + + IMustLock(dir); + + /* todo: support rcu-walk? */ + ret = ERR_PTR(-ECHILD); + if (flags & LOOKUP_RCU) + goto out; + + ret = ERR_PTR(-ENAMETOOLONG); + if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN)) + goto out; + + sb = dir->i_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + ret = ERR_PTR(err); + if (unlikely(err)) + goto out; + + err = au_di_init(dentry); + ret = ERR_PTR(err); + if (unlikely(err)) + goto out_si; + + inode = NULL; + npositive = 0; /* suppress a warning */ + parent = dentry->d_parent; /* dir inode is locked */ + di_read_lock_parent(parent, AuLock_IR); + err = au_alive_dir(parent); + if (!err) + err = au_digen_test(parent, au_sigen(sb)); + if (!err) { + /* regardless LOOKUP_CREATE, always ALLOW_NEG */ + npositive = au_lkup_dentry(dentry, au_dbtop(parent), + AuLkup_ALLOW_NEG); + err = npositive; + } + di_read_unlock(parent, AuLock_IR); + ret = ERR_PTR(err); + if (unlikely(err < 0)) + goto out_unlock; + + if (npositive) { + inode = au_new_inode(dentry, /*must_new*/0); + if (IS_ERR(inode)) { + ret = (void *)inode; + inode = NULL; + goto out_unlock; + } + } + + if (inode) + atomic_inc(&inode->i_count); + ret = d_splice_alias(inode, dentry); +#if 0 /* reserved for future use */ + if (unlikely(d_need_lookup(dentry))) { + spin_lock(&dentry->d_lock); + dentry->d_flags &= ~DCACHE_NEED_LOOKUP; + spin_unlock(&dentry->d_lock); + } else +#endif + if (inode) { + if (!IS_ERR(ret)) { + iput(inode); + if (ret && ret != dentry) + ii_write_unlock(inode); + } else { + ii_write_unlock(inode); + iput(inode); + inode = NULL; + } + } + +out_unlock: + di_write_unlock(dentry); +out_si: + si_read_unlock(sb); +out: + return ret; +} + +/* ---------------------------------------------------------------------- */ + +/* + * very dirty and complicated aufs ->atomic_open(). + * aufs_atomic_open() + * + au_aopen_or_create() + * + add_simple() + * + vfsub_atomic_open() + * + branch fs ->atomic_open() + * may call the actual 'open' for h_file + * + inc br_nfiles only if opened + * + au_aopen_no_open() or au_aopen_do_open() + * + * au_aopen_do_open() + * + finish_open() + * + au_do_aopen() + * + au_do_open() the body of all 'open' + * + au_do_open_nondir() + * set the passed h_file + * + * au_aopen_no_open() + * + finish_no_open() + */ + +struct aopen_node { + struct hlist_bl_node hblist; + struct file *file, *h_file; +}; + +static int au_do_aopen(struct inode *inode, struct file *file) +{ + struct hlist_bl_head *aopen; + struct hlist_bl_node *pos; + struct aopen_node *node; + struct au_do_open_args args = { + .aopen = 1, + .open = au_do_open_nondir + }; + + aopen = &au_sbi(inode->i_sb)->si_aopen; + hlist_bl_lock(aopen); + hlist_bl_for_each_entry(node, pos, aopen, hblist) + if (node->file == file) { + args.h_file = node->h_file; + break; + } + hlist_bl_unlock(aopen); + /* AuDebugOn(!args.h_file); */ + + return au_do_open(file, &args); +} + +static int au_aopen_do_open(struct file *file, struct dentry *dentry, + struct aopen_node *aopen_node) +{ + int err; + struct hlist_bl_head *aopen; + + AuLabel(here); + aopen = &au_sbi(dentry->d_sb)->si_aopen; + au_hbl_add(&aopen_node->hblist, aopen); + err = finish_open(file, dentry, au_do_aopen); + au_hbl_del(&aopen_node->hblist, aopen); + /* AuDbgFile(file); */ + AuDbg("%pd%s%s\n", dentry, + (file->f_mode & FMODE_CREATED) ? " created" : "", + (file->f_mode & FMODE_OPENED) ? " opened" : ""); + + AuTraceErr(err); + return err; +} + +static int au_aopen_no_open(struct file *file, struct dentry *dentry) +{ + int err; + + AuLabel(here); + dget(dentry); + err = finish_no_open(file, dentry); + + AuTraceErr(err); + return err; +} + +static int aufs_atomic_open(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned int open_flag, + umode_t create_mode) +{ + int err, did_open; + unsigned int lkup_flags; + aufs_bindex_t bindex; + struct super_block *sb; + struct dentry *parent, *d; + struct vfsub_aopen_args args = { + .open_flag = open_flag, + .create_mode = create_mode + }; + struct aopen_node aopen_node = { + .file = file + }; + + IMustLock(dir); + AuDbg("open_flag 0%o\n", open_flag); + AuDbgDentry(dentry); + + err = 0; + if (!au_di(dentry)) { + lkup_flags = LOOKUP_OPEN; + if (open_flag & O_CREAT) + lkup_flags |= LOOKUP_CREATE; + d = aufs_lookup(dir, dentry, lkup_flags); + if (IS_ERR(d)) { + err = PTR_ERR(d); + AuTraceErr(err); + goto out; + } else if (d) { + /* + * obsoleted dentry found. + * another error will be returned later. + */ + d_drop(d); + AuDbgDentry(d); + dput(d); + } + AuDbgDentry(dentry); + } + + if (d_is_positive(dentry) + || d_unhashed(dentry) + || d_unlinked(dentry) + || !(open_flag & O_CREAT)) { + err = au_aopen_no_open(file, dentry); + goto out; /* success */ + } + + err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_GEN); + if (unlikely(err)) + goto out; + + sb = dentry->d_sb; + parent = dentry->d_parent; /* dir is locked */ + di_write_lock_parent(parent); + err = au_lkup_dentry(dentry, /*btop*/0, AuLkup_ALLOW_NEG); + if (unlikely(err < 0)) + goto out_parent; + + AuDbgDentry(dentry); + if (d_is_positive(dentry)) { + err = au_aopen_no_open(file, dentry); + goto out_parent; /* success */ + } + + args.file = alloc_empty_file(file->f_flags, current_cred()); + err = PTR_ERR(args.file); + if (IS_ERR(args.file)) + goto out_parent; + + bindex = au_dbtop(dentry); + err = au_aopen_or_create(dir, dentry, &args); + AuTraceErr(err); + AuDbgFile(args.file); + file->f_mode = args.file->f_mode & ~FMODE_OPENED; + did_open = !!(args.file->f_mode & FMODE_OPENED); + if (!did_open) { + fput(args.file); + args.file = NULL; + } + di_write_unlock(parent); + di_write_unlock(dentry); + if (unlikely(err < 0)) { + if (args.file) + fput(args.file); + goto out_sb; + } + + if (!did_open) + err = au_aopen_no_open(file, dentry); + else { + aopen_node.h_file = args.file; + err = au_aopen_do_open(file, dentry, &aopen_node); + } + if (unlikely(err < 0)) { + if (args.file) + fput(args.file); + if (did_open) + au_lcnt_dec(&args.br->br_nfiles); + } + goto out_sb; /* success */ + +out_parent: + di_write_unlock(parent); + di_write_unlock(dentry); +out_sb: + si_read_unlock(sb); +out: + AuTraceErr(err); + AuDbgFile(file); + return err; +} + + +/* ---------------------------------------------------------------------- */ + +static int au_wr_dir_cpup(struct dentry *dentry, struct dentry *parent, + const unsigned char add_entry, aufs_bindex_t bcpup, + aufs_bindex_t btop) +{ + int err; + struct dentry *h_parent; + struct inode *h_dir; + + if (add_entry) + IMustLock(d_inode(parent)); + else + di_write_lock_parent(parent); + + err = 0; + if (!au_h_dptr(parent, bcpup)) { + if (btop > bcpup) + err = au_cpup_dirs(dentry, bcpup); + else if (btop < bcpup) + err = au_cpdown_dirs(dentry, bcpup); + else + BUG(); + } + if (!err && add_entry && !au_ftest_wrdir(add_entry, TMPFILE)) { + h_parent = au_h_dptr(parent, bcpup); + h_dir = d_inode(h_parent); + inode_lock_shared_nested(h_dir, AuLsc_I_PARENT); + err = au_lkup_neg(dentry, bcpup, /*wh*/0); + /* todo: no unlock here */ + inode_unlock_shared(h_dir); + + AuDbg("bcpup %d\n", bcpup); + if (!err) { + if (d_really_is_negative(dentry)) + au_set_h_dptr(dentry, btop, NULL); + au_update_dbrange(dentry, /*do_put_zero*/0); + } + } + + if (!add_entry) + di_write_unlock(parent); + if (!err) + err = bcpup; /* success */ + + AuTraceErr(err); + return err; +} + +/* + * decide the branch and the parent dir where we will create a new entry. + * returns new bindex or an error. + * copyup the parent dir if needed. + */ +int au_wr_dir(struct dentry *dentry, struct dentry *src_dentry, + struct au_wr_dir_args *args) +{ + int err; + unsigned int flags; + aufs_bindex_t bcpup, btop, src_btop; + const unsigned char add_entry + = au_ftest_wrdir(args->flags, ADD_ENTRY) + | au_ftest_wrdir(args->flags, TMPFILE); + struct super_block *sb; + struct dentry *parent; + struct au_sbinfo *sbinfo; + + sb = dentry->d_sb; + sbinfo = au_sbi(sb); + parent = dget_parent(dentry); + btop = au_dbtop(dentry); + bcpup = btop; + if (args->force_btgt < 0) { + if (src_dentry) { + src_btop = au_dbtop(src_dentry); + if (src_btop < btop) + bcpup = src_btop; + } else if (add_entry) { + flags = 0; + if (au_ftest_wrdir(args->flags, ISDIR)) + au_fset_wbr(flags, DIR); + err = AuWbrCreate(sbinfo, dentry, flags); + bcpup = err; + } + + if (bcpup < 0 || au_test_ro(sb, bcpup, d_inode(dentry))) { + if (add_entry) + err = AuWbrCopyup(sbinfo, dentry); + else { + if (!IS_ROOT(dentry)) { + di_read_lock_parent(parent, !AuLock_IR); + err = AuWbrCopyup(sbinfo, dentry); + di_read_unlock(parent, !AuLock_IR); + } else + err = AuWbrCopyup(sbinfo, dentry); + } + bcpup = err; + if (unlikely(err < 0)) + goto out; + } + } else { + bcpup = args->force_btgt; + AuDebugOn(au_test_ro(sb, bcpup, d_inode(dentry))); + } + + AuDbg("btop %d, bcpup %d\n", btop, bcpup); + err = bcpup; + if (bcpup == btop) + goto out; /* success */ + + /* copyup the new parent into the branch we process */ + err = au_wr_dir_cpup(dentry, parent, add_entry, bcpup, btop); + if (err >= 0) { + if (d_really_is_negative(dentry)) { + au_set_h_dptr(dentry, btop, NULL); + au_set_dbtop(dentry, bcpup); + au_set_dbbot(dentry, bcpup); + } + AuDebugOn(add_entry + && !au_ftest_wrdir(args->flags, TMPFILE) + && !au_h_dptr(dentry, bcpup)); + } + +out: + dput(parent); + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_pin_hdir_unlock(struct au_pin *p) +{ + if (p->hdir) + au_hn_inode_unlock(p->hdir); +} + +int au_pin_hdir_lock(struct au_pin *p) +{ + int err; + + err = 0; + if (!p->hdir) + goto out; + + /* even if an error happens later, keep this lock */ + au_hn_inode_lock_nested(p->hdir, p->lsc_hi); + + err = -EBUSY; + if (unlikely(p->hdir->hi_inode != d_inode(p->h_parent))) + goto out; + + err = 0; + if (p->h_dentry) + err = au_h_verify(p->h_dentry, p->udba, p->hdir->hi_inode, + p->h_parent, p->br); + +out: + return err; +} + +int au_pin_hdir_relock(struct au_pin *p) +{ + int err, i; + struct inode *h_i; + struct dentry *h_d[] = { + p->h_dentry, + p->h_parent + }; + + err = au_pin_hdir_lock(p); + if (unlikely(err)) + goto out; + + for (i = 0; !err && i < sizeof(h_d)/sizeof(*h_d); i++) { + if (!h_d[i]) + continue; + if (d_is_positive(h_d[i])) { + h_i = d_inode(h_d[i]); + err = !h_i->i_nlink; + } + } + +out: + return err; +} + +static void au_pin_hdir_set_owner(struct au_pin *p, struct task_struct *task) +{ +#if !defined(CONFIG_RWSEM_GENERIC_SPINLOCK) \ + && defined(CONFIG_RWSEM_SPIN_ON_OWNER) + p->hdir->hi_inode->i_rwsem.owner = task; +#endif +} + +void au_pin_hdir_acquire_nest(struct au_pin *p) +{ + if (p->hdir) { + rwsem_acquire_nest(&p->hdir->hi_inode->i_rwsem.dep_map, + p->lsc_hi, 0, NULL, _RET_IP_); + au_pin_hdir_set_owner(p, current); + } +} + +void au_pin_hdir_release(struct au_pin *p) +{ + if (p->hdir) { + au_pin_hdir_set_owner(p, p->task); + rwsem_release(&p->hdir->hi_inode->i_rwsem.dep_map, 1, _RET_IP_); + } +} + +struct dentry *au_pinned_h_parent(struct au_pin *pin) +{ + if (pin && pin->parent) + return au_h_dptr(pin->parent, pin->bindex); + return NULL; +} + +void au_unpin(struct au_pin *p) +{ + if (p->hdir) + au_pin_hdir_unlock(p); + if (p->h_mnt && au_ftest_pin(p->flags, MNT_WRITE)) + vfsub_mnt_drop_write(p->h_mnt); + if (!p->hdir) + return; + + if (!au_ftest_pin(p->flags, DI_LOCKED)) + di_read_unlock(p->parent, AuLock_IR); + iput(p->hdir->hi_inode); + dput(p->parent); + p->parent = NULL; + p->hdir = NULL; + p->h_mnt = NULL; + /* do not clear p->task */ +} + +int au_do_pin(struct au_pin *p) +{ + int err; + struct super_block *sb; + struct inode *h_dir; + + err = 0; + sb = p->dentry->d_sb; + p->br = au_sbr(sb, p->bindex); + if (IS_ROOT(p->dentry)) { + if (au_ftest_pin(p->flags, MNT_WRITE)) { + p->h_mnt = au_br_mnt(p->br); + err = vfsub_mnt_want_write(p->h_mnt); + if (unlikely(err)) { + au_fclr_pin(p->flags, MNT_WRITE); + goto out_err; + } + } + goto out; + } + + p->h_dentry = NULL; + if (p->bindex <= au_dbbot(p->dentry)) + p->h_dentry = au_h_dptr(p->dentry, p->bindex); + + p->parent = dget_parent(p->dentry); + if (!au_ftest_pin(p->flags, DI_LOCKED)) + di_read_lock(p->parent, AuLock_IR, p->lsc_di); + + h_dir = NULL; + p->h_parent = au_h_dptr(p->parent, p->bindex); + p->hdir = au_hi(d_inode(p->parent), p->bindex); + if (p->hdir) + h_dir = p->hdir->hi_inode; + + /* + * udba case, or + * if DI_LOCKED is not set, then p->parent may be different + * and h_parent can be NULL. + */ + if (unlikely(!p->hdir || !h_dir || !p->h_parent)) { + err = -EBUSY; + if (!au_ftest_pin(p->flags, DI_LOCKED)) + di_read_unlock(p->parent, AuLock_IR); + dput(p->parent); + p->parent = NULL; + goto out_err; + } + + if (au_ftest_pin(p->flags, MNT_WRITE)) { + p->h_mnt = au_br_mnt(p->br); + err = vfsub_mnt_want_write(p->h_mnt); + if (unlikely(err)) { + au_fclr_pin(p->flags, MNT_WRITE); + if (!au_ftest_pin(p->flags, DI_LOCKED)) + di_read_unlock(p->parent, AuLock_IR); + dput(p->parent); + p->parent = NULL; + goto out_err; + } + } + + au_igrab(h_dir); + err = au_pin_hdir_lock(p); + if (!err) + goto out; /* success */ + + au_unpin(p); + +out_err: + pr_err("err %d\n", err); + err = au_busy_or_stale(); +out: + return err; +} + +void au_pin_init(struct au_pin *p, struct dentry *dentry, + aufs_bindex_t bindex, int lsc_di, int lsc_hi, + unsigned int udba, unsigned char flags) +{ + p->dentry = dentry; + p->udba = udba; + p->lsc_di = lsc_di; + p->lsc_hi = lsc_hi; + p->flags = flags; + p->bindex = bindex; + + p->parent = NULL; + p->hdir = NULL; + p->h_mnt = NULL; + + p->h_dentry = NULL; + p->h_parent = NULL; + p->br = NULL; + p->task = current; +} + +int au_pin(struct au_pin *pin, struct dentry *dentry, aufs_bindex_t bindex, + unsigned int udba, unsigned char flags) +{ + au_pin_init(pin, dentry, bindex, AuLsc_DI_PARENT, AuLsc_I_PARENT2, + udba, flags); + return au_do_pin(pin); +} + +/* ---------------------------------------------------------------------- */ + +/* + * ->setattr() and ->getattr() are called in various cases. + * chmod, stat: dentry is revalidated. + * fchmod, fstat: file and dentry are not revalidated, additionally they may be + * unhashed. + * for ->setattr(), ia->ia_file is passed from ftruncate only. + */ +/* todo: consolidate with do_refresh() and simple_reval_dpath() */ +int au_reval_for_attr(struct dentry *dentry, unsigned int sigen) +{ + int err; + struct dentry *parent; + + err = 0; + if (au_digen_test(dentry, sigen)) { + parent = dget_parent(dentry); + di_read_lock_parent(parent, AuLock_IR); + err = au_refresh_dentry(dentry, parent); + di_read_unlock(parent, AuLock_IR); + dput(parent); + } + + AuTraceErr(err); + return err; +} + +int au_pin_and_icpup(struct dentry *dentry, struct iattr *ia, + struct au_icpup_args *a) +{ + int err; + loff_t sz; + aufs_bindex_t btop, ibtop; + struct dentry *hi_wh, *parent; + struct inode *inode; + struct au_wr_dir_args wr_dir_args = { + .force_btgt = -1, + .flags = 0 + }; + + if (d_is_dir(dentry)) + au_fset_wrdir(wr_dir_args.flags, ISDIR); + /* plink or hi_wh() case */ + btop = au_dbtop(dentry); + inode = d_inode(dentry); + ibtop = au_ibtop(inode); + if (btop != ibtop && !au_test_ro(inode->i_sb, ibtop, inode)) + wr_dir_args.force_btgt = ibtop; + err = au_wr_dir(dentry, /*src_dentry*/NULL, &wr_dir_args); + if (unlikely(err < 0)) + goto out; + a->btgt = err; + if (err != btop) + au_fset_icpup(a->flags, DID_CPUP); + + err = 0; + a->pin_flags = AuPin_MNT_WRITE; + parent = NULL; + if (!IS_ROOT(dentry)) { + au_fset_pin(a->pin_flags, DI_LOCKED); + parent = dget_parent(dentry); + di_write_lock_parent(parent); + } + + err = au_pin(&a->pin, dentry, a->btgt, a->udba, a->pin_flags); + if (unlikely(err)) + goto out_parent; + + sz = -1; + a->h_path.dentry = au_h_dptr(dentry, btop); + a->h_inode = d_inode(a->h_path.dentry); + if (ia && (ia->ia_valid & ATTR_SIZE)) { + inode_lock_shared_nested(a->h_inode, AuLsc_I_CHILD); + if (ia->ia_size < i_size_read(a->h_inode)) + sz = ia->ia_size; + inode_unlock_shared(a->h_inode); + } + + hi_wh = NULL; + if (au_ftest_icpup(a->flags, DID_CPUP) && d_unlinked(dentry)) { + hi_wh = au_hi_wh(inode, a->btgt); + if (!hi_wh) { + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = a->btgt, + .bsrc = -1, + .len = sz, + .pin = &a->pin + }; + err = au_sio_cpup_wh(&cpg, /*file*/NULL); + if (unlikely(err)) + goto out_unlock; + hi_wh = au_hi_wh(inode, a->btgt); + /* todo: revalidate hi_wh? */ + } + } + + if (parent) { + au_pin_set_parent_lflag(&a->pin, /*lflag*/0); + di_downgrade_lock(parent, AuLock_IR); + dput(parent); + parent = NULL; + } + if (!au_ftest_icpup(a->flags, DID_CPUP)) + goto out; /* success */ + + if (!d_unhashed(dentry)) { + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = a->btgt, + .bsrc = btop, + .len = sz, + .pin = &a->pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + err = au_sio_cpup_simple(&cpg); + if (!err) + a->h_path.dentry = au_h_dptr(dentry, a->btgt); + } else if (!hi_wh) + a->h_path.dentry = au_h_dptr(dentry, a->btgt); + else + a->h_path.dentry = hi_wh; /* do not dget here */ + +out_unlock: + a->h_inode = d_inode(a->h_path.dentry); + if (!err) + goto out; /* success */ + au_unpin(&a->pin); +out_parent: + if (parent) { + di_write_unlock(parent); + dput(parent); + } +out: + if (!err) + inode_lock_nested(a->h_inode, AuLsc_I_CHILD); + return err; +} + +static int aufs_setattr(struct dentry *dentry, struct iattr *ia) +{ + int err; + struct inode *inode, *delegated; + struct super_block *sb; + struct file *file; + struct au_icpup_args *a; + + inode = d_inode(dentry); + IMustLock(inode); + + err = setattr_prepare(dentry, ia); + if (unlikely(err)) + goto out; + + err = -ENOMEM; + a = kzalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + if (ia->ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) + ia->ia_valid &= ~ATTR_MODE; + + file = NULL; + sb = dentry->d_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out_kfree; + + if (ia->ia_valid & ATTR_FILE) { + /* currently ftruncate(2) only */ + AuDebugOn(!d_is_reg(dentry)); + file = ia->ia_file; + err = au_reval_and_lock_fdi(file, au_reopen_nondir, /*wlock*/1, + /*fi_lsc*/0); + if (unlikely(err)) + goto out_si; + ia->ia_file = au_hf_top(file); + a->udba = AuOpt_UDBA_NONE; + } else { + /* fchmod() doesn't pass ia_file */ + a->udba = au_opt_udba(sb); + di_write_lock_child(dentry); + /* no d_unlinked(), to set UDBA_NONE for root */ + if (d_unhashed(dentry)) + a->udba = AuOpt_UDBA_NONE; + if (a->udba != AuOpt_UDBA_NONE) { + AuDebugOn(IS_ROOT(dentry)); + err = au_reval_for_attr(dentry, au_sigen(sb)); + if (unlikely(err)) + goto out_dentry; + } + } + + err = au_pin_and_icpup(dentry, ia, a); + if (unlikely(err < 0)) + goto out_dentry; + if (au_ftest_icpup(a->flags, DID_CPUP)) { + ia->ia_file = NULL; + ia->ia_valid &= ~ATTR_FILE; + } + + a->h_path.mnt = au_sbr_mnt(sb, a->btgt); + if ((ia->ia_valid & (ATTR_MODE | ATTR_CTIME)) + == (ATTR_MODE | ATTR_CTIME)) { + err = security_path_chmod(&a->h_path, ia->ia_mode); + if (unlikely(err)) + goto out_unlock; + } else if ((ia->ia_valid & (ATTR_UID | ATTR_GID)) + && (ia->ia_valid & ATTR_CTIME)) { + err = security_path_chown(&a->h_path, ia->ia_uid, ia->ia_gid); + if (unlikely(err)) + goto out_unlock; + } + + if (ia->ia_valid & ATTR_SIZE) { + struct file *f; + + if (ia->ia_size < i_size_read(inode)) + /* unmap only */ + truncate_setsize(inode, ia->ia_size); + + f = NULL; + if (ia->ia_valid & ATTR_FILE) + f = ia->ia_file; + inode_unlock(a->h_inode); + err = vfsub_trunc(&a->h_path, ia->ia_size, ia->ia_valid, f); + inode_lock_nested(a->h_inode, AuLsc_I_CHILD); + } else { + delegated = NULL; + while (1) { + err = vfsub_notify_change(&a->h_path, ia, &delegated); + if (delegated) { + err = break_deleg_wait(&delegated); + if (!err) + continue; + } + break; + } + } + /* + * regardless aufs 'acl' option setting. + * why don't all acl-aware fs call this func from their ->setattr()? + */ + if (!err && (ia->ia_valid & ATTR_MODE)) + err = vfsub_acl_chmod(a->h_inode, ia->ia_mode); + if (!err) + au_cpup_attr_changeable(inode); + +out_unlock: + inode_unlock(a->h_inode); + au_unpin(&a->pin); + if (unlikely(err)) + au_update_dbtop(dentry); +out_dentry: + di_write_unlock(dentry); + if (file) { + fi_write_unlock(file); + ia->ia_file = file; + ia->ia_valid |= ATTR_FILE; + } +out_si: + si_read_unlock(sb); +out_kfree: + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} + +#if IS_ENABLED(CONFIG_AUFS_XATTR) || IS_ENABLED(CONFIG_FS_POSIX_ACL) +static int au_h_path_to_set_attr(struct dentry *dentry, + struct au_icpup_args *a, struct path *h_path) +{ + int err; + struct super_block *sb; + + sb = dentry->d_sb; + a->udba = au_opt_udba(sb); + /* no d_unlinked(), to set UDBA_NONE for root */ + if (d_unhashed(dentry)) + a->udba = AuOpt_UDBA_NONE; + if (a->udba != AuOpt_UDBA_NONE) { + AuDebugOn(IS_ROOT(dentry)); + err = au_reval_for_attr(dentry, au_sigen(sb)); + if (unlikely(err)) + goto out; + } + err = au_pin_and_icpup(dentry, /*ia*/NULL, a); + if (unlikely(err < 0)) + goto out; + + h_path->dentry = a->h_path.dentry; + h_path->mnt = au_sbr_mnt(sb, a->btgt); + +out: + return err; +} + +ssize_t au_sxattr(struct dentry *dentry, struct inode *inode, + struct au_sxattr *arg) +{ + int err; + struct path h_path; + struct super_block *sb; + struct au_icpup_args *a; + struct inode *h_inode; + + IMustLock(inode); + + err = -ENOMEM; + a = kzalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + sb = dentry->d_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out_kfree; + + h_path.dentry = NULL; /* silence gcc */ + di_write_lock_child(dentry); + err = au_h_path_to_set_attr(dentry, a, &h_path); + if (unlikely(err)) + goto out_di; + + inode_unlock(a->h_inode); + switch (arg->type) { + case AU_XATTR_SET: + AuDebugOn(d_is_negative(h_path.dentry)); + err = vfsub_setxattr(h_path.dentry, + arg->u.set.name, arg->u.set.value, + arg->u.set.size, arg->u.set.flags); + break; + case AU_ACL_SET: + err = -EOPNOTSUPP; + h_inode = d_inode(h_path.dentry); + if (h_inode->i_op->set_acl) + /* this will call posix_acl_update_mode */ + err = h_inode->i_op->set_acl(h_inode, + arg->u.acl_set.acl, + arg->u.acl_set.type); + break; + } + if (!err) + au_cpup_attr_timesizes(inode); + + au_unpin(&a->pin); + if (unlikely(err)) + au_update_dbtop(dentry); + +out_di: + di_write_unlock(dentry); + si_read_unlock(sb); +out_kfree: + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} +#endif + +static void au_refresh_iattr(struct inode *inode, struct kstat *st, + unsigned int nlink) +{ + unsigned int n; + + inode->i_mode = st->mode; + /* don't i_[ug]id_write() here */ + inode->i_uid = st->uid; + inode->i_gid = st->gid; + inode->i_atime = st->atime; + inode->i_mtime = st->mtime; + inode->i_ctime = st->ctime; + + au_cpup_attr_nlink(inode, /*force*/0); + if (S_ISDIR(inode->i_mode)) { + n = inode->i_nlink; + n -= nlink; + n += st->nlink; + smp_mb(); /* for i_nlink */ + /* 0 can happen */ + set_nlink(inode, n); + } + + spin_lock(&inode->i_lock); + inode->i_blocks = st->blocks; + i_size_write(inode, st->size); + spin_unlock(&inode->i_lock); +} + +/* + * common routine for aufs_getattr() and au_getxattr(). + * returns zero or negative (an error). + * @dentry will be read-locked in success. + */ +int au_h_path_getattr(struct dentry *dentry, struct inode *inode, int force, + struct path *h_path, int locked) +{ + int err; + unsigned int mnt_flags, sigen; + unsigned char udba_none; + aufs_bindex_t bindex; + struct super_block *sb, *h_sb; + + h_path->mnt = NULL; + h_path->dentry = NULL; + + err = 0; + sb = dentry->d_sb; + mnt_flags = au_mntflags(sb); + udba_none = !!au_opt_test(mnt_flags, UDBA_NONE); + + if (unlikely(locked)) + goto body; /* skip locking dinfo */ + + /* support fstat(2) */ + if (!d_unlinked(dentry) && !udba_none) { + sigen = au_sigen(sb); + err = au_digen_test(dentry, sigen); + if (!err) { + di_read_lock_child(dentry, AuLock_IR); + err = au_dbrange_test(dentry); + if (unlikely(err)) { + di_read_unlock(dentry, AuLock_IR); + goto out; + } + } else { + AuDebugOn(IS_ROOT(dentry)); + di_write_lock_child(dentry); + err = au_dbrange_test(dentry); + if (!err) + err = au_reval_for_attr(dentry, sigen); + if (!err) + di_downgrade_lock(dentry, AuLock_IR); + else { + di_write_unlock(dentry); + goto out; + } + } + } else + di_read_lock_child(dentry, AuLock_IR); + +body: + if (!inode) { + inode = d_inode(dentry); + if (unlikely(!inode)) + goto out; + } + bindex = au_ibtop(inode); + h_path->mnt = au_sbr_mnt(sb, bindex); + h_sb = h_path->mnt->mnt_sb; + if (!force + && !au_test_fs_bad_iattr(h_sb) + && udba_none) + goto out; /* success */ + + if (au_dbtop(dentry) == bindex) + h_path->dentry = au_h_dptr(dentry, bindex); + else if (au_opt_test(mnt_flags, PLINK) && au_plink_test(inode)) { + h_path->dentry = au_plink_lkup(inode, bindex); + if (IS_ERR(h_path->dentry)) + /* pretending success */ + h_path->dentry = NULL; + else + dput(h_path->dentry); + } + +out: + return err; +} + +static int aufs_getattr(const struct path *path, struct kstat *st, + u32 request, unsigned int query) +{ + int err; + unsigned char positive; + struct path h_path; + struct dentry *dentry; + struct inode *inode; + struct super_block *sb; + + dentry = path->dentry; + inode = d_inode(dentry); + sb = dentry->d_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out; + err = au_h_path_getattr(dentry, /*inode*/NULL, /*force*/0, &h_path, + /*locked*/0); + if (unlikely(err)) + goto out_si; + if (unlikely(!h_path.dentry)) + /* illegally overlapped or something */ + goto out_fill; /* pretending success */ + + positive = d_is_positive(h_path.dentry); + if (positive) + /* no vfsub version */ + err = vfs_getattr(&h_path, st, request, query); + if (!err) { + if (positive) + au_refresh_iattr(inode, st, + d_inode(h_path.dentry)->i_nlink); + goto out_fill; /* success */ + } + AuTraceErr(err); + goto out_di; + +out_fill: + generic_fillattr(inode, st); +out_di: + di_read_unlock(dentry, AuLock_IR); +out_si: + si_read_unlock(sb); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static const char *aufs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + const char *ret; + struct dentry *h_dentry; + struct inode *h_inode; + int err; + aufs_bindex_t bindex; + + ret = NULL; /* suppress a warning */ + err = -ECHILD; + if (!dentry) + goto out; + + err = aufs_read_lock(dentry, AuLock_IR | AuLock_GEN); + if (unlikely(err)) + goto out; + + err = au_d_hashed_positive(dentry); + if (unlikely(err)) + goto out_unlock; + + err = -EINVAL; + inode = d_inode(dentry); + bindex = au_ibtop(inode); + h_inode = au_h_iptr(inode, bindex); + if (unlikely(!h_inode->i_op->get_link)) + goto out_unlock; + + err = -EBUSY; + h_dentry = NULL; + if (au_dbtop(dentry) <= bindex) { + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry) + dget(h_dentry); + } + if (!h_dentry) { + h_dentry = d_find_any_alias(h_inode); + if (IS_ERR(h_dentry)) { + err = PTR_ERR(h_dentry); + goto out_unlock; + } + } + if (unlikely(!h_dentry)) + goto out_unlock; + + err = 0; + AuDbg("%ps\n", h_inode->i_op->get_link); + AuDbgDentry(h_dentry); + ret = vfs_get_link(h_dentry, done); + dput(h_dentry); + if (IS_ERR(ret)) + err = PTR_ERR(ret); + +out_unlock: + aufs_read_unlock(dentry, AuLock_IR); +out: + if (unlikely(err)) + ret = ERR_PTR(err); + AuTraceErrPtr(ret); + return ret; +} + +/* ---------------------------------------------------------------------- */ + +static int au_is_special(struct inode *inode) +{ + return (inode->i_mode & (S_IFBLK | S_IFCHR | S_IFIFO | S_IFSOCK)); +} + +static int aufs_update_time(struct inode *inode, struct timespec64 *ts, + int flags) +{ + int err; + aufs_bindex_t bindex; + struct super_block *sb; + struct inode *h_inode; + struct vfsmount *h_mnt; + + sb = inode->i_sb; + WARN_ONCE((flags & S_ATIME) && !IS_NOATIME(inode), + "unexpected s_flags 0x%lx", sb->s_flags); + + /* mmap_sem might be acquired already, cf. aufs_mmap() */ + lockdep_off(); + si_read_lock(sb, AuLock_FLUSH); + ii_write_lock_child(inode); + + err = 0; + bindex = au_ibtop(inode); + h_inode = au_h_iptr(inode, bindex); + if (!au_test_ro(sb, bindex, inode)) { + h_mnt = au_sbr_mnt(sb, bindex); + err = vfsub_mnt_want_write(h_mnt); + if (!err) { + err = vfsub_update_time(h_inode, ts, flags); + vfsub_mnt_drop_write(h_mnt); + } + } else if (au_is_special(h_inode)) { + /* + * Never copy-up here. + * These special files may already be opened and used for + * communicating. If we copied it up, then the communication + * would be corrupted. + */ + AuWarn1("timestamps for i%lu are ignored " + "since it is on readonly branch (hi%lu).\n", + inode->i_ino, h_inode->i_ino); + } else if (flags & ~S_ATIME) { + err = -EIO; + AuIOErr1("unexpected flags 0x%x\n", flags); + AuDebugOn(1); + } + + if (!err) + au_cpup_attr_timesizes(inode); + ii_write_unlock(inode); + si_read_unlock(sb); + lockdep_on(); + + if (!err && (flags & S_VERSION)) + inode_inc_iversion(inode); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* no getattr version will be set by module.c:aufs_init() */ +struct inode_operations aufs_iop_nogetattr[AuIop_Last], + aufs_iop[] = { + [AuIop_SYMLINK] = { + .permission = aufs_permission, +#ifdef CONFIG_FS_POSIX_ACL + .get_acl = aufs_get_acl, + .set_acl = aufs_set_acl, /* unsupport for symlink? */ +#endif + + .setattr = aufs_setattr, + .getattr = aufs_getattr, + +#ifdef CONFIG_AUFS_XATTR + .listxattr = aufs_listxattr, +#endif + + .get_link = aufs_get_link, + + /* .update_time = aufs_update_time */ + }, + [AuIop_DIR] = { + .create = aufs_create, + .lookup = aufs_lookup, + .link = aufs_link, + .unlink = aufs_unlink, + .symlink = aufs_symlink, + .mkdir = aufs_mkdir, + .rmdir = aufs_rmdir, + .mknod = aufs_mknod, + .rename = aufs_rename, + + .permission = aufs_permission, +#ifdef CONFIG_FS_POSIX_ACL + .get_acl = aufs_get_acl, + .set_acl = aufs_set_acl, +#endif + + .setattr = aufs_setattr, + .getattr = aufs_getattr, + +#ifdef CONFIG_AUFS_XATTR + .listxattr = aufs_listxattr, +#endif + + .update_time = aufs_update_time, + .atomic_open = aufs_atomic_open, + .tmpfile = aufs_tmpfile + }, + [AuIop_OTHER] = { + .permission = aufs_permission, +#ifdef CONFIG_FS_POSIX_ACL + .get_acl = aufs_get_acl, + .set_acl = aufs_set_acl, +#endif + + .setattr = aufs_setattr, + .getattr = aufs_getattr, + +#ifdef CONFIG_AUFS_XATTR + .listxattr = aufs_listxattr, +#endif + + .update_time = aufs_update_time + } +}; diff --git a/fs/aufs/i_op_add.c b/fs/aufs/i_op_add.c new file mode 100644 index 0000000000000..80f2cfeeaaa0e --- /dev/null +++ b/fs/aufs/i_op_add.c @@ -0,0 +1,936 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operations (add entry) + */ + +#include +#include "aufs.h" + +/* + * final procedure of adding a new entry, except link(2). + * remove whiteout, instantiate, copyup the parent dir's times and size + * and update version. + * if it failed, re-create the removed whiteout. + */ +static int epilog(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct dentry *dentry) +{ + int err, rerr; + aufs_bindex_t bwh; + struct path h_path; + struct super_block *sb; + struct inode *inode, *h_dir; + struct dentry *wh; + + bwh = -1; + sb = dir->i_sb; + if (wh_dentry) { + h_dir = d_inode(wh_dentry->d_parent); /* dir inode is locked */ + IMustLock(h_dir); + AuDebugOn(au_h_iptr(dir, bindex) != h_dir); + bwh = au_dbwh(dentry); + h_path.dentry = wh_dentry; + h_path.mnt = au_sbr_mnt(sb, bindex); + err = au_wh_unlink_dentry(au_h_iptr(dir, bindex), &h_path, + dentry); + if (unlikely(err)) + goto out; + } + + inode = au_new_inode(dentry, /*must_new*/1); + if (!IS_ERR(inode)) { + d_instantiate(dentry, inode); + dir = d_inode(dentry->d_parent); /* dir inode is locked */ + IMustLock(dir); + au_dir_ts(dir, bindex); + inode_inc_iversion(dir); + au_fhsm_wrote(sb, bindex, /*force*/0); + return 0; /* success */ + } + + err = PTR_ERR(inode); + if (!wh_dentry) + goto out; + + /* revert */ + /* dir inode is locked */ + wh = au_wh_create(dentry, bwh, wh_dentry->d_parent); + rerr = PTR_ERR(wh); + if (IS_ERR(wh)) { + AuIOErr("%pd reverting whiteout failed(%d, %d)\n", + dentry, err, rerr); + err = -EIO; + } else + dput(wh); + +out: + return err; +} + +static int au_d_may_add(struct dentry *dentry) +{ + int err; + + err = 0; + if (unlikely(d_unhashed(dentry))) + err = -ENOENT; + if (unlikely(d_really_is_positive(dentry))) + err = -EEXIST; + return err; +} + +/* + * simple tests for the adding inode operations. + * following the checks in vfs, plus the parent-child relationship. + */ +int au_may_add(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent, int isdir) +{ + int err; + umode_t h_mode; + struct dentry *h_dentry; + struct inode *h_inode; + + err = -ENAMETOOLONG; + if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN)) + goto out; + + h_dentry = au_h_dptr(dentry, bindex); + if (d_really_is_negative(dentry)) { + err = -EEXIST; + if (unlikely(d_is_positive(h_dentry))) + goto out; + } else { + /* rename(2) case */ + err = -EIO; + if (unlikely(d_is_negative(h_dentry))) + goto out; + h_inode = d_inode(h_dentry); + if (unlikely(!h_inode->i_nlink)) + goto out; + + h_mode = h_inode->i_mode; + if (!isdir) { + err = -EISDIR; + if (unlikely(S_ISDIR(h_mode))) + goto out; + } else if (unlikely(!S_ISDIR(h_mode))) { + err = -ENOTDIR; + goto out; + } + } + + err = 0; + /* expected parent dir is locked */ + if (unlikely(h_parent != h_dentry->d_parent)) + err = -EIO; + +out: + AuTraceErr(err); + return err; +} + +/* + * initial procedure of adding a new entry. + * prepare writable branch and the parent dir, lock it, + * and lookup whiteout for the new entry. + */ +static struct dentry* +lock_hdir_lkup_wh(struct dentry *dentry, struct au_dtime *dt, + struct dentry *src_dentry, struct au_pin *pin, + struct au_wr_dir_args *wr_dir_args) +{ + struct dentry *wh_dentry, *h_parent; + struct super_block *sb; + struct au_branch *br; + int err; + unsigned int udba; + aufs_bindex_t bcpup; + + AuDbg("%pd\n", dentry); + + err = au_wr_dir(dentry, src_dentry, wr_dir_args); + bcpup = err; + wh_dentry = ERR_PTR(err); + if (unlikely(err < 0)) + goto out; + + sb = dentry->d_sb; + udba = au_opt_udba(sb); + err = au_pin(pin, dentry, bcpup, udba, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + wh_dentry = ERR_PTR(err); + if (unlikely(err)) + goto out; + + h_parent = au_pinned_h_parent(pin); + if (udba != AuOpt_UDBA_NONE + && au_dbtop(dentry) == bcpup) + err = au_may_add(dentry, bcpup, h_parent, + au_ftest_wrdir(wr_dir_args->flags, ISDIR)); + else if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN)) + err = -ENAMETOOLONG; + wh_dentry = ERR_PTR(err); + if (unlikely(err)) + goto out_unpin; + + br = au_sbr(sb, bcpup); + if (dt) { + struct path tmp = { + .dentry = h_parent, + .mnt = au_br_mnt(br) + }; + au_dtime_store(dt, au_pinned_parent(pin), &tmp); + } + + wh_dentry = NULL; + if (bcpup != au_dbwh(dentry)) + goto out; /* success */ + + /* + * ENAMETOOLONG here means that if we allowed create such name, then it + * would not be able to removed in the future. So we don't allow such + * name here and we don't handle ENAMETOOLONG differently here. + */ + wh_dentry = au_wh_lkup(h_parent, &dentry->d_name, br); + +out_unpin: + if (IS_ERR(wh_dentry)) + au_unpin(pin); +out: + return wh_dentry; +} + +/* ---------------------------------------------------------------------- */ + +enum { Mknod, Symlink, Creat }; +struct simple_arg { + int type; + union { + struct { + umode_t mode; + bool want_excl; + bool try_aopen; + struct vfsub_aopen_args *aopen; + } c; + struct { + const char *symname; + } s; + struct { + umode_t mode; + dev_t dev; + } m; + } u; +}; + +static int add_simple(struct inode *dir, struct dentry *dentry, + struct simple_arg *arg) +{ + int err, rerr; + aufs_bindex_t btop; + unsigned char created; + const unsigned char try_aopen + = (arg->type == Creat && arg->u.c.try_aopen); + struct vfsub_aopen_args *aopen = arg->u.c.aopen; + struct dentry *wh_dentry, *parent; + struct inode *h_dir; + struct super_block *sb; + struct au_branch *br; + /* to reduce stack size */ + struct { + struct au_dtime dt; + struct au_pin pin; + struct path h_path; + struct au_wr_dir_args wr_dir_args; + } *a; + + AuDbg("%pd\n", dentry); + IMustLock(dir); + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + a->wr_dir_args.force_btgt = -1; + a->wr_dir_args.flags = AuWrDir_ADD_ENTRY; + + parent = dentry->d_parent; /* dir inode is locked */ + if (!try_aopen) { + err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN); + if (unlikely(err)) + goto out_free; + } + err = au_d_may_add(dentry); + if (unlikely(err)) + goto out_unlock; + if (!try_aopen) + di_write_lock_parent(parent); + wh_dentry = lock_hdir_lkup_wh(dentry, &a->dt, /*src_dentry*/NULL, + &a->pin, &a->wr_dir_args); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + btop = au_dbtop(dentry); + sb = dentry->d_sb; + br = au_sbr(sb, btop); + a->h_path.dentry = au_h_dptr(dentry, btop); + a->h_path.mnt = au_br_mnt(br); + h_dir = au_pinned_h_dir(&a->pin); + switch (arg->type) { + case Creat: + if (!try_aopen || !h_dir->i_op->atomic_open) { + err = vfsub_create(h_dir, &a->h_path, arg->u.c.mode, + arg->u.c.want_excl); + created = !err; + if (!err && try_aopen) + aopen->file->f_mode |= FMODE_CREATED; + } else { + aopen->br = br; + err = vfsub_atomic_open(h_dir, a->h_path.dentry, aopen); + AuDbg("err %d\n", err); + AuDbgFile(aopen->file); + created = err >= 0 + && !!(aopen->file->f_mode & FMODE_CREATED); + } + break; + case Symlink: + err = vfsub_symlink(h_dir, &a->h_path, arg->u.s.symname); + created = !err; + break; + case Mknod: + err = vfsub_mknod(h_dir, &a->h_path, arg->u.m.mode, + arg->u.m.dev); + created = !err; + break; + default: + BUG(); + } + if (unlikely(err < 0)) + goto out_unpin; + + err = epilog(dir, btop, wh_dentry, dentry); + if (!err) + goto out_unpin; /* success */ + + /* revert */ + if (created /* && d_is_positive(a->h_path.dentry) */) { + /* no delegation since it is just created */ + rerr = vfsub_unlink(h_dir, &a->h_path, /*delegated*/NULL, + /*force*/0); + if (rerr) { + AuIOErr("%pd revert failure(%d, %d)\n", + dentry, err, rerr); + err = -EIO; + } + au_dtime_revert(&a->dt); + } + if (try_aopen && h_dir->i_op->atomic_open + && (aopen->file->f_mode & FMODE_OPENED)) + /* aopen->file is still opened */ + au_lcnt_dec(&aopen->br->br_nfiles); + +out_unpin: + au_unpin(&a->pin); + dput(wh_dentry); +out_parent: + if (!try_aopen) + di_write_unlock(parent); +out_unlock: + if (unlikely(err)) { + au_update_dbtop(dentry); + d_drop(dentry); + } + if (!try_aopen) + aufs_read_unlock(dentry, AuLock_DW); +out_free: + au_kfree_rcu(a); +out: + return err; +} + +int aufs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, + dev_t dev) +{ + struct simple_arg arg = { + .type = Mknod, + .u.m = { + .mode = mode, + .dev = dev + } + }; + return add_simple(dir, dentry, &arg); +} + +int aufs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +{ + struct simple_arg arg = { + .type = Symlink, + .u.s.symname = symname + }; + return add_simple(dir, dentry, &arg); +} + +int aufs_create(struct inode *dir, struct dentry *dentry, umode_t mode, + bool want_excl) +{ + struct simple_arg arg = { + .type = Creat, + .u.c = { + .mode = mode, + .want_excl = want_excl + } + }; + return add_simple(dir, dentry, &arg); +} + +int au_aopen_or_create(struct inode *dir, struct dentry *dentry, + struct vfsub_aopen_args *aopen_args) +{ + struct simple_arg arg = { + .type = Creat, + .u.c = { + .mode = aopen_args->create_mode, + .want_excl = aopen_args->open_flag & O_EXCL, + .try_aopen = true, + .aopen = aopen_args + } + }; + return add_simple(dir, dentry, &arg); +} + +int aufs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + int err; + aufs_bindex_t bindex; + struct super_block *sb; + struct dentry *parent, *h_parent, *h_dentry; + struct inode *h_dir, *inode; + struct vfsmount *h_mnt; + struct au_wr_dir_args wr_dir_args = { + .force_btgt = -1, + .flags = AuWrDir_TMPFILE + }; + + /* copy-up may happen */ + inode_lock(dir); + + sb = dir->i_sb; + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out; + + err = au_di_init(dentry); + if (unlikely(err)) + goto out_si; + + err = -EBUSY; + parent = d_find_any_alias(dir); + AuDebugOn(!parent); + di_write_lock_parent(parent); + if (unlikely(d_inode(parent) != dir)) + goto out_parent; + + err = au_digen_test(parent, au_sigen(sb)); + if (unlikely(err)) + goto out_parent; + + bindex = au_dbtop(parent); + au_set_dbtop(dentry, bindex); + au_set_dbbot(dentry, bindex); + err = au_wr_dir(dentry, /*src_dentry*/NULL, &wr_dir_args); + bindex = err; + if (unlikely(err < 0)) + goto out_parent; + + err = -EOPNOTSUPP; + h_dir = au_h_iptr(dir, bindex); + if (unlikely(!h_dir->i_op->tmpfile)) + goto out_parent; + + h_mnt = au_sbr_mnt(sb, bindex); + err = vfsub_mnt_want_write(h_mnt); + if (unlikely(err)) + goto out_parent; + + h_parent = au_h_dptr(parent, bindex); + h_dentry = vfs_tmpfile(h_parent, mode, /*open_flag*/0); + if (IS_ERR(h_dentry)) { + err = PTR_ERR(h_dentry); + goto out_mnt; + } + + au_set_dbtop(dentry, bindex); + au_set_dbbot(dentry, bindex); + au_set_h_dptr(dentry, bindex, dget(h_dentry)); + inode = au_new_inode(dentry, /*must_new*/1); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + au_set_h_dptr(dentry, bindex, NULL); + au_set_dbtop(dentry, -1); + au_set_dbbot(dentry, -1); + } else { + if (!inode->i_nlink) + set_nlink(inode, 1); + d_tmpfile(dentry, inode); + au_di(dentry)->di_tmpfile = 1; + + /* update without i_mutex */ + if (au_ibtop(dir) == au_dbtop(dentry)) + au_cpup_attr_timesizes(dir); + } + dput(h_dentry); + +out_mnt: + vfsub_mnt_drop_write(h_mnt); +out_parent: + di_write_unlock(parent); + dput(parent); + di_write_unlock(dentry); + if (unlikely(err)) { + au_di_fin(dentry); + dentry->d_fsdata = NULL; + } +out_si: + si_read_unlock(sb); +out: + inode_unlock(dir); + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct au_link_args { + aufs_bindex_t bdst, bsrc; + struct au_pin pin; + struct path h_path; + struct dentry *src_parent, *parent; +}; + +static int au_cpup_before_link(struct dentry *src_dentry, + struct au_link_args *a) +{ + int err; + struct dentry *h_src_dentry; + struct au_cp_generic cpg = { + .dentry = src_dentry, + .bdst = a->bdst, + .bsrc = a->bsrc, + .len = -1, + .pin = &a->pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN /* | AuCpup_KEEPLINO */ + }; + + di_read_lock_parent(a->src_parent, AuLock_IR); + err = au_test_and_cpup_dirs(src_dentry, a->bdst); + if (unlikely(err)) + goto out; + + h_src_dentry = au_h_dptr(src_dentry, a->bsrc); + err = au_pin(&a->pin, src_dentry, a->bdst, + au_opt_udba(src_dentry->d_sb), + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out; + + err = au_sio_cpup_simple(&cpg); + au_unpin(&a->pin); + +out: + di_read_unlock(a->src_parent, AuLock_IR); + return err; +} + +static int au_cpup_or_link(struct dentry *src_dentry, struct dentry *dentry, + struct au_link_args *a) +{ + int err; + unsigned char plink; + aufs_bindex_t bbot; + struct dentry *h_src_dentry; + struct inode *h_inode, *inode, *delegated; + struct super_block *sb; + struct file *h_file; + + plink = 0; + h_inode = NULL; + sb = src_dentry->d_sb; + inode = d_inode(src_dentry); + if (au_ibtop(inode) <= a->bdst) + h_inode = au_h_iptr(inode, a->bdst); + if (!h_inode || !h_inode->i_nlink) { + /* copyup src_dentry as the name of dentry. */ + bbot = au_dbbot(dentry); + if (bbot < a->bsrc) + au_set_dbbot(dentry, a->bsrc); + au_set_h_dptr(dentry, a->bsrc, + dget(au_h_dptr(src_dentry, a->bsrc))); + dget(a->h_path.dentry); + au_set_h_dptr(dentry, a->bdst, NULL); + AuDbg("temporary d_inode...\n"); + spin_lock(&dentry->d_lock); + dentry->d_inode = d_inode(src_dentry); /* tmp */ + spin_unlock(&dentry->d_lock); + h_file = au_h_open_pre(dentry, a->bsrc, /*force_wr*/0); + if (IS_ERR(h_file)) + err = PTR_ERR(h_file); + else { + struct au_cp_generic cpg = { + .dentry = dentry, + .bdst = a->bdst, + .bsrc = -1, + .len = -1, + .pin = &a->pin, + .flags = AuCpup_KEEPLINO + }; + err = au_sio_cpup_simple(&cpg); + au_h_open_post(dentry, a->bsrc, h_file); + if (!err) { + dput(a->h_path.dentry); + a->h_path.dentry = au_h_dptr(dentry, a->bdst); + } else + au_set_h_dptr(dentry, a->bdst, + a->h_path.dentry); + } + spin_lock(&dentry->d_lock); + dentry->d_inode = NULL; /* restore */ + spin_unlock(&dentry->d_lock); + AuDbg("temporary d_inode...done\n"); + au_set_h_dptr(dentry, a->bsrc, NULL); + au_set_dbbot(dentry, bbot); + } else { + /* the inode of src_dentry already exists on a.bdst branch */ + h_src_dentry = d_find_alias(h_inode); + if (!h_src_dentry && au_plink_test(inode)) { + plink = 1; + h_src_dentry = au_plink_lkup(inode, a->bdst); + err = PTR_ERR(h_src_dentry); + if (IS_ERR(h_src_dentry)) + goto out; + + if (unlikely(d_is_negative(h_src_dentry))) { + dput(h_src_dentry); + h_src_dentry = NULL; + } + + } + if (h_src_dentry) { + delegated = NULL; + err = vfsub_link(h_src_dentry, au_pinned_h_dir(&a->pin), + &a->h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + dput(h_src_dentry); + } else { + AuIOErr("no dentry found for hi%lu on b%d\n", + h_inode->i_ino, a->bdst); + err = -EIO; + } + } + + if (!err && !plink) + au_plink_append(inode, a->bdst, a->h_path.dentry); + +out: + AuTraceErr(err); + return err; +} + +int aufs_link(struct dentry *src_dentry, struct inode *dir, + struct dentry *dentry) +{ + int err, rerr; + struct au_dtime dt; + struct au_link_args *a; + struct dentry *wh_dentry, *h_src_dentry; + struct inode *inode, *delegated; + struct super_block *sb; + struct au_wr_dir_args wr_dir_args = { + /* .force_btgt = -1, */ + .flags = AuWrDir_ADD_ENTRY + }; + + IMustLock(dir); + inode = d_inode(src_dentry); + IMustLock(inode); + + err = -ENOMEM; + a = kzalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + a->parent = dentry->d_parent; /* dir inode is locked */ + err = aufs_read_and_write_lock2(dentry, src_dentry, + AuLock_NOPLM | AuLock_GEN); + if (unlikely(err)) + goto out_kfree; + err = au_d_linkable(src_dentry); + if (unlikely(err)) + goto out_unlock; + err = au_d_may_add(dentry); + if (unlikely(err)) + goto out_unlock; + + a->src_parent = dget_parent(src_dentry); + wr_dir_args.force_btgt = au_ibtop(inode); + + di_write_lock_parent(a->parent); + wr_dir_args.force_btgt = au_wbr(dentry, wr_dir_args.force_btgt); + wh_dentry = lock_hdir_lkup_wh(dentry, &dt, src_dentry, &a->pin, + &wr_dir_args); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + err = 0; + sb = dentry->d_sb; + a->bdst = au_dbtop(dentry); + a->h_path.dentry = au_h_dptr(dentry, a->bdst); + a->h_path.mnt = au_sbr_mnt(sb, a->bdst); + a->bsrc = au_ibtop(inode); + h_src_dentry = au_h_d_alias(src_dentry, a->bsrc); + if (!h_src_dentry && au_di(src_dentry)->di_tmpfile) + h_src_dentry = dget(au_hi_wh(inode, a->bsrc)); + if (!h_src_dentry) { + a->bsrc = au_dbtop(src_dentry); + h_src_dentry = au_h_d_alias(src_dentry, a->bsrc); + AuDebugOn(!h_src_dentry); + } else if (IS_ERR(h_src_dentry)) { + err = PTR_ERR(h_src_dentry); + goto out_parent; + } + + /* + * aufs doesn't touch the credential so + * security_dentry_create_files_as() is unnecessary. + */ + if (au_opt_test(au_mntflags(sb), PLINK)) { + if (a->bdst < a->bsrc + /* && h_src_dentry->d_sb != a->h_path.dentry->d_sb */) + err = au_cpup_or_link(src_dentry, dentry, a); + else { + delegated = NULL; + err = vfsub_link(h_src_dentry, au_pinned_h_dir(&a->pin), + &a->h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + } + dput(h_src_dentry); + } else { + /* + * copyup src_dentry to the branch we process, + * and then link(2) to it. + */ + dput(h_src_dentry); + if (a->bdst < a->bsrc + /* && h_src_dentry->d_sb != a->h_path.dentry->d_sb */) { + au_unpin(&a->pin); + di_write_unlock(a->parent); + err = au_cpup_before_link(src_dentry, a); + di_write_lock_parent(a->parent); + if (!err) + err = au_pin(&a->pin, dentry, a->bdst, + au_opt_udba(sb), + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (unlikely(err)) + goto out_wh; + } + if (!err) { + h_src_dentry = au_h_dptr(src_dentry, a->bdst); + err = -ENOENT; + if (h_src_dentry && d_is_positive(h_src_dentry)) { + delegated = NULL; + err = vfsub_link(h_src_dentry, + au_pinned_h_dir(&a->pin), + &a->h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry" + " for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + } + } + } + if (unlikely(err)) + goto out_unpin; + + if (wh_dentry) { + a->h_path.dentry = wh_dentry; + err = au_wh_unlink_dentry(au_pinned_h_dir(&a->pin), &a->h_path, + dentry); + if (unlikely(err)) + goto out_revert; + } + + au_dir_ts(dir, a->bdst); + inode_inc_iversion(dir); + inc_nlink(inode); + inode->i_ctime = dir->i_ctime; + d_instantiate(dentry, au_igrab(inode)); + if (d_unhashed(a->h_path.dentry)) + /* some filesystem calls d_drop() */ + d_drop(dentry); + /* some filesystems consume an inode even hardlink */ + au_fhsm_wrote(sb, a->bdst, /*force*/0); + goto out_unpin; /* success */ + +out_revert: + /* no delegation since it is just created */ + rerr = vfsub_unlink(au_pinned_h_dir(&a->pin), &a->h_path, + /*delegated*/NULL, /*force*/0); + if (unlikely(rerr)) { + AuIOErr("%pd reverting failed(%d, %d)\n", dentry, err, rerr); + err = -EIO; + } + au_dtime_revert(&dt); +out_unpin: + au_unpin(&a->pin); +out_wh: + dput(wh_dentry); +out_parent: + di_write_unlock(a->parent); + dput(a->src_parent); +out_unlock: + if (unlikely(err)) { + au_update_dbtop(dentry); + d_drop(dentry); + } + aufs_read_and_write_unlock2(dentry, src_dentry); +out_kfree: + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} + +int aufs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + int err, rerr; + aufs_bindex_t bindex; + unsigned char diropq; + struct path h_path; + struct dentry *wh_dentry, *parent, *opq_dentry; + struct inode *h_inode; + struct super_block *sb; + struct { + struct au_pin pin; + struct au_dtime dt; + } *a; /* reduce the stack usage */ + struct au_wr_dir_args wr_dir_args = { + .force_btgt = -1, + .flags = AuWrDir_ADD_ENTRY | AuWrDir_ISDIR + }; + + IMustLock(dir); + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN); + if (unlikely(err)) + goto out_free; + err = au_d_may_add(dentry); + if (unlikely(err)) + goto out_unlock; + + parent = dentry->d_parent; /* dir inode is locked */ + di_write_lock_parent(parent); + wh_dentry = lock_hdir_lkup_wh(dentry, &a->dt, /*src_dentry*/NULL, + &a->pin, &wr_dir_args); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + sb = dentry->d_sb; + bindex = au_dbtop(dentry); + h_path.dentry = au_h_dptr(dentry, bindex); + h_path.mnt = au_sbr_mnt(sb, bindex); + err = vfsub_mkdir(au_pinned_h_dir(&a->pin), &h_path, mode); + if (unlikely(err)) + goto out_unpin; + + /* make the dir opaque */ + diropq = 0; + h_inode = d_inode(h_path.dentry); + if (wh_dentry + || au_opt_test(au_mntflags(sb), ALWAYS_DIROPQ)) { + inode_lock_nested(h_inode, AuLsc_I_CHILD); + opq_dentry = au_diropq_create(dentry, bindex); + inode_unlock(h_inode); + err = PTR_ERR(opq_dentry); + if (IS_ERR(opq_dentry)) + goto out_dir; + dput(opq_dentry); + diropq = 1; + } + + err = epilog(dir, bindex, wh_dentry, dentry); + if (!err) { + inc_nlink(dir); + goto out_unpin; /* success */ + } + + /* revert */ + if (diropq) { + AuLabel(revert opq); + inode_lock_nested(h_inode, AuLsc_I_CHILD); + rerr = au_diropq_remove(dentry, bindex); + inode_unlock(h_inode); + if (rerr) { + AuIOErr("%pd reverting diropq failed(%d, %d)\n", + dentry, err, rerr); + err = -EIO; + } + } + +out_dir: + AuLabel(revert dir); + rerr = vfsub_rmdir(au_pinned_h_dir(&a->pin), &h_path); + if (rerr) { + AuIOErr("%pd reverting dir failed(%d, %d)\n", + dentry, err, rerr); + err = -EIO; + } + au_dtime_revert(&a->dt); +out_unpin: + au_unpin(&a->pin); + dput(wh_dentry); +out_parent: + di_write_unlock(parent); +out_unlock: + if (unlikely(err)) { + au_update_dbtop(dentry); + d_drop(dentry); + } + aufs_read_unlock(dentry, AuLock_DW); +out_free: + au_kfree_rcu(a); +out: + return err; +} diff --git a/fs/aufs/i_op_del.c b/fs/aufs/i_op_del.c new file mode 100644 index 0000000000000..9beba2ba8db02 --- /dev/null +++ b/fs/aufs/i_op_del.c @@ -0,0 +1,513 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operations (del entry) + */ + +#include +#include "aufs.h" + +/* + * decide if a new whiteout for @dentry is necessary or not. + * when it is necessary, prepare the parent dir for the upper branch whose + * branch index is @bcpup for creation. the actual creation of the whiteout will + * be done by caller. + * return value: + * 0: wh is unnecessary + * plus: wh is necessary + * minus: error + */ +int au_wr_dir_need_wh(struct dentry *dentry, int isdir, aufs_bindex_t *bcpup) +{ + int need_wh, err; + aufs_bindex_t btop; + struct super_block *sb; + + sb = dentry->d_sb; + btop = au_dbtop(dentry); + if (*bcpup < 0) { + *bcpup = btop; + if (au_test_ro(sb, btop, d_inode(dentry))) { + err = AuWbrCopyup(au_sbi(sb), dentry); + *bcpup = err; + if (unlikely(err < 0)) + goto out; + } + } else + AuDebugOn(btop < *bcpup + || au_test_ro(sb, *bcpup, d_inode(dentry))); + AuDbg("bcpup %d, btop %d\n", *bcpup, btop); + + if (*bcpup != btop) { + err = au_cpup_dirs(dentry, *bcpup); + if (unlikely(err)) + goto out; + need_wh = 1; + } else { + struct au_dinfo *dinfo, *tmp; + + need_wh = -ENOMEM; + dinfo = au_di(dentry); + tmp = au_di_alloc(sb, AuLsc_DI_TMP); + if (tmp) { + au_di_cp(tmp, dinfo); + au_di_swap(tmp, dinfo); + /* returns the number of positive dentries */ + need_wh = au_lkup_dentry(dentry, btop + 1, + /* AuLkup_IGNORE_PERM */ 0); + au_di_swap(tmp, dinfo); + au_rw_write_unlock(&tmp->di_rwsem); + au_di_free(tmp); + } + } + AuDbg("need_wh %d\n", need_wh); + err = need_wh; + +out: + return err; +} + +/* + * simple tests for the del-entry operations. + * following the checks in vfs, plus the parent-child relationship. + */ +int au_may_del(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent, int isdir) +{ + int err; + umode_t h_mode; + struct dentry *h_dentry, *h_latest; + struct inode *h_inode; + + h_dentry = au_h_dptr(dentry, bindex); + if (d_really_is_positive(dentry)) { + err = -ENOENT; + if (unlikely(d_is_negative(h_dentry))) + goto out; + h_inode = d_inode(h_dentry); + if (unlikely(!h_inode->i_nlink)) + goto out; + + h_mode = h_inode->i_mode; + if (!isdir) { + err = -EISDIR; + if (unlikely(S_ISDIR(h_mode))) + goto out; + } else if (unlikely(!S_ISDIR(h_mode))) { + err = -ENOTDIR; + goto out; + } + } else { + /* rename(2) case */ + err = -EIO; + if (unlikely(d_is_positive(h_dentry))) + goto out; + } + + err = -ENOENT; + /* expected parent dir is locked */ + if (unlikely(h_parent != h_dentry->d_parent)) + goto out; + err = 0; + + /* + * rmdir a dir may break the consistency on some filesystem. + * let's try heavy test. + */ + err = -EACCES; + if (unlikely(!au_opt_test(au_mntflags(dentry->d_sb), DIRPERM1) + && au_test_h_perm(d_inode(h_parent), + MAY_EXEC | MAY_WRITE))) + goto out; + + h_latest = au_sio_lkup_one(&dentry->d_name, h_parent); + err = -EIO; + if (IS_ERR(h_latest)) + goto out; + if (h_latest == h_dentry) + err = 0; + dput(h_latest); + +out: + return err; +} + +/* + * decide the branch where we operate for @dentry. the branch index will be set + * @rbcpup. after deciding it, 'pin' it and store the timestamps of the parent + * dir for reverting. + * when a new whiteout is necessary, create it. + */ +static struct dentry* +lock_hdir_create_wh(struct dentry *dentry, int isdir, aufs_bindex_t *rbcpup, + struct au_dtime *dt, struct au_pin *pin) +{ + struct dentry *wh_dentry; + struct super_block *sb; + struct path h_path; + int err, need_wh; + unsigned int udba; + aufs_bindex_t bcpup; + + need_wh = au_wr_dir_need_wh(dentry, isdir, rbcpup); + wh_dentry = ERR_PTR(need_wh); + if (unlikely(need_wh < 0)) + goto out; + + sb = dentry->d_sb; + udba = au_opt_udba(sb); + bcpup = *rbcpup; + err = au_pin(pin, dentry, bcpup, udba, + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + wh_dentry = ERR_PTR(err); + if (unlikely(err)) + goto out; + + h_path.dentry = au_pinned_h_parent(pin); + if (udba != AuOpt_UDBA_NONE + && au_dbtop(dentry) == bcpup) { + err = au_may_del(dentry, bcpup, h_path.dentry, isdir); + wh_dentry = ERR_PTR(err); + if (unlikely(err)) + goto out_unpin; + } + + h_path.mnt = au_sbr_mnt(sb, bcpup); + au_dtime_store(dt, au_pinned_parent(pin), &h_path); + wh_dentry = NULL; + if (!need_wh) + goto out; /* success, no need to create whiteout */ + + wh_dentry = au_wh_create(dentry, bcpup, h_path.dentry); + if (IS_ERR(wh_dentry)) + goto out_unpin; + + /* returns with the parent is locked and wh_dentry is dget-ed */ + goto out; /* success */ + +out_unpin: + au_unpin(pin); +out: + return wh_dentry; +} + +/* + * when removing a dir, rename it to a unique temporary whiteout-ed name first + * in order to be revertible and save time for removing many child whiteouts + * under the dir. + * returns 1 when there are too many child whiteout and caller should remove + * them asynchronously. returns 0 when the number of children is enough small to + * remove now or the branch fs is a remote fs. + * otherwise return an error. + */ +static int renwh_and_rmdir(struct dentry *dentry, aufs_bindex_t bindex, + struct au_nhash *whlist, struct inode *dir) +{ + int rmdir_later, err, dirwh; + struct dentry *h_dentry; + struct super_block *sb; + struct inode *inode; + + sb = dentry->d_sb; + SiMustAnyLock(sb); + h_dentry = au_h_dptr(dentry, bindex); + err = au_whtmp_ren(h_dentry, au_sbr(sb, bindex)); + if (unlikely(err)) + goto out; + + /* stop monitoring */ + inode = d_inode(dentry); + au_hn_free(au_hi(inode, bindex)); + + if (!au_test_fs_remote(h_dentry->d_sb)) { + dirwh = au_sbi(sb)->si_dirwh; + rmdir_later = (dirwh <= 1); + if (!rmdir_later) + rmdir_later = au_nhash_test_longer_wh(whlist, bindex, + dirwh); + if (rmdir_later) + return rmdir_later; + } + + err = au_whtmp_rmdir(dir, bindex, h_dentry, whlist); + if (unlikely(err)) { + AuIOErr("rmdir %pd, b%d failed, %d. ignored\n", + h_dentry, bindex, err); + err = 0; + } + +out: + AuTraceErr(err); + return err; +} + +/* + * final procedure for deleting a entry. + * maintain dentry and iattr. + */ +static void epilog(struct inode *dir, struct dentry *dentry, + aufs_bindex_t bindex) +{ + struct inode *inode; + + inode = d_inode(dentry); + d_drop(dentry); + inode->i_ctime = dir->i_ctime; + + au_dir_ts(dir, bindex); + inode_inc_iversion(dir); +} + +/* + * when an error happened, remove the created whiteout and revert everything. + */ +static int do_revert(int err, struct inode *dir, aufs_bindex_t bindex, + aufs_bindex_t bwh, struct dentry *wh_dentry, + struct dentry *dentry, struct au_dtime *dt) +{ + int rerr; + struct path h_path = { + .dentry = wh_dentry, + .mnt = au_sbr_mnt(dir->i_sb, bindex) + }; + + rerr = au_wh_unlink_dentry(au_h_iptr(dir, bindex), &h_path, dentry); + if (!rerr) { + au_set_dbwh(dentry, bwh); + au_dtime_revert(dt); + return 0; + } + + AuIOErr("%pd reverting whiteout failed(%d, %d)\n", dentry, err, rerr); + return -EIO; +} + +/* ---------------------------------------------------------------------- */ + +int aufs_unlink(struct inode *dir, struct dentry *dentry) +{ + int err; + aufs_bindex_t bwh, bindex, btop; + struct inode *inode, *h_dir, *delegated; + struct dentry *parent, *wh_dentry; + /* to reduce stack size */ + struct { + struct au_dtime dt; + struct au_pin pin; + struct path h_path; + } *a; + + IMustLock(dir); + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN); + if (unlikely(err)) + goto out_free; + err = au_d_hashed_positive(dentry); + if (unlikely(err)) + goto out_unlock; + inode = d_inode(dentry); + IMustLock(inode); + err = -EISDIR; + if (unlikely(d_is_dir(dentry))) + goto out_unlock; /* possible? */ + + btop = au_dbtop(dentry); + bwh = au_dbwh(dentry); + bindex = -1; + parent = dentry->d_parent; /* dir inode is locked */ + di_write_lock_parent(parent); + wh_dentry = lock_hdir_create_wh(dentry, /*isdir*/0, &bindex, &a->dt, + &a->pin); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + a->h_path.mnt = au_sbr_mnt(dentry->d_sb, btop); + a->h_path.dentry = au_h_dptr(dentry, btop); + dget(a->h_path.dentry); + if (bindex == btop) { + h_dir = au_pinned_h_dir(&a->pin); + delegated = NULL; + err = vfsub_unlink(h_dir, &a->h_path, &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + } else { + /* dir inode is locked */ + h_dir = d_inode(wh_dentry->d_parent); + IMustLock(h_dir); + err = 0; + } + + if (!err) { + vfsub_drop_nlink(inode); + epilog(dir, dentry, bindex); + + /* update target timestamps */ + if (bindex == btop) { + vfsub_update_h_iattr(&a->h_path, /*did*/NULL); + /*ignore*/ + inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime; + } else + /* todo: this timestamp may be reverted later */ + inode->i_ctime = h_dir->i_ctime; + goto out_unpin; /* success */ + } + + /* revert */ + if (wh_dentry) { + int rerr; + + rerr = do_revert(err, dir, bindex, bwh, wh_dentry, dentry, + &a->dt); + if (rerr) + err = rerr; + } + +out_unpin: + au_unpin(&a->pin); + dput(wh_dentry); + dput(a->h_path.dentry); +out_parent: + di_write_unlock(parent); +out_unlock: + aufs_read_unlock(dentry, AuLock_DW); +out_free: + au_kfree_rcu(a); +out: + return err; +} + +int aufs_rmdir(struct inode *dir, struct dentry *dentry) +{ + int err, rmdir_later; + aufs_bindex_t bwh, bindex, btop; + struct inode *inode; + struct dentry *parent, *wh_dentry, *h_dentry; + struct au_whtmp_rmdir *args; + /* to reduce stack size */ + struct { + struct au_dtime dt; + struct au_pin pin; + } *a; + + IMustLock(dir); + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_GEN); + if (unlikely(err)) + goto out_free; + err = au_alive_dir(dentry); + if (unlikely(err)) + goto out_unlock; + inode = d_inode(dentry); + IMustLock(inode); + err = -ENOTDIR; + if (unlikely(!d_is_dir(dentry))) + goto out_unlock; /* possible? */ + + err = -ENOMEM; + args = au_whtmp_rmdir_alloc(dir->i_sb, GFP_NOFS); + if (unlikely(!args)) + goto out_unlock; + + parent = dentry->d_parent; /* dir inode is locked */ + di_write_lock_parent(parent); + err = au_test_empty(dentry, &args->whlist); + if (unlikely(err)) + goto out_parent; + + btop = au_dbtop(dentry); + bwh = au_dbwh(dentry); + bindex = -1; + wh_dentry = lock_hdir_create_wh(dentry, /*isdir*/1, &bindex, &a->dt, + &a->pin); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) + goto out_parent; + + h_dentry = au_h_dptr(dentry, btop); + dget(h_dentry); + rmdir_later = 0; + if (bindex == btop) { + err = renwh_and_rmdir(dentry, btop, &args->whlist, dir); + if (err > 0) { + rmdir_later = err; + err = 0; + } + } else { + /* stop monitoring */ + au_hn_free(au_hi(inode, btop)); + + /* dir inode is locked */ + IMustLock(d_inode(wh_dentry->d_parent)); + err = 0; + } + + if (!err) { + vfsub_dead_dir(inode); + au_set_dbdiropq(dentry, -1); + epilog(dir, dentry, bindex); + + if (rmdir_later) { + au_whtmp_kick_rmdir(dir, btop, h_dentry, args); + args = NULL; + } + + goto out_unpin; /* success */ + } + + /* revert */ + AuLabel(revert); + if (wh_dentry) { + int rerr; + + rerr = do_revert(err, dir, bindex, bwh, wh_dentry, dentry, + &a->dt); + if (rerr) + err = rerr; + } + +out_unpin: + au_unpin(&a->pin); + dput(wh_dentry); + dput(h_dentry); +out_parent: + di_write_unlock(parent); + if (args) + au_whtmp_rmdir_free(args); +out_unlock: + aufs_read_unlock(dentry, AuLock_DW); +out_free: + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/i_op_ren.c b/fs/aufs/i_op_ren.c new file mode 100644 index 0000000000000..435fa50811ffc --- /dev/null +++ b/fs/aufs/i_op_ren.c @@ -0,0 +1,1250 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operation (rename entry) + * todo: this is crazy monster + */ + +#include +#include "aufs.h" + +enum { AuSRC, AuDST, AuSrcDst }; +enum { AuPARENT, AuCHILD, AuParentChild }; + +#define AuRen_ISDIR_SRC 1 +#define AuRen_ISDIR_DST (1 << 1) +#define AuRen_ISSAMEDIR (1 << 2) +#define AuRen_WHSRC (1 << 3) +#define AuRen_WHDST (1 << 4) +#define AuRen_MNT_WRITE (1 << 5) +#define AuRen_DT_DSTDIR (1 << 6) +#define AuRen_DIROPQ_SRC (1 << 7) +#define AuRen_DIROPQ_DST (1 << 8) +#define AuRen_DIRREN (1 << 9) +#define AuRen_DROPPED_SRC (1 << 10) +#define AuRen_DROPPED_DST (1 << 11) +#define au_ftest_ren(flags, name) ((flags) & AuRen_##name) +#define au_fset_ren(flags, name) \ + do { (flags) |= AuRen_##name; } while (0) +#define au_fclr_ren(flags, name) \ + do { (flags) &= ~AuRen_##name; } while (0) + +#ifndef CONFIG_AUFS_DIRREN +#undef AuRen_DIRREN +#define AuRen_DIRREN 0 +#endif + +struct au_ren_args { + struct { + struct dentry *dentry, *h_dentry, *parent, *h_parent, + *wh_dentry; + struct inode *dir, *inode; + struct au_hinode *hdir, *hinode; + struct au_dtime dt[AuParentChild]; + aufs_bindex_t btop, bdiropq; + } sd[AuSrcDst]; + +#define src_dentry sd[AuSRC].dentry +#define src_dir sd[AuSRC].dir +#define src_inode sd[AuSRC].inode +#define src_h_dentry sd[AuSRC].h_dentry +#define src_parent sd[AuSRC].parent +#define src_h_parent sd[AuSRC].h_parent +#define src_wh_dentry sd[AuSRC].wh_dentry +#define src_hdir sd[AuSRC].hdir +#define src_hinode sd[AuSRC].hinode +#define src_h_dir sd[AuSRC].hdir->hi_inode +#define src_dt sd[AuSRC].dt +#define src_btop sd[AuSRC].btop +#define src_bdiropq sd[AuSRC].bdiropq + +#define dst_dentry sd[AuDST].dentry +#define dst_dir sd[AuDST].dir +#define dst_inode sd[AuDST].inode +#define dst_h_dentry sd[AuDST].h_dentry +#define dst_parent sd[AuDST].parent +#define dst_h_parent sd[AuDST].h_parent +#define dst_wh_dentry sd[AuDST].wh_dentry +#define dst_hdir sd[AuDST].hdir +#define dst_hinode sd[AuDST].hinode +#define dst_h_dir sd[AuDST].hdir->hi_inode +#define dst_dt sd[AuDST].dt +#define dst_btop sd[AuDST].btop +#define dst_bdiropq sd[AuDST].bdiropq + + struct dentry *h_trap; + struct au_branch *br; + struct path h_path; + struct au_nhash whlist; + aufs_bindex_t btgt, src_bwh; + + struct { + unsigned short auren_flags; + unsigned char flags; /* syscall parameter */ + unsigned char exchange; + } __packed; + + struct au_whtmp_rmdir *thargs; + struct dentry *h_dst; + struct au_hinode *h_root; +}; + +/* ---------------------------------------------------------------------- */ + +/* + * functions for reverting. + * when an error happened in a single rename systemcall, we should revert + * everything as if nothing happened. + * we don't need to revert the copied-up/down the parent dir since they are + * harmless. + */ + +#define RevertFailure(fmt, ...) do { \ + AuIOErr("revert failure: " fmt " (%d, %d)\n", \ + ##__VA_ARGS__, err, rerr); \ + err = -EIO; \ +} while (0) + +static void au_ren_do_rev_diropq(int err, struct au_ren_args *a, int idx) +{ + int rerr; + struct dentry *d; +#define src_or_dst(member) a->sd[idx].member + + d = src_or_dst(dentry); /* {src,dst}_dentry */ + au_hn_inode_lock_nested(src_or_dst(hinode), AuLsc_I_CHILD); + rerr = au_diropq_remove(d, a->btgt); + au_hn_inode_unlock(src_or_dst(hinode)); + au_set_dbdiropq(d, src_or_dst(bdiropq)); + if (rerr) + RevertFailure("remove diropq %pd", d); + +#undef src_or_dst_ +} + +static void au_ren_rev_diropq(int err, struct au_ren_args *a) +{ + if (au_ftest_ren(a->auren_flags, DIROPQ_SRC)) + au_ren_do_rev_diropq(err, a, AuSRC); + if (au_ftest_ren(a->auren_flags, DIROPQ_DST)) + au_ren_do_rev_diropq(err, a, AuDST); +} + +static void au_ren_rev_rename(int err, struct au_ren_args *a) +{ + int rerr; + struct inode *delegated; + + a->h_path.dentry = vfsub_lkup_one(&a->src_dentry->d_name, + a->src_h_parent); + rerr = PTR_ERR(a->h_path.dentry); + if (IS_ERR(a->h_path.dentry)) { + RevertFailure("lkup one %pd", a->src_dentry); + return; + } + + delegated = NULL; + rerr = vfsub_rename(a->dst_h_dir, + au_h_dptr(a->src_dentry, a->btgt), + a->src_h_dir, &a->h_path, &delegated, a->flags); + if (unlikely(rerr == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal rename\n"); + iput(delegated); + } + d_drop(a->h_path.dentry); + dput(a->h_path.dentry); + /* au_set_h_dptr(a->src_dentry, a->btgt, NULL); */ + if (rerr) + RevertFailure("rename %pd", a->src_dentry); +} + +static void au_ren_rev_whtmp(int err, struct au_ren_args *a) +{ + int rerr; + struct inode *delegated; + + a->h_path.dentry = vfsub_lkup_one(&a->dst_dentry->d_name, + a->dst_h_parent); + rerr = PTR_ERR(a->h_path.dentry); + if (IS_ERR(a->h_path.dentry)) { + RevertFailure("lkup one %pd", a->dst_dentry); + return; + } + if (d_is_positive(a->h_path.dentry)) { + d_drop(a->h_path.dentry); + dput(a->h_path.dentry); + return; + } + + delegated = NULL; + rerr = vfsub_rename(a->dst_h_dir, a->h_dst, a->dst_h_dir, &a->h_path, + &delegated, a->flags); + if (unlikely(rerr == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal rename\n"); + iput(delegated); + } + d_drop(a->h_path.dentry); + dput(a->h_path.dentry); + if (!rerr) + au_set_h_dptr(a->dst_dentry, a->btgt, dget(a->h_dst)); + else + RevertFailure("rename %pd", a->h_dst); +} + +static void au_ren_rev_whsrc(int err, struct au_ren_args *a) +{ + int rerr; + + a->h_path.dentry = a->src_wh_dentry; + rerr = au_wh_unlink_dentry(a->src_h_dir, &a->h_path, a->src_dentry); + au_set_dbwh(a->src_dentry, a->src_bwh); + if (rerr) + RevertFailure("unlink %pd", a->src_wh_dentry); +} +#undef RevertFailure + +/* ---------------------------------------------------------------------- */ + +/* + * when we have to copyup the renaming entry, do it with the rename-target name + * in order to minimize the cost (the later actual rename is unnecessary). + * otherwise rename it on the target branch. + */ +static int au_ren_or_cpup(struct au_ren_args *a) +{ + int err; + struct dentry *d; + struct inode *delegated; + + d = a->src_dentry; + if (au_dbtop(d) == a->btgt) { + a->h_path.dentry = a->dst_h_dentry; + AuDebugOn(au_dbtop(d) != a->btgt); + delegated = NULL; + err = vfsub_rename(a->src_h_dir, au_h_dptr(d, a->btgt), + a->dst_h_dir, &a->h_path, &delegated, + a->flags); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal rename\n"); + iput(delegated); + } + } else + BUG(); + + if (!err && a->h_dst) + /* it will be set to dinfo later */ + dget(a->h_dst); + + return err; +} + +/* cf. aufs_rmdir() */ +static int au_ren_del_whtmp(struct au_ren_args *a) +{ + int err; + struct inode *dir; + + dir = a->dst_dir; + SiMustAnyLock(dir->i_sb); + if (!au_nhash_test_longer_wh(&a->whlist, a->btgt, + au_sbi(dir->i_sb)->si_dirwh) + || au_test_fs_remote(a->h_dst->d_sb)) { + err = au_whtmp_rmdir(dir, a->btgt, a->h_dst, &a->whlist); + if (unlikely(err)) + pr_warn("failed removing whtmp dir %pd (%d), " + "ignored.\n", a->h_dst, err); + } else { + au_nhash_wh_free(&a->thargs->whlist); + a->thargs->whlist = a->whlist; + a->whlist.nh_num = 0; + au_whtmp_kick_rmdir(dir, a->btgt, a->h_dst, a->thargs); + dput(a->h_dst); + a->thargs = NULL; + } + + return 0; +} + +/* make it 'opaque' dir. */ +static int au_ren_do_diropq(struct au_ren_args *a, int idx) +{ + int err; + struct dentry *d, *diropq; +#define src_or_dst(member) a->sd[idx].member + + err = 0; + d = src_or_dst(dentry); /* {src,dst}_dentry */ + src_or_dst(bdiropq) = au_dbdiropq(d); + src_or_dst(hinode) = au_hi(src_or_dst(inode), a->btgt); + au_hn_inode_lock_nested(src_or_dst(hinode), AuLsc_I_CHILD); + diropq = au_diropq_create(d, a->btgt); + au_hn_inode_unlock(src_or_dst(hinode)); + if (IS_ERR(diropq)) + err = PTR_ERR(diropq); + else + dput(diropq); + +#undef src_or_dst_ + return err; +} + +static int au_ren_diropq(struct au_ren_args *a) +{ + int err; + unsigned char always; + struct dentry *d; + + err = 0; + d = a->dst_dentry; /* already renamed on the branch */ + always = !!au_opt_test(au_mntflags(d->d_sb), ALWAYS_DIROPQ); + if (au_ftest_ren(a->auren_flags, ISDIR_SRC) + && !au_ftest_ren(a->auren_flags, DIRREN) + && a->btgt != au_dbdiropq(a->src_dentry) + && (a->dst_wh_dentry + || a->btgt <= au_dbdiropq(d) + /* hide the lower to keep xino */ + /* the lowers may not be a dir, but we hide them anyway */ + || a->btgt < au_dbbot(d) + || always)) { + AuDbg("here\n"); + err = au_ren_do_diropq(a, AuSRC); + if (unlikely(err)) + goto out; + au_fset_ren(a->auren_flags, DIROPQ_SRC); + } + if (!a->exchange) + goto out; /* success */ + + d = a->src_dentry; /* already renamed on the branch */ + if (au_ftest_ren(a->auren_flags, ISDIR_DST) + && a->btgt != au_dbdiropq(a->dst_dentry) + && (a->btgt < au_dbdiropq(d) + || a->btgt < au_dbbot(d) + || always)) { + AuDbgDentry(a->src_dentry); + AuDbgDentry(a->dst_dentry); + err = au_ren_do_diropq(a, AuDST); + if (unlikely(err)) + goto out_rev_src; + au_fset_ren(a->auren_flags, DIROPQ_DST); + } + goto out; /* success */ + +out_rev_src: + AuDbg("err %d, reverting src\n", err); + au_ren_rev_diropq(err, a); +out: + return err; +} + +static int do_rename(struct au_ren_args *a) +{ + int err; + struct dentry *d, *h_d; + + if (!a->exchange) { + /* prepare workqueue args for asynchronous rmdir */ + h_d = a->dst_h_dentry; + if (au_ftest_ren(a->auren_flags, ISDIR_DST) + /* && !au_ftest_ren(a->auren_flags, DIRREN) */ + && d_is_positive(h_d)) { + err = -ENOMEM; + a->thargs = au_whtmp_rmdir_alloc(a->src_dentry->d_sb, + GFP_NOFS); + if (unlikely(!a->thargs)) + goto out; + a->h_dst = dget(h_d); + } + + /* create whiteout for src_dentry */ + if (au_ftest_ren(a->auren_flags, WHSRC)) { + a->src_bwh = au_dbwh(a->src_dentry); + AuDebugOn(a->src_bwh >= 0); + a->src_wh_dentry = au_wh_create(a->src_dentry, a->btgt, + a->src_h_parent); + err = PTR_ERR(a->src_wh_dentry); + if (IS_ERR(a->src_wh_dentry)) + goto out_thargs; + } + + /* lookup whiteout for dentry */ + if (au_ftest_ren(a->auren_flags, WHDST)) { + h_d = au_wh_lkup(a->dst_h_parent, + &a->dst_dentry->d_name, a->br); + err = PTR_ERR(h_d); + if (IS_ERR(h_d)) + goto out_whsrc; + if (d_is_negative(h_d)) + dput(h_d); + else + a->dst_wh_dentry = h_d; + } + + /* rename dentry to tmpwh */ + if (a->thargs) { + err = au_whtmp_ren(a->dst_h_dentry, a->br); + if (unlikely(err)) + goto out_whdst; + + d = a->dst_dentry; + au_set_h_dptr(d, a->btgt, NULL); + err = au_lkup_neg(d, a->btgt, /*wh*/0); + if (unlikely(err)) + goto out_whtmp; + a->dst_h_dentry = au_h_dptr(d, a->btgt); + } + } + + BUG_ON(d_is_positive(a->dst_h_dentry) && a->src_btop != a->btgt); +#if 0 /* debugging */ + BUG_ON(!au_ftest_ren(a->auren_flags, DIRREN) + && d_is_positive(a->dst_h_dentry) + && a->src_btop != a->btgt); +#endif + + /* rename by vfs_rename or cpup */ + err = au_ren_or_cpup(a); + if (unlikely(err)) + /* leave the copied-up one */ + goto out_whtmp; + + /* make dir opaque */ + err = au_ren_diropq(a); + if (unlikely(err)) + goto out_rename; + + /* update target timestamps */ + if (a->exchange) { + AuDebugOn(au_dbtop(a->dst_dentry) != a->btgt); + a->h_path.dentry = au_h_dptr(a->dst_dentry, a->btgt); + vfsub_update_h_iattr(&a->h_path, /*did*/NULL); /*ignore*/ + a->dst_inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime; + } + AuDebugOn(au_dbtop(a->src_dentry) != a->btgt); + a->h_path.dentry = au_h_dptr(a->src_dentry, a->btgt); + vfsub_update_h_iattr(&a->h_path, /*did*/NULL); /*ignore*/ + a->src_inode->i_ctime = d_inode(a->h_path.dentry)->i_ctime; + + if (!a->exchange) { + /* remove whiteout for dentry */ + if (a->dst_wh_dentry) { + a->h_path.dentry = a->dst_wh_dentry; + err = au_wh_unlink_dentry(a->dst_h_dir, &a->h_path, + a->dst_dentry); + if (unlikely(err)) + goto out_diropq; + } + + /* remove whtmp */ + if (a->thargs) + au_ren_del_whtmp(a); /* ignore this error */ + + au_fhsm_wrote(a->src_dentry->d_sb, a->btgt, /*force*/0); + } + err = 0; + goto out_success; + +out_diropq: + au_ren_rev_diropq(err, a); +out_rename: + au_ren_rev_rename(err, a); + dput(a->h_dst); +out_whtmp: + if (a->thargs) + au_ren_rev_whtmp(err, a); +out_whdst: + dput(a->dst_wh_dentry); + a->dst_wh_dentry = NULL; +out_whsrc: + if (a->src_wh_dentry) + au_ren_rev_whsrc(err, a); +out_success: + dput(a->src_wh_dentry); + dput(a->dst_wh_dentry); +out_thargs: + if (a->thargs) { + dput(a->h_dst); + au_whtmp_rmdir_free(a->thargs); + a->thargs = NULL; + } +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * test if @dentry dir can be rename destination or not. + * success means, it is a logically empty dir. + */ +static int may_rename_dstdir(struct dentry *dentry, struct au_nhash *whlist) +{ + return au_test_empty(dentry, whlist); +} + +/* + * test if @a->src_dentry dir can be rename source or not. + * if it can, return 0. + * success means, + * - it is a logically empty dir. + * - or, it exists on writable branch and has no children including whiteouts + * on the lower branch unless DIRREN is on. + */ +static int may_rename_srcdir(struct au_ren_args *a) +{ + int err; + unsigned int rdhash; + aufs_bindex_t btop, btgt; + struct dentry *dentry; + struct super_block *sb; + struct au_sbinfo *sbinfo; + + dentry = a->src_dentry; + sb = dentry->d_sb; + sbinfo = au_sbi(sb); + if (au_opt_test(sbinfo->si_mntflags, DIRREN)) + au_fset_ren(a->auren_flags, DIRREN); + + btgt = a->btgt; + btop = au_dbtop(dentry); + if (btop != btgt) { + struct au_nhash whlist; + + SiMustAnyLock(sb); + rdhash = sbinfo->si_rdhash; + if (!rdhash) + rdhash = au_rdhash_est(au_dir_size(/*file*/NULL, + dentry)); + err = au_nhash_alloc(&whlist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_test_empty(dentry, &whlist); + au_nhash_wh_free(&whlist); + goto out; + } + + if (btop == au_dbtaildir(dentry)) + return 0; /* success */ + + err = au_test_empty_lower(dentry); + +out: + if (err == -ENOTEMPTY) { + if (au_ftest_ren(a->auren_flags, DIRREN)) { + err = 0; + } else { + AuWarn1("renaming dir who has child(ren) on multiple " + "branches, is not supported\n"); + err = -EXDEV; + } + } + return err; +} + +/* side effect: sets whlist and h_dentry */ +static int au_ren_may_dir(struct au_ren_args *a) +{ + int err; + unsigned int rdhash; + struct dentry *d; + + d = a->dst_dentry; + SiMustAnyLock(d->d_sb); + + err = 0; + if (au_ftest_ren(a->auren_flags, ISDIR_DST) && a->dst_inode) { + rdhash = au_sbi(d->d_sb)->si_rdhash; + if (!rdhash) + rdhash = au_rdhash_est(au_dir_size(/*file*/NULL, d)); + err = au_nhash_alloc(&a->whlist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out; + + if (!a->exchange) { + au_set_dbtop(d, a->dst_btop); + err = may_rename_dstdir(d, &a->whlist); + au_set_dbtop(d, a->btgt); + } else + err = may_rename_srcdir(a); + } + a->dst_h_dentry = au_h_dptr(d, au_dbtop(d)); + if (unlikely(err)) + goto out; + + d = a->src_dentry; + a->src_h_dentry = au_h_dptr(d, au_dbtop(d)); + if (au_ftest_ren(a->auren_flags, ISDIR_SRC)) { + err = may_rename_srcdir(a); + if (unlikely(err)) { + au_nhash_wh_free(&a->whlist); + a->whlist.nh_num = 0; + } + } +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * simple tests for rename. + * following the checks in vfs, plus the parent-child relationship. + */ +static int au_may_ren(struct au_ren_args *a) +{ + int err, isdir; + struct inode *h_inode; + + if (a->src_btop == a->btgt) { + err = au_may_del(a->src_dentry, a->btgt, a->src_h_parent, + au_ftest_ren(a->auren_flags, ISDIR_SRC)); + if (unlikely(err)) + goto out; + err = -EINVAL; + if (unlikely(a->src_h_dentry == a->h_trap)) + goto out; + } + + err = 0; + if (a->dst_btop != a->btgt) + goto out; + + err = -ENOTEMPTY; + if (unlikely(a->dst_h_dentry == a->h_trap)) + goto out; + + err = -EIO; + isdir = !!au_ftest_ren(a->auren_flags, ISDIR_DST); + if (d_really_is_negative(a->dst_dentry)) { + if (d_is_negative(a->dst_h_dentry)) + err = au_may_add(a->dst_dentry, a->btgt, + a->dst_h_parent, isdir); + } else { + if (unlikely(d_is_negative(a->dst_h_dentry))) + goto out; + h_inode = d_inode(a->dst_h_dentry); + if (h_inode->i_nlink) + err = au_may_del(a->dst_dentry, a->btgt, + a->dst_h_parent, isdir); + } + +out: + if (unlikely(err == -ENOENT || err == -EEXIST)) + err = -EIO; + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * locking order + * (VFS) + * - src_dir and dir by lock_rename() + * - inode if exists + * (aufs) + * - lock all + * + src_dentry and dentry by aufs_read_and_write_lock2() which calls, + * + si_read_lock + * + di_write_lock2_child() + * + di_write_lock_child() + * + ii_write_lock_child() + * + di_write_lock_child2() + * + ii_write_lock_child2() + * + src_parent and parent + * + di_write_lock_parent() + * + ii_write_lock_parent() + * + di_write_lock_parent2() + * + ii_write_lock_parent2() + * + lower src_dir and dir by vfsub_lock_rename() + * + verify the every relationships between child and parent. if any + * of them failed, unlock all and return -EBUSY. + */ +static void au_ren_unlock(struct au_ren_args *a) +{ + vfsub_unlock_rename(a->src_h_parent, a->src_hdir, + a->dst_h_parent, a->dst_hdir); + if (au_ftest_ren(a->auren_flags, DIRREN) + && a->h_root) + au_hn_inode_unlock(a->h_root); + if (au_ftest_ren(a->auren_flags, MNT_WRITE)) + vfsub_mnt_drop_write(au_br_mnt(a->br)); +} + +static int au_ren_lock(struct au_ren_args *a) +{ + int err; + unsigned int udba; + + err = 0; + a->src_h_parent = au_h_dptr(a->src_parent, a->btgt); + a->src_hdir = au_hi(a->src_dir, a->btgt); + a->dst_h_parent = au_h_dptr(a->dst_parent, a->btgt); + a->dst_hdir = au_hi(a->dst_dir, a->btgt); + + err = vfsub_mnt_want_write(au_br_mnt(a->br)); + if (unlikely(err)) + goto out; + au_fset_ren(a->auren_flags, MNT_WRITE); + if (au_ftest_ren(a->auren_flags, DIRREN)) { + struct dentry *root; + struct inode *dir; + + /* + * sbinfo is already locked, so this ii_read_lock is + * unnecessary. but our debugging feature checks it. + */ + root = a->src_inode->i_sb->s_root; + if (root != a->src_parent && root != a->dst_parent) { + dir = d_inode(root); + ii_read_lock_parent3(dir); + a->h_root = au_hi(dir, a->btgt); + ii_read_unlock(dir); + au_hn_inode_lock_nested(a->h_root, AuLsc_I_PARENT3); + } + } + a->h_trap = vfsub_lock_rename(a->src_h_parent, a->src_hdir, + a->dst_h_parent, a->dst_hdir); + udba = au_opt_udba(a->src_dentry->d_sb); + if (unlikely(a->src_hdir->hi_inode != d_inode(a->src_h_parent) + || a->dst_hdir->hi_inode != d_inode(a->dst_h_parent))) + err = au_busy_or_stale(); + if (!err && au_dbtop(a->src_dentry) == a->btgt) + err = au_h_verify(a->src_h_dentry, udba, + d_inode(a->src_h_parent), a->src_h_parent, + a->br); + if (!err && au_dbtop(a->dst_dentry) == a->btgt) + err = au_h_verify(a->dst_h_dentry, udba, + d_inode(a->dst_h_parent), a->dst_h_parent, + a->br); + if (!err) + goto out; /* success */ + + err = au_busy_or_stale(); + au_ren_unlock(a); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +static void au_ren_refresh_dir(struct au_ren_args *a) +{ + struct inode *dir; + + dir = a->dst_dir; + inode_inc_iversion(dir); + if (au_ftest_ren(a->auren_flags, ISDIR_SRC)) { + /* is this updating defined in POSIX? */ + au_cpup_attr_timesizes(a->src_inode); + au_cpup_attr_nlink(dir, /*force*/1); + } + au_dir_ts(dir, a->btgt); + + if (a->exchange) { + dir = a->src_dir; + inode_inc_iversion(dir); + if (au_ftest_ren(a->auren_flags, ISDIR_DST)) { + /* is this updating defined in POSIX? */ + au_cpup_attr_timesizes(a->dst_inode); + au_cpup_attr_nlink(dir, /*force*/1); + } + au_dir_ts(dir, a->btgt); + } + + if (au_ftest_ren(a->auren_flags, ISSAMEDIR)) + return; + + dir = a->src_dir; + inode_inc_iversion(dir); + if (au_ftest_ren(a->auren_flags, ISDIR_SRC)) + au_cpup_attr_nlink(dir, /*force*/1); + au_dir_ts(dir, a->btgt); +} + +static void au_ren_refresh(struct au_ren_args *a) +{ + aufs_bindex_t bbot, bindex; + struct dentry *d, *h_d; + struct inode *i, *h_i; + struct super_block *sb; + + d = a->dst_dentry; + d_drop(d); + if (a->h_dst) + /* already dget-ed by au_ren_or_cpup() */ + au_set_h_dptr(d, a->btgt, a->h_dst); + + i = a->dst_inode; + if (i) { + if (!a->exchange) { + if (!au_ftest_ren(a->auren_flags, ISDIR_DST)) + vfsub_drop_nlink(i); + else { + vfsub_dead_dir(i); + au_cpup_attr_timesizes(i); + } + au_update_dbrange(d, /*do_put_zero*/1); + } else + au_cpup_attr_nlink(i, /*force*/1); + } else { + bbot = a->btgt; + for (bindex = au_dbtop(d); bindex < bbot; bindex++) + au_set_h_dptr(d, bindex, NULL); + bbot = au_dbbot(d); + for (bindex = a->btgt + 1; bindex <= bbot; bindex++) + au_set_h_dptr(d, bindex, NULL); + au_update_dbrange(d, /*do_put_zero*/0); + } + + if (a->exchange + || au_ftest_ren(a->auren_flags, DIRREN)) { + d_drop(a->src_dentry); + if (au_ftest_ren(a->auren_flags, DIRREN)) + au_set_dbwh(a->src_dentry, -1); + return; + } + + d = a->src_dentry; + au_set_dbwh(d, -1); + bbot = au_dbbot(d); + for (bindex = a->btgt + 1; bindex <= bbot; bindex++) { + h_d = au_h_dptr(d, bindex); + if (h_d) + au_set_h_dptr(d, bindex, NULL); + } + au_set_dbbot(d, a->btgt); + + sb = d->d_sb; + i = a->src_inode; + if (au_opt_test(au_mntflags(sb), PLINK) && au_plink_test(i)) + return; /* success */ + + bbot = au_ibbot(i); + for (bindex = a->btgt + 1; bindex <= bbot; bindex++) { + h_i = au_h_iptr(i, bindex); + if (h_i) { + au_xino_write(sb, bindex, h_i->i_ino, /*ino*/0); + /* ignore this error */ + au_set_h_iptr(i, bindex, NULL, 0); + } + } + au_set_ibbot(i, a->btgt); +} + +/* ---------------------------------------------------------------------- */ + +/* mainly for link(2) and rename(2) */ +int au_wbr(struct dentry *dentry, aufs_bindex_t btgt) +{ + aufs_bindex_t bdiropq, bwh; + struct dentry *parent; + struct au_branch *br; + + parent = dentry->d_parent; + IMustLock(d_inode(parent)); /* dir is locked */ + + bdiropq = au_dbdiropq(parent); + bwh = au_dbwh(dentry); + br = au_sbr(dentry->d_sb, btgt); + if (au_br_rdonly(br) + || (0 <= bdiropq && bdiropq < btgt) + || (0 <= bwh && bwh < btgt)) + btgt = -1; + + AuDbg("btgt %d\n", btgt); + return btgt; +} + +/* sets src_btop, dst_btop and btgt */ +static int au_ren_wbr(struct au_ren_args *a) +{ + int err; + struct au_wr_dir_args wr_dir_args = { + /* .force_btgt = -1, */ + .flags = AuWrDir_ADD_ENTRY + }; + + a->src_btop = au_dbtop(a->src_dentry); + a->dst_btop = au_dbtop(a->dst_dentry); + if (au_ftest_ren(a->auren_flags, ISDIR_SRC) + || au_ftest_ren(a->auren_flags, ISDIR_DST)) + au_fset_wrdir(wr_dir_args.flags, ISDIR); + wr_dir_args.force_btgt = a->src_btop; + if (a->dst_inode && a->dst_btop < a->src_btop) + wr_dir_args.force_btgt = a->dst_btop; + wr_dir_args.force_btgt = au_wbr(a->dst_dentry, wr_dir_args.force_btgt); + err = au_wr_dir(a->dst_dentry, a->src_dentry, &wr_dir_args); + a->btgt = err; + if (a->exchange) + au_update_dbtop(a->dst_dentry); + + return err; +} + +static void au_ren_dt(struct au_ren_args *a) +{ + a->h_path.dentry = a->src_h_parent; + au_dtime_store(a->src_dt + AuPARENT, a->src_parent, &a->h_path); + if (!au_ftest_ren(a->auren_flags, ISSAMEDIR)) { + a->h_path.dentry = a->dst_h_parent; + au_dtime_store(a->dst_dt + AuPARENT, a->dst_parent, &a->h_path); + } + + au_fclr_ren(a->auren_flags, DT_DSTDIR); + if (!au_ftest_ren(a->auren_flags, ISDIR_SRC) + && !a->exchange) + return; + + a->h_path.dentry = a->src_h_dentry; + au_dtime_store(a->src_dt + AuCHILD, a->src_dentry, &a->h_path); + if (d_is_positive(a->dst_h_dentry)) { + au_fset_ren(a->auren_flags, DT_DSTDIR); + a->h_path.dentry = a->dst_h_dentry; + au_dtime_store(a->dst_dt + AuCHILD, a->dst_dentry, &a->h_path); + } +} + +static void au_ren_rev_dt(int err, struct au_ren_args *a) +{ + struct dentry *h_d; + struct inode *h_inode; + + au_dtime_revert(a->src_dt + AuPARENT); + if (!au_ftest_ren(a->auren_flags, ISSAMEDIR)) + au_dtime_revert(a->dst_dt + AuPARENT); + + if (au_ftest_ren(a->auren_flags, ISDIR_SRC) && err != -EIO) { + h_d = a->src_dt[AuCHILD].dt_h_path.dentry; + h_inode = d_inode(h_d); + inode_lock_nested(h_inode, AuLsc_I_CHILD); + au_dtime_revert(a->src_dt + AuCHILD); + inode_unlock(h_inode); + + if (au_ftest_ren(a->auren_flags, DT_DSTDIR)) { + h_d = a->dst_dt[AuCHILD].dt_h_path.dentry; + h_inode = d_inode(h_d); + inode_lock_nested(h_inode, AuLsc_I_CHILD); + au_dtime_revert(a->dst_dt + AuCHILD); + inode_unlock(h_inode); + } + } +} + +/* ---------------------------------------------------------------------- */ + +int aufs_rename(struct inode *_src_dir, struct dentry *_src_dentry, + struct inode *_dst_dir, struct dentry *_dst_dentry, + unsigned int _flags) +{ + int err, lock_flags; + void *rev; + /* reduce stack space */ + struct au_ren_args *a; + struct au_pin pin; + + AuDbg("%pd, %pd, 0x%x\n", _src_dentry, _dst_dentry, _flags); + IMustLock(_src_dir); + IMustLock(_dst_dir); + + err = -EINVAL; + if (unlikely(_flags & RENAME_WHITEOUT)) + goto out; + + err = -ENOMEM; + BUILD_BUG_ON(sizeof(*a) > PAGE_SIZE); + a = kzalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + a->flags = _flags; + BUILD_BUG_ON(sizeof(a->exchange) == sizeof(u8) + && RENAME_EXCHANGE > U8_MAX); + a->exchange = _flags & RENAME_EXCHANGE; + a->src_dir = _src_dir; + a->src_dentry = _src_dentry; + a->src_inode = NULL; + if (d_really_is_positive(a->src_dentry)) + a->src_inode = d_inode(a->src_dentry); + a->src_parent = a->src_dentry->d_parent; /* dir inode is locked */ + a->dst_dir = _dst_dir; + a->dst_dentry = _dst_dentry; + a->dst_inode = NULL; + if (d_really_is_positive(a->dst_dentry)) + a->dst_inode = d_inode(a->dst_dentry); + a->dst_parent = a->dst_dentry->d_parent; /* dir inode is locked */ + if (a->dst_inode) { + /* + * if EXCHANGE && src is non-dir && dst is dir, + * dst is not locked. + */ + /* IMustLock(a->dst_inode); */ + au_igrab(a->dst_inode); + } + + err = -ENOTDIR; + lock_flags = AuLock_FLUSH | AuLock_NOPLM | AuLock_GEN; + if (d_is_dir(a->src_dentry)) { + au_fset_ren(a->auren_flags, ISDIR_SRC); + if (unlikely(!a->exchange + && d_really_is_positive(a->dst_dentry) + && !d_is_dir(a->dst_dentry))) + goto out_free; + lock_flags |= AuLock_DIRS; + } + if (a->dst_inode && d_is_dir(a->dst_dentry)) { + au_fset_ren(a->auren_flags, ISDIR_DST); + if (unlikely(!a->exchange + && d_really_is_positive(a->src_dentry) + && !d_is_dir(a->src_dentry))) + goto out_free; + lock_flags |= AuLock_DIRS; + } + err = aufs_read_and_write_lock2(a->dst_dentry, a->src_dentry, + lock_flags); + if (unlikely(err)) + goto out_free; + + err = au_d_hashed_positive(a->src_dentry); + if (unlikely(err)) + goto out_unlock; + err = -ENOENT; + if (a->dst_inode) { + /* + * If it is a dir, VFS unhash it before this + * function. It means we cannot rely upon d_unhashed(). + */ + if (unlikely(!a->dst_inode->i_nlink)) + goto out_unlock; + if (!au_ftest_ren(a->auren_flags, ISDIR_DST)) { + err = au_d_hashed_positive(a->dst_dentry); + if (unlikely(err && !a->exchange)) + goto out_unlock; + } else if (unlikely(IS_DEADDIR(a->dst_inode))) + goto out_unlock; + } else if (unlikely(d_unhashed(a->dst_dentry))) + goto out_unlock; + + /* + * is it possible? + * yes, it happened (in linux-3.3-rcN) but I don't know why. + * there may exist a problem somewhere else. + */ + err = -EINVAL; + if (unlikely(d_inode(a->dst_parent) == d_inode(a->src_dentry))) + goto out_unlock; + + au_fset_ren(a->auren_flags, ISSAMEDIR); /* temporary */ + di_write_lock_parent(a->dst_parent); + + /* which branch we process */ + err = au_ren_wbr(a); + if (unlikely(err < 0)) + goto out_parent; + a->br = au_sbr(a->dst_dentry->d_sb, a->btgt); + a->h_path.mnt = au_br_mnt(a->br); + + /* are they available to be renamed */ + err = au_ren_may_dir(a); + if (unlikely(err)) + goto out_children; + + /* prepare the writable parent dir on the same branch */ + if (a->dst_btop == a->btgt) { + au_fset_ren(a->auren_flags, WHDST); + } else { + err = au_cpup_dirs(a->dst_dentry, a->btgt); + if (unlikely(err)) + goto out_children; + } + + err = 0; + if (!a->exchange) { + if (a->src_dir != a->dst_dir) { + /* + * this temporary unlock is safe, + * because both dir->i_mutex are locked. + */ + di_write_unlock(a->dst_parent); + di_write_lock_parent(a->src_parent); + err = au_wr_dir_need_wh(a->src_dentry, + au_ftest_ren(a->auren_flags, + ISDIR_SRC), + &a->btgt); + di_write_unlock(a->src_parent); + di_write_lock2_parent(a->src_parent, a->dst_parent, + /*isdir*/1); + au_fclr_ren(a->auren_flags, ISSAMEDIR); + } else + err = au_wr_dir_need_wh(a->src_dentry, + au_ftest_ren(a->auren_flags, + ISDIR_SRC), + &a->btgt); + } + if (unlikely(err < 0)) + goto out_children; + if (err) + au_fset_ren(a->auren_flags, WHSRC); + + /* cpup src */ + if (a->src_btop != a->btgt) { + err = au_pin(&pin, a->src_dentry, a->btgt, + au_opt_udba(a->src_dentry->d_sb), + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (!err) { + struct au_cp_generic cpg = { + .dentry = a->src_dentry, + .bdst = a->btgt, + .bsrc = a->src_btop, + .len = -1, + .pin = &pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + AuDebugOn(au_dbtop(a->src_dentry) != a->src_btop); + err = au_sio_cpup_simple(&cpg); + au_unpin(&pin); + } + if (unlikely(err)) + goto out_children; + a->src_btop = a->btgt; + a->src_h_dentry = au_h_dptr(a->src_dentry, a->btgt); + if (!a->exchange) + au_fset_ren(a->auren_flags, WHSRC); + } + + /* cpup dst */ + if (a->exchange && a->dst_inode + && a->dst_btop != a->btgt) { + err = au_pin(&pin, a->dst_dentry, a->btgt, + au_opt_udba(a->dst_dentry->d_sb), + AuPin_DI_LOCKED | AuPin_MNT_WRITE); + if (!err) { + struct au_cp_generic cpg = { + .dentry = a->dst_dentry, + .bdst = a->btgt, + .bsrc = a->dst_btop, + .len = -1, + .pin = &pin, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + err = au_sio_cpup_simple(&cpg); + au_unpin(&pin); + } + if (unlikely(err)) + goto out_children; + a->dst_btop = a->btgt; + a->dst_h_dentry = au_h_dptr(a->dst_dentry, a->btgt); + } + + /* lock them all */ + err = au_ren_lock(a); + if (unlikely(err)) + /* leave the copied-up one */ + goto out_children; + + if (!a->exchange) { + if (!au_opt_test(au_mntflags(a->dst_dir->i_sb), UDBA_NONE)) + err = au_may_ren(a); + else if (unlikely(a->dst_dentry->d_name.len > AUFS_MAX_NAMELEN)) + err = -ENAMETOOLONG; + if (unlikely(err)) + goto out_hdir; + } + + /* store timestamps to be revertible */ + au_ren_dt(a); + + /* store dirren info */ + if (au_ftest_ren(a->auren_flags, DIRREN)) { + err = au_dr_rename(a->src_dentry, a->btgt, + &a->dst_dentry->d_name, &rev); + AuTraceErr(err); + if (unlikely(err)) + goto out_dt; + } + + /* here we go */ + err = do_rename(a); + if (unlikely(err)) + goto out_dirren; + + if (au_ftest_ren(a->auren_flags, DIRREN)) + au_dr_rename_fin(a->src_dentry, a->btgt, rev); + + /* update dir attributes */ + au_ren_refresh_dir(a); + + /* dput/iput all lower dentries */ + au_ren_refresh(a); + + goto out_hdir; /* success */ + +out_dirren: + if (au_ftest_ren(a->auren_flags, DIRREN)) + au_dr_rename_rev(a->src_dentry, a->btgt, rev); +out_dt: + au_ren_rev_dt(err, a); +out_hdir: + au_ren_unlock(a); +out_children: + au_nhash_wh_free(&a->whlist); + if (err && a->dst_inode && a->dst_btop != a->btgt) { + AuDbg("btop %d, btgt %d\n", a->dst_btop, a->btgt); + au_set_h_dptr(a->dst_dentry, a->btgt, NULL); + au_set_dbtop(a->dst_dentry, a->dst_btop); + } +out_parent: + if (!err) { + if (d_unhashed(a->src_dentry)) + au_fset_ren(a->auren_flags, DROPPED_SRC); + if (d_unhashed(a->dst_dentry)) + au_fset_ren(a->auren_flags, DROPPED_DST); + if (!a->exchange) + d_move(a->src_dentry, a->dst_dentry); + else { + d_exchange(a->src_dentry, a->dst_dentry); + if (au_ftest_ren(a->auren_flags, DROPPED_DST)) + d_drop(a->dst_dentry); + } + if (au_ftest_ren(a->auren_flags, DROPPED_SRC)) + d_drop(a->src_dentry); + } else { + au_update_dbtop(a->dst_dentry); + if (!a->dst_inode) + d_drop(a->dst_dentry); + } + if (au_ftest_ren(a->auren_flags, ISSAMEDIR)) + di_write_unlock(a->dst_parent); + else + di_write_unlock2(a->src_parent, a->dst_parent); +out_unlock: + aufs_read_and_write_unlock2(a->dst_dentry, a->src_dentry); +out_free: + iput(a->dst_inode); + if (a->thargs) + au_whtmp_rmdir_free(a->thargs); + au_kfree_rcu(a); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/iinfo.c b/fs/aufs/iinfo.c new file mode 100644 index 0000000000000..016db39451cd9 --- /dev/null +++ b/fs/aufs/iinfo.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode private data + */ + +#include "aufs.h" + +struct inode *au_h_iptr(struct inode *inode, aufs_bindex_t bindex) +{ + struct inode *h_inode; + struct au_hinode *hinode; + + IiMustAnyLock(inode); + + hinode = au_hinode(au_ii(inode), bindex); + h_inode = hinode->hi_inode; + AuDebugOn(h_inode && atomic_read(&h_inode->i_count) <= 0); + return h_inode; +} + +/* todo: hard/soft set? */ +void au_hiput(struct au_hinode *hinode) +{ + au_hn_free(hinode); + dput(hinode->hi_whdentry); + iput(hinode->hi_inode); +} + +unsigned int au_hi_flags(struct inode *inode, int isdir) +{ + unsigned int flags; + const unsigned int mnt_flags = au_mntflags(inode->i_sb); + + flags = 0; + if (au_opt_test(mnt_flags, XINO)) + au_fset_hi(flags, XINO); + if (isdir && au_opt_test(mnt_flags, UDBA_HNOTIFY)) + au_fset_hi(flags, HNOTIFY); + return flags; +} + +void au_set_h_iptr(struct inode *inode, aufs_bindex_t bindex, + struct inode *h_inode, unsigned int flags) +{ + struct au_hinode *hinode; + struct inode *hi; + struct au_iinfo *iinfo = au_ii(inode); + + IiMustWriteLock(inode); + + hinode = au_hinode(iinfo, bindex); + hi = hinode->hi_inode; + AuDebugOn(h_inode && atomic_read(&h_inode->i_count) <= 0); + + if (hi) + au_hiput(hinode); + hinode->hi_inode = h_inode; + if (h_inode) { + int err; + struct super_block *sb = inode->i_sb; + struct au_branch *br; + + AuDebugOn(inode->i_mode + && (h_inode->i_mode & S_IFMT) + != (inode->i_mode & S_IFMT)); + if (bindex == iinfo->ii_btop) + au_cpup_igen(inode, h_inode); + br = au_sbr(sb, bindex); + hinode->hi_id = br->br_id; + if (au_ftest_hi(flags, XINO)) { + err = au_xino_write(sb, bindex, h_inode->i_ino, + inode->i_ino); + if (unlikely(err)) + AuIOErr1("failed au_xino_write() %d\n", err); + } + + if (au_ftest_hi(flags, HNOTIFY) + && au_br_hnotifyable(br->br_perm)) { + err = au_hn_alloc(hinode, inode); + if (unlikely(err)) + AuIOErr1("au_hn_alloc() %d\n", err); + } + } +} + +void au_set_hi_wh(struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_wh) +{ + struct au_hinode *hinode; + + IiMustWriteLock(inode); + + hinode = au_hinode(au_ii(inode), bindex); + AuDebugOn(hinode->hi_whdentry); + hinode->hi_whdentry = h_wh; +} + +void au_update_iigen(struct inode *inode, int half) +{ + struct au_iinfo *iinfo; + struct au_iigen *iigen; + unsigned int sigen; + + sigen = au_sigen(inode->i_sb); + iinfo = au_ii(inode); + iigen = &iinfo->ii_generation; + spin_lock(&iigen->ig_spin); + iigen->ig_generation = sigen; + if (half) + au_ig_fset(iigen->ig_flags, HALF_REFRESHED); + else + au_ig_fclr(iigen->ig_flags, HALF_REFRESHED); + spin_unlock(&iigen->ig_spin); +} + +/* it may be called at remount time, too */ +void au_update_ibrange(struct inode *inode, int do_put_zero) +{ + struct au_iinfo *iinfo; + aufs_bindex_t bindex, bbot; + + AuDebugOn(au_is_bad_inode(inode)); + IiMustWriteLock(inode); + + iinfo = au_ii(inode); + if (do_put_zero && iinfo->ii_btop >= 0) { + for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot; + bindex++) { + struct inode *h_i; + + h_i = au_hinode(iinfo, bindex)->hi_inode; + if (h_i + && !h_i->i_nlink + && !(h_i->i_state & I_LINKABLE)) + au_set_h_iptr(inode, bindex, NULL, 0); + } + } + + iinfo->ii_btop = -1; + iinfo->ii_bbot = -1; + bbot = au_sbbot(inode->i_sb); + for (bindex = 0; bindex <= bbot; bindex++) + if (au_hinode(iinfo, bindex)->hi_inode) { + iinfo->ii_btop = bindex; + break; + } + if (iinfo->ii_btop >= 0) + for (bindex = bbot; bindex >= iinfo->ii_btop; bindex--) + if (au_hinode(iinfo, bindex)->hi_inode) { + iinfo->ii_bbot = bindex; + break; + } + AuDebugOn(iinfo->ii_btop > iinfo->ii_bbot); +} + +/* ---------------------------------------------------------------------- */ + +void au_icntnr_init_once(void *_c) +{ + struct au_icntnr *c = _c; + struct au_iinfo *iinfo = &c->iinfo; + + spin_lock_init(&iinfo->ii_generation.ig_spin); + au_rw_init(&iinfo->ii_rwsem); + inode_init_once(&c->vfs_inode); +} + +void au_hinode_init(struct au_hinode *hinode) +{ + hinode->hi_inode = NULL; + hinode->hi_id = -1; + au_hn_init(hinode); + hinode->hi_whdentry = NULL; +} + +int au_iinfo_init(struct inode *inode) +{ + struct au_iinfo *iinfo; + struct super_block *sb; + struct au_hinode *hi; + int nbr, i; + + sb = inode->i_sb; + iinfo = &(container_of(inode, struct au_icntnr, vfs_inode)->iinfo); + nbr = au_sbbot(sb) + 1; + if (unlikely(nbr <= 0)) + nbr = 1; + hi = kmalloc_array(nbr, sizeof(*iinfo->ii_hinode), GFP_NOFS); + if (hi) { + au_lcnt_inc(&au_sbi(sb)->si_ninodes); + + iinfo->ii_hinode = hi; + for (i = 0; i < nbr; i++, hi++) + au_hinode_init(hi); + + iinfo->ii_generation.ig_generation = au_sigen(sb); + iinfo->ii_btop = -1; + iinfo->ii_bbot = -1; + iinfo->ii_vdir = NULL; + return 0; + } + return -ENOMEM; +} + +int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink) +{ + int err, i; + struct au_hinode *hip; + + AuRwMustWriteLock(&iinfo->ii_rwsem); + + err = -ENOMEM; + hip = au_krealloc(iinfo->ii_hinode, sizeof(*hip) * nbr, GFP_NOFS, + may_shrink); + if (hip) { + iinfo->ii_hinode = hip; + i = iinfo->ii_bbot + 1; + hip += i; + for (; i < nbr; i++, hip++) + au_hinode_init(hip); + err = 0; + } + + return err; +} + +void au_iinfo_fin(struct inode *inode) +{ + struct au_iinfo *iinfo; + struct au_hinode *hi; + struct super_block *sb; + aufs_bindex_t bindex, bbot; + const unsigned char unlinked = !inode->i_nlink; + + AuDebugOn(au_is_bad_inode(inode)); + + sb = inode->i_sb; + au_lcnt_dec(&au_sbi(sb)->si_ninodes); + if (si_pid_test(sb)) + au_xino_delete_inode(inode, unlinked); + else { + /* + * it is safe to hide the dependency between sbinfo and + * sb->s_umount. + */ + lockdep_off(); + si_noflush_read_lock(sb); + au_xino_delete_inode(inode, unlinked); + si_read_unlock(sb); + lockdep_on(); + } + + iinfo = au_ii(inode); + if (iinfo->ii_vdir) + au_vdir_free(iinfo->ii_vdir); + + bindex = iinfo->ii_btop; + if (bindex >= 0) { + hi = au_hinode(iinfo, bindex); + bbot = iinfo->ii_bbot; + while (bindex++ <= bbot) { + if (hi->hi_inode) + au_hiput(hi); + hi++; + } + } + au_kfree_rcu(iinfo->ii_hinode); + AuRwDestroy(&iinfo->ii_rwsem); +} diff --git a/fs/aufs/inode.c b/fs/aufs/inode.c new file mode 100644 index 0000000000000..4de1b7507c5b3 --- /dev/null +++ b/fs/aufs/inode.c @@ -0,0 +1,529 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode functions + */ + +#include +#include "aufs.h" + +struct inode *au_igrab(struct inode *inode) +{ + if (inode) { + AuDebugOn(!atomic_read(&inode->i_count)); + ihold(inode); + } + return inode; +} + +static void au_refresh_hinode_attr(struct inode *inode, int do_version) +{ + au_cpup_attr_all(inode, /*force*/0); + au_update_iigen(inode, /*half*/1); + if (do_version) + inode_inc_iversion(inode); +} + +static int au_ii_refresh(struct inode *inode, int *update) +{ + int err, e, nbr; + umode_t type; + aufs_bindex_t bindex, new_bindex; + struct super_block *sb; + struct au_iinfo *iinfo; + struct au_hinode *p, *q, tmp; + + AuDebugOn(au_is_bad_inode(inode)); + IiMustWriteLock(inode); + + *update = 0; + sb = inode->i_sb; + nbr = au_sbbot(sb) + 1; + type = inode->i_mode & S_IFMT; + iinfo = au_ii(inode); + err = au_hinode_realloc(iinfo, nbr, /*may_shrink*/0); + if (unlikely(err)) + goto out; + + AuDebugOn(iinfo->ii_btop < 0); + p = au_hinode(iinfo, iinfo->ii_btop); + for (bindex = iinfo->ii_btop; bindex <= iinfo->ii_bbot; + bindex++, p++) { + if (!p->hi_inode) + continue; + + AuDebugOn(type != (p->hi_inode->i_mode & S_IFMT)); + new_bindex = au_br_index(sb, p->hi_id); + if (new_bindex == bindex) + continue; + + if (new_bindex < 0) { + *update = 1; + au_hiput(p); + p->hi_inode = NULL; + continue; + } + + if (new_bindex < iinfo->ii_btop) + iinfo->ii_btop = new_bindex; + if (iinfo->ii_bbot < new_bindex) + iinfo->ii_bbot = new_bindex; + /* swap two lower inode, and loop again */ + q = au_hinode(iinfo, new_bindex); + tmp = *q; + *q = *p; + *p = tmp; + if (tmp.hi_inode) { + bindex--; + p--; + } + } + au_update_ibrange(inode, /*do_put_zero*/0); + au_hinode_realloc(iinfo, nbr, /*may_shrink*/1); /* harmless if err */ + e = au_dy_irefresh(inode); + if (unlikely(e && !err)) + err = e; + +out: + AuTraceErr(err); + return err; +} + +void au_refresh_iop(struct inode *inode, int force_getattr) +{ + int type; + struct au_sbinfo *sbi = au_sbi(inode->i_sb); + const struct inode_operations *iop + = force_getattr ? aufs_iop : sbi->si_iop_array; + + if (inode->i_op == iop) + return; + + switch (inode->i_mode & S_IFMT) { + case S_IFDIR: + type = AuIop_DIR; + break; + case S_IFLNK: + type = AuIop_SYMLINK; + break; + default: + type = AuIop_OTHER; + break; + } + + inode->i_op = iop + type; + /* unnecessary smp_wmb() */ +} + +int au_refresh_hinode_self(struct inode *inode) +{ + int err, update; + + err = au_ii_refresh(inode, &update); + if (!err) + au_refresh_hinode_attr(inode, update && S_ISDIR(inode->i_mode)); + + AuTraceErr(err); + return err; +} + +int au_refresh_hinode(struct inode *inode, struct dentry *dentry) +{ + int err, e, update; + unsigned int flags; + umode_t mode; + aufs_bindex_t bindex, bbot; + unsigned char isdir; + struct au_hinode *p; + struct au_iinfo *iinfo; + + err = au_ii_refresh(inode, &update); + if (unlikely(err)) + goto out; + + update = 0; + iinfo = au_ii(inode); + p = au_hinode(iinfo, iinfo->ii_btop); + mode = (inode->i_mode & S_IFMT); + isdir = S_ISDIR(mode); + flags = au_hi_flags(inode, isdir); + bbot = au_dbbot(dentry); + for (bindex = au_dbtop(dentry); bindex <= bbot; bindex++) { + struct inode *h_i, *h_inode; + struct dentry *h_d; + + h_d = au_h_dptr(dentry, bindex); + if (!h_d || d_is_negative(h_d)) + continue; + + h_inode = d_inode(h_d); + AuDebugOn(mode != (h_inode->i_mode & S_IFMT)); + if (iinfo->ii_btop <= bindex && bindex <= iinfo->ii_bbot) { + h_i = au_h_iptr(inode, bindex); + if (h_i) { + if (h_i == h_inode) + continue; + err = -EIO; + break; + } + } + if (bindex < iinfo->ii_btop) + iinfo->ii_btop = bindex; + if (iinfo->ii_bbot < bindex) + iinfo->ii_bbot = bindex; + au_set_h_iptr(inode, bindex, au_igrab(h_inode), flags); + update = 1; + } + au_update_ibrange(inode, /*do_put_zero*/0); + e = au_dy_irefresh(inode); + if (unlikely(e && !err)) + err = e; + if (!err) + au_refresh_hinode_attr(inode, update && isdir); + +out: + AuTraceErr(err); + return err; +} + +static int set_inode(struct inode *inode, struct dentry *dentry) +{ + int err; + unsigned int flags; + umode_t mode; + aufs_bindex_t bindex, btop, btail; + unsigned char isdir; + struct dentry *h_dentry; + struct inode *h_inode; + struct au_iinfo *iinfo; + const struct inode_operations *iop; + + IiMustWriteLock(inode); + + err = 0; + isdir = 0; + iop = au_sbi(inode->i_sb)->si_iop_array; + btop = au_dbtop(dentry); + h_dentry = au_h_dptr(dentry, btop); + h_inode = d_inode(h_dentry); + mode = h_inode->i_mode; + switch (mode & S_IFMT) { + case S_IFREG: + btail = au_dbtail(dentry); + inode->i_op = iop + AuIop_OTHER; + inode->i_fop = &aufs_file_fop; + err = au_dy_iaop(inode, btop, h_inode); + if (unlikely(err)) + goto out; + break; + case S_IFDIR: + isdir = 1; + btail = au_dbtaildir(dentry); + inode->i_op = iop + AuIop_DIR; + inode->i_fop = &aufs_dir_fop; + break; + case S_IFLNK: + btail = au_dbtail(dentry); + inode->i_op = iop + AuIop_SYMLINK; + break; + case S_IFBLK: + case S_IFCHR: + case S_IFIFO: + case S_IFSOCK: + btail = au_dbtail(dentry); + inode->i_op = iop + AuIop_OTHER; + init_special_inode(inode, mode, h_inode->i_rdev); + break; + default: + AuIOErr("Unknown file type 0%o\n", mode); + err = -EIO; + goto out; + } + + /* do not set hnotify for whiteouted dirs (SHWH mode) */ + flags = au_hi_flags(inode, isdir); + if (au_opt_test(au_mntflags(dentry->d_sb), SHWH) + && au_ftest_hi(flags, HNOTIFY) + && dentry->d_name.len > AUFS_WH_PFX_LEN + && !memcmp(dentry->d_name.name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) + au_fclr_hi(flags, HNOTIFY); + iinfo = au_ii(inode); + iinfo->ii_btop = btop; + iinfo->ii_bbot = btail; + for (bindex = btop; bindex <= btail; bindex++) { + h_dentry = au_h_dptr(dentry, bindex); + if (h_dentry) + au_set_h_iptr(inode, bindex, + au_igrab(d_inode(h_dentry)), flags); + } + au_cpup_attr_all(inode, /*force*/1); + /* + * to force calling aufs_get_acl() every time, + * do not call cache_no_acl() for aufs inode. + */ + +out: + return err; +} + +/* + * successful returns with iinfo write_locked + * minus: errno + * zero: success, matched + * plus: no error, but unmatched + */ +static int reval_inode(struct inode *inode, struct dentry *dentry) +{ + int err; + unsigned int gen, igflags; + aufs_bindex_t bindex, bbot; + struct inode *h_inode, *h_dinode; + struct dentry *h_dentry; + + /* + * before this function, if aufs got any iinfo lock, it must be only + * one, the parent dir. + * it can happen by UDBA and the obsoleted inode number. + */ + err = -EIO; + if (unlikely(inode->i_ino == parent_ino(dentry))) + goto out; + + err = 1; + ii_write_lock_new_child(inode); + h_dentry = au_h_dptr(dentry, au_dbtop(dentry)); + h_dinode = d_inode(h_dentry); + bbot = au_ibbot(inode); + for (bindex = au_ibtop(inode); bindex <= bbot; bindex++) { + h_inode = au_h_iptr(inode, bindex); + if (!h_inode || h_inode != h_dinode) + continue; + + err = 0; + gen = au_iigen(inode, &igflags); + if (gen == au_digen(dentry) + && !au_ig_ftest(igflags, HALF_REFRESHED)) + break; + + /* fully refresh inode using dentry */ + err = au_refresh_hinode(inode, dentry); + if (!err) + au_update_iigen(inode, /*half*/0); + break; + } + + if (unlikely(err)) + ii_write_unlock(inode); +out: + return err; +} + +int au_ino(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + unsigned int d_type, ino_t *ino) +{ + int err, idx; + const int isnondir = d_type != DT_DIR; + + /* prevent hardlinked inode number from race condition */ + if (isnondir) { + err = au_xinondir_enter(sb, bindex, h_ino, &idx); + if (unlikely(err)) + goto out; + } + + err = au_xino_read(sb, bindex, h_ino, ino); + if (unlikely(err)) + goto out_xinondir; + + if (!*ino) { + err = -EIO; + *ino = au_xino_new_ino(sb); + if (unlikely(!*ino)) + goto out_xinondir; + err = au_xino_write(sb, bindex, h_ino, *ino); + if (unlikely(err)) + goto out_xinondir; + } + +out_xinondir: + if (isnondir && idx >= 0) + au_xinondir_leave(sb, bindex, h_ino, idx); +out: + return err; +} + +/* successful returns with iinfo write_locked */ +/* todo: return with unlocked? */ +struct inode *au_new_inode(struct dentry *dentry, int must_new) +{ + struct inode *inode, *h_inode; + struct dentry *h_dentry; + struct super_block *sb; + ino_t h_ino, ino; + int err, idx, hlinked; + aufs_bindex_t btop; + + sb = dentry->d_sb; + btop = au_dbtop(dentry); + h_dentry = au_h_dptr(dentry, btop); + h_inode = d_inode(h_dentry); + h_ino = h_inode->i_ino; + hlinked = !d_is_dir(h_dentry) && h_inode->i_nlink > 1; + +new_ino: + /* + * stop 'race'-ing between hardlinks under different + * parents. + */ + if (hlinked) { + err = au_xinondir_enter(sb, btop, h_ino, &idx); + inode = ERR_PTR(err); + if (unlikely(err)) + goto out; + } + + err = au_xino_read(sb, btop, h_ino, &ino); + inode = ERR_PTR(err); + if (unlikely(err)) + goto out_xinondir; + + if (!ino) { + ino = au_xino_new_ino(sb); + if (unlikely(!ino)) { + inode = ERR_PTR(-EIO); + goto out_xinondir; + } + } + + AuDbg("i%lu\n", (unsigned long)ino); + inode = au_iget_locked(sb, ino); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out_xinondir; + + AuDbg("%lx, new %d\n", inode->i_state, !!(inode->i_state & I_NEW)); + if (inode->i_state & I_NEW) { + ii_write_lock_new_child(inode); + err = set_inode(inode, dentry); + if (!err) { + unlock_new_inode(inode); + goto out_xinondir; /* success */ + } + + /* + * iget_failed() calls iput(), but we need to call + * ii_write_unlock() after iget_failed(). so dirty hack for + * i_count. + */ + atomic_inc(&inode->i_count); + iget_failed(inode); + ii_write_unlock(inode); + au_xino_write(sb, btop, h_ino, /*ino*/0); + /* ignore this error */ + goto out_iput; + } else if (!must_new && !IS_DEADDIR(inode) && inode->i_nlink) { + /* + * horrible race condition between lookup, readdir and copyup + * (or something). + */ + if (hlinked && idx >= 0) + au_xinondir_leave(sb, btop, h_ino, idx); + err = reval_inode(inode, dentry); + if (unlikely(err < 0)) { + hlinked = 0; + goto out_iput; + } + if (!err) + goto out; /* success */ + else if (hlinked && idx >= 0) { + err = au_xinondir_enter(sb, btop, h_ino, &idx); + if (unlikely(err)) { + iput(inode); + inode = ERR_PTR(err); + goto out; + } + } + } + + if (unlikely(au_test_fs_unique_ino(h_inode))) + AuWarn1("Warning: Un-notified UDBA or repeatedly renamed dir," + " b%d, %s, %pd, hi%lu, i%lu.\n", + btop, au_sbtype(h_dentry->d_sb), dentry, + (unsigned long)h_ino, (unsigned long)ino); + ino = 0; + err = au_xino_write(sb, btop, h_ino, /*ino*/0); + if (!err) { + iput(inode); + if (hlinked && idx >= 0) + au_xinondir_leave(sb, btop, h_ino, idx); + goto new_ino; + } + +out_iput: + iput(inode); + inode = ERR_PTR(err); +out_xinondir: + if (hlinked && idx >= 0) + au_xinondir_leave(sb, btop, h_ino, idx); +out: + return inode; +} + +/* ---------------------------------------------------------------------- */ + +int au_test_ro(struct super_block *sb, aufs_bindex_t bindex, + struct inode *inode) +{ + int err; + struct inode *hi; + + err = au_br_rdonly(au_sbr(sb, bindex)); + + /* pseudo-link after flushed may happen out of bounds */ + if (!err + && inode + && au_ibtop(inode) <= bindex + && bindex <= au_ibbot(inode)) { + /* + * permission check is unnecessary since vfsub routine + * will be called later + */ + hi = au_h_iptr(inode, bindex); + if (hi) + err = IS_IMMUTABLE(hi) ? -EROFS : 0; + } + + return err; +} + +int au_test_h_perm(struct inode *h_inode, int mask) +{ + if (uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) + return 0; + return inode_permission(h_inode, mask); +} + +int au_test_h_perm_sio(struct inode *h_inode, int mask) +{ + if (au_test_nfs(h_inode->i_sb) + && (mask & MAY_WRITE) + && S_ISDIR(h_inode->i_mode)) + mask |= MAY_READ; /* force permission check */ + return au_test_h_perm(h_inode, mask); +} diff --git a/fs/aufs/inode.h b/fs/aufs/inode.h new file mode 100644 index 0000000000000..2342f74eb097c --- /dev/null +++ b/fs/aufs/inode.h @@ -0,0 +1,698 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * inode operations + */ + +#ifndef __AUFS_INODE_H__ +#define __AUFS_INODE_H__ + +#ifdef __KERNEL__ + +#include +#include "rwsem.h" + +struct vfsmount; + +struct au_hnotify { +#ifdef CONFIG_AUFS_HNOTIFY +#ifdef CONFIG_AUFS_HFSNOTIFY + /* never use fsnotify_add_vfsmount_mark() */ + struct fsnotify_mark hn_mark; +#endif + struct inode *hn_aufs_inode; /* no get/put */ + struct rcu_head rcu; +#endif +} ____cacheline_aligned_in_smp; + +struct au_hinode { + struct inode *hi_inode; + aufs_bindex_t hi_id; +#ifdef CONFIG_AUFS_HNOTIFY + struct au_hnotify *hi_notify; +#endif + + /* reference to the copied-up whiteout with get/put */ + struct dentry *hi_whdentry; +}; + +/* ig_flags */ +#define AuIG_HALF_REFRESHED 1 +#define au_ig_ftest(flags, name) ((flags) & AuIG_##name) +#define au_ig_fset(flags, name) \ + do { (flags) |= AuIG_##name; } while (0) +#define au_ig_fclr(flags, name) \ + do { (flags) &= ~AuIG_##name; } while (0) + +struct au_iigen { + spinlock_t ig_spin; + __u32 ig_generation, ig_flags; +}; + +struct au_vdir; +struct au_iinfo { + struct au_iigen ii_generation; + struct super_block *ii_hsb1; /* no get/put */ + + struct au_rwsem ii_rwsem; + aufs_bindex_t ii_btop, ii_bbot; + __u32 ii_higen; + struct au_hinode *ii_hinode; + struct au_vdir *ii_vdir; +}; + +struct au_icntnr { + struct au_iinfo iinfo; + struct inode vfs_inode; + struct hlist_bl_node plink; + struct rcu_head rcu; +} ____cacheline_aligned_in_smp; + +/* au_pin flags */ +#define AuPin_DI_LOCKED 1 +#define AuPin_MNT_WRITE (1 << 1) +#define au_ftest_pin(flags, name) ((flags) & AuPin_##name) +#define au_fset_pin(flags, name) \ + do { (flags) |= AuPin_##name; } while (0) +#define au_fclr_pin(flags, name) \ + do { (flags) &= ~AuPin_##name; } while (0) + +struct au_pin { + /* input */ + struct dentry *dentry; + unsigned int udba; + unsigned char lsc_di, lsc_hi, flags; + aufs_bindex_t bindex; + + /* output */ + struct dentry *parent; + struct au_hinode *hdir; + struct vfsmount *h_mnt; + + /* temporary unlock/relock for copyup */ + struct dentry *h_dentry, *h_parent; + struct au_branch *br; + struct task_struct *task; +}; + +void au_pin_hdir_unlock(struct au_pin *p); +int au_pin_hdir_lock(struct au_pin *p); +int au_pin_hdir_relock(struct au_pin *p); +void au_pin_hdir_acquire_nest(struct au_pin *p); +void au_pin_hdir_release(struct au_pin *p); + +/* ---------------------------------------------------------------------- */ + +static inline struct au_iinfo *au_ii(struct inode *inode) +{ + BUG_ON(is_bad_inode(inode)); + return &(container_of(inode, struct au_icntnr, vfs_inode)->iinfo); +} + +/* ---------------------------------------------------------------------- */ + +/* inode.c */ +struct inode *au_igrab(struct inode *inode); +void au_refresh_iop(struct inode *inode, int force_getattr); +int au_refresh_hinode_self(struct inode *inode); +int au_refresh_hinode(struct inode *inode, struct dentry *dentry); +int au_ino(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + unsigned int d_type, ino_t *ino); +struct inode *au_new_inode(struct dentry *dentry, int must_new); +int au_test_ro(struct super_block *sb, aufs_bindex_t bindex, + struct inode *inode); +int au_test_h_perm(struct inode *h_inode, int mask); +int au_test_h_perm_sio(struct inode *h_inode, int mask); + +static inline int au_wh_ino(struct super_block *sb, aufs_bindex_t bindex, + ino_t h_ino, unsigned int d_type, ino_t *ino) +{ +#ifdef CONFIG_AUFS_SHWH + return au_ino(sb, bindex, h_ino, d_type, ino); +#else + return 0; +#endif +} + +/* i_op.c */ +enum { + AuIop_SYMLINK, + AuIop_DIR, + AuIop_OTHER, + AuIop_Last +}; +extern struct inode_operations aufs_iop[AuIop_Last], /* not const */ + aufs_iop_nogetattr[AuIop_Last]; + +/* au_wr_dir flags */ +#define AuWrDir_ADD_ENTRY 1 +#define AuWrDir_ISDIR (1 << 1) +#define AuWrDir_TMPFILE (1 << 2) +#define au_ftest_wrdir(flags, name) ((flags) & AuWrDir_##name) +#define au_fset_wrdir(flags, name) \ + do { (flags) |= AuWrDir_##name; } while (0) +#define au_fclr_wrdir(flags, name) \ + do { (flags) &= ~AuWrDir_##name; } while (0) + +struct au_wr_dir_args { + aufs_bindex_t force_btgt; + unsigned char flags; +}; +int au_wr_dir(struct dentry *dentry, struct dentry *src_dentry, + struct au_wr_dir_args *args); + +struct dentry *au_pinned_h_parent(struct au_pin *pin); +void au_pin_init(struct au_pin *pin, struct dentry *dentry, + aufs_bindex_t bindex, int lsc_di, int lsc_hi, + unsigned int udba, unsigned char flags); +int au_pin(struct au_pin *pin, struct dentry *dentry, aufs_bindex_t bindex, + unsigned int udba, unsigned char flags) __must_check; +int au_do_pin(struct au_pin *pin) __must_check; +void au_unpin(struct au_pin *pin); +int au_reval_for_attr(struct dentry *dentry, unsigned int sigen); + +#define AuIcpup_DID_CPUP 1 +#define au_ftest_icpup(flags, name) ((flags) & AuIcpup_##name) +#define au_fset_icpup(flags, name) \ + do { (flags) |= AuIcpup_##name; } while (0) +#define au_fclr_icpup(flags, name) \ + do { (flags) &= ~AuIcpup_##name; } while (0) + +struct au_icpup_args { + unsigned char flags; + unsigned char pin_flags; + aufs_bindex_t btgt; + unsigned int udba; + struct au_pin pin; + struct path h_path; + struct inode *h_inode; +}; + +int au_pin_and_icpup(struct dentry *dentry, struct iattr *ia, + struct au_icpup_args *a); + +int au_h_path_getattr(struct dentry *dentry, struct inode *inode, int force, + struct path *h_path, int locked); + +/* i_op_add.c */ +int au_may_add(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent, int isdir); +int aufs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, + dev_t dev); +int aufs_symlink(struct inode *dir, struct dentry *dentry, const char *symname); +int aufs_create(struct inode *dir, struct dentry *dentry, umode_t mode, + bool want_excl); +struct vfsub_aopen_args; +int au_aopen_or_create(struct inode *dir, struct dentry *dentry, + struct vfsub_aopen_args *args); +int aufs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode); +int aufs_link(struct dentry *src_dentry, struct inode *dir, + struct dentry *dentry); +int aufs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode); + +/* i_op_del.c */ +int au_wr_dir_need_wh(struct dentry *dentry, int isdir, aufs_bindex_t *bcpup); +int au_may_del(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent, int isdir); +int aufs_unlink(struct inode *dir, struct dentry *dentry); +int aufs_rmdir(struct inode *dir, struct dentry *dentry); + +/* i_op_ren.c */ +int au_wbr(struct dentry *dentry, aufs_bindex_t btgt); +int aufs_rename(struct inode *src_dir, struct dentry *src_dentry, + struct inode *dir, struct dentry *dentry, + unsigned int flags); + +/* iinfo.c */ +struct inode *au_h_iptr(struct inode *inode, aufs_bindex_t bindex); +void au_hiput(struct au_hinode *hinode); +void au_set_hi_wh(struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_wh); +unsigned int au_hi_flags(struct inode *inode, int isdir); + +/* hinode flags */ +#define AuHi_XINO 1 +#define AuHi_HNOTIFY (1 << 1) +#define au_ftest_hi(flags, name) ((flags) & AuHi_##name) +#define au_fset_hi(flags, name) \ + do { (flags) |= AuHi_##name; } while (0) +#define au_fclr_hi(flags, name) \ + do { (flags) &= ~AuHi_##name; } while (0) + +#ifndef CONFIG_AUFS_HNOTIFY +#undef AuHi_HNOTIFY +#define AuHi_HNOTIFY 0 +#endif + +void au_set_h_iptr(struct inode *inode, aufs_bindex_t bindex, + struct inode *h_inode, unsigned int flags); + +void au_update_iigen(struct inode *inode, int half); +void au_update_ibrange(struct inode *inode, int do_put_zero); + +void au_icntnr_init_once(void *_c); +void au_hinode_init(struct au_hinode *hinode); +int au_iinfo_init(struct inode *inode); +void au_iinfo_fin(struct inode *inode); +int au_hinode_realloc(struct au_iinfo *iinfo, int nbr, int may_shrink); + +#ifdef CONFIG_PROC_FS +/* plink.c */ +int au_plink_maint(struct super_block *sb, int flags); +struct au_sbinfo; +void au_plink_maint_leave(struct au_sbinfo *sbinfo); +int au_plink_maint_enter(struct super_block *sb); +#ifdef CONFIG_AUFS_DEBUG +void au_plink_list(struct super_block *sb); +#else +AuStubVoid(au_plink_list, struct super_block *sb) +#endif +int au_plink_test(struct inode *inode); +struct dentry *au_plink_lkup(struct inode *inode, aufs_bindex_t bindex); +void au_plink_append(struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_dentry); +void au_plink_put(struct super_block *sb, int verbose); +void au_plink_clean(struct super_block *sb, int verbose); +void au_plink_half_refresh(struct super_block *sb, aufs_bindex_t br_id); +#else +AuStubInt0(au_plink_maint, struct super_block *sb, int flags); +AuStubVoid(au_plink_maint_leave, struct au_sbinfo *sbinfo); +AuStubInt0(au_plink_maint_enter, struct super_block *sb); +AuStubVoid(au_plink_list, struct super_block *sb); +AuStubInt0(au_plink_test, struct inode *inode); +AuStub(struct dentry *, au_plink_lkup, return NULL, + struct inode *inode, aufs_bindex_t bindex); +AuStubVoid(au_plink_append, struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_dentry); +AuStubVoid(au_plink_put, struct super_block *sb, int verbose); +AuStubVoid(au_plink_clean, struct super_block *sb, int verbose); +AuStubVoid(au_plink_half_refresh, struct super_block *sb, aufs_bindex_t br_id); +#endif /* CONFIG_PROC_FS */ + +#ifdef CONFIG_AUFS_XATTR +/* xattr.c */ +int au_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, int ignore_flags, + unsigned int verbose); +ssize_t aufs_listxattr(struct dentry *dentry, char *list, size_t size); +void au_xattr_init(struct super_block *sb); +#else +AuStubInt0(au_cpup_xattr, struct dentry *h_dst, struct dentry *h_src, + int ignore_flags, unsigned int verbose); +AuStubVoid(au_xattr_init, struct super_block *sb); +#endif + +#ifdef CONFIG_FS_POSIX_ACL +struct posix_acl *aufs_get_acl(struct inode *inode, int type); +int aufs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +#endif + +#if IS_ENABLED(CONFIG_AUFS_XATTR) || IS_ENABLED(CONFIG_FS_POSIX_ACL) +enum { + AU_XATTR_SET, + AU_ACL_SET +}; + +struct au_sxattr { + int type; + union { + struct { + const char *name; + const void *value; + size_t size; + int flags; + } set; + struct { + struct posix_acl *acl; + int type; + } acl_set; + } u; +}; +ssize_t au_sxattr(struct dentry *dentry, struct inode *inode, + struct au_sxattr *arg); +#endif + +/* ---------------------------------------------------------------------- */ + +/* lock subclass for iinfo */ +enum { + AuLsc_II_CHILD, /* child first */ + AuLsc_II_CHILD2, /* rename(2), link(2), and cpup at hnotify */ + AuLsc_II_CHILD3, /* copyup dirs */ + AuLsc_II_PARENT, /* see AuLsc_I_PARENT in vfsub.h */ + AuLsc_II_PARENT2, + AuLsc_II_PARENT3, /* copyup dirs */ + AuLsc_II_NEW_CHILD +}; + +/* + * ii_read_lock_child, ii_write_lock_child, + * ii_read_lock_child2, ii_write_lock_child2, + * ii_read_lock_child3, ii_write_lock_child3, + * ii_read_lock_parent, ii_write_lock_parent, + * ii_read_lock_parent2, ii_write_lock_parent2, + * ii_read_lock_parent3, ii_write_lock_parent3, + * ii_read_lock_new_child, ii_write_lock_new_child, + */ +#define AuReadLockFunc(name, lsc) \ +static inline void ii_read_lock_##name(struct inode *i) \ +{ \ + au_rw_read_lock_nested(&au_ii(i)->ii_rwsem, AuLsc_II_##lsc); \ +} + +#define AuWriteLockFunc(name, lsc) \ +static inline void ii_write_lock_##name(struct inode *i) \ +{ \ + au_rw_write_lock_nested(&au_ii(i)->ii_rwsem, AuLsc_II_##lsc); \ +} + +#define AuRWLockFuncs(name, lsc) \ + AuReadLockFunc(name, lsc) \ + AuWriteLockFunc(name, lsc) + +AuRWLockFuncs(child, CHILD); +AuRWLockFuncs(child2, CHILD2); +AuRWLockFuncs(child3, CHILD3); +AuRWLockFuncs(parent, PARENT); +AuRWLockFuncs(parent2, PARENT2); +AuRWLockFuncs(parent3, PARENT3); +AuRWLockFuncs(new_child, NEW_CHILD); + +#undef AuReadLockFunc +#undef AuWriteLockFunc +#undef AuRWLockFuncs + +#define ii_read_unlock(i) au_rw_read_unlock(&au_ii(i)->ii_rwsem) +#define ii_write_unlock(i) au_rw_write_unlock(&au_ii(i)->ii_rwsem) +#define ii_downgrade_lock(i) au_rw_dgrade_lock(&au_ii(i)->ii_rwsem) + +#define IiMustNoWaiters(i) AuRwMustNoWaiters(&au_ii(i)->ii_rwsem) +#define IiMustAnyLock(i) AuRwMustAnyLock(&au_ii(i)->ii_rwsem) +#define IiMustWriteLock(i) AuRwMustWriteLock(&au_ii(i)->ii_rwsem) + +/* ---------------------------------------------------------------------- */ + +static inline void au_icntnr_init(struct au_icntnr *c) +{ +#ifdef CONFIG_AUFS_DEBUG + c->vfs_inode.i_mode = 0; +#endif +} + +static inline unsigned int au_iigen(struct inode *inode, unsigned int *igflags) +{ + unsigned int gen; + struct au_iinfo *iinfo; + struct au_iigen *iigen; + + iinfo = au_ii(inode); + iigen = &iinfo->ii_generation; + spin_lock(&iigen->ig_spin); + if (igflags) + *igflags = iigen->ig_flags; + gen = iigen->ig_generation; + spin_unlock(&iigen->ig_spin); + + return gen; +} + +/* tiny test for inode number */ +/* tmpfs generation is too rough */ +static inline int au_test_higen(struct inode *inode, struct inode *h_inode) +{ + struct au_iinfo *iinfo; + + iinfo = au_ii(inode); + AuRwMustAnyLock(&iinfo->ii_rwsem); + return !(iinfo->ii_hsb1 == h_inode->i_sb + && iinfo->ii_higen == h_inode->i_generation); +} + +static inline void au_iigen_dec(struct inode *inode) +{ + struct au_iinfo *iinfo; + struct au_iigen *iigen; + + iinfo = au_ii(inode); + iigen = &iinfo->ii_generation; + spin_lock(&iigen->ig_spin); + iigen->ig_generation--; + spin_unlock(&iigen->ig_spin); +} + +static inline int au_iigen_test(struct inode *inode, unsigned int sigen) +{ + int err; + + err = 0; + if (unlikely(inode && au_iigen(inode, NULL) != sigen)) + err = -EIO; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static inline struct au_hinode *au_hinode(struct au_iinfo *iinfo, + aufs_bindex_t bindex) +{ + return iinfo->ii_hinode + bindex; +} + +static inline int au_is_bad_inode(struct inode *inode) +{ + return !!(is_bad_inode(inode) || !au_hinode(au_ii(inode), 0)); +} + +static inline aufs_bindex_t au_ii_br_id(struct inode *inode, + aufs_bindex_t bindex) +{ + IiMustAnyLock(inode); + return au_hinode(au_ii(inode), bindex)->hi_id; +} + +static inline aufs_bindex_t au_ibtop(struct inode *inode) +{ + IiMustAnyLock(inode); + return au_ii(inode)->ii_btop; +} + +static inline aufs_bindex_t au_ibbot(struct inode *inode) +{ + IiMustAnyLock(inode); + return au_ii(inode)->ii_bbot; +} + +static inline struct au_vdir *au_ivdir(struct inode *inode) +{ + IiMustAnyLock(inode); + return au_ii(inode)->ii_vdir; +} + +static inline struct dentry *au_hi_wh(struct inode *inode, aufs_bindex_t bindex) +{ + IiMustAnyLock(inode); + return au_hinode(au_ii(inode), bindex)->hi_whdentry; +} + +static inline void au_set_ibtop(struct inode *inode, aufs_bindex_t bindex) +{ + IiMustWriteLock(inode); + au_ii(inode)->ii_btop = bindex; +} + +static inline void au_set_ibbot(struct inode *inode, aufs_bindex_t bindex) +{ + IiMustWriteLock(inode); + au_ii(inode)->ii_bbot = bindex; +} + +static inline void au_set_ivdir(struct inode *inode, struct au_vdir *vdir) +{ + IiMustWriteLock(inode); + au_ii(inode)->ii_vdir = vdir; +} + +static inline struct au_hinode *au_hi(struct inode *inode, aufs_bindex_t bindex) +{ + IiMustAnyLock(inode); + return au_hinode(au_ii(inode), bindex); +} + +/* ---------------------------------------------------------------------- */ + +static inline struct dentry *au_pinned_parent(struct au_pin *pin) +{ + if (pin) + return pin->parent; + return NULL; +} + +static inline struct inode *au_pinned_h_dir(struct au_pin *pin) +{ + if (pin && pin->hdir) + return pin->hdir->hi_inode; + return NULL; +} + +static inline struct au_hinode *au_pinned_hdir(struct au_pin *pin) +{ + if (pin) + return pin->hdir; + return NULL; +} + +static inline void au_pin_set_dentry(struct au_pin *pin, struct dentry *dentry) +{ + if (pin) + pin->dentry = dentry; +} + +static inline void au_pin_set_parent_lflag(struct au_pin *pin, + unsigned char lflag) +{ + if (pin) { + if (lflag) + au_fset_pin(pin->flags, DI_LOCKED); + else + au_fclr_pin(pin->flags, DI_LOCKED); + } +} + +#if 0 /* reserved */ +static inline void au_pin_set_parent(struct au_pin *pin, struct dentry *parent) +{ + if (pin) { + dput(pin->parent); + pin->parent = dget(parent); + } +} +#endif + +/* ---------------------------------------------------------------------- */ + +struct au_branch; +#ifdef CONFIG_AUFS_HNOTIFY +struct au_hnotify_op { + void (*ctl)(struct au_hinode *hinode, int do_set); + int (*alloc)(struct au_hinode *hinode); + + /* + * if it returns true, the the caller should free hinode->hi_notify, + * otherwise ->free() frees it. + */ + int (*free)(struct au_hinode *hinode, + struct au_hnotify *hn) __must_check; + + void (*fin)(void); + int (*init)(void); + + int (*reset_br)(unsigned int udba, struct au_branch *br, int perm); + void (*fin_br)(struct au_branch *br); + int (*init_br)(struct au_branch *br, int perm); +}; + +/* hnotify.c */ +int au_hn_alloc(struct au_hinode *hinode, struct inode *inode); +void au_hn_free(struct au_hinode *hinode); +void au_hn_ctl(struct au_hinode *hinode, int do_set); +void au_hn_reset(struct inode *inode, unsigned int flags); +int au_hnotify(struct inode *h_dir, struct au_hnotify *hnotify, u32 mask, + struct qstr *h_child_qstr, struct inode *h_child_inode); +int au_hnotify_reset_br(unsigned int udba, struct au_branch *br, int perm); +int au_hnotify_init_br(struct au_branch *br, int perm); +void au_hnotify_fin_br(struct au_branch *br); +int __init au_hnotify_init(void); +void au_hnotify_fin(void); + +/* hfsnotify.c */ +extern const struct au_hnotify_op au_hnotify_op; + +static inline +void au_hn_init(struct au_hinode *hinode) +{ + hinode->hi_notify = NULL; +} + +static inline struct au_hnotify *au_hn(struct au_hinode *hinode) +{ + return hinode->hi_notify; +} + +#else +AuStub(int, au_hn_alloc, return -EOPNOTSUPP, + struct au_hinode *hinode __maybe_unused, + struct inode *inode __maybe_unused) +AuStub(struct au_hnotify *, au_hn, return NULL, struct au_hinode *hinode) +AuStubVoid(au_hn_free, struct au_hinode *hinode __maybe_unused) +AuStubVoid(au_hn_ctl, struct au_hinode *hinode __maybe_unused, + int do_set __maybe_unused) +AuStubVoid(au_hn_reset, struct inode *inode __maybe_unused, + unsigned int flags __maybe_unused) +AuStubInt0(au_hnotify_reset_br, unsigned int udba __maybe_unused, + struct au_branch *br __maybe_unused, + int perm __maybe_unused) +AuStubInt0(au_hnotify_init_br, struct au_branch *br __maybe_unused, + int perm __maybe_unused) +AuStubVoid(au_hnotify_fin_br, struct au_branch *br __maybe_unused) +AuStubInt0(__init au_hnotify_init, void) +AuStubVoid(au_hnotify_fin, void) +AuStubVoid(au_hn_init, struct au_hinode *hinode __maybe_unused) +#endif /* CONFIG_AUFS_HNOTIFY */ + +static inline void au_hn_suspend(struct au_hinode *hdir) +{ + au_hn_ctl(hdir, /*do_set*/0); +} + +static inline void au_hn_resume(struct au_hinode *hdir) +{ + au_hn_ctl(hdir, /*do_set*/1); +} + +static inline void au_hn_inode_lock(struct au_hinode *hdir) +{ + inode_lock(hdir->hi_inode); + au_hn_suspend(hdir); +} + +static inline void au_hn_inode_lock_nested(struct au_hinode *hdir, + unsigned int sc __maybe_unused) +{ + inode_lock_nested(hdir->hi_inode, sc); + au_hn_suspend(hdir); +} + +#if 0 /* unused */ +#include "vfsub.h" +static inline void au_hn_inode_lock_shared_nested(struct au_hinode *hdir, + unsigned int sc) +{ + inode_lock_shared_nested(hdir->hi_inode, sc); + au_hn_suspend(hdir); +} +#endif + +static inline void au_hn_inode_unlock(struct au_hinode *hdir) +{ + au_hn_resume(hdir); + inode_unlock(hdir->hi_inode); +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_INODE_H__ */ diff --git a/fs/aufs/ioctl.c b/fs/aufs/ioctl.c new file mode 100644 index 0000000000000..ae4f73386b907 --- /dev/null +++ b/fs/aufs/ioctl.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * ioctl + * plink-management and readdir in userspace. + * assist the pathconf(3) wrapper library. + * move-down + * File-based Hierarchical Storage Management. + */ + +#include +#include +#include "aufs.h" + +static int au_wbr_fd(struct path *path, struct aufs_wbr_fd __user *arg) +{ + int err, fd; + aufs_bindex_t wbi, bindex, bbot; + struct file *h_file; + struct super_block *sb; + struct dentry *root; + struct au_branch *br; + struct aufs_wbr_fd wbrfd = { + .oflags = au_dir_roflags, + .brid = -1 + }; + const int valid = O_RDONLY | O_NONBLOCK | O_LARGEFILE | O_DIRECTORY + | O_NOATIME | O_CLOEXEC; + + AuDebugOn(wbrfd.oflags & ~valid); + + if (arg) { + err = copy_from_user(&wbrfd, arg, sizeof(wbrfd)); + if (unlikely(err)) { + err = -EFAULT; + goto out; + } + + err = -EINVAL; + AuDbg("wbrfd{0%o, %d}\n", wbrfd.oflags, wbrfd.brid); + wbrfd.oflags |= au_dir_roflags; + AuDbg("0%o\n", wbrfd.oflags); + if (unlikely(wbrfd.oflags & ~valid)) + goto out; + } + + fd = get_unused_fd_flags(0); + err = fd; + if (unlikely(fd < 0)) + goto out; + + h_file = ERR_PTR(-EINVAL); + wbi = 0; + br = NULL; + sb = path->dentry->d_sb; + root = sb->s_root; + aufs_read_lock(root, AuLock_IR); + bbot = au_sbbot(sb); + if (wbrfd.brid >= 0) { + wbi = au_br_index(sb, wbrfd.brid); + if (unlikely(wbi < 0 || wbi > bbot)) + goto out_unlock; + } + + h_file = ERR_PTR(-ENOENT); + br = au_sbr(sb, wbi); + if (!au_br_writable(br->br_perm)) { + if (arg) + goto out_unlock; + + bindex = wbi + 1; + wbi = -1; + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_writable(br->br_perm)) { + wbi = bindex; + br = au_sbr(sb, wbi); + break; + } + } + } + AuDbg("wbi %d\n", wbi); + if (wbi >= 0) + h_file = au_h_open(root, wbi, wbrfd.oflags, NULL, + /*force_wr*/0); + +out_unlock: + aufs_read_unlock(root, AuLock_IR); + err = PTR_ERR(h_file); + if (IS_ERR(h_file)) + goto out_fd; + + au_lcnt_dec(&br->br_nfiles); /* cf. au_h_open() */ + fd_install(fd, h_file); + err = fd; + goto out; /* success */ + +out_fd: + put_unused_fd(fd); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +long aufs_ioctl_dir(struct file *file, unsigned int cmd, unsigned long arg) +{ + long err; + struct dentry *dentry; + + switch (cmd) { + case AUFS_CTL_RDU: + case AUFS_CTL_RDU_INO: + err = au_rdu_ioctl(file, cmd, arg); + break; + + case AUFS_CTL_WBR_FD: + err = au_wbr_fd(&file->f_path, (void __user *)arg); + break; + + case AUFS_CTL_IBUSY: + err = au_ibusy_ioctl(file, arg); + break; + + case AUFS_CTL_BRINFO: + err = au_brinfo_ioctl(file, arg); + break; + + case AUFS_CTL_FHSM_FD: + dentry = file->f_path.dentry; + if (IS_ROOT(dentry)) + err = au_fhsm_fd(dentry->d_sb, arg); + else + err = -ENOTTY; + break; + + default: + /* do not call the lower */ + AuDbg("0x%x\n", cmd); + err = -ENOTTY; + } + + AuTraceErr(err); + return err; +} + +long aufs_ioctl_nondir(struct file *file, unsigned int cmd, unsigned long arg) +{ + long err; + + switch (cmd) { + case AUFS_CTL_MVDOWN: + err = au_mvdown(file->f_path.dentry, (void __user *)arg); + break; + + case AUFS_CTL_WBR_FD: + err = au_wbr_fd(&file->f_path, (void __user *)arg); + break; + + default: + /* do not call the lower */ + AuDbg("0x%x\n", cmd); + err = -ENOTTY; + } + + AuTraceErr(err); + return err; +} + +#ifdef CONFIG_COMPAT +long aufs_compat_ioctl_dir(struct file *file, unsigned int cmd, + unsigned long arg) +{ + long err; + + switch (cmd) { + case AUFS_CTL_RDU: + case AUFS_CTL_RDU_INO: + err = au_rdu_compat_ioctl(file, cmd, arg); + break; + + case AUFS_CTL_IBUSY: + err = au_ibusy_compat_ioctl(file, arg); + break; + + case AUFS_CTL_BRINFO: + err = au_brinfo_compat_ioctl(file, arg); + break; + + default: + err = aufs_ioctl_dir(file, cmd, arg); + } + + AuTraceErr(err); + return err; +} + +long aufs_compat_ioctl_nondir(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return aufs_ioctl_nondir(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif diff --git a/fs/aufs/lcnt.h b/fs/aufs/lcnt.h new file mode 100644 index 0000000000000..8afcabe558263 --- /dev/null +++ b/fs/aufs/lcnt.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * simple long counter wrapper + */ + +#ifndef __AUFS_LCNT_H__ +#define __AUFS_LCNT_H__ + +#ifdef __KERNEL__ + +#include "debug.h" + +#define AuLCntATOMIC 1 +#define AuLCntPCPUCNT 2 +/* + * why does percpu_refcount require extra synchronize_rcu()s in + * au_br_do_free() + */ +#define AuLCntPCPUREF 3 + +/* #define AuLCntChosen AuLCntATOMIC */ +#define AuLCntChosen AuLCntPCPUCNT +/* #define AuLCntChosen AuLCntPCPUREF */ + +#if AuLCntChosen == AuLCntATOMIC +#include + +typedef atomic_long_t au_lcnt_t; + +static inline int au_lcnt_init(au_lcnt_t *cnt, void *release __maybe_unused) +{ + atomic_long_set(cnt, 0); + return 0; +} + +static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused) +{ + /* empty */ +} + +static inline void au_lcnt_fin(au_lcnt_t *cnt __maybe_unused, + int do_sync __maybe_unused) +{ + /* empty */ +} + +static inline void au_lcnt_inc(au_lcnt_t *cnt) +{ + atomic_long_inc(cnt); +} + +static inline void au_lcnt_dec(au_lcnt_t *cnt) +{ + atomic_long_dec(cnt); +} + +static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev __maybe_unused) +{ + return atomic_long_read(cnt); +} +#endif + +#if AuLCntChosen == AuLCntPCPUCNT +#include + +typedef struct percpu_counter au_lcnt_t; + +static inline int au_lcnt_init(au_lcnt_t *cnt, void *release __maybe_unused) +{ + return percpu_counter_init(cnt, 0, GFP_NOFS); +} + +static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused) +{ + /* empty */ +} + +static inline void au_lcnt_fin(au_lcnt_t *cnt, int do_sync __maybe_unused) +{ + percpu_counter_destroy(cnt); +} + +static inline void au_lcnt_inc(au_lcnt_t *cnt) +{ + percpu_counter_inc(cnt); +} + +static inline void au_lcnt_dec(au_lcnt_t *cnt) +{ + percpu_counter_dec(cnt); +} + +static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev __maybe_unused) +{ + s64 n; + + n = percpu_counter_sum(cnt); + BUG_ON(n < 0); + if (LONG_MAX != LLONG_MAX + && n > LONG_MAX) + AuWarn1("%s\n", "wrap-around"); + + return n; +} +#endif + +#if AuLCntChosen == AuLCntPCPUREF +#include + +typedef struct percpu_ref au_lcnt_t; + +static inline int au_lcnt_init(au_lcnt_t *cnt, percpu_ref_func_t *release) +{ + if (!release) + release = percpu_ref_exit; + return percpu_ref_init(cnt, release, /*percpu mode*/0, GFP_NOFS); +} + +static inline void au_lcnt_wait_for_fin(au_lcnt_t *cnt __maybe_unused) +{ + synchronize_rcu(); +} + +static inline void au_lcnt_fin(au_lcnt_t *cnt, int do_sync) +{ + percpu_ref_kill(cnt); + if (do_sync) + au_lcnt_wait_for_fin(cnt); +} + +static inline void au_lcnt_inc(au_lcnt_t *cnt) +{ + percpu_ref_get(cnt); +} + +static inline void au_lcnt_dec(au_lcnt_t *cnt) +{ + percpu_ref_put(cnt); +} + +/* + * avoid calling this func as possible. + */ +static inline long au_lcnt_read(au_lcnt_t *cnt, int do_rev) +{ + long l; + + percpu_ref_switch_to_atomic_sync(cnt); + l = atomic_long_read(&cnt->count); + if (do_rev) + percpu_ref_switch_to_percpu(cnt); + + /* percpu_ref is initialized by 1 instead of 0 */ + return l - 1; +} +#endif + +#ifdef CONFIG_AUFS_DEBUG +#define AuLCntZero(val) do { \ + long l = val; \ + if (l) \ + AuDbg("%s = %ld\n", #val, l); \ +} while (0) +#else +#define AuLCntZero(val) do {} while (0) +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_LCNT_H__ */ diff --git a/fs/aufs/loop.c b/fs/aufs/loop.c new file mode 100644 index 0000000000000..81a9b2807d1f8 --- /dev/null +++ b/fs/aufs/loop.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * support for loopback block device as a branch + */ + +#include "aufs.h" + +/* added into drivers/block/loop.c */ +static struct file *(*backing_file_func)(struct super_block *sb); + +/* + * test if two lower dentries have overlapping branches. + */ +int au_test_loopback_overlap(struct super_block *sb, struct dentry *h_adding) +{ + struct super_block *h_sb; + struct file *backing_file; + + if (unlikely(!backing_file_func)) { + /* don't load "loop" module here */ + backing_file_func = symbol_get(loop_backing_file); + if (unlikely(!backing_file_func)) + /* "loop" module is not loaded */ + return 0; + } + + h_sb = h_adding->d_sb; + backing_file = backing_file_func(h_sb); + if (!backing_file) + return 0; + + h_adding = backing_file->f_path.dentry; + /* + * h_adding can be local NFS. + * in this case aufs cannot detect the loop. + */ + if (unlikely(h_adding->d_sb == sb)) + return 1; + return !!au_test_subdir(h_adding, sb->s_root); +} + +/* true if a kernel thread named 'loop[0-9].*' accesses a file */ +int au_test_loopback_kthread(void) +{ + int ret; + struct task_struct *tsk = current; + char c, comm[sizeof(tsk->comm)]; + + ret = 0; + if (tsk->flags & PF_KTHREAD) { + get_task_comm(comm, tsk); + c = comm[4]; + ret = ('0' <= c && c <= '9' + && !strncmp(comm, "loop", 4)); + } + + return ret; +} + +/* ---------------------------------------------------------------------- */ + +#define au_warn_loopback_step 16 +static int au_warn_loopback_nelem = au_warn_loopback_step; +static unsigned long *au_warn_loopback_array; + +void au_warn_loopback(struct super_block *h_sb) +{ + int i, new_nelem; + unsigned long *a, magic; + static DEFINE_SPINLOCK(spin); + + magic = h_sb->s_magic; + spin_lock(&spin); + a = au_warn_loopback_array; + for (i = 0; i < au_warn_loopback_nelem && *a; i++) + if (a[i] == magic) { + spin_unlock(&spin); + return; + } + + /* h_sb is new to us, print it */ + if (i < au_warn_loopback_nelem) { + a[i] = magic; + goto pr; + } + + /* expand the array */ + new_nelem = au_warn_loopback_nelem + au_warn_loopback_step; + a = au_kzrealloc(au_warn_loopback_array, + au_warn_loopback_nelem * sizeof(unsigned long), + new_nelem * sizeof(unsigned long), GFP_ATOMIC, + /*may_shrink*/0); + if (a) { + au_warn_loopback_nelem = new_nelem; + au_warn_loopback_array = a; + a[i] = magic; + goto pr; + } + + spin_unlock(&spin); + AuWarn1("realloc failed, ignored\n"); + return; + +pr: + spin_unlock(&spin); + pr_warn("you may want to try another patch for loopback file " + "on %s(0x%lx) branch\n", au_sbtype(h_sb), magic); +} + +int au_loopback_init(void) +{ + int err; + struct super_block *sb __maybe_unused; + + BUILD_BUG_ON(sizeof(sb->s_magic) != sizeof(unsigned long)); + + err = 0; + au_warn_loopback_array = kcalloc(au_warn_loopback_step, + sizeof(unsigned long), GFP_NOFS); + if (unlikely(!au_warn_loopback_array)) + err = -ENOMEM; + + return err; +} + +void au_loopback_fin(void) +{ + if (backing_file_func) + symbol_put(loop_backing_file); + au_kfree_try_rcu(au_warn_loopback_array); +} diff --git a/fs/aufs/loop.h b/fs/aufs/loop.h new file mode 100644 index 0000000000000..65c38bbb5d561 --- /dev/null +++ b/fs/aufs/loop.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * support for loopback mount as a branch + */ + +#ifndef __AUFS_LOOP_H__ +#define __AUFS_LOOP_H__ + +#ifdef __KERNEL__ + +struct dentry; +struct super_block; + +#ifdef CONFIG_AUFS_BDEV_LOOP +/* drivers/block/loop.c */ +struct file *loop_backing_file(struct super_block *sb); + +/* loop.c */ +int au_test_loopback_overlap(struct super_block *sb, struct dentry *h_adding); +int au_test_loopback_kthread(void); +void au_warn_loopback(struct super_block *h_sb); + +int au_loopback_init(void); +void au_loopback_fin(void); +#else +AuStub(struct file *, loop_backing_file, return NULL, struct super_block *sb) + +AuStubInt0(au_test_loopback_overlap, struct super_block *sb, + struct dentry *h_adding) +AuStubInt0(au_test_loopback_kthread, void) +AuStubVoid(au_warn_loopback, struct super_block *h_sb) + +AuStubInt0(au_loopback_init, void) +AuStubVoid(au_loopback_fin, void) +#endif /* BLK_DEV_LOOP */ + +#endif /* __KERNEL__ */ +#endif /* __AUFS_LOOP_H__ */ diff --git a/fs/aufs/magic.mk b/fs/aufs/magic.mk new file mode 100644 index 0000000000000..7bc9eef3ffec5 --- /dev/null +++ b/fs/aufs/magic.mk @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0 + +# defined in ${srctree}/fs/fuse/inode.c +# tristate +ifdef CONFIG_FUSE_FS +ccflags-y += -DFUSE_SUPER_MAGIC=0x65735546 +endif + +# defined in ${srctree}/fs/xfs/xfs_sb.h +# tristate +ifdef CONFIG_XFS_FS +ccflags-y += -DXFS_SB_MAGIC=0x58465342 +endif + +# defined in ${srctree}/fs/configfs/mount.c +# tristate +ifdef CONFIG_CONFIGFS_FS +ccflags-y += -DCONFIGFS_MAGIC=0x62656570 +endif + +# defined in ${srctree}/fs/ubifs/ubifs.h +# tristate +ifdef CONFIG_UBIFS_FS +ccflags-y += -DUBIFS_SUPER_MAGIC=0x24051905 +endif + +# defined in ${srctree}/fs/hfsplus/hfsplus_raw.h +# tristate +ifdef CONFIG_HFSPLUS_FS +ccflags-y += -DHFSPLUS_SUPER_MAGIC=0x482b +endif diff --git a/fs/aufs/module.c b/fs/aufs/module.c new file mode 100644 index 0000000000000..7245197a6cbf0 --- /dev/null +++ b/fs/aufs/module.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * module global variables and operations + */ + +#include +#include +#include "aufs.h" + +/* shrinkable realloc */ +void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink) +{ + size_t sz; + int diff; + + sz = 0; + diff = -1; + if (p) { +#if 0 /* unused */ + if (!new_sz) { + au_kfree_rcu(p); + p = NULL; + goto out; + } +#else + AuDebugOn(!new_sz); +#endif + sz = ksize(p); + diff = au_kmidx_sub(sz, new_sz); + } + if (sz && !diff) + goto out; + + if (sz < new_sz) + /* expand or SLOB */ + p = krealloc(p, new_sz, gfp); + else if (new_sz < sz && may_shrink) { + /* shrink */ + void *q; + + q = kmalloc(new_sz, gfp); + if (q) { + if (p) { + memcpy(q, p, new_sz); + au_kfree_try_rcu(p); + } + p = q; + } else + p = NULL; + } + +out: + return p; +} + +void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp, + int may_shrink) +{ + p = au_krealloc(p, new_sz, gfp, may_shrink); + if (p && new_sz > nused) + memset(p + nused, 0, new_sz - nused); + return p; +} + +/* ---------------------------------------------------------------------- */ +/* + * aufs caches + */ +struct kmem_cache *au_cache[AuCache_Last]; + +static void au_cache_fin(void) +{ + int i; + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + + /* excluding AuCache_HNOTIFY */ + BUILD_BUG_ON(AuCache_HNOTIFY + 1 != AuCache_Last); + for (i = 0; i < AuCache_HNOTIFY; i++) { + kmem_cache_destroy(au_cache[i]); + au_cache[i] = NULL; + } +} + +static int __init au_cache_init(void) +{ + au_cache[AuCache_DINFO] = AuCacheCtor(au_dinfo, au_di_init_once); + if (au_cache[AuCache_DINFO]) + /* SLAB_DESTROY_BY_RCU */ + au_cache[AuCache_ICNTNR] = AuCacheCtor(au_icntnr, + au_icntnr_init_once); + if (au_cache[AuCache_ICNTNR]) + au_cache[AuCache_FINFO] = AuCacheCtor(au_finfo, + au_fi_init_once); + if (au_cache[AuCache_FINFO]) + au_cache[AuCache_VDIR] = AuCache(au_vdir); + if (au_cache[AuCache_VDIR]) + au_cache[AuCache_DEHSTR] = AuCache(au_vdir_dehstr); + if (au_cache[AuCache_DEHSTR]) + return 0; + + au_cache_fin(); + return -ENOMEM; +} + +/* ---------------------------------------------------------------------- */ + +int au_dir_roflags; + +#ifdef CONFIG_AUFS_SBILIST +/* + * iterate_supers_type() doesn't protect us from + * remounting (branch management) + */ +struct hlist_bl_head au_sbilist; +#endif + +/* + * functions for module interface. + */ +MODULE_LICENSE("GPL"); +/* MODULE_LICENSE("GPL v2"); */ +MODULE_AUTHOR("Junjiro R. Okajima "); +MODULE_DESCRIPTION(AUFS_NAME + " -- Advanced multi layered unification filesystem"); +MODULE_VERSION(AUFS_VERSION); +MODULE_ALIAS_FS(AUFS_NAME); + +/* this module parameter has no meaning when SYSFS is disabled */ +int sysaufs_brs = 1; +MODULE_PARM_DESC(brs, "use /fs/aufs/si_*/brN"); +module_param_named(brs, sysaufs_brs, int, 0444); + +/* this module parameter has no meaning when USER_NS is disabled */ +bool au_userns; +MODULE_PARM_DESC(allow_userns, "allow unprivileged to mount under userns"); +module_param_named(allow_userns, au_userns, bool, 0444); + +/* ---------------------------------------------------------------------- */ + +static char au_esc_chars[0x20 + 3]; /* 0x01-0x20, backslash, del, and NULL */ + +int au_seq_path(struct seq_file *seq, struct path *path) +{ + int err; + + err = seq_path(seq, path, au_esc_chars); + if (err >= 0) + err = 0; + else + err = -ENOMEM; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int __init aufs_init(void) +{ + int err, i; + char *p; + + p = au_esc_chars; + for (i = 1; i <= ' '; i++) + *p++ = i; + *p++ = '\\'; + *p++ = '\x7f'; + *p = 0; + + au_dir_roflags = au_file_roflags(O_DIRECTORY | O_LARGEFILE); + + memcpy(aufs_iop_nogetattr, aufs_iop, sizeof(aufs_iop)); + for (i = 0; i < AuIop_Last; i++) + aufs_iop_nogetattr[i].getattr = NULL; + + memset(au_cache, 0, sizeof(au_cache)); /* including hnotify */ + + au_sbilist_init(); + sysaufs_brs_init(); + au_debug_init(); + au_dy_init(); + err = sysaufs_init(); + if (unlikely(err)) + goto out; + err = dbgaufs_init(); + if (unlikely(err)) + goto out_sysaufs; + err = au_procfs_init(); + if (unlikely(err)) + goto out_dbgaufs; + err = au_wkq_init(); + if (unlikely(err)) + goto out_procfs; + err = au_loopback_init(); + if (unlikely(err)) + goto out_wkq; + err = au_hnotify_init(); + if (unlikely(err)) + goto out_loopback; + err = au_sysrq_init(); + if (unlikely(err)) + goto out_hin; + err = au_cache_init(); + if (unlikely(err)) + goto out_sysrq; + + aufs_fs_type.fs_flags |= au_userns ? FS_USERNS_MOUNT : 0; + err = register_filesystem(&aufs_fs_type); + if (unlikely(err)) + goto out_cache; + + /* since we define pr_fmt, call printk directly */ + printk(KERN_INFO AUFS_NAME " " AUFS_VERSION "\n"); + goto out; /* success */ + +out_cache: + au_cache_fin(); +out_sysrq: + au_sysrq_fin(); +out_hin: + au_hnotify_fin(); +out_loopback: + au_loopback_fin(); +out_wkq: + au_wkq_fin(); +out_procfs: + au_procfs_fin(); +out_dbgaufs: + dbgaufs_fin(); +out_sysaufs: + sysaufs_fin(); + au_dy_fin(); +out: + return err; +} + +static void __exit aufs_exit(void) +{ + unregister_filesystem(&aufs_fs_type); + au_cache_fin(); + au_sysrq_fin(); + au_hnotify_fin(); + au_loopback_fin(); + au_wkq_fin(); + au_procfs_fin(); + dbgaufs_fin(); + sysaufs_fin(); + au_dy_fin(); +} + +module_init(aufs_init); +module_exit(aufs_exit); diff --git a/fs/aufs/module.h b/fs/aufs/module.h new file mode 100644 index 0000000000000..d6a76788d2ffe --- /dev/null +++ b/fs/aufs/module.h @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * module initialization and module-global + */ + +#ifndef __AUFS_MODULE_H__ +#define __AUFS_MODULE_H__ + +#ifdef __KERNEL__ + +#include +#include "debug.h" +#include "dentry.h" +#include "dir.h" +#include "file.h" +#include "inode.h" + +struct path; +struct seq_file; + +/* module parameters */ +extern int sysaufs_brs; +extern bool au_userns; + +/* ---------------------------------------------------------------------- */ + +extern int au_dir_roflags; + +void *au_krealloc(void *p, unsigned int new_sz, gfp_t gfp, int may_shrink); +void *au_kzrealloc(void *p, unsigned int nused, unsigned int new_sz, gfp_t gfp, + int may_shrink); + +/* + * Comparing the size of the object with sizeof(struct rcu_head) + * case 1: object is always larger + * --> au_kfree_rcu() or au_kfree_do_rcu() + * case 2: object is always smaller + * --> au_kfree_small() + * case 3: object can be any size + * --> au_kfree_try_rcu() + */ + +static inline void au_kfree_do_rcu(const void *p) +{ + struct { + struct rcu_head rcu; + } *a = (void *)p; + + kfree_rcu(a, rcu); +} + +#define au_kfree_rcu(_p) do { \ + typeof(_p) p = (_p); \ + BUILD_BUG_ON(sizeof(*p) < sizeof(struct rcu_head)); \ + if (p) \ + au_kfree_do_rcu(p); \ + } while (0) + +#define au_kfree_do_sz_test(sz) (sz >= sizeof(struct rcu_head)) +#define au_kfree_sz_test(p) (p && au_kfree_do_sz_test(ksize(p))) + +static inline void au_kfree_try_rcu(const void *p) +{ + if (!p) + return; + if (au_kfree_sz_test(p)) + au_kfree_do_rcu(p); + else + kfree(p); +} + +static inline void au_kfree_small(const void *p) +{ + if (!p) + return; + AuDebugOn(au_kfree_sz_test(p)); + kfree(p); +} + +static inline int au_kmidx_sub(size_t sz, size_t new_sz) +{ +#ifndef CONFIG_SLOB + return kmalloc_index(sz) - kmalloc_index(new_sz); +#else + return -1; /* SLOB is untested */ +#endif +} + +int au_seq_path(struct seq_file *seq, struct path *path); + +#ifdef CONFIG_PROC_FS +/* procfs.c */ +int __init au_procfs_init(void); +void au_procfs_fin(void); +#else +AuStubInt0(au_procfs_init, void); +AuStubVoid(au_procfs_fin, void); +#endif + +/* ---------------------------------------------------------------------- */ + +/* kmem cache */ +enum { + AuCache_DINFO, + AuCache_ICNTNR, + AuCache_FINFO, + AuCache_VDIR, + AuCache_DEHSTR, + AuCache_HNOTIFY, /* must be last */ + AuCache_Last +}; + +extern struct kmem_cache *au_cache[AuCache_Last]; + +#define AuCacheFlags (SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD) +#define AuCache(type) KMEM_CACHE(type, AuCacheFlags) +#define AuCacheCtor(type, ctor) \ + kmem_cache_create(#type, sizeof(struct type), \ + __alignof__(struct type), AuCacheFlags, ctor) + +#define AuCacheFuncs(name, index) \ + static inline struct au_##name *au_cache_alloc_##name(void) \ + { return kmem_cache_alloc(au_cache[AuCache_##index], GFP_NOFS); } \ + static inline void au_cache_free_##name##_norcu(struct au_##name *p) \ + { kmem_cache_free(au_cache[AuCache_##index], p); } \ + \ + static inline void au_cache_free_##name##_rcu_cb(struct rcu_head *rcu) \ + { void *p = rcu; \ + p -= offsetof(struct au_##name, rcu); \ + kmem_cache_free(au_cache[AuCache_##index], p); } \ + static inline void au_cache_free_##name##_rcu(struct au_##name *p) \ + { BUILD_BUG_ON(sizeof(struct au_##name) < sizeof(struct rcu_head)); \ + call_rcu(&p->rcu, au_cache_free_##name##_rcu_cb); } \ + \ + static inline void au_cache_free_##name(struct au_##name *p) \ + { /* au_cache_free_##name##_norcu(p); */ \ + au_cache_free_##name##_rcu(p); } + +AuCacheFuncs(dinfo, DINFO); +AuCacheFuncs(icntnr, ICNTNR); +AuCacheFuncs(finfo, FINFO); +AuCacheFuncs(vdir, VDIR); +AuCacheFuncs(vdir_dehstr, DEHSTR); +#ifdef CONFIG_AUFS_HNOTIFY +AuCacheFuncs(hnotify, HNOTIFY); +#endif + +#endif /* __KERNEL__ */ +#endif /* __AUFS_MODULE_H__ */ diff --git a/fs/aufs/mvdown.c b/fs/aufs/mvdown.c new file mode 100644 index 0000000000000..17cc60cdca277 --- /dev/null +++ b/fs/aufs/mvdown.c @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * move-down, opposite of copy-up + */ + +#include "aufs.h" + +struct au_mvd_args { + struct { + struct super_block *h_sb; + struct dentry *h_parent; + struct au_hinode *hdir; + struct inode *h_dir, *h_inode; + struct au_pin pin; + } info[AUFS_MVDOWN_NARRAY]; + + struct aufs_mvdown mvdown; + struct dentry *dentry, *parent; + struct inode *inode, *dir; + struct super_block *sb; + aufs_bindex_t bopq, bwh, bfound; + unsigned char rename_lock; +}; + +#define mvd_errno mvdown.au_errno +#define mvd_bsrc mvdown.stbr[AUFS_MVDOWN_UPPER].bindex +#define mvd_src_brid mvdown.stbr[AUFS_MVDOWN_UPPER].brid +#define mvd_bdst mvdown.stbr[AUFS_MVDOWN_LOWER].bindex +#define mvd_dst_brid mvdown.stbr[AUFS_MVDOWN_LOWER].brid + +#define mvd_h_src_sb info[AUFS_MVDOWN_UPPER].h_sb +#define mvd_h_src_parent info[AUFS_MVDOWN_UPPER].h_parent +#define mvd_hdir_src info[AUFS_MVDOWN_UPPER].hdir +#define mvd_h_src_dir info[AUFS_MVDOWN_UPPER].h_dir +#define mvd_h_src_inode info[AUFS_MVDOWN_UPPER].h_inode +#define mvd_pin_src info[AUFS_MVDOWN_UPPER].pin + +#define mvd_h_dst_sb info[AUFS_MVDOWN_LOWER].h_sb +#define mvd_h_dst_parent info[AUFS_MVDOWN_LOWER].h_parent +#define mvd_hdir_dst info[AUFS_MVDOWN_LOWER].hdir +#define mvd_h_dst_dir info[AUFS_MVDOWN_LOWER].h_dir +#define mvd_h_dst_inode info[AUFS_MVDOWN_LOWER].h_inode +#define mvd_pin_dst info[AUFS_MVDOWN_LOWER].pin + +#define AU_MVD_PR(flag, ...) do { \ + if (flag) \ + pr_err(__VA_ARGS__); \ + } while (0) + +static int find_lower_writable(struct au_mvd_args *a) +{ + struct super_block *sb; + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + sb = a->sb; + bindex = a->mvd_bsrc; + bbot = au_sbbot(sb); + if (a->mvdown.flags & AUFS_MVDOWN_FHSM_LOWER) + for (bindex++; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_fhsm(br->br_perm) + && !sb_rdonly(au_br_sb(br))) + return bindex; + } + else if (!(a->mvdown.flags & AUFS_MVDOWN_ROLOWER)) + for (bindex++; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (!au_br_rdonly(br)) + return bindex; + } + else + for (bindex++; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (!sb_rdonly(au_br_sb(br))) { + if (au_br_rdonly(br)) + a->mvdown.flags + |= AUFS_MVDOWN_ROLOWER_R; + return bindex; + } + } + + return -1; +} + +/* make the parent dir on bdst */ +static int au_do_mkdir(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + + err = 0; + a->mvd_hdir_src = au_hi(a->dir, a->mvd_bsrc); + a->mvd_hdir_dst = au_hi(a->dir, a->mvd_bdst); + a->mvd_h_src_parent = au_h_dptr(a->parent, a->mvd_bsrc); + a->mvd_h_dst_parent = NULL; + if (au_dbbot(a->parent) >= a->mvd_bdst) + a->mvd_h_dst_parent = au_h_dptr(a->parent, a->mvd_bdst); + if (!a->mvd_h_dst_parent) { + err = au_cpdown_dirs(a->dentry, a->mvd_bdst); + if (unlikely(err)) { + AU_MVD_PR(dmsg, "cpdown_dirs failed\n"); + goto out; + } + a->mvd_h_dst_parent = au_h_dptr(a->parent, a->mvd_bdst); + } + +out: + AuTraceErr(err); + return err; +} + +/* lock them all */ +static int au_do_lock(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct dentry *h_trap; + + a->mvd_h_src_sb = au_sbr_sb(a->sb, a->mvd_bsrc); + a->mvd_h_dst_sb = au_sbr_sb(a->sb, a->mvd_bdst); + err = au_pin(&a->mvd_pin_dst, a->dentry, a->mvd_bdst, + au_opt_udba(a->sb), + AuPin_MNT_WRITE | AuPin_DI_LOCKED); + AuTraceErr(err); + if (unlikely(err)) { + AU_MVD_PR(dmsg, "pin_dst failed\n"); + goto out; + } + + if (a->mvd_h_src_sb != a->mvd_h_dst_sb) { + a->rename_lock = 0; + au_pin_init(&a->mvd_pin_src, a->dentry, a->mvd_bsrc, + AuLsc_DI_PARENT, AuLsc_I_PARENT3, + au_opt_udba(a->sb), + AuPin_MNT_WRITE | AuPin_DI_LOCKED); + err = au_do_pin(&a->mvd_pin_src); + AuTraceErr(err); + a->mvd_h_src_dir = d_inode(a->mvd_h_src_parent); + if (unlikely(err)) { + AU_MVD_PR(dmsg, "pin_src failed\n"); + goto out_dst; + } + goto out; /* success */ + } + + a->rename_lock = 1; + au_pin_hdir_unlock(&a->mvd_pin_dst); + err = au_pin(&a->mvd_pin_src, a->dentry, a->mvd_bsrc, + au_opt_udba(a->sb), + AuPin_MNT_WRITE | AuPin_DI_LOCKED); + AuTraceErr(err); + a->mvd_h_src_dir = d_inode(a->mvd_h_src_parent); + if (unlikely(err)) { + AU_MVD_PR(dmsg, "pin_src failed\n"); + au_pin_hdir_lock(&a->mvd_pin_dst); + goto out_dst; + } + au_pin_hdir_unlock(&a->mvd_pin_src); + h_trap = vfsub_lock_rename(a->mvd_h_src_parent, a->mvd_hdir_src, + a->mvd_h_dst_parent, a->mvd_hdir_dst); + if (h_trap) { + err = (h_trap != a->mvd_h_src_parent); + if (err) + err = (h_trap != a->mvd_h_dst_parent); + } + BUG_ON(err); /* it should never happen */ + if (unlikely(a->mvd_h_src_dir != au_pinned_h_dir(&a->mvd_pin_src))) { + err = -EBUSY; + AuTraceErr(err); + vfsub_unlock_rename(a->mvd_h_src_parent, a->mvd_hdir_src, + a->mvd_h_dst_parent, a->mvd_hdir_dst); + au_pin_hdir_lock(&a->mvd_pin_src); + au_unpin(&a->mvd_pin_src); + au_pin_hdir_lock(&a->mvd_pin_dst); + goto out_dst; + } + goto out; /* success */ + +out_dst: + au_unpin(&a->mvd_pin_dst); +out: + AuTraceErr(err); + return err; +} + +static void au_do_unlock(const unsigned char dmsg, struct au_mvd_args *a) +{ + if (!a->rename_lock) + au_unpin(&a->mvd_pin_src); + else { + vfsub_unlock_rename(a->mvd_h_src_parent, a->mvd_hdir_src, + a->mvd_h_dst_parent, a->mvd_hdir_dst); + au_pin_hdir_lock(&a->mvd_pin_src); + au_unpin(&a->mvd_pin_src); + au_pin_hdir_lock(&a->mvd_pin_dst); + } + au_unpin(&a->mvd_pin_dst); +} + +/* copy-down the file */ +static int au_do_cpdown(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct au_cp_generic cpg = { + .dentry = a->dentry, + .bdst = a->mvd_bdst, + .bsrc = a->mvd_bsrc, + .len = -1, + .pin = &a->mvd_pin_dst, + .flags = AuCpup_DTIME | AuCpup_HOPEN + }; + + AuDbg("b%d, b%d\n", cpg.bsrc, cpg.bdst); + if (a->mvdown.flags & AUFS_MVDOWN_OWLOWER) + au_fset_cpup(cpg.flags, OVERWRITE); + if (a->mvdown.flags & AUFS_MVDOWN_ROLOWER) + au_fset_cpup(cpg.flags, RWDST); + err = au_sio_cpdown_simple(&cpg); + if (unlikely(err)) + AU_MVD_PR(dmsg, "cpdown failed\n"); + + AuTraceErr(err); + return err; +} + +/* + * unlink the whiteout on bdst if exist which may be created by UDBA while we + * were sleeping + */ +static int au_do_unlink_wh(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct path h_path; + struct au_branch *br; + struct inode *delegated; + + br = au_sbr(a->sb, a->mvd_bdst); + h_path.dentry = au_wh_lkup(a->mvd_h_dst_parent, &a->dentry->d_name, br); + err = PTR_ERR(h_path.dentry); + if (IS_ERR(h_path.dentry)) { + AU_MVD_PR(dmsg, "wh_lkup failed\n"); + goto out; + } + + err = 0; + if (d_is_positive(h_path.dentry)) { + h_path.mnt = au_br_mnt(br); + delegated = NULL; + err = vfsub_unlink(d_inode(a->mvd_h_dst_parent), &h_path, + &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + if (unlikely(err)) + AU_MVD_PR(dmsg, "wh_unlink failed\n"); + } + dput(h_path.dentry); + +out: + AuTraceErr(err); + return err; +} + +/* + * unlink the topmost h_dentry + */ +static int au_do_unlink(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct path h_path; + struct inode *delegated; + + h_path.mnt = au_sbr_mnt(a->sb, a->mvd_bsrc); + h_path.dentry = au_h_dptr(a->dentry, a->mvd_bsrc); + delegated = NULL; + err = vfsub_unlink(a->mvd_h_src_dir, &h_path, &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + if (unlikely(err)) + AU_MVD_PR(dmsg, "unlink failed\n"); + + AuTraceErr(err); + return err; +} + +/* Since mvdown succeeded, we ignore an error of this function */ +static void au_do_stfs(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct au_branch *br; + + a->mvdown.flags |= AUFS_MVDOWN_STFS_FAILED; + br = au_sbr(a->sb, a->mvd_bsrc); + err = au_br_stfs(br, &a->mvdown.stbr[AUFS_MVDOWN_UPPER].stfs); + if (!err) { + br = au_sbr(a->sb, a->mvd_bdst); + a->mvdown.stbr[AUFS_MVDOWN_LOWER].brid = br->br_id; + err = au_br_stfs(br, &a->mvdown.stbr[AUFS_MVDOWN_LOWER].stfs); + } + if (!err) + a->mvdown.flags &= ~AUFS_MVDOWN_STFS_FAILED; + else + AU_MVD_PR(dmsg, "statfs failed (%d), ignored\n", err); +} + +/* + * copy-down the file and unlink the bsrc file. + * - unlink the bdst whout if exist + * - copy-down the file (with whtmp name and rename) + * - unlink the bsrc file + */ +static int au_do_mvdown(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + + err = au_do_mkdir(dmsg, a); + if (!err) + err = au_do_lock(dmsg, a); + if (unlikely(err)) + goto out; + + /* + * do not revert the activities we made on bdst since they should be + * harmless in aufs. + */ + + err = au_do_cpdown(dmsg, a); + if (!err) + err = au_do_unlink_wh(dmsg, a); + if (!err && !(a->mvdown.flags & AUFS_MVDOWN_KUPPER)) + err = au_do_unlink(dmsg, a); + if (unlikely(err)) + goto out_unlock; + + AuDbg("%pd2, 0x%x, %d --> %d\n", + a->dentry, a->mvdown.flags, a->mvd_bsrc, a->mvd_bdst); + if (find_lower_writable(a) < 0) + a->mvdown.flags |= AUFS_MVDOWN_BOTTOM; + + if (a->mvdown.flags & AUFS_MVDOWN_STFS) + au_do_stfs(dmsg, a); + + /* maintain internal array */ + if (!(a->mvdown.flags & AUFS_MVDOWN_KUPPER)) { + au_set_h_dptr(a->dentry, a->mvd_bsrc, NULL); + au_set_dbtop(a->dentry, a->mvd_bdst); + au_set_h_iptr(a->inode, a->mvd_bsrc, NULL, /*flags*/0); + au_set_ibtop(a->inode, a->mvd_bdst); + } else { + /* hide the lower */ + au_set_h_dptr(a->dentry, a->mvd_bdst, NULL); + au_set_dbbot(a->dentry, a->mvd_bsrc); + au_set_h_iptr(a->inode, a->mvd_bdst, NULL, /*flags*/0); + au_set_ibbot(a->inode, a->mvd_bsrc); + } + if (au_dbbot(a->dentry) < a->mvd_bdst) + au_set_dbbot(a->dentry, a->mvd_bdst); + if (au_ibbot(a->inode) < a->mvd_bdst) + au_set_ibbot(a->inode, a->mvd_bdst); + +out_unlock: + au_do_unlock(dmsg, a); +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* make sure the file is idle */ +static int au_mvd_args_busy(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err, plinked; + + err = 0; + plinked = !!au_opt_test(au_mntflags(a->sb), PLINK); + if (au_dbtop(a->dentry) == a->mvd_bsrc + && au_dcount(a->dentry) == 1 + && atomic_read(&a->inode->i_count) == 1 + /* && a->mvd_h_src_inode->i_nlink == 1 */ + && (!plinked || !au_plink_test(a->inode)) + && a->inode->i_nlink == 1) + goto out; + + err = -EBUSY; + AU_MVD_PR(dmsg, + "b%d, d{b%d, c%d?}, i{c%d?, l%u}, hi{l%u}, p{%d, %d}\n", + a->mvd_bsrc, au_dbtop(a->dentry), au_dcount(a->dentry), + atomic_read(&a->inode->i_count), a->inode->i_nlink, + a->mvd_h_src_inode->i_nlink, + plinked, plinked ? au_plink_test(a->inode) : 0); + +out: + AuTraceErr(err); + return err; +} + +/* make sure the parent dir is fine */ +static int au_mvd_args_parent(const unsigned char dmsg, + struct au_mvd_args *a) +{ + int err; + aufs_bindex_t bindex; + + err = 0; + if (unlikely(au_alive_dir(a->parent))) { + err = -ENOENT; + AU_MVD_PR(dmsg, "parent dir is dead\n"); + goto out; + } + + a->bopq = au_dbdiropq(a->parent); + bindex = au_wbr_nonopq(a->dentry, a->mvd_bdst); + AuDbg("b%d\n", bindex); + if (unlikely((bindex >= 0 && bindex < a->mvd_bdst) + || (a->bopq != -1 && a->bopq < a->mvd_bdst))) { + err = -EINVAL; + a->mvd_errno = EAU_MVDOWN_OPAQUE; + AU_MVD_PR(dmsg, "ancestor is opaque b%d, b%d\n", + a->bopq, a->mvd_bdst); + } + +out: + AuTraceErr(err); + return err; +} + +static int au_mvd_args_intermediate(const unsigned char dmsg, + struct au_mvd_args *a) +{ + int err; + struct au_dinfo *dinfo, *tmp; + + /* lookup the next lower positive entry */ + err = -ENOMEM; + tmp = au_di_alloc(a->sb, AuLsc_DI_TMP); + if (unlikely(!tmp)) + goto out; + + a->bfound = -1; + a->bwh = -1; + dinfo = au_di(a->dentry); + au_di_cp(tmp, dinfo); + au_di_swap(tmp, dinfo); + + /* returns the number of positive dentries */ + err = au_lkup_dentry(a->dentry, a->mvd_bsrc + 1, + /* AuLkup_IGNORE_PERM */ 0); + if (!err) + a->bwh = au_dbwh(a->dentry); + else if (err > 0) + a->bfound = au_dbtop(a->dentry); + + au_di_swap(tmp, dinfo); + au_rw_write_unlock(&tmp->di_rwsem); + au_di_free(tmp); + if (unlikely(err < 0)) + AU_MVD_PR(dmsg, "failed look-up lower\n"); + + /* + * here, we have these cases. + * bfound == -1 + * no positive dentry under bsrc. there are more sub-cases. + * bwh < 0 + * there no whiteout, we can safely move-down. + * bwh <= bsrc + * impossible + * bsrc < bwh && bwh < bdst + * there is a whiteout on RO branch. cannot proceed. + * bwh == bdst + * there is a whiteout on the RW target branch. it should + * be removed. + * bdst < bwh + * there is a whiteout somewhere unrelated branch. + * -1 < bfound && bfound <= bsrc + * impossible. + * bfound < bdst + * found, but it is on RO branch between bsrc and bdst. cannot + * proceed. + * bfound == bdst + * found, replace it if AUFS_MVDOWN_FORCE is set. otherwise return + * error. + * bdst < bfound + * found, after we create the file on bdst, it will be hidden. + */ + + AuDebugOn(a->bfound == -1 + && a->bwh != -1 + && a->bwh <= a->mvd_bsrc); + AuDebugOn(-1 < a->bfound + && a->bfound <= a->mvd_bsrc); + + err = -EINVAL; + if (a->bfound == -1 + && a->mvd_bsrc < a->bwh + && a->bwh != -1 + && a->bwh < a->mvd_bdst) { + a->mvd_errno = EAU_MVDOWN_WHITEOUT; + AU_MVD_PR(dmsg, "bsrc %d, bdst %d, bfound %d, bwh %d\n", + a->mvd_bsrc, a->mvd_bdst, a->bfound, a->bwh); + goto out; + } else if (a->bfound != -1 && a->bfound < a->mvd_bdst) { + a->mvd_errno = EAU_MVDOWN_UPPER; + AU_MVD_PR(dmsg, "bdst %d, bfound %d\n", + a->mvd_bdst, a->bfound); + goto out; + } + + err = 0; /* success */ + +out: + AuTraceErr(err); + return err; +} + +static int au_mvd_args_exist(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + + err = 0; + if (!(a->mvdown.flags & AUFS_MVDOWN_OWLOWER) + && a->bfound == a->mvd_bdst) + err = -EEXIST; + AuTraceErr(err); + return err; +} + +static int au_mvd_args(const unsigned char dmsg, struct au_mvd_args *a) +{ + int err; + struct au_branch *br; + + err = -EISDIR; + if (unlikely(S_ISDIR(a->inode->i_mode))) + goto out; + + err = -EINVAL; + if (!(a->mvdown.flags & AUFS_MVDOWN_BRID_UPPER)) + a->mvd_bsrc = au_ibtop(a->inode); + else { + a->mvd_bsrc = au_br_index(a->sb, a->mvd_src_brid); + if (unlikely(a->mvd_bsrc < 0 + || (a->mvd_bsrc < au_dbtop(a->dentry) + || au_dbbot(a->dentry) < a->mvd_bsrc + || !au_h_dptr(a->dentry, a->mvd_bsrc)) + || (a->mvd_bsrc < au_ibtop(a->inode) + || au_ibbot(a->inode) < a->mvd_bsrc + || !au_h_iptr(a->inode, a->mvd_bsrc)))) { + a->mvd_errno = EAU_MVDOWN_NOUPPER; + AU_MVD_PR(dmsg, "no upper\n"); + goto out; + } + } + if (unlikely(a->mvd_bsrc == au_sbbot(a->sb))) { + a->mvd_errno = EAU_MVDOWN_BOTTOM; + AU_MVD_PR(dmsg, "on the bottom\n"); + goto out; + } + a->mvd_h_src_inode = au_h_iptr(a->inode, a->mvd_bsrc); + br = au_sbr(a->sb, a->mvd_bsrc); + err = au_br_rdonly(br); + if (!(a->mvdown.flags & AUFS_MVDOWN_ROUPPER)) { + if (unlikely(err)) + goto out; + } else if (!(vfsub_native_ro(a->mvd_h_src_inode) + || IS_APPEND(a->mvd_h_src_inode))) { + if (err) + a->mvdown.flags |= AUFS_MVDOWN_ROUPPER_R; + /* go on */ + } else + goto out; + + err = -EINVAL; + if (!(a->mvdown.flags & AUFS_MVDOWN_BRID_LOWER)) { + a->mvd_bdst = find_lower_writable(a); + if (unlikely(a->mvd_bdst < 0)) { + a->mvd_errno = EAU_MVDOWN_BOTTOM; + AU_MVD_PR(dmsg, "no writable lower branch\n"); + goto out; + } + } else { + a->mvd_bdst = au_br_index(a->sb, a->mvd_dst_brid); + if (unlikely(a->mvd_bdst < 0 + || au_sbbot(a->sb) < a->mvd_bdst)) { + a->mvd_errno = EAU_MVDOWN_NOLOWERBR; + AU_MVD_PR(dmsg, "no lower brid\n"); + goto out; + } + } + + err = au_mvd_args_busy(dmsg, a); + if (!err) + err = au_mvd_args_parent(dmsg, a); + if (!err) + err = au_mvd_args_intermediate(dmsg, a); + if (!err) + err = au_mvd_args_exist(dmsg, a); + if (!err) + AuDbg("b%d, b%d\n", a->mvd_bsrc, a->mvd_bdst); + +out: + AuTraceErr(err); + return err; +} + +int au_mvdown(struct dentry *dentry, struct aufs_mvdown __user *uarg) +{ + int err, e; + unsigned char dmsg; + struct au_mvd_args *args; + struct inode *inode; + + inode = d_inode(dentry); + err = -EPERM; + if (unlikely(!capable(CAP_SYS_ADMIN))) + goto out; + + err = -ENOMEM; + args = kmalloc(sizeof(*args), GFP_NOFS); + if (unlikely(!args)) + goto out; + + err = copy_from_user(&args->mvdown, uarg, sizeof(args->mvdown)); + if (!err) + err = !access_ok(VERIFY_WRITE, uarg, sizeof(*uarg)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out_free; + } + AuDbg("flags 0x%x\n", args->mvdown.flags); + args->mvdown.flags &= ~(AUFS_MVDOWN_ROLOWER_R | AUFS_MVDOWN_ROUPPER_R); + args->mvdown.au_errno = 0; + args->dentry = dentry; + args->inode = inode; + args->sb = dentry->d_sb; + + err = -ENOENT; + dmsg = !!(args->mvdown.flags & AUFS_MVDOWN_DMSG); + args->parent = dget_parent(dentry); + args->dir = d_inode(args->parent); + inode_lock_nested(args->dir, I_MUTEX_PARENT); + dput(args->parent); + if (unlikely(args->parent != dentry->d_parent)) { + AU_MVD_PR(dmsg, "parent dir is moved\n"); + goto out_dir; + } + + inode_lock_nested(inode, I_MUTEX_CHILD); + err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_NOPLMW); + if (unlikely(err)) + goto out_inode; + + di_write_lock_parent(args->parent); + err = au_mvd_args(dmsg, args); + if (unlikely(err)) + goto out_parent; + + err = au_do_mvdown(dmsg, args); + if (unlikely(err)) + goto out_parent; + + au_cpup_attr_timesizes(args->dir); + au_cpup_attr_timesizes(inode); + if (!(args->mvdown.flags & AUFS_MVDOWN_KUPPER)) + au_cpup_igen(inode, au_h_iptr(inode, args->mvd_bdst)); + /* au_digen_dec(dentry); */ + +out_parent: + di_write_unlock(args->parent); + aufs_read_unlock(dentry, AuLock_DW); +out_inode: + inode_unlock(inode); +out_dir: + inode_unlock(args->dir); +out_free: + e = copy_to_user(uarg, &args->mvdown, sizeof(args->mvdown)); + if (unlikely(e)) + err = -EFAULT; + au_kfree_rcu(args); +out: + AuTraceErr(err); + return err; +} diff --git a/fs/aufs/opts.c b/fs/aufs/opts.c new file mode 100644 index 0000000000000..16e785facf9b9 --- /dev/null +++ b/fs/aufs/opts.c @@ -0,0 +1,1880 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * mount options/flags + */ + +#include +#include /* a distribution requires */ +#include +#include "aufs.h" + +/* ---------------------------------------------------------------------- */ + +enum { + Opt_br, + Opt_add, Opt_del, Opt_mod, Opt_append, Opt_prepend, + Opt_idel, Opt_imod, + Opt_dirwh, Opt_rdcache, Opt_rdblk, Opt_rdhash, + Opt_rdblk_def, Opt_rdhash_def, + Opt_xino, Opt_noxino, + Opt_trunc_xino, Opt_trunc_xino_v, Opt_notrunc_xino, + Opt_trunc_xino_path, Opt_itrunc_xino, + Opt_trunc_xib, Opt_notrunc_xib, + Opt_shwh, Opt_noshwh, + Opt_plink, Opt_noplink, Opt_list_plink, + Opt_udba, + Opt_dio, Opt_nodio, + Opt_diropq_a, Opt_diropq_w, + Opt_warn_perm, Opt_nowarn_perm, + Opt_wbr_copyup, Opt_wbr_create, + Opt_fhsm_sec, + Opt_verbose, Opt_noverbose, + Opt_sum, Opt_nosum, Opt_wsum, + Opt_dirperm1, Opt_nodirperm1, + Opt_dirren, Opt_nodirren, + Opt_acl, Opt_noacl, + Opt_tail, Opt_ignore, Opt_ignore_silent, Opt_err +}; + +static match_table_t options = { + {Opt_br, "br=%s"}, + {Opt_br, "br:%s"}, + + {Opt_add, "add=%d:%s"}, + {Opt_add, "add:%d:%s"}, + {Opt_add, "ins=%d:%s"}, + {Opt_add, "ins:%d:%s"}, + {Opt_append, "append=%s"}, + {Opt_append, "append:%s"}, + {Opt_prepend, "prepend=%s"}, + {Opt_prepend, "prepend:%s"}, + + {Opt_del, "del=%s"}, + {Opt_del, "del:%s"}, + /* {Opt_idel, "idel:%d"}, */ + {Opt_mod, "mod=%s"}, + {Opt_mod, "mod:%s"}, + /* {Opt_imod, "imod:%d:%s"}, */ + + {Opt_dirwh, "dirwh=%d"}, + + {Opt_xino, "xino=%s"}, + {Opt_noxino, "noxino"}, + {Opt_trunc_xino, "trunc_xino"}, + {Opt_trunc_xino_v, "trunc_xino_v=%d:%d"}, + {Opt_notrunc_xino, "notrunc_xino"}, + {Opt_trunc_xino_path, "trunc_xino=%s"}, + {Opt_itrunc_xino, "itrunc_xino=%d"}, + /* {Opt_zxino, "zxino=%s"}, */ + {Opt_trunc_xib, "trunc_xib"}, + {Opt_notrunc_xib, "notrunc_xib"}, + +#ifdef CONFIG_PROC_FS + {Opt_plink, "plink"}, +#else + {Opt_ignore_silent, "plink"}, +#endif + + {Opt_noplink, "noplink"}, + +#ifdef CONFIG_AUFS_DEBUG + {Opt_list_plink, "list_plink"}, +#endif + + {Opt_udba, "udba=%s"}, + + {Opt_dio, "dio"}, + {Opt_nodio, "nodio"}, + +#ifdef CONFIG_AUFS_DIRREN + {Opt_dirren, "dirren"}, + {Opt_nodirren, "nodirren"}, +#else + {Opt_ignore, "dirren"}, + {Opt_ignore_silent, "nodirren"}, +#endif + +#ifdef CONFIG_AUFS_FHSM + {Opt_fhsm_sec, "fhsm_sec=%d"}, +#else + {Opt_ignore, "fhsm_sec=%d"}, +#endif + + {Opt_diropq_a, "diropq=always"}, + {Opt_diropq_a, "diropq=a"}, + {Opt_diropq_w, "diropq=whiteouted"}, + {Opt_diropq_w, "diropq=w"}, + + {Opt_warn_perm, "warn_perm"}, + {Opt_nowarn_perm, "nowarn_perm"}, + + /* keep them temporary */ + {Opt_ignore_silent, "nodlgt"}, + {Opt_ignore, "clean_plink"}, + +#ifdef CONFIG_AUFS_SHWH + {Opt_shwh, "shwh"}, +#endif + {Opt_noshwh, "noshwh"}, + + {Opt_dirperm1, "dirperm1"}, + {Opt_nodirperm1, "nodirperm1"}, + + {Opt_verbose, "verbose"}, + {Opt_verbose, "v"}, + {Opt_noverbose, "noverbose"}, + {Opt_noverbose, "quiet"}, + {Opt_noverbose, "q"}, + {Opt_noverbose, "silent"}, + + {Opt_sum, "sum"}, + {Opt_nosum, "nosum"}, + {Opt_wsum, "wsum"}, + + {Opt_rdcache, "rdcache=%d"}, + {Opt_rdblk, "rdblk=%d"}, + {Opt_rdblk_def, "rdblk=def"}, + {Opt_rdhash, "rdhash=%d"}, + {Opt_rdhash_def, "rdhash=def"}, + + {Opt_wbr_create, "create=%s"}, + {Opt_wbr_create, "create_policy=%s"}, + {Opt_wbr_copyup, "cpup=%s"}, + {Opt_wbr_copyup, "copyup=%s"}, + {Opt_wbr_copyup, "copyup_policy=%s"}, + + /* generic VFS flag */ +#ifdef CONFIG_FS_POSIX_ACL + {Opt_acl, "acl"}, + {Opt_noacl, "noacl"}, +#else + {Opt_ignore, "acl"}, + {Opt_ignore_silent, "noacl"}, +#endif + + /* internal use for the scripts */ + {Opt_ignore_silent, "si=%s"}, + + {Opt_br, "dirs=%s"}, + {Opt_ignore, "debug=%d"}, + {Opt_ignore, "delete=whiteout"}, + {Opt_ignore, "delete=all"}, + {Opt_ignore, "imap=%s"}, + + /* temporary workaround, due to old mount(8)? */ + {Opt_ignore_silent, "relatime"}, + + {Opt_err, NULL} +}; + +/* ---------------------------------------------------------------------- */ + +static const char *au_parser_pattern(int val, match_table_t tbl) +{ + struct match_token *p; + + p = tbl; + while (p->pattern) { + if (p->token == val) + return p->pattern; + p++; + } + BUG(); + return "??"; +} + +static const char *au_optstr(int *val, match_table_t tbl) +{ + struct match_token *p; + int v; + + v = *val; + if (!v) + goto out; + p = tbl; + while (p->pattern) { + if (p->token + && (v & p->token) == p->token) { + *val &= ~p->token; + return p->pattern; + } + p++; + } + +out: + return NULL; +} + +/* ---------------------------------------------------------------------- */ + +static match_table_t brperm = { + {AuBrPerm_RO, AUFS_BRPERM_RO}, + {AuBrPerm_RR, AUFS_BRPERM_RR}, + {AuBrPerm_RW, AUFS_BRPERM_RW}, + {0, NULL} +}; + +static match_table_t brattr = { + /* general */ + {AuBrAttr_COO_REG, AUFS_BRATTR_COO_REG}, + {AuBrAttr_COO_ALL, AUFS_BRATTR_COO_ALL}, + /* 'unpin' attrib is meaningless since linux-3.18-rc1 */ + {AuBrAttr_UNPIN, AUFS_BRATTR_UNPIN}, +#ifdef CONFIG_AUFS_FHSM + {AuBrAttr_FHSM, AUFS_BRATTR_FHSM}, +#endif +#ifdef CONFIG_AUFS_XATTR + {AuBrAttr_ICEX, AUFS_BRATTR_ICEX}, + {AuBrAttr_ICEX_SEC, AUFS_BRATTR_ICEX_SEC}, + {AuBrAttr_ICEX_SYS, AUFS_BRATTR_ICEX_SYS}, + {AuBrAttr_ICEX_TR, AUFS_BRATTR_ICEX_TR}, + {AuBrAttr_ICEX_USR, AUFS_BRATTR_ICEX_USR}, + {AuBrAttr_ICEX_OTH, AUFS_BRATTR_ICEX_OTH}, +#endif + + /* ro/rr branch */ + {AuBrRAttr_WH, AUFS_BRRATTR_WH}, + + /* rw branch */ + {AuBrWAttr_MOO, AUFS_BRWATTR_MOO}, + {AuBrWAttr_NoLinkWH, AUFS_BRWATTR_NLWH}, + + {0, NULL} +}; + +static int br_attr_val(char *str, match_table_t table, substring_t args[]) +{ + int attr, v; + char *p; + + attr = 0; + do { + p = strchr(str, '+'); + if (p) + *p = 0; + v = match_token(str, table, args); + if (v) { + if (v & AuBrAttr_CMOO_Mask) + attr &= ~AuBrAttr_CMOO_Mask; + attr |= v; + } else { + if (p) + *p = '+'; + pr_warn("ignored branch attribute %s\n", str); + break; + } + if (p) + str = p + 1; + } while (p); + + return attr; +} + +static int au_do_optstr_br_attr(au_br_perm_str_t *str, int perm) +{ + int sz; + const char *p; + char *q; + + q = str->a; + *q = 0; + p = au_optstr(&perm, brattr); + if (p) { + sz = strlen(p); + memcpy(q, p, sz + 1); + q += sz; + } else + goto out; + + do { + p = au_optstr(&perm, brattr); + if (p) { + *q++ = '+'; + sz = strlen(p); + memcpy(q, p, sz + 1); + q += sz; + } + } while (p); + +out: + return q - str->a; +} + +static int noinline_for_stack br_perm_val(char *perm) +{ + int val, bad, sz; + char *p; + substring_t args[MAX_OPT_ARGS]; + au_br_perm_str_t attr; + + p = strchr(perm, '+'); + if (p) + *p = 0; + val = match_token(perm, brperm, args); + if (!val) { + if (p) + *p = '+'; + pr_warn("ignored branch permission %s\n", perm); + val = AuBrPerm_RO; + goto out; + } + if (!p) + goto out; + + val |= br_attr_val(p + 1, brattr, args); + + bad = 0; + switch (val & AuBrPerm_Mask) { + case AuBrPerm_RO: + case AuBrPerm_RR: + bad = val & AuBrWAttr_Mask; + val &= ~AuBrWAttr_Mask; + break; + case AuBrPerm_RW: + bad = val & AuBrRAttr_Mask; + val &= ~AuBrRAttr_Mask; + break; + } + + /* + * 'unpin' attrib becomes meaningless since linux-3.18-rc1, but aufs + * does not treat it as an error, just warning. + * this is a tiny guard for the user operation. + */ + if (val & AuBrAttr_UNPIN) { + bad |= AuBrAttr_UNPIN; + val &= ~AuBrAttr_UNPIN; + } + + if (unlikely(bad)) { + sz = au_do_optstr_br_attr(&attr, bad); + AuDebugOn(!sz); + pr_warn("ignored branch attribute %s\n", attr.a); + } + +out: + return val; +} + +void au_optstr_br_perm(au_br_perm_str_t *str, int perm) +{ + au_br_perm_str_t attr; + const char *p; + char *q; + int sz; + + q = str->a; + p = au_optstr(&perm, brperm); + AuDebugOn(!p || !*p); + sz = strlen(p); + memcpy(q, p, sz + 1); + q += sz; + + sz = au_do_optstr_br_attr(&attr, perm); + if (sz) { + *q++ = '+'; + memcpy(q, attr.a, sz + 1); + } + + AuDebugOn(strlen(str->a) >= sizeof(str->a)); +} + +/* ---------------------------------------------------------------------- */ + +static match_table_t udbalevel = { + {AuOpt_UDBA_REVAL, "reval"}, + {AuOpt_UDBA_NONE, "none"}, +#ifdef CONFIG_AUFS_HNOTIFY + {AuOpt_UDBA_HNOTIFY, "notify"}, /* abstraction */ +#ifdef CONFIG_AUFS_HFSNOTIFY + {AuOpt_UDBA_HNOTIFY, "fsnotify"}, +#endif +#endif + {-1, NULL} +}; + +static int noinline_for_stack udba_val(char *str) +{ + substring_t args[MAX_OPT_ARGS]; + + return match_token(str, udbalevel, args); +} + +const char *au_optstr_udba(int udba) +{ + return au_parser_pattern(udba, udbalevel); +} + +/* ---------------------------------------------------------------------- */ + +static match_table_t au_wbr_create_policy = { + {AuWbrCreate_TDP, "tdp"}, + {AuWbrCreate_TDP, "top-down-parent"}, + {AuWbrCreate_RR, "rr"}, + {AuWbrCreate_RR, "round-robin"}, + {AuWbrCreate_MFS, "mfs"}, + {AuWbrCreate_MFS, "most-free-space"}, + {AuWbrCreate_MFSV, "mfs:%d"}, + {AuWbrCreate_MFSV, "most-free-space:%d"}, + + /* top-down regardless the parent, and then mfs */ + {AuWbrCreate_TDMFS, "tdmfs:%d"}, + {AuWbrCreate_TDMFSV, "tdmfs:%d:%d"}, + + {AuWbrCreate_MFSRR, "mfsrr:%d"}, + {AuWbrCreate_MFSRRV, "mfsrr:%d:%d"}, + {AuWbrCreate_PMFS, "pmfs"}, + {AuWbrCreate_PMFSV, "pmfs:%d"}, + {AuWbrCreate_PMFSRR, "pmfsrr:%d"}, + {AuWbrCreate_PMFSRRV, "pmfsrr:%d:%d"}, + + {-1, NULL} +}; + +static int au_wbr_mfs_wmark(substring_t *arg, char *str, + struct au_opt_wbr_create *create) +{ + int err; + unsigned long long ull; + + err = 0; + if (!match_u64(arg, &ull)) + create->mfsrr_watermark = ull; + else { + pr_err("bad integer in %s\n", str); + err = -EINVAL; + } + + return err; +} + +static int au_wbr_mfs_sec(substring_t *arg, char *str, + struct au_opt_wbr_create *create) +{ + int n, err; + + err = 0; + if (!match_int(arg, &n) && 0 <= n && n <= AUFS_MFS_MAX_SEC) + create->mfs_second = n; + else { + pr_err("bad integer in %s\n", str); + err = -EINVAL; + } + + return err; +} + +static int noinline_for_stack +au_wbr_create_val(char *str, struct au_opt_wbr_create *create) +{ + int err, e; + substring_t args[MAX_OPT_ARGS]; + + err = match_token(str, au_wbr_create_policy, args); + create->wbr_create = err; + switch (err) { + case AuWbrCreate_MFSRRV: + case AuWbrCreate_TDMFSV: + case AuWbrCreate_PMFSRRV: + e = au_wbr_mfs_wmark(&args[0], str, create); + if (!e) + e = au_wbr_mfs_sec(&args[1], str, create); + if (unlikely(e)) + err = e; + break; + case AuWbrCreate_MFSRR: + case AuWbrCreate_TDMFS: + case AuWbrCreate_PMFSRR: + e = au_wbr_mfs_wmark(&args[0], str, create); + if (unlikely(e)) { + err = e; + break; + } + /*FALLTHROUGH*/ + case AuWbrCreate_MFS: + case AuWbrCreate_PMFS: + create->mfs_second = AUFS_MFS_DEF_SEC; + break; + case AuWbrCreate_MFSV: + case AuWbrCreate_PMFSV: + e = au_wbr_mfs_sec(&args[0], str, create); + if (unlikely(e)) + err = e; + break; + } + + return err; +} + +const char *au_optstr_wbr_create(int wbr_create) +{ + return au_parser_pattern(wbr_create, au_wbr_create_policy); +} + +static match_table_t au_wbr_copyup_policy = { + {AuWbrCopyup_TDP, "tdp"}, + {AuWbrCopyup_TDP, "top-down-parent"}, + {AuWbrCopyup_BUP, "bup"}, + {AuWbrCopyup_BUP, "bottom-up-parent"}, + {AuWbrCopyup_BU, "bu"}, + {AuWbrCopyup_BU, "bottom-up"}, + {-1, NULL} +}; + +static int noinline_for_stack au_wbr_copyup_val(char *str) +{ + substring_t args[MAX_OPT_ARGS]; + + return match_token(str, au_wbr_copyup_policy, args); +} + +const char *au_optstr_wbr_copyup(int wbr_copyup) +{ + return au_parser_pattern(wbr_copyup, au_wbr_copyup_policy); +} + +/* ---------------------------------------------------------------------- */ + +static const int lkup_dirflags = LOOKUP_FOLLOW | LOOKUP_DIRECTORY; + +static void dump_opts(struct au_opts *opts) +{ +#ifdef CONFIG_AUFS_DEBUG + /* reduce stack space */ + union { + struct au_opt_add *add; + struct au_opt_del *del; + struct au_opt_mod *mod; + struct au_opt_xino *xino; + struct au_opt_xino_itrunc *xino_itrunc; + struct au_opt_wbr_create *create; + } u; + struct au_opt *opt; + + opt = opts->opt; + while (opt->type != Opt_tail) { + switch (opt->type) { + case Opt_add: + u.add = &opt->add; + AuDbg("add {b%d, %s, 0x%x, %p}\n", + u.add->bindex, u.add->pathname, u.add->perm, + u.add->path.dentry); + break; + case Opt_del: + case Opt_idel: + u.del = &opt->del; + AuDbg("del {%s, %p}\n", + u.del->pathname, u.del->h_path.dentry); + break; + case Opt_mod: + case Opt_imod: + u.mod = &opt->mod; + AuDbg("mod {%s, 0x%x, %p}\n", + u.mod->path, u.mod->perm, u.mod->h_root); + break; + case Opt_append: + u.add = &opt->add; + AuDbg("append {b%d, %s, 0x%x, %p}\n", + u.add->bindex, u.add->pathname, u.add->perm, + u.add->path.dentry); + break; + case Opt_prepend: + u.add = &opt->add; + AuDbg("prepend {b%d, %s, 0x%x, %p}\n", + u.add->bindex, u.add->pathname, u.add->perm, + u.add->path.dentry); + break; + case Opt_dirwh: + AuDbg("dirwh %d\n", opt->dirwh); + break; + case Opt_rdcache: + AuDbg("rdcache %d\n", opt->rdcache); + break; + case Opt_rdblk: + AuDbg("rdblk %u\n", opt->rdblk); + break; + case Opt_rdblk_def: + AuDbg("rdblk_def\n"); + break; + case Opt_rdhash: + AuDbg("rdhash %u\n", opt->rdhash); + break; + case Opt_rdhash_def: + AuDbg("rdhash_def\n"); + break; + case Opt_xino: + u.xino = &opt->xino; + AuDbg("xino {%s %pD}\n", u.xino->path, u.xino->file); + break; + case Opt_trunc_xino: + AuLabel(trunc_xino); + break; + case Opt_notrunc_xino: + AuLabel(notrunc_xino); + break; + case Opt_trunc_xino_path: + case Opt_itrunc_xino: + u.xino_itrunc = &opt->xino_itrunc; + AuDbg("trunc_xino %d\n", u.xino_itrunc->bindex); + break; + case Opt_noxino: + AuLabel(noxino); + break; + case Opt_trunc_xib: + AuLabel(trunc_xib); + break; + case Opt_notrunc_xib: + AuLabel(notrunc_xib); + break; + case Opt_shwh: + AuLabel(shwh); + break; + case Opt_noshwh: + AuLabel(noshwh); + break; + case Opt_dirperm1: + AuLabel(dirperm1); + break; + case Opt_nodirperm1: + AuLabel(nodirperm1); + break; + case Opt_plink: + AuLabel(plink); + break; + case Opt_noplink: + AuLabel(noplink); + break; + case Opt_list_plink: + AuLabel(list_plink); + break; + case Opt_udba: + AuDbg("udba %d, %s\n", + opt->udba, au_optstr_udba(opt->udba)); + break; + case Opt_dio: + AuLabel(dio); + break; + case Opt_nodio: + AuLabel(nodio); + break; + case Opt_diropq_a: + AuLabel(diropq_a); + break; + case Opt_diropq_w: + AuLabel(diropq_w); + break; + case Opt_warn_perm: + AuLabel(warn_perm); + break; + case Opt_nowarn_perm: + AuLabel(nowarn_perm); + break; + case Opt_verbose: + AuLabel(verbose); + break; + case Opt_noverbose: + AuLabel(noverbose); + break; + case Opt_sum: + AuLabel(sum); + break; + case Opt_nosum: + AuLabel(nosum); + break; + case Opt_wsum: + AuLabel(wsum); + break; + case Opt_wbr_create: + u.create = &opt->wbr_create; + AuDbg("create %d, %s\n", u.create->wbr_create, + au_optstr_wbr_create(u.create->wbr_create)); + switch (u.create->wbr_create) { + case AuWbrCreate_MFSV: + case AuWbrCreate_PMFSV: + AuDbg("%d sec\n", u.create->mfs_second); + break; + case AuWbrCreate_MFSRR: + case AuWbrCreate_TDMFS: + AuDbg("%llu watermark\n", + u.create->mfsrr_watermark); + break; + case AuWbrCreate_MFSRRV: + case AuWbrCreate_TDMFSV: + case AuWbrCreate_PMFSRRV: + AuDbg("%llu watermark, %d sec\n", + u.create->mfsrr_watermark, + u.create->mfs_second); + break; + } + break; + case Opt_wbr_copyup: + AuDbg("copyup %d, %s\n", opt->wbr_copyup, + au_optstr_wbr_copyup(opt->wbr_copyup)); + break; + case Opt_fhsm_sec: + AuDbg("fhsm_sec %u\n", opt->fhsm_second); + break; + case Opt_dirren: + AuLabel(dirren); + break; + case Opt_nodirren: + AuLabel(nodirren); + break; + case Opt_acl: + AuLabel(acl); + break; + case Opt_noacl: + AuLabel(noacl); + break; + default: + BUG(); + } + opt++; + } +#endif +} + +void au_opts_free(struct au_opts *opts) +{ + struct au_opt *opt; + + opt = opts->opt; + while (opt->type != Opt_tail) { + switch (opt->type) { + case Opt_add: + case Opt_append: + case Opt_prepend: + path_put(&opt->add.path); + break; + case Opt_del: + case Opt_idel: + path_put(&opt->del.h_path); + break; + case Opt_mod: + case Opt_imod: + dput(opt->mod.h_root); + break; + case Opt_xino: + fput(opt->xino.file); + break; + } + opt++; + } +} + +static int opt_add(struct au_opt *opt, char *opt_str, unsigned long sb_flags, + aufs_bindex_t bindex) +{ + int err; + struct au_opt_add *add = &opt->add; + char *p; + + add->bindex = bindex; + add->perm = AuBrPerm_RO; + add->pathname = opt_str; + p = strchr(opt_str, '='); + if (p) { + *p++ = 0; + if (*p) + add->perm = br_perm_val(p); + } + + err = vfsub_kern_path(add->pathname, lkup_dirflags, &add->path); + if (!err) { + if (!p) { + add->perm = AuBrPerm_RO; + if (au_test_fs_rr(add->path.dentry->d_sb)) + add->perm = AuBrPerm_RR; + else if (!bindex && !(sb_flags & SB_RDONLY)) + add->perm = AuBrPerm_RW; + } + opt->type = Opt_add; + goto out; + } + pr_err("lookup failed %s (%d)\n", add->pathname, err); + err = -EINVAL; + +out: + return err; +} + +static int au_opts_parse_del(struct au_opt_del *del, substring_t args[]) +{ + int err; + + del->pathname = args[0].from; + AuDbg("del path %s\n", del->pathname); + + err = vfsub_kern_path(del->pathname, lkup_dirflags, &del->h_path); + if (unlikely(err)) + pr_err("lookup failed %s (%d)\n", del->pathname, err); + + return err; +} + +#if 0 /* reserved for future use */ +static int au_opts_parse_idel(struct super_block *sb, aufs_bindex_t bindex, + struct au_opt_del *del, substring_t args[]) +{ + int err; + struct dentry *root; + + err = -EINVAL; + root = sb->s_root; + aufs_read_lock(root, AuLock_FLUSH); + if (bindex < 0 || au_sbbot(sb) < bindex) { + pr_err("out of bounds, %d\n", bindex); + goto out; + } + + err = 0; + del->h_path.dentry = dget(au_h_dptr(root, bindex)); + del->h_path.mnt = mntget(au_sbr_mnt(sb, bindex)); + +out: + aufs_read_unlock(root, !AuLock_IR); + return err; +} +#endif + +static int noinline_for_stack +au_opts_parse_mod(struct au_opt_mod *mod, substring_t args[]) +{ + int err; + struct path path; + char *p; + + err = -EINVAL; + mod->path = args[0].from; + p = strchr(mod->path, '='); + if (unlikely(!p)) { + pr_err("no permission %s\n", args[0].from); + goto out; + } + + *p++ = 0; + err = vfsub_kern_path(mod->path, lkup_dirflags, &path); + if (unlikely(err)) { + pr_err("lookup failed %s (%d)\n", mod->path, err); + goto out; + } + + mod->perm = br_perm_val(p); + AuDbg("mod path %s, perm 0x%x, %s\n", mod->path, mod->perm, p); + mod->h_root = dget(path.dentry); + path_put(&path); + +out: + return err; +} + +#if 0 /* reserved for future use */ +static int au_opts_parse_imod(struct super_block *sb, aufs_bindex_t bindex, + struct au_opt_mod *mod, substring_t args[]) +{ + int err; + struct dentry *root; + + err = -EINVAL; + root = sb->s_root; + aufs_read_lock(root, AuLock_FLUSH); + if (bindex < 0 || au_sbbot(sb) < bindex) { + pr_err("out of bounds, %d\n", bindex); + goto out; + } + + err = 0; + mod->perm = br_perm_val(args[1].from); + AuDbg("mod path %s, perm 0x%x, %s\n", + mod->path, mod->perm, args[1].from); + mod->h_root = dget(au_h_dptr(root, bindex)); + +out: + aufs_read_unlock(root, !AuLock_IR); + return err; +} +#endif + +static int au_opts_parse_xino(struct super_block *sb, struct au_opt_xino *xino, + substring_t args[]) +{ + int err; + struct file *file; + + file = au_xino_create(sb, args[0].from, /*silent*/0, /*wbrtop*/0); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + + err = -EINVAL; + if (unlikely(file->f_path.dentry->d_sb == sb)) { + fput(file); + pr_err("%s must be outside\n", args[0].from); + goto out; + } + + err = 0; + xino->file = file; + xino->path = args[0].from; + +out: + return err; +} + +static int noinline_for_stack +au_opts_parse_xino_itrunc_path(struct super_block *sb, + struct au_opt_xino_itrunc *xino_itrunc, + substring_t args[]) +{ + int err; + aufs_bindex_t bbot, bindex; + struct path path; + struct dentry *root; + + err = vfsub_kern_path(args[0].from, lkup_dirflags, &path); + if (unlikely(err)) { + pr_err("lookup failed %s (%d)\n", args[0].from, err); + goto out; + } + + xino_itrunc->bindex = -1; + root = sb->s_root; + aufs_read_lock(root, AuLock_FLUSH); + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + if (au_h_dptr(root, bindex) == path.dentry) { + xino_itrunc->bindex = bindex; + break; + } + } + aufs_read_unlock(root, !AuLock_IR); + path_put(&path); + + if (unlikely(xino_itrunc->bindex < 0)) { + pr_err("no such branch %s\n", args[0].from); + err = -EINVAL; + } + +out: + return err; +} + +/* called without aufs lock */ +int au_opts_parse(struct super_block *sb, char *str, struct au_opts *opts) +{ + int err, n, token; + aufs_bindex_t bindex; + unsigned char skipped; + struct dentry *root; + struct au_opt *opt, *opt_tail; + char *opt_str; + /* reduce the stack space */ + union { + struct au_opt_xino_itrunc *xino_itrunc; + struct au_opt_wbr_create *create; + } u; + struct { + substring_t args[MAX_OPT_ARGS]; + } *a; + + err = -ENOMEM; + a = kmalloc(sizeof(*a), GFP_NOFS); + if (unlikely(!a)) + goto out; + + root = sb->s_root; + err = 0; + bindex = 0; + opt = opts->opt; + opt_tail = opt + opts->max_opt - 1; + opt->type = Opt_tail; + while (!err && (opt_str = strsep(&str, ",")) && *opt_str) { + err = -EINVAL; + skipped = 0; + token = match_token(opt_str, options, a->args); + switch (token) { + case Opt_br: + err = 0; + while (!err && (opt_str = strsep(&a->args[0].from, ":")) + && *opt_str) { + err = opt_add(opt, opt_str, opts->sb_flags, + bindex++); + if (unlikely(!err && ++opt > opt_tail)) { + err = -E2BIG; + break; + } + opt->type = Opt_tail; + skipped = 1; + } + break; + case Opt_add: + if (unlikely(match_int(&a->args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + bindex = n; + err = opt_add(opt, a->args[1].from, opts->sb_flags, + bindex); + if (!err) + opt->type = token; + break; + case Opt_append: + err = opt_add(opt, a->args[0].from, opts->sb_flags, + /*dummy bindex*/1); + if (!err) + opt->type = token; + break; + case Opt_prepend: + err = opt_add(opt, a->args[0].from, opts->sb_flags, + /*bindex*/0); + if (!err) + opt->type = token; + break; + case Opt_del: + err = au_opts_parse_del(&opt->del, a->args); + if (!err) + opt->type = token; + break; +#if 0 /* reserved for future use */ + case Opt_idel: + del->pathname = "(indexed)"; + if (unlikely(match_int(&args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + err = au_opts_parse_idel(sb, n, &opt->del, a->args); + if (!err) + opt->type = token; + break; +#endif + case Opt_mod: + err = au_opts_parse_mod(&opt->mod, a->args); + if (!err) + opt->type = token; + break; +#ifdef IMOD /* reserved for future use */ + case Opt_imod: + u.mod->path = "(indexed)"; + if (unlikely(match_int(&a->args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + err = au_opts_parse_imod(sb, n, &opt->mod, a->args); + if (!err) + opt->type = token; + break; +#endif + case Opt_xino: + err = au_opts_parse_xino(sb, &opt->xino, a->args); + if (!err) + opt->type = token; + break; + + case Opt_trunc_xino_path: + err = au_opts_parse_xino_itrunc_path + (sb, &opt->xino_itrunc, a->args); + if (!err) + opt->type = token; + break; + + case Opt_itrunc_xino: + u.xino_itrunc = &opt->xino_itrunc; + if (unlikely(match_int(&a->args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + u.xino_itrunc->bindex = n; + aufs_read_lock(root, AuLock_FLUSH); + if (n < 0 || au_sbbot(sb) < n) { + pr_err("out of bounds, %d\n", n); + aufs_read_unlock(root, !AuLock_IR); + break; + } + aufs_read_unlock(root, !AuLock_IR); + err = 0; + opt->type = token; + break; + + case Opt_dirwh: + if (unlikely(match_int(&a->args[0], &opt->dirwh))) + break; + err = 0; + opt->type = token; + break; + + case Opt_rdcache: + if (unlikely(match_int(&a->args[0], &n))) { + pr_err("bad integer in %s\n", opt_str); + break; + } + if (unlikely(n > AUFS_RDCACHE_MAX)) { + pr_err("rdcache must be smaller than %d\n", + AUFS_RDCACHE_MAX); + break; + } + opt->rdcache = n; + err = 0; + opt->type = token; + break; + case Opt_rdblk: + if (unlikely(match_int(&a->args[0], &n) + || n < 0 + || n > KMALLOC_MAX_SIZE)) { + pr_err("bad integer in %s\n", opt_str); + break; + } + if (unlikely(n && n < NAME_MAX)) { + pr_err("rdblk must be larger than %d\n", + NAME_MAX); + break; + } + opt->rdblk = n; + err = 0; + opt->type = token; + break; + case Opt_rdhash: + if (unlikely(match_int(&a->args[0], &n) + || n < 0 + || n * sizeof(struct hlist_head) + > KMALLOC_MAX_SIZE)) { + pr_err("bad integer in %s\n", opt_str); + break; + } + opt->rdhash = n; + err = 0; + opt->type = token; + break; + + case Opt_trunc_xino: + case Opt_notrunc_xino: + case Opt_noxino: + case Opt_trunc_xib: + case Opt_notrunc_xib: + case Opt_shwh: + case Opt_noshwh: + case Opt_dirperm1: + case Opt_nodirperm1: + case Opt_plink: + case Opt_noplink: + case Opt_list_plink: + case Opt_dio: + case Opt_nodio: + case Opt_diropq_a: + case Opt_diropq_w: + case Opt_warn_perm: + case Opt_nowarn_perm: + case Opt_verbose: + case Opt_noverbose: + case Opt_sum: + case Opt_nosum: + case Opt_wsum: + case Opt_rdblk_def: + case Opt_rdhash_def: + case Opt_dirren: + case Opt_nodirren: + case Opt_acl: + case Opt_noacl: + err = 0; + opt->type = token; + break; + + case Opt_udba: + opt->udba = udba_val(a->args[0].from); + if (opt->udba >= 0) { + err = 0; + opt->type = token; + } else + pr_err("wrong value, %s\n", opt_str); + break; + + case Opt_wbr_create: + u.create = &opt->wbr_create; + u.create->wbr_create + = au_wbr_create_val(a->args[0].from, u.create); + if (u.create->wbr_create >= 0) { + err = 0; + opt->type = token; + } else + pr_err("wrong value, %s\n", opt_str); + break; + case Opt_wbr_copyup: + opt->wbr_copyup = au_wbr_copyup_val(a->args[0].from); + if (opt->wbr_copyup >= 0) { + err = 0; + opt->type = token; + } else + pr_err("wrong value, %s\n", opt_str); + break; + + case Opt_fhsm_sec: + if (unlikely(match_int(&a->args[0], &n) + || n < 0)) { + pr_err("bad integer in %s\n", opt_str); + break; + } + if (sysaufs_brs) { + opt->fhsm_second = n; + opt->type = token; + } else + pr_warn("ignored %s\n", opt_str); + err = 0; + break; + + case Opt_ignore: + pr_warn("ignored %s\n", opt_str); + /*FALLTHROUGH*/ + case Opt_ignore_silent: + skipped = 1; + err = 0; + break; + case Opt_err: + pr_err("unknown option %s\n", opt_str); + break; + } + + if (!err && !skipped) { + if (unlikely(++opt > opt_tail)) { + err = -E2BIG; + opt--; + opt->type = Opt_tail; + break; + } + opt->type = Opt_tail; + } + } + + au_kfree_rcu(a); + dump_opts(opts); + if (unlikely(err)) + au_opts_free(opts); + +out: + return err; +} + +static int au_opt_wbr_create(struct super_block *sb, + struct au_opt_wbr_create *create) +{ + int err; + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + err = 1; /* handled */ + sbinfo = au_sbi(sb); + if (sbinfo->si_wbr_create_ops->fin) { + err = sbinfo->si_wbr_create_ops->fin(sb); + if (!err) + err = 1; + } + + sbinfo->si_wbr_create = create->wbr_create; + sbinfo->si_wbr_create_ops = au_wbr_create_ops + create->wbr_create; + switch (create->wbr_create) { + case AuWbrCreate_MFSRRV: + case AuWbrCreate_MFSRR: + case AuWbrCreate_TDMFS: + case AuWbrCreate_TDMFSV: + case AuWbrCreate_PMFSRR: + case AuWbrCreate_PMFSRRV: + sbinfo->si_wbr_mfs.mfsrr_watermark = create->mfsrr_watermark; + /*FALLTHROUGH*/ + case AuWbrCreate_MFS: + case AuWbrCreate_MFSV: + case AuWbrCreate_PMFS: + case AuWbrCreate_PMFSV: + sbinfo->si_wbr_mfs.mfs_expire + = msecs_to_jiffies(create->mfs_second * MSEC_PER_SEC); + break; + } + + if (sbinfo->si_wbr_create_ops->init) + sbinfo->si_wbr_create_ops->init(sb); /* ignore */ + + return err; +} + +/* + * returns, + * plus: processed without an error + * zero: unprocessed + */ +static int au_opt_simple(struct super_block *sb, struct au_opt *opt, + struct au_opts *opts) +{ + int err; + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + err = 1; /* handled */ + sbinfo = au_sbi(sb); + switch (opt->type) { + case Opt_udba: + sbinfo->si_mntflags &= ~AuOptMask_UDBA; + sbinfo->si_mntflags |= opt->udba; + opts->given_udba |= opt->udba; + break; + + case Opt_plink: + au_opt_set(sbinfo->si_mntflags, PLINK); + break; + case Opt_noplink: + if (au_opt_test(sbinfo->si_mntflags, PLINK)) + au_plink_put(sb, /*verbose*/1); + au_opt_clr(sbinfo->si_mntflags, PLINK); + break; + case Opt_list_plink: + if (au_opt_test(sbinfo->si_mntflags, PLINK)) + au_plink_list(sb); + break; + + case Opt_dio: + au_opt_set(sbinfo->si_mntflags, DIO); + au_fset_opts(opts->flags, REFRESH_DYAOP); + break; + case Opt_nodio: + au_opt_clr(sbinfo->si_mntflags, DIO); + au_fset_opts(opts->flags, REFRESH_DYAOP); + break; + + case Opt_fhsm_sec: + au_fhsm_set(sbinfo, opt->fhsm_second); + break; + + case Opt_diropq_a: + au_opt_set(sbinfo->si_mntflags, ALWAYS_DIROPQ); + break; + case Opt_diropq_w: + au_opt_clr(sbinfo->si_mntflags, ALWAYS_DIROPQ); + break; + + case Opt_warn_perm: + au_opt_set(sbinfo->si_mntflags, WARN_PERM); + break; + case Opt_nowarn_perm: + au_opt_clr(sbinfo->si_mntflags, WARN_PERM); + break; + + case Opt_verbose: + au_opt_set(sbinfo->si_mntflags, VERBOSE); + break; + case Opt_noverbose: + au_opt_clr(sbinfo->si_mntflags, VERBOSE); + break; + + case Opt_sum: + au_opt_set(sbinfo->si_mntflags, SUM); + break; + case Opt_wsum: + au_opt_clr(sbinfo->si_mntflags, SUM); + au_opt_set(sbinfo->si_mntflags, SUM_W); + break; + case Opt_nosum: + au_opt_clr(sbinfo->si_mntflags, SUM); + au_opt_clr(sbinfo->si_mntflags, SUM_W); + break; + + case Opt_wbr_create: + err = au_opt_wbr_create(sb, &opt->wbr_create); + break; + case Opt_wbr_copyup: + sbinfo->si_wbr_copyup = opt->wbr_copyup; + sbinfo->si_wbr_copyup_ops = au_wbr_copyup_ops + opt->wbr_copyup; + break; + + case Opt_dirwh: + sbinfo->si_dirwh = opt->dirwh; + break; + + case Opt_rdcache: + sbinfo->si_rdcache + = msecs_to_jiffies(opt->rdcache * MSEC_PER_SEC); + break; + case Opt_rdblk: + sbinfo->si_rdblk = opt->rdblk; + break; + case Opt_rdblk_def: + sbinfo->si_rdblk = AUFS_RDBLK_DEF; + break; + case Opt_rdhash: + sbinfo->si_rdhash = opt->rdhash; + break; + case Opt_rdhash_def: + sbinfo->si_rdhash = AUFS_RDHASH_DEF; + break; + + case Opt_shwh: + au_opt_set(sbinfo->si_mntflags, SHWH); + break; + case Opt_noshwh: + au_opt_clr(sbinfo->si_mntflags, SHWH); + break; + + case Opt_dirperm1: + au_opt_set(sbinfo->si_mntflags, DIRPERM1); + break; + case Opt_nodirperm1: + au_opt_clr(sbinfo->si_mntflags, DIRPERM1); + break; + + case Opt_trunc_xino: + au_opt_set(sbinfo->si_mntflags, TRUNC_XINO); + break; + case Opt_notrunc_xino: + au_opt_clr(sbinfo->si_mntflags, TRUNC_XINO); + break; + + case Opt_trunc_xino_path: + case Opt_itrunc_xino: + err = au_xino_trunc(sb, opt->xino_itrunc.bindex, + /*idx_begin*/0); + if (!err) + err = 1; + break; + + case Opt_trunc_xib: + au_fset_opts(opts->flags, TRUNC_XIB); + break; + case Opt_notrunc_xib: + au_fclr_opts(opts->flags, TRUNC_XIB); + break; + + case Opt_dirren: + err = 1; + if (!au_opt_test(sbinfo->si_mntflags, DIRREN)) { + err = au_dr_opt_set(sb); + if (!err) + err = 1; + } + if (err == 1) + au_opt_set(sbinfo->si_mntflags, DIRREN); + break; + case Opt_nodirren: + err = 1; + if (au_opt_test(sbinfo->si_mntflags, DIRREN)) { + err = au_dr_opt_clr(sb, au_ftest_opts(opts->flags, + DR_FLUSHED)); + if (!err) + err = 1; + } + if (err == 1) + au_opt_clr(sbinfo->si_mntflags, DIRREN); + break; + + case Opt_acl: + sb->s_flags |= SB_POSIXACL; + break; + case Opt_noacl: + sb->s_flags &= ~SB_POSIXACL; + break; + + default: + err = 0; + break; + } + + return err; +} + +/* + * returns tri-state. + * plus: processed without an error + * zero: unprocessed + * minus: error + */ +static int au_opt_br(struct super_block *sb, struct au_opt *opt, + struct au_opts *opts) +{ + int err, do_refresh; + + err = 0; + switch (opt->type) { + case Opt_append: + opt->add.bindex = au_sbbot(sb) + 1; + if (opt->add.bindex < 0) + opt->add.bindex = 0; + goto add; + /* Always goto add, not fallthrough */ + case Opt_prepend: + opt->add.bindex = 0; + /* fallthrough */ + add: /* indented label */ + case Opt_add: + err = au_br_add(sb, &opt->add, + au_ftest_opts(opts->flags, REMOUNT)); + if (!err) { + err = 1; + au_fset_opts(opts->flags, REFRESH); + } + break; + + case Opt_del: + case Opt_idel: + err = au_br_del(sb, &opt->del, + au_ftest_opts(opts->flags, REMOUNT)); + if (!err) { + err = 1; + au_fset_opts(opts->flags, TRUNC_XIB); + au_fset_opts(opts->flags, REFRESH); + } + break; + + case Opt_mod: + case Opt_imod: + err = au_br_mod(sb, &opt->mod, + au_ftest_opts(opts->flags, REMOUNT), + &do_refresh); + if (!err) { + err = 1; + if (do_refresh) + au_fset_opts(opts->flags, REFRESH); + } + break; + } + return err; +} + +static int au_opt_xino(struct super_block *sb, struct au_opt *opt, + struct au_opt_xino **opt_xino, + struct au_opts *opts) +{ + int err; + + err = 0; + switch (opt->type) { + case Opt_xino: + err = au_xino_set(sb, &opt->xino, + !!au_ftest_opts(opts->flags, REMOUNT)); + if (unlikely(err)) + break; + + *opt_xino = &opt->xino; + break; + + case Opt_noxino: + au_xino_clr(sb); + *opt_xino = (void *)-1; + break; + } + + return err; +} + +int au_opts_verify(struct super_block *sb, unsigned long sb_flags, + unsigned int pending) +{ + int err, fhsm; + aufs_bindex_t bindex, bbot; + unsigned char do_plink, skip, do_free, can_no_dreval; + struct au_branch *br; + struct au_wbr *wbr; + struct dentry *root, *dentry; + struct inode *dir, *h_dir; + struct au_sbinfo *sbinfo; + struct au_hinode *hdir; + + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + AuDebugOn(!(sbinfo->si_mntflags & AuOptMask_UDBA)); + + if (!(sb_flags & SB_RDONLY)) { + if (unlikely(!au_br_writable(au_sbr_perm(sb, 0)))) + pr_warn("first branch should be rw\n"); + if (unlikely(au_opt_test(sbinfo->si_mntflags, SHWH))) + pr_warn_once("shwh should be used with ro\n"); + } + + if (au_opt_test((sbinfo->si_mntflags | pending), UDBA_HNOTIFY) + && !au_opt_test(sbinfo->si_mntflags, XINO)) + pr_warn_once("udba=*notify requires xino\n"); + + if (au_opt_test(sbinfo->si_mntflags, DIRPERM1)) + pr_warn_once("dirperm1 breaks the protection" + " by the permission bits on the lower branch\n"); + + err = 0; + fhsm = 0; + root = sb->s_root; + dir = d_inode(root); + do_plink = !!au_opt_test(sbinfo->si_mntflags, PLINK); + can_no_dreval = !!au_opt_test((sbinfo->si_mntflags | pending), + UDBA_NONE); + bbot = au_sbbot(sb); + for (bindex = 0; !err && bindex <= bbot; bindex++) { + skip = 0; + h_dir = au_h_iptr(dir, bindex); + br = au_sbr(sb, bindex); + + if ((br->br_perm & AuBrAttr_ICEX) + && !h_dir->i_op->listxattr) + br->br_perm &= ~AuBrAttr_ICEX; +#if 0 /* untested */ + if ((br->br_perm & AuBrAttr_ICEX_SEC) + && (au_br_sb(br)->s_flags & SB_NOSEC)) + br->br_perm &= ~AuBrAttr_ICEX_SEC; +#endif + + do_free = 0; + wbr = br->br_wbr; + if (wbr) + wbr_wh_read_lock(wbr); + + if (!au_br_writable(br->br_perm)) { + do_free = !!wbr; + skip = (!wbr + || (!wbr->wbr_whbase + && !wbr->wbr_plink + && !wbr->wbr_orph)); + } else if (!au_br_wh_linkable(br->br_perm)) { + /* skip = (!br->br_whbase && !br->br_orph); */ + skip = (!wbr || !wbr->wbr_whbase); + if (skip && wbr) { + if (do_plink) + skip = !!wbr->wbr_plink; + else + skip = !wbr->wbr_plink; + } + } else { + /* skip = (br->br_whbase && br->br_ohph); */ + skip = (wbr && wbr->wbr_whbase); + if (skip) { + if (do_plink) + skip = !!wbr->wbr_plink; + else + skip = !wbr->wbr_plink; + } + } + if (wbr) + wbr_wh_read_unlock(wbr); + + if (can_no_dreval) { + dentry = br->br_path.dentry; + spin_lock(&dentry->d_lock); + if (dentry->d_flags & + (DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE)) + can_no_dreval = 0; + spin_unlock(&dentry->d_lock); + } + + if (au_br_fhsm(br->br_perm)) { + fhsm++; + AuDebugOn(!br->br_fhsm); + } + + if (skip) + continue; + + hdir = au_hi(dir, bindex); + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + if (wbr) + wbr_wh_write_lock(wbr); + err = au_wh_init(br, sb); + if (wbr) + wbr_wh_write_unlock(wbr); + au_hn_inode_unlock(hdir); + + if (!err && do_free) { + au_kfree_rcu(wbr); + br->br_wbr = NULL; + } + } + + if (can_no_dreval) + au_fset_si(sbinfo, NO_DREVAL); + else + au_fclr_si(sbinfo, NO_DREVAL); + + if (fhsm >= 2) { + au_fset_si(sbinfo, FHSM); + for (bindex = bbot; bindex >= 0; bindex--) { + br = au_sbr(sb, bindex); + if (au_br_fhsm(br->br_perm)) { + au_fhsm_set_bottom(sb, bindex); + break; + } + } + } else { + au_fclr_si(sbinfo, FHSM); + au_fhsm_set_bottom(sb, -1); + } + + return err; +} + +int au_opts_mount(struct super_block *sb, struct au_opts *opts) +{ + int err; + unsigned int tmp; + aufs_bindex_t bindex, bbot; + struct au_opt *opt; + struct au_opt_xino *opt_xino, xino; + struct au_sbinfo *sbinfo; + struct au_branch *br; + struct inode *dir; + + SiMustWriteLock(sb); + + err = 0; + opt_xino = NULL; + opt = opts->opt; + while (err >= 0 && opt->type != Opt_tail) + err = au_opt_simple(sb, opt++, opts); + if (err > 0) + err = 0; + else if (unlikely(err < 0)) + goto out; + + /* disable xino and udba temporary */ + sbinfo = au_sbi(sb); + tmp = sbinfo->si_mntflags; + au_opt_clr(sbinfo->si_mntflags, XINO); + au_opt_set_udba(sbinfo->si_mntflags, UDBA_REVAL); + + opt = opts->opt; + while (err >= 0 && opt->type != Opt_tail) + err = au_opt_br(sb, opt++, opts); + if (err > 0) + err = 0; + else if (unlikely(err < 0)) + goto out; + + bbot = au_sbbot(sb); + if (unlikely(bbot < 0)) { + err = -EINVAL; + pr_err("no branches\n"); + goto out; + } + + if (au_opt_test(tmp, XINO)) + au_opt_set(sbinfo->si_mntflags, XINO); + opt = opts->opt; + while (!err && opt->type != Opt_tail) + err = au_opt_xino(sb, opt++, &opt_xino, opts); + if (unlikely(err)) + goto out; + + err = au_opts_verify(sb, sb->s_flags, tmp); + if (unlikely(err)) + goto out; + + /* restore xino */ + if (au_opt_test(tmp, XINO) && !opt_xino) { + xino.file = au_xino_def(sb); + err = PTR_ERR(xino.file); + if (IS_ERR(xino.file)) + goto out; + + err = au_xino_set(sb, &xino, /*remount*/0); + fput(xino.file); + if (unlikely(err)) + goto out; + } + + /* restore udba */ + tmp &= AuOptMask_UDBA; + sbinfo->si_mntflags &= ~AuOptMask_UDBA; + sbinfo->si_mntflags |= tmp; + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + err = au_hnotify_reset_br(tmp, br, br->br_perm); + if (unlikely(err)) + AuIOErr("hnotify failed on br %d, %d, ignored\n", + bindex, err); + /* go on even if err */ + } + if (au_opt_test(tmp, UDBA_HNOTIFY)) { + dir = d_inode(sb->s_root); + au_hn_reset(dir, au_hi_flags(dir, /*isdir*/1) & ~AuHi_XINO); + } + +out: + return err; +} + +int au_opts_remount(struct super_block *sb, struct au_opts *opts) +{ + int err, rerr; + unsigned char no_dreval; + struct inode *dir; + struct au_opt_xino *opt_xino; + struct au_opt *opt; + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + err = au_dr_opt_flush(sb); + if (unlikely(err)) + goto out; + au_fset_opts(opts->flags, DR_FLUSHED); + + dir = d_inode(sb->s_root); + sbinfo = au_sbi(sb); + opt_xino = NULL; + opt = opts->opt; + while (err >= 0 && opt->type != Opt_tail) { + err = au_opt_simple(sb, opt, opts); + if (!err) + err = au_opt_br(sb, opt, opts); + if (!err) + err = au_opt_xino(sb, opt, &opt_xino, opts); + opt++; + } + if (err > 0) + err = 0; + AuTraceErr(err); + /* go on even err */ + + no_dreval = !!au_ftest_si(sbinfo, NO_DREVAL); + rerr = au_opts_verify(sb, opts->sb_flags, /*pending*/0); + if (unlikely(rerr && !err)) + err = rerr; + + if (no_dreval != !!au_ftest_si(sbinfo, NO_DREVAL)) + au_fset_opts(opts->flags, REFRESH_IDOP); + + if (au_ftest_opts(opts->flags, TRUNC_XIB)) { + rerr = au_xib_trunc(sb); + if (unlikely(rerr && !err)) + err = rerr; + } + + /* will be handled by the caller */ + if (!au_ftest_opts(opts->flags, REFRESH) + && (opts->given_udba + || au_opt_test(sbinfo->si_mntflags, XINO) + || au_ftest_opts(opts->flags, REFRESH_IDOP) + )) + au_fset_opts(opts->flags, REFRESH); + + AuDbg("status 0x%x\n", opts->flags); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +unsigned int au_opt_udba(struct super_block *sb) +{ + return au_mntflags(sb) & AuOptMask_UDBA; +} diff --git a/fs/aufs/opts.h b/fs/aufs/opts.h new file mode 100644 index 0000000000000..79f3ea56f8bb8 --- /dev/null +++ b/fs/aufs/opts.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * mount options/flags + */ + +#ifndef __AUFS_OPTS_H__ +#define __AUFS_OPTS_H__ + +#ifdef __KERNEL__ + +#include + +struct file; + +/* ---------------------------------------------------------------------- */ + +/* mount flags */ +#define AuOpt_XINO 1 /* external inode number bitmap + and translation table */ +#define AuOpt_TRUNC_XINO (1 << 1) /* truncate xino files */ +#define AuOpt_UDBA_NONE (1 << 2) /* users direct branch access */ +#define AuOpt_UDBA_REVAL (1 << 3) +#define AuOpt_UDBA_HNOTIFY (1 << 4) +#define AuOpt_SHWH (1 << 5) /* show whiteout */ +#define AuOpt_PLINK (1 << 6) /* pseudo-link */ +#define AuOpt_DIRPERM1 (1 << 7) /* ignore the lower dir's perm + bits */ +#define AuOpt_ALWAYS_DIROPQ (1 << 9) /* policy to creating diropq */ +#define AuOpt_SUM (1 << 10) /* summation for statfs(2) */ +#define AuOpt_SUM_W (1 << 11) /* unimplemented */ +#define AuOpt_WARN_PERM (1 << 12) /* warn when add-branch */ +#define AuOpt_VERBOSE (1 << 13) /* print the cause of error */ +#define AuOpt_DIO (1 << 14) /* direct io */ +#define AuOpt_DIRREN (1 << 15) /* directory rename */ + +#ifndef CONFIG_AUFS_HNOTIFY +#undef AuOpt_UDBA_HNOTIFY +#define AuOpt_UDBA_HNOTIFY 0 +#endif +#ifndef CONFIG_AUFS_DIRREN +#undef AuOpt_DIRREN +#define AuOpt_DIRREN 0 +#endif +#ifndef CONFIG_AUFS_SHWH +#undef AuOpt_SHWH +#define AuOpt_SHWH 0 +#endif + +#define AuOpt_Def (AuOpt_XINO \ + | AuOpt_UDBA_REVAL \ + | AuOpt_PLINK \ + /* | AuOpt_DIRPERM1 */ \ + | AuOpt_WARN_PERM) +#define AuOptMask_UDBA (AuOpt_UDBA_NONE \ + | AuOpt_UDBA_REVAL \ + | AuOpt_UDBA_HNOTIFY) + +#define au_opt_test(flags, name) (flags & AuOpt_##name) +#define au_opt_set(flags, name) do { \ + BUILD_BUG_ON(AuOpt_##name & AuOptMask_UDBA); \ + ((flags) |= AuOpt_##name); \ +} while (0) +#define au_opt_set_udba(flags, name) do { \ + (flags) &= ~AuOptMask_UDBA; \ + ((flags) |= AuOpt_##name); \ +} while (0) +#define au_opt_clr(flags, name) do { \ + ((flags) &= ~AuOpt_##name); \ +} while (0) + +static inline unsigned int au_opts_plink(unsigned int mntflags) +{ +#ifdef CONFIG_PROC_FS + return mntflags; +#else + return mntflags & ~AuOpt_PLINK; +#endif +} + +/* ---------------------------------------------------------------------- */ + +/* policies to select one among multiple writable branches */ +enum { + AuWbrCreate_TDP, /* top down parent */ + AuWbrCreate_RR, /* round robin */ + AuWbrCreate_MFS, /* most free space */ + AuWbrCreate_MFSV, /* mfs with seconds */ + AuWbrCreate_MFSRR, /* mfs then rr */ + AuWbrCreate_MFSRRV, /* mfs then rr with seconds */ + AuWbrCreate_TDMFS, /* top down regardless parent and mfs */ + AuWbrCreate_TDMFSV, /* top down regardless parent and mfs */ + AuWbrCreate_PMFS, /* parent and mfs */ + AuWbrCreate_PMFSV, /* parent and mfs with seconds */ + AuWbrCreate_PMFSRR, /* parent, mfs and round-robin */ + AuWbrCreate_PMFSRRV, /* plus seconds */ + + AuWbrCreate_Def = AuWbrCreate_TDP +}; + +enum { + AuWbrCopyup_TDP, /* top down parent */ + AuWbrCopyup_BUP, /* bottom up parent */ + AuWbrCopyup_BU, /* bottom up */ + + AuWbrCopyup_Def = AuWbrCopyup_TDP +}; + +/* ---------------------------------------------------------------------- */ + +struct au_opt_add { + aufs_bindex_t bindex; + char *pathname; + int perm; + struct path path; +}; + +struct au_opt_del { + char *pathname; + struct path h_path; +}; + +struct au_opt_mod { + char *path; + int perm; + struct dentry *h_root; +}; + +struct au_opt_xino { + char *path; + struct file *file; +}; + +struct au_opt_xino_itrunc { + aufs_bindex_t bindex; +}; + +struct au_opt_wbr_create { + int wbr_create; + int mfs_second; + unsigned long long mfsrr_watermark; +}; + +struct au_opt { + int type; + union { + struct au_opt_xino xino; + struct au_opt_xino_itrunc xino_itrunc; + struct au_opt_add add; + struct au_opt_del del; + struct au_opt_mod mod; + int dirwh; + int rdcache; + unsigned int rdblk; + unsigned int rdhash; + int udba; + struct au_opt_wbr_create wbr_create; + int wbr_copyup; + unsigned int fhsm_second; + }; +}; + +/* opts flags */ +#define AuOpts_REMOUNT 1 +#define AuOpts_REFRESH (1 << 1) +#define AuOpts_TRUNC_XIB (1 << 2) +#define AuOpts_REFRESH_DYAOP (1 << 3) +#define AuOpts_REFRESH_IDOP (1 << 4) +#define AuOpts_DR_FLUSHED (1 << 5) +#define au_ftest_opts(flags, name) ((flags) & AuOpts_##name) +#define au_fset_opts(flags, name) \ + do { (flags) |= AuOpts_##name; } while (0) +#define au_fclr_opts(flags, name) \ + do { (flags) &= ~AuOpts_##name; } while (0) + +#ifndef CONFIG_AUFS_DIRREN +#undef AuOpts_DR_FLUSHED +#define AuOpts_DR_FLUSHED 0 +#endif + +struct au_opts { + struct au_opt *opt; + int max_opt; + + unsigned int given_udba; + unsigned int flags; + unsigned long sb_flags; +}; + +/* ---------------------------------------------------------------------- */ + +/* opts.c */ +void au_optstr_br_perm(au_br_perm_str_t *str, int perm); +const char *au_optstr_udba(int udba); +const char *au_optstr_wbr_copyup(int wbr_copyup); +const char *au_optstr_wbr_create(int wbr_create); + +void au_opts_free(struct au_opts *opts); +struct super_block; +int au_opts_parse(struct super_block *sb, char *str, struct au_opts *opts); +int au_opts_verify(struct super_block *sb, unsigned long sb_flags, + unsigned int pending); +int au_opts_mount(struct super_block *sb, struct au_opts *opts); +int au_opts_remount(struct super_block *sb, struct au_opts *opts); + +unsigned int au_opt_udba(struct super_block *sb); + +#endif /* __KERNEL__ */ +#endif /* __AUFS_OPTS_H__ */ diff --git a/fs/aufs/plink.c b/fs/aufs/plink.c new file mode 100644 index 0000000000000..850829b218889 --- /dev/null +++ b/fs/aufs/plink.c @@ -0,0 +1,516 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * pseudo-link + */ + +#include "aufs.h" + +/* + * the pseudo-link maintenance mode. + * during a user process maintains the pseudo-links, + * prohibit adding a new plink and branch manipulation. + * + * Flags + * NOPLM: + * For entry functions which will handle plink, and i_mutex is already held + * in VFS. + * They cannot wait and should return an error at once. + * Callers has to check the error. + * NOPLMW: + * For entry functions which will handle plink, but i_mutex is not held + * in VFS. + * They can wait the plink maintenance mode to finish. + * + * They behave like F_SETLK and F_SETLKW. + * If the caller never handle plink, then both flags are unnecessary. + */ + +int au_plink_maint(struct super_block *sb, int flags) +{ + int err; + pid_t pid, ppid; + struct task_struct *parent, *prev; + struct au_sbinfo *sbi; + + SiMustAnyLock(sb); + + err = 0; + if (!au_opt_test(au_mntflags(sb), PLINK)) + goto out; + + sbi = au_sbi(sb); + pid = sbi->si_plink_maint_pid; + if (!pid || pid == current->pid) + goto out; + + /* todo: it highly depends upon /sbin/mount.aufs */ + prev = NULL; + parent = current; + ppid = 0; + rcu_read_lock(); + while (1) { + parent = rcu_dereference(parent->real_parent); + if (parent == prev) + break; + ppid = task_pid_vnr(parent); + if (pid == ppid) { + rcu_read_unlock(); + goto out; + } + prev = parent; + } + rcu_read_unlock(); + + if (au_ftest_lock(flags, NOPLMW)) { + /* if there is no i_mutex lock in VFS, we don't need to wait */ + /* AuDebugOn(!lockdep_depth(current)); */ + while (sbi->si_plink_maint_pid) { + si_read_unlock(sb); + /* gave up wake_up_bit() */ + wait_event(sbi->si_plink_wq, !sbi->si_plink_maint_pid); + + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&sbi->si_nowait); + si_noflush_read_lock(sb); + } + } else if (au_ftest_lock(flags, NOPLM)) { + AuDbg("ppid %d, pid %d\n", ppid, pid); + err = -EAGAIN; + } + +out: + return err; +} + +void au_plink_maint_leave(struct au_sbinfo *sbinfo) +{ + spin_lock(&sbinfo->si_plink_maint_lock); + sbinfo->si_plink_maint_pid = 0; + spin_unlock(&sbinfo->si_plink_maint_lock); + wake_up_all(&sbinfo->si_plink_wq); +} + +int au_plink_maint_enter(struct super_block *sb) +{ + int err; + struct au_sbinfo *sbinfo; + + err = 0; + sbinfo = au_sbi(sb); + /* make sure i am the only one in this fs */ + si_write_lock(sb, AuLock_FLUSH); + if (au_opt_test(au_mntflags(sb), PLINK)) { + spin_lock(&sbinfo->si_plink_maint_lock); + if (!sbinfo->si_plink_maint_pid) + sbinfo->si_plink_maint_pid = current->pid; + else + err = -EBUSY; + spin_unlock(&sbinfo->si_plink_maint_lock); + } + si_write_unlock(sb); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_DEBUG +void au_plink_list(struct super_block *sb) +{ + int i; + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_icntnr *icntnr; + + SiMustAnyLock(sb); + + sbinfo = au_sbi(sb); + AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK)); + AuDebugOn(au_plink_maint(sb, AuLock_NOPLM)); + + for (i = 0; i < AuPlink_NHASH; i++) { + hbl = sbinfo->si_plink + i; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(icntnr, pos, hbl, plink) + AuDbg("%lu\n", icntnr->vfs_inode.i_ino); + hlist_bl_unlock(hbl); + } +} +#endif + +/* is the inode pseudo-linked? */ +int au_plink_test(struct inode *inode) +{ + int found, i; + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_icntnr *icntnr; + + sbinfo = au_sbi(inode->i_sb); + AuRwMustAnyLock(&sbinfo->si_rwsem); + AuDebugOn(!au_opt_test(au_mntflags(inode->i_sb), PLINK)); + AuDebugOn(au_plink_maint(inode->i_sb, AuLock_NOPLM)); + + found = 0; + i = au_plink_hash(inode->i_ino); + hbl = sbinfo->si_plink + i; + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(icntnr, pos, hbl, plink) + if (&icntnr->vfs_inode == inode) { + found = 1; + break; + } + hlist_bl_unlock(hbl); + return found; +} + +/* ---------------------------------------------------------------------- */ + +/* + * generate a name for plink. + * the file will be stored under AUFS_WH_PLINKDIR. + */ +/* 20 is max digits length of ulong 64 */ +#define PLINK_NAME_LEN ((20 + 1) * 2) + +static int plink_name(char *name, int len, struct inode *inode, + aufs_bindex_t bindex) +{ + int rlen; + struct inode *h_inode; + + h_inode = au_h_iptr(inode, bindex); + rlen = snprintf(name, len, "%lu.%lu", inode->i_ino, h_inode->i_ino); + return rlen; +} + +struct au_do_plink_lkup_args { + struct dentry **errp; + struct qstr *tgtname; + struct dentry *h_parent; + struct au_branch *br; +}; + +static struct dentry *au_do_plink_lkup(struct qstr *tgtname, + struct dentry *h_parent, + struct au_branch *br) +{ + struct dentry *h_dentry; + struct inode *h_inode; + + h_inode = d_inode(h_parent); + inode_lock_shared_nested(h_inode, AuLsc_I_CHILD2); + h_dentry = vfsub_lkup_one(tgtname, h_parent); + inode_unlock_shared(h_inode); + return h_dentry; +} + +static void au_call_do_plink_lkup(void *args) +{ + struct au_do_plink_lkup_args *a = args; + *a->errp = au_do_plink_lkup(a->tgtname, a->h_parent, a->br); +} + +/* lookup the plink-ed @inode under the branch at @bindex */ +struct dentry *au_plink_lkup(struct inode *inode, aufs_bindex_t bindex) +{ + struct dentry *h_dentry, *h_parent; + struct au_branch *br; + int wkq_err; + char a[PLINK_NAME_LEN]; + struct qstr tgtname = QSTR_INIT(a, 0); + + AuDebugOn(au_plink_maint(inode->i_sb, AuLock_NOPLM)); + + br = au_sbr(inode->i_sb, bindex); + h_parent = br->br_wbr->wbr_plink; + tgtname.len = plink_name(a, sizeof(a), inode, bindex); + + if (!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) { + struct au_do_plink_lkup_args args = { + .errp = &h_dentry, + .tgtname = &tgtname, + .h_parent = h_parent, + .br = br + }; + + wkq_err = au_wkq_wait(au_call_do_plink_lkup, &args); + if (unlikely(wkq_err)) + h_dentry = ERR_PTR(wkq_err); + } else + h_dentry = au_do_plink_lkup(&tgtname, h_parent, br); + + return h_dentry; +} + +/* create a pseudo-link */ +static int do_whplink(struct qstr *tgt, struct dentry *h_parent, + struct dentry *h_dentry, struct au_branch *br) +{ + int err; + struct path h_path = { + .mnt = au_br_mnt(br) + }; + struct inode *h_dir, *delegated; + + h_dir = d_inode(h_parent); + inode_lock_nested(h_dir, AuLsc_I_CHILD2); +again: + h_path.dentry = vfsub_lkup_one(tgt, h_parent); + err = PTR_ERR(h_path.dentry); + if (IS_ERR(h_path.dentry)) + goto out; + + err = 0; + /* wh.plink dir is not monitored */ + /* todo: is it really safe? */ + if (d_is_positive(h_path.dentry) + && d_inode(h_path.dentry) != d_inode(h_dentry)) { + delegated = NULL; + err = vfsub_unlink(h_dir, &h_path, &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + dput(h_path.dentry); + h_path.dentry = NULL; + if (!err) + goto again; + } + if (!err && d_is_negative(h_path.dentry)) { + delegated = NULL; + err = vfsub_link(h_dentry, h_dir, &h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + } + dput(h_path.dentry); + +out: + inode_unlock(h_dir); + return err; +} + +struct do_whplink_args { + int *errp; + struct qstr *tgt; + struct dentry *h_parent; + struct dentry *h_dentry; + struct au_branch *br; +}; + +static void call_do_whplink(void *args) +{ + struct do_whplink_args *a = args; + *a->errp = do_whplink(a->tgt, a->h_parent, a->h_dentry, a->br); +} + +static int whplink(struct dentry *h_dentry, struct inode *inode, + aufs_bindex_t bindex, struct au_branch *br) +{ + int err, wkq_err; + struct au_wbr *wbr; + struct dentry *h_parent; + char a[PLINK_NAME_LEN]; + struct qstr tgtname = QSTR_INIT(a, 0); + + wbr = au_sbr(inode->i_sb, bindex)->br_wbr; + h_parent = wbr->wbr_plink; + tgtname.len = plink_name(a, sizeof(a), inode, bindex); + + /* always superio. */ + if (!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) { + struct do_whplink_args args = { + .errp = &err, + .tgt = &tgtname, + .h_parent = h_parent, + .h_dentry = h_dentry, + .br = br + }; + wkq_err = au_wkq_wait(call_do_whplink, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } else + err = do_whplink(&tgtname, h_parent, h_dentry, br); + + return err; +} + +/* + * create a new pseudo-link for @h_dentry on @bindex. + * the linked inode is held in aufs @inode. + */ +void au_plink_append(struct inode *inode, aufs_bindex_t bindex, + struct dentry *h_dentry) +{ + struct super_block *sb; + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_icntnr *icntnr; + int found, err, cnt, i; + + sb = inode->i_sb; + sbinfo = au_sbi(sb); + AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK)); + AuDebugOn(au_plink_maint(sb, AuLock_NOPLM)); + + found = au_plink_test(inode); + if (found) + return; + + i = au_plink_hash(inode->i_ino); + hbl = sbinfo->si_plink + i; + au_igrab(inode); + + hlist_bl_lock(hbl); + hlist_bl_for_each_entry(icntnr, pos, hbl, plink) { + if (&icntnr->vfs_inode == inode) { + found = 1; + break; + } + } + if (!found) { + icntnr = container_of(inode, struct au_icntnr, vfs_inode); + hlist_bl_add_head(&icntnr->plink, hbl); + } + hlist_bl_unlock(hbl); + if (!found) { + cnt = au_hbl_count(hbl); +#define msg "unexpectedly unbalanced or too many pseudo-links" + if (cnt > AUFS_PLINK_WARN) + AuWarn1(msg ", %d\n", cnt); +#undef msg + err = whplink(h_dentry, inode, bindex, au_sbr(sb, bindex)); + if (unlikely(err)) { + pr_warn("err %d, damaged pseudo link.\n", err); + au_hbl_del(&icntnr->plink, hbl); + iput(&icntnr->vfs_inode); + } + } else + iput(&icntnr->vfs_inode); +} + +/* free all plinks */ +void au_plink_put(struct super_block *sb, int verbose) +{ + int i, warned; + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos, *tmp; + struct au_icntnr *icntnr; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK)); + AuDebugOn(au_plink_maint(sb, AuLock_NOPLM)); + + /* no spin_lock since sbinfo is write-locked */ + warned = 0; + for (i = 0; i < AuPlink_NHASH; i++) { + hbl = sbinfo->si_plink + i; + if (!warned && verbose && !hlist_bl_empty(hbl)) { + pr_warn("pseudo-link is not flushed"); + warned = 1; + } + hlist_bl_for_each_entry_safe(icntnr, pos, tmp, hbl, plink) + iput(&icntnr->vfs_inode); + INIT_HLIST_BL_HEAD(hbl); + } +} + +void au_plink_clean(struct super_block *sb, int verbose) +{ + struct dentry *root; + + root = sb->s_root; + aufs_write_lock(root); + if (au_opt_test(au_mntflags(sb), PLINK)) + au_plink_put(sb, verbose); + aufs_write_unlock(root); +} + +static int au_plink_do_half_refresh(struct inode *inode, aufs_bindex_t br_id) +{ + int do_put; + aufs_bindex_t btop, bbot, bindex; + + do_put = 0; + btop = au_ibtop(inode); + bbot = au_ibbot(inode); + if (btop >= 0) { + for (bindex = btop; bindex <= bbot; bindex++) { + if (!au_h_iptr(inode, bindex) + || au_ii_br_id(inode, bindex) != br_id) + continue; + au_set_h_iptr(inode, bindex, NULL, 0); + do_put = 1; + break; + } + if (do_put) + for (bindex = btop; bindex <= bbot; bindex++) + if (au_h_iptr(inode, bindex)) { + do_put = 0; + break; + } + } else + do_put = 1; + + return do_put; +} + +/* free the plinks on a branch specified by @br_id */ +void au_plink_half_refresh(struct super_block *sb, aufs_bindex_t br_id) +{ + struct au_sbinfo *sbinfo; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos, *tmp; + struct au_icntnr *icntnr; + struct inode *inode; + int i, do_put; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + AuDebugOn(!au_opt_test(au_mntflags(sb), PLINK)); + AuDebugOn(au_plink_maint(sb, AuLock_NOPLM)); + + /* no bit_lock since sbinfo is write-locked */ + for (i = 0; i < AuPlink_NHASH; i++) { + hbl = sbinfo->si_plink + i; + hlist_bl_for_each_entry_safe(icntnr, pos, tmp, hbl, plink) { + inode = au_igrab(&icntnr->vfs_inode); + ii_write_lock_child(inode); + do_put = au_plink_do_half_refresh(inode, br_id); + if (do_put) { + hlist_bl_del(&icntnr->plink); + iput(inode); + } + ii_write_unlock(inode); + iput(inode); + } + } +} diff --git a/fs/aufs/poll.c b/fs/aufs/poll.c new file mode 100644 index 0000000000000..6975d3ab2d299 --- /dev/null +++ b/fs/aufs/poll.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * poll operation + * There is only one filesystem which implements ->poll operation, currently. + */ + +#include "aufs.h" + +__poll_t aufs_poll(struct file *file, struct poll_table_struct *pt) +{ + __poll_t mask; + struct file *h_file; + struct super_block *sb; + + /* We should pretend an error happened. */ + mask = EPOLLERR /* | EPOLLIN | EPOLLOUT */; + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLMW); + + h_file = au_read_pre(file, /*keep_fi*/0, /*lsc*/0); + if (IS_ERR(h_file)) { + AuDbg("h_file %ld\n", PTR_ERR(h_file)); + goto out; + } + + mask = vfs_poll(h_file, pt); + fput(h_file); /* instead of au_read_post() */ + +out: + si_read_unlock(sb); + if (mask & EPOLLERR) + AuDbg("mask 0x%x\n", mask); + return mask; +} diff --git a/fs/aufs/posix_acl.c b/fs/aufs/posix_acl.c new file mode 100644 index 0000000000000..0789335650de2 --- /dev/null +++ b/fs/aufs/posix_acl.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2014-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * posix acl operations + */ + +#include +#include "aufs.h" + +struct posix_acl *aufs_get_acl(struct inode *inode, int type) +{ + struct posix_acl *acl; + int err; + aufs_bindex_t bindex; + struct inode *h_inode; + struct super_block *sb; + + acl = NULL; + sb = inode->i_sb; + si_read_lock(sb, AuLock_FLUSH); + ii_read_lock_child(inode); + if (!(sb->s_flags & SB_POSIXACL)) + goto out; + + bindex = au_ibtop(inode); + h_inode = au_h_iptr(inode, bindex); + if (unlikely(!h_inode + || ((h_inode->i_mode & S_IFMT) + != (inode->i_mode & S_IFMT)))) { + err = au_busy_or_stale(); + acl = ERR_PTR(err); + goto out; + } + + /* always topmost only */ + acl = get_acl(h_inode, type); + if (IS_ERR(acl)) + forget_cached_acl(inode, type); + else + set_cached_acl(inode, type, acl); + +out: + ii_read_unlock(inode); + si_read_unlock(sb); + + AuTraceErrPtr(acl); + return acl; +} + +int aufs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +{ + int err; + ssize_t ssz; + struct dentry *dentry; + struct au_sxattr arg = { + .type = AU_ACL_SET, + .u.acl_set = { + .acl = acl, + .type = type + }, + }; + + IMustLock(inode); + + if (inode->i_ino == AUFS_ROOT_INO) + dentry = dget(inode->i_sb->s_root); + else { + dentry = d_find_alias(inode); + if (!dentry) + dentry = d_find_any_alias(inode); + if (!dentry) { + pr_warn("cannot handle this inode, " + "please report to aufs-users ML\n"); + err = -ENOENT; + goto out; + } + } + + ssz = au_sxattr(dentry, inode, &arg); + /* forget even it if succeeds since the branch might set differently */ + forget_cached_acl(inode, type); + dput(dentry); + err = ssz; + if (ssz >= 0) + err = 0; + +out: + return err; +} diff --git a/fs/aufs/procfs.c b/fs/aufs/procfs.c new file mode 100644 index 0000000000000..20bb15d454724 --- /dev/null +++ b/fs/aufs/procfs.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * procfs interfaces + */ + +#include +#include "aufs.h" + +static int au_procfs_plm_release(struct inode *inode, struct file *file) +{ + struct au_sbinfo *sbinfo; + + sbinfo = file->private_data; + if (sbinfo) { + au_plink_maint_leave(sbinfo); + kobject_put(&sbinfo->si_kobj); + } + + return 0; +} + +static void au_procfs_plm_write_clean(struct file *file) +{ + struct au_sbinfo *sbinfo; + + sbinfo = file->private_data; + if (sbinfo) + au_plink_clean(sbinfo->si_sb, /*verbose*/0); +} + +static int au_procfs_plm_write_si(struct file *file, unsigned long id) +{ + int err; + struct super_block *sb; + struct au_sbinfo *sbinfo; + struct hlist_bl_node *pos; + + err = -EBUSY; + if (unlikely(file->private_data)) + goto out; + + sb = NULL; + /* don't use au_sbilist_lock() here */ + hlist_bl_lock(&au_sbilist); + hlist_bl_for_each_entry(sbinfo, pos, &au_sbilist, si_list) + if (id == sysaufs_si_id(sbinfo)) { + if (kobject_get_unless_zero(&sbinfo->si_kobj)) + sb = sbinfo->si_sb; + break; + } + hlist_bl_unlock(&au_sbilist); + + err = -EINVAL; + if (unlikely(!sb)) + goto out; + + err = au_plink_maint_enter(sb); + if (!err) + /* keep kobject_get() */ + file->private_data = sbinfo; + else + kobject_put(&sbinfo->si_kobj); +out: + return err; +} + +/* + * Accept a valid "si=xxxx" only. + * Once it is accepted successfully, accept "clean" too. + */ +static ssize_t au_procfs_plm_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + ssize_t err; + unsigned long id; + /* last newline is allowed */ + char buf[3 + sizeof(unsigned long) * 2 + 1]; + + err = -EACCES; + if (unlikely(!capable(CAP_SYS_ADMIN))) + goto out; + + err = -EINVAL; + if (unlikely(count > sizeof(buf))) + goto out; + + err = copy_from_user(buf, ubuf, count); + if (unlikely(err)) { + err = -EFAULT; + goto out; + } + buf[count] = 0; + + err = -EINVAL; + if (!strcmp("clean", buf)) { + au_procfs_plm_write_clean(file); + goto out_success; + } else if (unlikely(strncmp("si=", buf, 3))) + goto out; + + err = kstrtoul(buf + 3, 16, &id); + if (unlikely(err)) + goto out; + + err = au_procfs_plm_write_si(file, id); + if (unlikely(err)) + goto out; + +out_success: + err = count; /* success */ +out: + return err; +} + +static const struct file_operations au_procfs_plm_fop = { + .write = au_procfs_plm_write, + .release = au_procfs_plm_release, + .owner = THIS_MODULE +}; + +/* ---------------------------------------------------------------------- */ + +static struct proc_dir_entry *au_procfs_dir; + +void au_procfs_fin(void) +{ + remove_proc_entry(AUFS_PLINK_MAINT_NAME, au_procfs_dir); + remove_proc_entry(AUFS_PLINK_MAINT_DIR, NULL); +} + +int __init au_procfs_init(void) +{ + int err; + struct proc_dir_entry *entry; + + err = -ENOMEM; + au_procfs_dir = proc_mkdir(AUFS_PLINK_MAINT_DIR, NULL); + if (unlikely(!au_procfs_dir)) + goto out; + + entry = proc_create(AUFS_PLINK_MAINT_NAME, S_IFREG | 0200, + au_procfs_dir, &au_procfs_plm_fop); + if (unlikely(!entry)) + goto out_dir; + + err = 0; + goto out; /* success */ + + +out_dir: + remove_proc_entry(AUFS_PLINK_MAINT_DIR, NULL); +out: + return err; +} diff --git a/fs/aufs/rdu.c b/fs/aufs/rdu.c new file mode 100644 index 0000000000000..54472f58898d2 --- /dev/null +++ b/fs/aufs/rdu.c @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * readdir in userspace. + */ + +#include +#include +#include +#include "aufs.h" + +/* bits for struct aufs_rdu.flags */ +#define AuRdu_CALLED 1 +#define AuRdu_CONT (1 << 1) +#define AuRdu_FULL (1 << 2) +#define au_ftest_rdu(flags, name) ((flags) & AuRdu_##name) +#define au_fset_rdu(flags, name) \ + do { (flags) |= AuRdu_##name; } while (0) +#define au_fclr_rdu(flags, name) \ + do { (flags) &= ~AuRdu_##name; } while (0) + +struct au_rdu_arg { + struct dir_context ctx; + struct aufs_rdu *rdu; + union au_rdu_ent_ul ent; + unsigned long end; + + struct super_block *sb; + int err; +}; + +static int au_rdu_fill(struct dir_context *ctx, const char *name, int nlen, + loff_t offset, u64 h_ino, unsigned int d_type) +{ + int err, len; + struct au_rdu_arg *arg = container_of(ctx, struct au_rdu_arg, ctx); + struct aufs_rdu *rdu = arg->rdu; + struct au_rdu_ent ent; + + err = 0; + arg->err = 0; + au_fset_rdu(rdu->cookie.flags, CALLED); + len = au_rdu_len(nlen); + if (arg->ent.ul + len < arg->end) { + ent.ino = h_ino; + ent.bindex = rdu->cookie.bindex; + ent.type = d_type; + ent.nlen = nlen; + if (unlikely(nlen > AUFS_MAX_NAMELEN)) + ent.type = DT_UNKNOWN; + + /* unnecessary to support mmap_sem since this is a dir */ + err = -EFAULT; + if (copy_to_user(arg->ent.e, &ent, sizeof(ent))) + goto out; + if (copy_to_user(arg->ent.e->name, name, nlen)) + goto out; + /* the terminating NULL */ + if (__put_user(0, arg->ent.e->name + nlen)) + goto out; + err = 0; + /* AuDbg("%p, %.*s\n", arg->ent.p, nlen, name); */ + arg->ent.ul += len; + rdu->rent++; + } else { + err = -EFAULT; + au_fset_rdu(rdu->cookie.flags, FULL); + rdu->full = 1; + rdu->tail = arg->ent; + } + +out: + /* AuTraceErr(err); */ + return err; +} + +static int au_rdu_do(struct file *h_file, struct au_rdu_arg *arg) +{ + int err; + loff_t offset; + struct au_rdu_cookie *cookie = &arg->rdu->cookie; + + /* we don't have to care (FMODE_32BITHASH | FMODE_64BITHASH) for ext4 */ + offset = vfsub_llseek(h_file, cookie->h_pos, SEEK_SET); + err = offset; + if (unlikely(offset != cookie->h_pos)) + goto out; + + err = 0; + do { + arg->err = 0; + au_fclr_rdu(cookie->flags, CALLED); + /* smp_mb(); */ + err = vfsub_iterate_dir(h_file, &arg->ctx); + if (err >= 0) + err = arg->err; + } while (!err + && au_ftest_rdu(cookie->flags, CALLED) + && !au_ftest_rdu(cookie->flags, FULL)); + cookie->h_pos = h_file->f_pos; + +out: + AuTraceErr(err); + return err; +} + +static int au_rdu(struct file *file, struct aufs_rdu *rdu) +{ + int err; + aufs_bindex_t bbot; + struct au_rdu_arg arg = { + .ctx = { + .actor = au_rdu_fill + } + }; + struct dentry *dentry; + struct inode *inode; + struct file *h_file; + struct au_rdu_cookie *cookie = &rdu->cookie; + + err = !access_ok(VERIFY_WRITE, rdu->ent.e, rdu->sz); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out; + } + rdu->rent = 0; + rdu->tail = rdu->ent; + rdu->full = 0; + arg.rdu = rdu; + arg.ent = rdu->ent; + arg.end = arg.ent.ul; + arg.end += rdu->sz; + + err = -ENOTDIR; + if (unlikely(!file->f_op->iterate && !file->f_op->iterate_shared)) + goto out; + + err = security_file_permission(file, MAY_READ); + AuTraceErr(err); + if (unlikely(err)) + goto out; + + dentry = file->f_path.dentry; + inode = d_inode(dentry); + inode_lock_shared(inode); + + arg.sb = inode->i_sb; + err = si_read_lock(arg.sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out_mtx; + err = au_alive_dir(dentry); + if (unlikely(err)) + goto out_si; + /* todo: reval? */ + fi_read_lock(file); + + err = -EAGAIN; + if (unlikely(au_ftest_rdu(cookie->flags, CONT) + && cookie->generation != au_figen(file))) + goto out_unlock; + + err = 0; + if (!rdu->blk) { + rdu->blk = au_sbi(arg.sb)->si_rdblk; + if (!rdu->blk) + rdu->blk = au_dir_size(file, /*dentry*/NULL); + } + bbot = au_fbtop(file); + if (cookie->bindex < bbot) + cookie->bindex = bbot; + bbot = au_fbbot_dir(file); + /* AuDbg("b%d, b%d\n", cookie->bindex, bbot); */ + for (; !err && cookie->bindex <= bbot; + cookie->bindex++, cookie->h_pos = 0) { + h_file = au_hf_dir(file, cookie->bindex); + if (!h_file) + continue; + + au_fclr_rdu(cookie->flags, FULL); + err = au_rdu_do(h_file, &arg); + AuTraceErr(err); + if (unlikely(au_ftest_rdu(cookie->flags, FULL) || err)) + break; + } + AuDbg("rent %llu\n", rdu->rent); + + if (!err && !au_ftest_rdu(cookie->flags, CONT)) { + rdu->shwh = !!au_opt_test(au_sbi(arg.sb)->si_mntflags, SHWH); + au_fset_rdu(cookie->flags, CONT); + cookie->generation = au_figen(file); + } + + ii_read_lock_child(inode); + fsstack_copy_attr_atime(inode, au_h_iptr(inode, au_ibtop(inode))); + ii_read_unlock(inode); + +out_unlock: + fi_read_unlock(file); +out_si: + si_read_unlock(arg.sb); +out_mtx: + inode_unlock_shared(inode); +out: + AuTraceErr(err); + return err; +} + +static int au_rdu_ino(struct file *file, struct aufs_rdu *rdu) +{ + int err; + ino_t ino; + unsigned long long nent; + union au_rdu_ent_ul *u; + struct au_rdu_ent ent; + struct super_block *sb; + + err = 0; + nent = rdu->nent; + u = &rdu->ent; + sb = file->f_path.dentry->d_sb; + si_read_lock(sb, AuLock_FLUSH); + while (nent-- > 0) { + /* unnecessary to support mmap_sem since this is a dir */ + err = copy_from_user(&ent, u->e, sizeof(ent)); + if (!err) + err = !access_ok(VERIFY_WRITE, &u->e->ino, sizeof(ino)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + break; + } + + /* AuDbg("b%d, i%llu\n", ent.bindex, ent.ino); */ + if (!ent.wh) + err = au_ino(sb, ent.bindex, ent.ino, ent.type, &ino); + else + err = au_wh_ino(sb, ent.bindex, ent.ino, ent.type, + &ino); + if (unlikely(err)) { + AuTraceErr(err); + break; + } + + err = __put_user(ino, &u->e->ino); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + break; + } + u->ul += au_rdu_len(ent.nlen); + } + si_read_unlock(sb); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_rdu_verify(struct aufs_rdu *rdu) +{ + AuDbg("rdu{%llu, %p, %u | %u | %llu, %u, %u | " + "%llu, b%d, 0x%x, g%u}\n", + rdu->sz, rdu->ent.e, rdu->verify[AufsCtlRduV_SZ], + rdu->blk, + rdu->rent, rdu->shwh, rdu->full, + rdu->cookie.h_pos, rdu->cookie.bindex, rdu->cookie.flags, + rdu->cookie.generation); + + if (rdu->verify[AufsCtlRduV_SZ] == sizeof(*rdu)) + return 0; + + AuDbg("%u:%u\n", + rdu->verify[AufsCtlRduV_SZ], (unsigned int)sizeof(*rdu)); + return -EINVAL; +} + +long au_rdu_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long err, e; + struct aufs_rdu rdu; + void __user *p = (void __user *)arg; + + err = copy_from_user(&rdu, p, sizeof(rdu)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out; + } + err = au_rdu_verify(&rdu); + if (unlikely(err)) + goto out; + + switch (cmd) { + case AUFS_CTL_RDU: + err = au_rdu(file, &rdu); + if (unlikely(err)) + break; + + e = copy_to_user(p, &rdu, sizeof(rdu)); + if (unlikely(e)) { + err = -EFAULT; + AuTraceErr(err); + } + break; + case AUFS_CTL_RDU_INO: + err = au_rdu_ino(file, &rdu); + break; + + default: + /* err = -ENOTTY; */ + err = -EINVAL; + } + +out: + AuTraceErr(err); + return err; +} + +#ifdef CONFIG_COMPAT +long au_rdu_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long err, e; + struct aufs_rdu rdu; + void __user *p = compat_ptr(arg); + + /* todo: get_user()? */ + err = copy_from_user(&rdu, p, sizeof(rdu)); + if (unlikely(err)) { + err = -EFAULT; + AuTraceErr(err); + goto out; + } + rdu.ent.e = compat_ptr(rdu.ent.ul); + err = au_rdu_verify(&rdu); + if (unlikely(err)) + goto out; + + switch (cmd) { + case AUFS_CTL_RDU: + err = au_rdu(file, &rdu); + if (unlikely(err)) + break; + + rdu.ent.ul = ptr_to_compat(rdu.ent.e); + rdu.tail.ul = ptr_to_compat(rdu.tail.e); + e = copy_to_user(p, &rdu, sizeof(rdu)); + if (unlikely(e)) { + err = -EFAULT; + AuTraceErr(err); + } + break; + case AUFS_CTL_RDU_INO: + err = au_rdu_ino(file, &rdu); + break; + + default: + /* err = -ENOTTY; */ + err = -EINVAL; + } + +out: + AuTraceErr(err); + return err; +} +#endif diff --git a/fs/aufs/rwsem.h b/fs/aufs/rwsem.h new file mode 100644 index 0000000000000..370eae1593489 --- /dev/null +++ b/fs/aufs/rwsem.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * simple read-write semaphore wrappers + */ + +#ifndef __AUFS_RWSEM_H__ +#define __AUFS_RWSEM_H__ + +#ifdef __KERNEL__ + +#include "debug.h" + +/* in the future, the name 'au_rwsem' will be totally gone */ +#define au_rwsem rw_semaphore + +/* to debug easier, do not make them inlined functions */ +#define AuRwMustNoWaiters(rw) AuDebugOn(rwsem_is_contended(rw)) +/* rwsem_is_locked() is unusable */ +#define AuRwMustReadLock(rw) AuDebugOn(!lockdep_recursing(current) \ + && debug_locks \ + && !lockdep_is_held_type(rw, 1)) +#define AuRwMustWriteLock(rw) AuDebugOn(!lockdep_recursing(current) \ + && debug_locks \ + && !lockdep_is_held_type(rw, 0)) +#define AuRwMustAnyLock(rw) AuDebugOn(!lockdep_recursing(current) \ + && debug_locks \ + && !lockdep_is_held(rw)) +#define AuRwDestroy(rw) AuDebugOn(!lockdep_recursing(current) \ + && debug_locks \ + && lockdep_is_held(rw)) + +#define au_rw_init(rw) init_rwsem(rw) + +#define au_rw_init_wlock(rw) do { \ + au_rw_init(rw); \ + down_write(rw); \ + } while (0) + +#define au_rw_init_wlock_nested(rw, lsc) do { \ + au_rw_init(rw); \ + down_write_nested(rw, lsc); \ + } while (0) + +#define au_rw_read_lock(rw) down_read(rw) +#define au_rw_read_lock_nested(rw, lsc) down_read_nested(rw, lsc) +#define au_rw_read_unlock(rw) up_read(rw) +#define au_rw_dgrade_lock(rw) downgrade_write(rw) +#define au_rw_write_lock(rw) down_write(rw) +#define au_rw_write_lock_nested(rw, lsc) down_write_nested(rw, lsc) +#define au_rw_write_unlock(rw) up_write(rw) +/* why is not _nested version defined? */ +#define au_rw_read_trylock(rw) down_read_trylock(rw) +#define au_rw_write_trylock(rw) down_write_trylock(rw) + +#endif /* __KERNEL__ */ +#endif /* __AUFS_RWSEM_H__ */ diff --git a/fs/aufs/sbinfo.c b/fs/aufs/sbinfo.c new file mode 100644 index 0000000000000..3835be61d9781 --- /dev/null +++ b/fs/aufs/sbinfo.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * superblock private data + */ + +#include +#include "aufs.h" + +/* + * they are necessary regardless sysfs is disabled. + */ +void au_si_free(struct kobject *kobj) +{ + int i; + struct au_sbinfo *sbinfo; + char *locked __maybe_unused; /* debug only */ + + sbinfo = container_of(kobj, struct au_sbinfo, si_kobj); + for (i = 0; i < AuPlink_NHASH; i++) + AuDebugOn(!hlist_bl_empty(sbinfo->si_plink + i)); + AuDebugOn(atomic_read(&sbinfo->si_nowait.nw_len)); + + AuLCntZero(au_lcnt_read(&sbinfo->si_ninodes, /*do_rev*/0)); + au_lcnt_fin(&sbinfo->si_ninodes, /*do_sync*/0); + AuLCntZero(au_lcnt_read(&sbinfo->si_nfiles, /*do_rev*/0)); + au_lcnt_fin(&sbinfo->si_nfiles, /*do_sync*/0); + + dbgaufs_si_fin(sbinfo); + au_rw_write_lock(&sbinfo->si_rwsem); + au_br_free(sbinfo); + au_rw_write_unlock(&sbinfo->si_rwsem); + + au_kfree_try_rcu(sbinfo->si_branch); + mutex_destroy(&sbinfo->si_xib_mtx); + AuRwDestroy(&sbinfo->si_rwsem); + + au_lcnt_wait_for_fin(&sbinfo->si_ninodes); + /* si_nfiles is waited too */ + au_kfree_rcu(sbinfo); +} + +int au_si_alloc(struct super_block *sb) +{ + int err, i; + struct au_sbinfo *sbinfo; + + err = -ENOMEM; + sbinfo = kzalloc(sizeof(*sbinfo), GFP_NOFS); + if (unlikely(!sbinfo)) + goto out; + + /* will be reallocated separately */ + sbinfo->si_branch = kzalloc(sizeof(*sbinfo->si_branch), GFP_NOFS); + if (unlikely(!sbinfo->si_branch)) + goto out_sbinfo; + + err = sysaufs_si_init(sbinfo); + if (!err) { + dbgaufs_si_null(sbinfo); + err = dbgaufs_si_init(sbinfo); + if (unlikely(err)) + kobject_put(&sbinfo->si_kobj); + } + if (unlikely(err)) + goto out_br; + + au_nwt_init(&sbinfo->si_nowait); + au_rw_init_wlock(&sbinfo->si_rwsem); + + au_lcnt_init(&sbinfo->si_ninodes, /*release*/NULL); + au_lcnt_init(&sbinfo->si_nfiles, /*release*/NULL); + + sbinfo->si_bbot = -1; + sbinfo->si_last_br_id = AUFS_BRANCH_MAX / 2; + + sbinfo->si_wbr_copyup = AuWbrCopyup_Def; + sbinfo->si_wbr_create = AuWbrCreate_Def; + sbinfo->si_wbr_copyup_ops = au_wbr_copyup_ops + sbinfo->si_wbr_copyup; + sbinfo->si_wbr_create_ops = au_wbr_create_ops + sbinfo->si_wbr_create; + + au_fhsm_init(sbinfo); + + sbinfo->si_mntflags = au_opts_plink(AuOpt_Def); + + sbinfo->si_xino_jiffy = jiffies; + sbinfo->si_xino_expire + = msecs_to_jiffies(AUFS_XINO_DEF_SEC * MSEC_PER_SEC); + mutex_init(&sbinfo->si_xib_mtx); + /* leave si_xib_last_pindex and si_xib_next_bit */ + + INIT_HLIST_BL_HEAD(&sbinfo->si_aopen); + + sbinfo->si_rdcache = msecs_to_jiffies(AUFS_RDCACHE_DEF * MSEC_PER_SEC); + sbinfo->si_rdblk = AUFS_RDBLK_DEF; + sbinfo->si_rdhash = AUFS_RDHASH_DEF; + sbinfo->si_dirwh = AUFS_DIRWH_DEF; + + for (i = 0; i < AuPlink_NHASH; i++) + INIT_HLIST_BL_HEAD(sbinfo->si_plink + i); + init_waitqueue_head(&sbinfo->si_plink_wq); + spin_lock_init(&sbinfo->si_plink_maint_lock); + + INIT_HLIST_BL_HEAD(&sbinfo->si_files); + + /* with getattr by default */ + sbinfo->si_iop_array = aufs_iop; + + /* leave other members for sysaufs and si_mnt. */ + sbinfo->si_sb = sb; + sb->s_fs_info = sbinfo; + si_pid_set(sb); + return 0; /* success */ + +out_br: + au_kfree_try_rcu(sbinfo->si_branch); +out_sbinfo: + au_kfree_rcu(sbinfo); +out: + return err; +} + +int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink) +{ + int err, sz; + struct au_branch **brp; + + AuRwMustWriteLock(&sbinfo->si_rwsem); + + err = -ENOMEM; + sz = sizeof(*brp) * (sbinfo->si_bbot + 1); + if (unlikely(!sz)) + sz = sizeof(*brp); + brp = au_kzrealloc(sbinfo->si_branch, sz, sizeof(*brp) * nbr, GFP_NOFS, + may_shrink); + if (brp) { + sbinfo->si_branch = brp; + err = 0; + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +unsigned int au_sigen_inc(struct super_block *sb) +{ + unsigned int gen; + struct inode *inode; + + SiMustWriteLock(sb); + + gen = ++au_sbi(sb)->si_generation; + au_update_digen(sb->s_root); + inode = d_inode(sb->s_root); + au_update_iigen(inode, /*half*/0); + inode_inc_iversion(inode); + return gen; +} + +aufs_bindex_t au_new_br_id(struct super_block *sb) +{ + aufs_bindex_t br_id; + int i; + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + for (i = 0; i <= AUFS_BRANCH_MAX; i++) { + br_id = ++sbinfo->si_last_br_id; + AuDebugOn(br_id < 0); + if (br_id && au_br_index(sb, br_id) < 0) + return br_id; + } + + return -1; +} + +/* ---------------------------------------------------------------------- */ + +/* it is ok that new 'nwt' tasks are appended while we are sleeping */ +int si_read_lock(struct super_block *sb, int flags) +{ + int err; + + err = 0; + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&au_sbi(sb)->si_nowait); + + si_noflush_read_lock(sb); + err = au_plink_maint(sb, flags); + if (unlikely(err)) + si_read_unlock(sb); + + return err; +} + +int si_write_lock(struct super_block *sb, int flags) +{ + int err; + + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&au_sbi(sb)->si_nowait); + + si_noflush_write_lock(sb); + err = au_plink_maint(sb, flags); + if (unlikely(err)) + si_write_unlock(sb); + + return err; +} + +/* dentry and super_block lock. call at entry point */ +int aufs_read_lock(struct dentry *dentry, int flags) +{ + int err; + struct super_block *sb; + + sb = dentry->d_sb; + err = si_read_lock(sb, flags); + if (unlikely(err)) + goto out; + + if (au_ftest_lock(flags, DW)) + di_write_lock_child(dentry); + else + di_read_lock_child(dentry, flags); + + if (au_ftest_lock(flags, GEN)) { + err = au_digen_test(dentry, au_sigen(sb)); + if (!au_opt_test(au_mntflags(sb), UDBA_NONE)) + AuDebugOn(!err && au_dbrange_test(dentry)); + else if (!err) + err = au_dbrange_test(dentry); + if (unlikely(err)) + aufs_read_unlock(dentry, flags); + } + +out: + return err; +} + +void aufs_read_unlock(struct dentry *dentry, int flags) +{ + if (au_ftest_lock(flags, DW)) + di_write_unlock(dentry); + else + di_read_unlock(dentry, flags); + si_read_unlock(dentry->d_sb); +} + +void aufs_write_lock(struct dentry *dentry) +{ + si_write_lock(dentry->d_sb, AuLock_FLUSH | AuLock_NOPLMW); + di_write_lock_child(dentry); +} + +void aufs_write_unlock(struct dentry *dentry) +{ + di_write_unlock(dentry); + si_write_unlock(dentry->d_sb); +} + +int aufs_read_and_write_lock2(struct dentry *d1, struct dentry *d2, int flags) +{ + int err; + unsigned int sigen; + struct super_block *sb; + + sb = d1->d_sb; + err = si_read_lock(sb, flags); + if (unlikely(err)) + goto out; + + di_write_lock2_child(d1, d2, au_ftest_lock(flags, DIRS)); + + if (au_ftest_lock(flags, GEN)) { + sigen = au_sigen(sb); + err = au_digen_test(d1, sigen); + AuDebugOn(!err && au_dbrange_test(d1)); + if (!err) { + err = au_digen_test(d2, sigen); + AuDebugOn(!err && au_dbrange_test(d2)); + } + if (unlikely(err)) + aufs_read_and_write_unlock2(d1, d2); + } + +out: + return err; +} + +void aufs_read_and_write_unlock2(struct dentry *d1, struct dentry *d2) +{ + di_write_unlock2(d1, d2); + si_read_unlock(d1->d_sb); +} diff --git a/fs/aufs/super.c b/fs/aufs/super.c new file mode 100644 index 0000000000000..359270fe879ac --- /dev/null +++ b/fs/aufs/super.c @@ -0,0 +1,1049 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * mount and super_block operations + */ + +#include +#include +#include +#include +#include +#include "aufs.h" + +/* + * super_operations + */ +static struct inode *aufs_alloc_inode(struct super_block *sb __maybe_unused) +{ + struct au_icntnr *c; + + c = au_cache_alloc_icntnr(); + if (c) { + au_icntnr_init(c); + inode_set_iversion(&c->vfs_inode, 1); /* sigen(sb); */ + c->iinfo.ii_hinode = NULL; + return &c->vfs_inode; + } + return NULL; +} + +static void aufs_destroy_inode_cb(struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + + au_cache_free_icntnr(container_of(inode, struct au_icntnr, vfs_inode)); +} + +static void aufs_destroy_inode(struct inode *inode) +{ + if (!au_is_bad_inode(inode)) + au_iinfo_fin(inode); + call_rcu(&inode->i_rcu, aufs_destroy_inode_cb); +} + +struct inode *au_iget_locked(struct super_block *sb, ino_t ino) +{ + struct inode *inode; + int err; + + inode = iget_locked(sb, ino); + if (unlikely(!inode)) { + inode = ERR_PTR(-ENOMEM); + goto out; + } + if (!(inode->i_state & I_NEW)) + goto out; + + err = au_xigen_new(inode); + if (!err) + err = au_iinfo_init(inode); + if (!err) + inode_inc_iversion(inode); + else { + iget_failed(inode); + inode = ERR_PTR(err); + } + +out: + /* never return NULL */ + AuDebugOn(!inode); + AuTraceErrPtr(inode); + return inode; +} + +/* lock free root dinfo */ +static int au_show_brs(struct seq_file *seq, struct super_block *sb) +{ + int err; + aufs_bindex_t bindex, bbot; + struct path path; + struct au_hdentry *hdp; + struct au_branch *br; + au_br_perm_str_t perm; + + err = 0; + bbot = au_sbbot(sb); + bindex = 0; + hdp = au_hdentry(au_di(sb->s_root), bindex); + for (; !err && bindex <= bbot; bindex++, hdp++) { + br = au_sbr(sb, bindex); + path.mnt = au_br_mnt(br); + path.dentry = hdp->hd_dentry; + err = au_seq_path(seq, &path); + if (!err) { + au_optstr_br_perm(&perm, br->br_perm); + seq_printf(seq, "=%s", perm.a); + if (bindex != bbot) + seq_putc(seq, ':'); + } + } + if (unlikely(err || seq_has_overflowed(seq))) + err = -E2BIG; + + return err; +} + +static void au_gen_fmt(char *fmt, int len __maybe_unused, const char *pat, + const char *append) +{ + char *p; + + p = fmt; + while (*pat != ':') + *p++ = *pat++; + *p++ = *pat++; + strcpy(p, append); + AuDebugOn(strlen(fmt) >= len); +} + +static void au_show_wbr_create(struct seq_file *m, int v, + struct au_sbinfo *sbinfo) +{ + const char *pat; + char fmt[32]; + struct au_wbr_mfs *mfs; + + AuRwMustAnyLock(&sbinfo->si_rwsem); + + seq_puts(m, ",create="); + pat = au_optstr_wbr_create(v); + mfs = &sbinfo->si_wbr_mfs; + switch (v) { + case AuWbrCreate_TDP: + case AuWbrCreate_RR: + case AuWbrCreate_MFS: + case AuWbrCreate_PMFS: + seq_puts(m, pat); + break; + case AuWbrCreate_MFSRR: + case AuWbrCreate_TDMFS: + case AuWbrCreate_PMFSRR: + au_gen_fmt(fmt, sizeof(fmt), pat, "%llu"); + seq_printf(m, fmt, mfs->mfsrr_watermark); + break; + case AuWbrCreate_MFSV: + case AuWbrCreate_PMFSV: + au_gen_fmt(fmt, sizeof(fmt), pat, "%lu"); + seq_printf(m, fmt, + jiffies_to_msecs(mfs->mfs_expire) + / MSEC_PER_SEC); + break; + case AuWbrCreate_MFSRRV: + case AuWbrCreate_TDMFSV: + case AuWbrCreate_PMFSRRV: + au_gen_fmt(fmt, sizeof(fmt), pat, "%llu:%lu"); + seq_printf(m, fmt, mfs->mfsrr_watermark, + jiffies_to_msecs(mfs->mfs_expire) / MSEC_PER_SEC); + break; + default: + BUG(); + } +} + +static int au_show_xino(struct seq_file *seq, struct super_block *sb) +{ +#ifdef CONFIG_SYSFS + return 0; +#else + int err; + const int len = sizeof(AUFS_XINO_FNAME) - 1; + aufs_bindex_t bindex, brid; + struct qstr *name; + struct file *f; + struct dentry *d, *h_root; + struct au_branch *br; + + AuRwMustAnyLock(&sbinfo->si_rwsem); + + err = 0; + f = au_sbi(sb)->si_xib; + if (!f) + goto out; + + /* stop printing the default xino path on the first writable branch */ + h_root = NULL; + bindex = au_xi_root(sb, f->f_path.dentry); + if (bindex >= 0) { + br = au_sbr_sb(sb, bindex); + h_root = au_br_dentry(br); + } + + d = f->f_path.dentry; + name = &d->d_name; + /* safe ->d_parent because the file is unlinked */ + if (d->d_parent == h_root + && name->len == len + && !memcmp(name->name, AUFS_XINO_FNAME, len)) + goto out; + + seq_puts(seq, ",xino="); + err = au_xino_path(seq, f); + +out: + return err; +#endif +} + +/* seq_file will re-call me in case of too long string */ +static int aufs_show_options(struct seq_file *m, struct dentry *dentry) +{ + int err; + unsigned int mnt_flags, v; + struct super_block *sb; + struct au_sbinfo *sbinfo; + +#define AuBool(name, str) do { \ + v = au_opt_test(mnt_flags, name); \ + if (v != au_opt_test(AuOpt_Def, name)) \ + seq_printf(m, ",%s" #str, v ? "" : "no"); \ +} while (0) + +#define AuStr(name, str) do { \ + v = mnt_flags & AuOptMask_##name; \ + if (v != (AuOpt_Def & AuOptMask_##name)) \ + seq_printf(m, "," #str "=%s", au_optstr_##str(v)); \ +} while (0) + +#define AuUInt(name, str, val) do { \ + if (val != AUFS_##name##_DEF) \ + seq_printf(m, "," #str "=%u", val); \ +} while (0) + + sb = dentry->d_sb; + if (sb->s_flags & SB_POSIXACL) + seq_puts(m, ",acl"); +#if 0 /* reserved for future use */ + if (sb->s_flags & SB_I_VERSION) + seq_puts(m, ",i_version"); +#endif + + /* lock free root dinfo */ + si_noflush_read_lock(sb); + sbinfo = au_sbi(sb); + seq_printf(m, ",si=%lx", sysaufs_si_id(sbinfo)); + + mnt_flags = au_mntflags(sb); + if (au_opt_test(mnt_flags, XINO)) { + err = au_show_xino(m, sb); + if (unlikely(err)) + goto out; + } else + seq_puts(m, ",noxino"); + + AuBool(TRUNC_XINO, trunc_xino); + AuStr(UDBA, udba); + AuBool(SHWH, shwh); + AuBool(PLINK, plink); + AuBool(DIO, dio); + AuBool(DIRPERM1, dirperm1); + + v = sbinfo->si_wbr_create; + if (v != AuWbrCreate_Def) + au_show_wbr_create(m, v, sbinfo); + + v = sbinfo->si_wbr_copyup; + if (v != AuWbrCopyup_Def) + seq_printf(m, ",cpup=%s", au_optstr_wbr_copyup(v)); + + v = au_opt_test(mnt_flags, ALWAYS_DIROPQ); + if (v != au_opt_test(AuOpt_Def, ALWAYS_DIROPQ)) + seq_printf(m, ",diropq=%c", v ? 'a' : 'w'); + + AuUInt(DIRWH, dirwh, sbinfo->si_dirwh); + + v = jiffies_to_msecs(sbinfo->si_rdcache) / MSEC_PER_SEC; + AuUInt(RDCACHE, rdcache, v); + + AuUInt(RDBLK, rdblk, sbinfo->si_rdblk); + AuUInt(RDHASH, rdhash, sbinfo->si_rdhash); + + au_fhsm_show(m, sbinfo); + + AuBool(DIRREN, dirren); + AuBool(SUM, sum); + /* AuBool(SUM_W, wsum); */ + AuBool(WARN_PERM, warn_perm); + AuBool(VERBOSE, verbose); + +out: + /* be sure to print "br:" last */ + if (!sysaufs_brs) { + seq_puts(m, ",br:"); + au_show_brs(m, sb); + } + si_read_unlock(sb); + return 0; + +#undef AuBool +#undef AuStr +#undef AuUInt +} + +/* ---------------------------------------------------------------------- */ + +/* sum mode which returns the summation for statfs(2) */ + +static u64 au_add_till_max(u64 a, u64 b) +{ + u64 old; + + old = a; + a += b; + if (old <= a) + return a; + return ULLONG_MAX; +} + +static u64 au_mul_till_max(u64 a, long mul) +{ + u64 old; + + old = a; + a *= mul; + if (old <= a) + return a; + return ULLONG_MAX; +} + +static int au_statfs_sum(struct super_block *sb, struct kstatfs *buf) +{ + int err; + long bsize, factor; + u64 blocks, bfree, bavail, files, ffree; + aufs_bindex_t bbot, bindex, i; + unsigned char shared; + struct path h_path; + struct super_block *h_sb; + + err = 0; + bsize = LONG_MAX; + files = 0; + ffree = 0; + blocks = 0; + bfree = 0; + bavail = 0; + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + h_path.mnt = au_sbr_mnt(sb, bindex); + h_sb = h_path.mnt->mnt_sb; + shared = 0; + for (i = 0; !shared && i < bindex; i++) + shared = (au_sbr_sb(sb, i) == h_sb); + if (shared) + continue; + + /* sb->s_root for NFS is unreliable */ + h_path.dentry = h_path.mnt->mnt_root; + err = vfs_statfs(&h_path, buf); + if (unlikely(err)) + goto out; + + if (bsize > buf->f_bsize) { + /* + * we will reduce bsize, so we have to expand blocks + * etc. to match them again + */ + factor = (bsize / buf->f_bsize); + blocks = au_mul_till_max(blocks, factor); + bfree = au_mul_till_max(bfree, factor); + bavail = au_mul_till_max(bavail, factor); + bsize = buf->f_bsize; + } + + factor = (buf->f_bsize / bsize); + blocks = au_add_till_max(blocks, + au_mul_till_max(buf->f_blocks, factor)); + bfree = au_add_till_max(bfree, + au_mul_till_max(buf->f_bfree, factor)); + bavail = au_add_till_max(bavail, + au_mul_till_max(buf->f_bavail, factor)); + files = au_add_till_max(files, buf->f_files); + ffree = au_add_till_max(ffree, buf->f_ffree); + } + + buf->f_bsize = bsize; + buf->f_blocks = blocks; + buf->f_bfree = bfree; + buf->f_bavail = bavail; + buf->f_files = files; + buf->f_ffree = ffree; + buf->f_frsize = 0; + +out: + return err; +} + +static int aufs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + int err; + struct path h_path; + struct super_block *sb; + + /* lock free root dinfo */ + sb = dentry->d_sb; + si_noflush_read_lock(sb); + if (!au_opt_test(au_mntflags(sb), SUM)) { + /* sb->s_root for NFS is unreliable */ + h_path.mnt = au_sbr_mnt(sb, 0); + h_path.dentry = h_path.mnt->mnt_root; + err = vfs_statfs(&h_path, buf); + } else + err = au_statfs_sum(sb, buf); + si_read_unlock(sb); + + if (!err) { + buf->f_type = AUFS_SUPER_MAGIC; + buf->f_namelen = AUFS_MAX_NAMELEN; + memset(&buf->f_fsid, 0, sizeof(buf->f_fsid)); + } + /* buf->f_bsize = buf->f_blocks = buf->f_bfree = buf->f_bavail = -1; */ + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int aufs_sync_fs(struct super_block *sb, int wait) +{ + int err, e; + aufs_bindex_t bbot, bindex; + struct au_branch *br; + struct super_block *h_sb; + + err = 0; + si_noflush_read_lock(sb); + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (!au_br_writable(br->br_perm)) + continue; + + h_sb = au_sbr_sb(sb, bindex); + e = vfsub_sync_filesystem(h_sb, wait); + if (unlikely(e && !err)) + err = e; + /* go on even if an error happens */ + } + si_read_unlock(sb); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* final actions when unmounting a file system */ +static void aufs_put_super(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + sbinfo = au_sbi(sb); + if (sbinfo) + kobject_put(&sbinfo->si_kobj); +} + +/* ---------------------------------------------------------------------- */ + +void *au_array_alloc(unsigned long long *hint, au_arraycb_t cb, + struct super_block *sb, void *arg) +{ + void *array; + unsigned long long n, sz; + + array = NULL; + n = 0; + if (!*hint) + goto out; + + if (*hint > ULLONG_MAX / sizeof(array)) { + array = ERR_PTR(-EMFILE); + pr_err("hint %llu\n", *hint); + goto out; + } + + sz = sizeof(array) * *hint; + array = kzalloc(sz, GFP_NOFS); + if (unlikely(!array)) + array = vzalloc(sz); + if (unlikely(!array)) { + array = ERR_PTR(-ENOMEM); + goto out; + } + + n = cb(sb, array, *hint, arg); + AuDebugOn(n > *hint); + +out: + *hint = n; + return array; +} + +static unsigned long long au_iarray_cb(struct super_block *sb, void *a, + unsigned long long max __maybe_unused, + void *arg) +{ + unsigned long long n; + struct inode **p, *inode; + struct list_head *head; + + n = 0; + p = a; + head = arg; + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(inode, head, i_sb_list) { + if (!au_is_bad_inode(inode) + && au_ii(inode)->ii_btop >= 0) { + spin_lock(&inode->i_lock); + if (atomic_read(&inode->i_count)) { + au_igrab(inode); + *p++ = inode; + n++; + AuDebugOn(n > max); + } + spin_unlock(&inode->i_lock); + } + } + spin_unlock(&sb->s_inode_list_lock); + + return n; +} + +struct inode **au_iarray_alloc(struct super_block *sb, unsigned long long *max) +{ + struct au_sbinfo *sbi; + + sbi = au_sbi(sb); + *max = au_lcnt_read(&sbi->si_ninodes, /*do_rev*/1); + return au_array_alloc(max, au_iarray_cb, sb, &sb->s_inodes); +} + +void au_iarray_free(struct inode **a, unsigned long long max) +{ + unsigned long long ull; + + for (ull = 0; ull < max; ull++) + iput(a[ull]); + kvfree(a); +} + +/* ---------------------------------------------------------------------- */ + +/* + * refresh dentry and inode at remount time. + */ +/* todo: consolidate with simple_reval_dpath() and au_reval_for_attr() */ +static int au_do_refresh(struct dentry *dentry, unsigned int dir_flags, + struct dentry *parent) +{ + int err; + + di_write_lock_child(dentry); + di_read_lock_parent(parent, AuLock_IR); + err = au_refresh_dentry(dentry, parent); + if (!err && dir_flags) + au_hn_reset(d_inode(dentry), dir_flags); + di_read_unlock(parent, AuLock_IR); + di_write_unlock(dentry); + + return err; +} + +static int au_do_refresh_d(struct dentry *dentry, unsigned int sigen, + struct au_sbinfo *sbinfo, + const unsigned int dir_flags, unsigned int do_idop) +{ + int err; + struct dentry *parent; + + err = 0; + parent = dget_parent(dentry); + if (!au_digen_test(parent, sigen) && au_digen_test(dentry, sigen)) { + if (d_really_is_positive(dentry)) { + if (!d_is_dir(dentry)) + err = au_do_refresh(dentry, /*dir_flags*/0, + parent); + else { + err = au_do_refresh(dentry, dir_flags, parent); + if (unlikely(err)) + au_fset_si(sbinfo, FAILED_REFRESH_DIR); + } + } else + err = au_do_refresh(dentry, /*dir_flags*/0, parent); + AuDbgDentry(dentry); + } + dput(parent); + + if (!err) { + if (do_idop) + au_refresh_dop(dentry, /*force_reval*/0); + } else + au_refresh_dop(dentry, /*force_reval*/1); + + AuTraceErr(err); + return err; +} + +static int au_refresh_d(struct super_block *sb, unsigned int do_idop) +{ + int err, i, j, ndentry, e; + unsigned int sigen; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry **dentries, *d; + struct au_sbinfo *sbinfo; + struct dentry *root = sb->s_root; + const unsigned int dir_flags = au_hi_flags(d_inode(root), /*isdir*/1); + + if (do_idop) + au_refresh_dop(root, /*force_reval*/0); + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_dcsub_pages(&dpages, root, NULL, NULL); + if (unlikely(err)) + goto out_dpages; + + sigen = au_sigen(sb); + sbinfo = au_sbi(sb); + for (i = 0; i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + dentries = dpage->dentries; + ndentry = dpage->ndentry; + for (j = 0; j < ndentry; j++) { + d = dentries[j]; + e = au_do_refresh_d(d, sigen, sbinfo, dir_flags, + do_idop); + if (unlikely(e && !err)) + err = e; + /* go on even err */ + } + } + +out_dpages: + au_dpages_free(&dpages); +out: + return err; +} + +static int au_refresh_i(struct super_block *sb, unsigned int do_idop) +{ + int err, e; + unsigned int sigen; + unsigned long long max, ull; + struct inode *inode, **array; + + array = au_iarray_alloc(sb, &max); + err = PTR_ERR(array); + if (IS_ERR(array)) + goto out; + + err = 0; + sigen = au_sigen(sb); + for (ull = 0; ull < max; ull++) { + inode = array[ull]; + if (unlikely(!inode)) + break; + + e = 0; + ii_write_lock_child(inode); + if (au_iigen(inode, NULL) != sigen) { + e = au_refresh_hinode_self(inode); + if (unlikely(e)) { + au_refresh_iop(inode, /*force_getattr*/1); + pr_err("error %d, i%lu\n", e, inode->i_ino); + if (!err) + err = e; + /* go on even if err */ + } + } + if (!e && do_idop) + au_refresh_iop(inode, /*force_getattr*/0); + ii_write_unlock(inode); + } + + au_iarray_free(array, max); + +out: + return err; +} + +static void au_remount_refresh(struct super_block *sb, unsigned int do_idop) +{ + int err, e; + unsigned int udba; + aufs_bindex_t bindex, bbot; + struct dentry *root; + struct inode *inode; + struct au_branch *br; + struct au_sbinfo *sbi; + + au_sigen_inc(sb); + sbi = au_sbi(sb); + au_fclr_si(sbi, FAILED_REFRESH_DIR); + + root = sb->s_root; + DiMustNoWaiters(root); + inode = d_inode(root); + IiMustNoWaiters(inode); + + udba = au_opt_udba(sb); + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + err = au_hnotify_reset_br(udba, br, br->br_perm); + if (unlikely(err)) + AuIOErr("hnotify failed on br %d, %d, ignored\n", + bindex, err); + /* go on even if err */ + } + au_hn_reset(inode, au_hi_flags(inode, /*isdir*/1)); + + if (do_idop) { + if (au_ftest_si(sbi, NO_DREVAL)) { + AuDebugOn(sb->s_d_op == &aufs_dop_noreval); + sb->s_d_op = &aufs_dop_noreval; + AuDebugOn(sbi->si_iop_array == aufs_iop_nogetattr); + sbi->si_iop_array = aufs_iop_nogetattr; + } else { + AuDebugOn(sb->s_d_op == &aufs_dop); + sb->s_d_op = &aufs_dop; + AuDebugOn(sbi->si_iop_array == aufs_iop); + sbi->si_iop_array = aufs_iop; + } + pr_info("reset to %ps and %ps\n", + sb->s_d_op, sbi->si_iop_array); + } + + di_write_unlock(root); + err = au_refresh_d(sb, do_idop); + e = au_refresh_i(sb, do_idop); + if (unlikely(e && !err)) + err = e; + /* aufs_write_lock() calls ..._child() */ + di_write_lock_child(root); + + au_cpup_attr_all(inode, /*force*/1); + + if (unlikely(err)) + AuIOErr("refresh failed, ignored, %d\n", err); +} + +/* stop extra interpretation of errno in mount(8), and strange error messages */ +static int cvt_err(int err) +{ + AuTraceErr(err); + + switch (err) { + case -ENOENT: + case -ENOTDIR: + case -EEXIST: + case -EIO: + err = -EINVAL; + } + return err; +} + +static int aufs_remount_fs(struct super_block *sb, int *flags, char *data) +{ + int err, do_dx; + unsigned int mntflags; + struct au_opts opts = { + .opt = NULL + }; + struct dentry *root; + struct inode *inode; + struct au_sbinfo *sbinfo; + + err = 0; + root = sb->s_root; + if (!data || !*data) { + err = si_write_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (!err) { + di_write_lock_child(root); + err = au_opts_verify(sb, *flags, /*pending*/0); + aufs_write_unlock(root); + } + goto out; + } + + err = -ENOMEM; + opts.opt = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!opts.opt)) + goto out; + opts.max_opt = PAGE_SIZE / sizeof(*opts.opt); + opts.flags = AuOpts_REMOUNT; + opts.sb_flags = *flags; + + /* parse it before aufs lock */ + err = au_opts_parse(sb, data, &opts); + if (unlikely(err)) + goto out_opts; + + sbinfo = au_sbi(sb); + inode = d_inode(root); + inode_lock(inode); + err = si_write_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out_mtx; + di_write_lock_child(root); + + /* au_opts_remount() may return an error */ + err = au_opts_remount(sb, &opts); + au_opts_free(&opts); + + if (au_ftest_opts(opts.flags, REFRESH)) + au_remount_refresh(sb, au_ftest_opts(opts.flags, REFRESH_IDOP)); + + if (au_ftest_opts(opts.flags, REFRESH_DYAOP)) { + mntflags = au_mntflags(sb); + do_dx = !!au_opt_test(mntflags, DIO); + au_dy_arefresh(do_dx); + } + + au_fhsm_wrote_all(sb, /*force*/1); /* ?? */ + aufs_write_unlock(root); + +out_mtx: + inode_unlock(inode); +out_opts: + free_page((unsigned long)opts.opt); +out: + err = cvt_err(err); + AuTraceErr(err); + return err; +} + +static const struct super_operations aufs_sop = { + .alloc_inode = aufs_alloc_inode, + .destroy_inode = aufs_destroy_inode, + /* always deleting, no clearing */ + .drop_inode = generic_delete_inode, + .show_options = aufs_show_options, + .statfs = aufs_statfs, + .put_super = aufs_put_super, + .sync_fs = aufs_sync_fs, + .remount_fs = aufs_remount_fs +}; + +/* ---------------------------------------------------------------------- */ + +static int alloc_root(struct super_block *sb) +{ + int err; + struct inode *inode; + struct dentry *root; + + err = -ENOMEM; + inode = au_iget_locked(sb, AUFS_ROOT_INO); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out; + + inode->i_op = aufs_iop + AuIop_DIR; /* with getattr by default */ + inode->i_fop = &aufs_dir_fop; + inode->i_mode = S_IFDIR; + set_nlink(inode, 2); + unlock_new_inode(inode); + + root = d_make_root(inode); + if (unlikely(!root)) + goto out; + err = PTR_ERR(root); + if (IS_ERR(root)) + goto out; + + err = au_di_init(root); + if (!err) { + sb->s_root = root; + return 0; /* success */ + } + dput(root); + +out: + return err; +} + +static int aufs_fill_super(struct super_block *sb, void *raw_data, + int silent __maybe_unused) +{ + int err; + struct au_opts opts = { + .opt = NULL + }; + struct au_sbinfo *sbinfo; + struct dentry *root; + struct inode *inode; + char *arg = raw_data; + + if (unlikely(!arg || !*arg)) { + err = -EINVAL; + pr_err("no arg\n"); + goto out; + } + + err = -ENOMEM; + opts.opt = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!opts.opt)) + goto out; + opts.max_opt = PAGE_SIZE / sizeof(*opts.opt); + opts.sb_flags = sb->s_flags; + + err = au_si_alloc(sb); + if (unlikely(err)) + goto out_opts; + sbinfo = au_sbi(sb); + + /* all timestamps always follow the ones on the branch */ + sb->s_flags |= SB_NOATIME | SB_NODIRATIME; + sb->s_flags |= SB_I_VERSION; /* do we really need this? */ + sb->s_op = &aufs_sop; + sb->s_d_op = &aufs_dop; + sb->s_magic = AUFS_SUPER_MAGIC; + sb->s_maxbytes = 0; + sb->s_stack_depth = 1; + au_export_init(sb); + au_xattr_init(sb); + + err = alloc_root(sb); + if (unlikely(err)) { + si_write_unlock(sb); + goto out_info; + } + root = sb->s_root; + inode = d_inode(root); + + /* + * actually we can parse options regardless aufs lock here. + * but at remount time, parsing must be done before aufs lock. + * so we follow the same rule. + */ + ii_write_lock_parent(inode); + aufs_write_unlock(root); + err = au_opts_parse(sb, arg, &opts); + if (unlikely(err)) + goto out_root; + + /* lock vfs_inode first, then aufs. */ + inode_lock(inode); + aufs_write_lock(root); + err = au_opts_mount(sb, &opts); + au_opts_free(&opts); + if (!err && au_ftest_si(sbinfo, NO_DREVAL)) { + sb->s_d_op = &aufs_dop_noreval; + pr_info("%ps\n", sb->s_d_op); + au_refresh_dop(root, /*force_reval*/0); + sbinfo->si_iop_array = aufs_iop_nogetattr; + au_refresh_iop(inode, /*force_getattr*/0); + } + aufs_write_unlock(root); + inode_unlock(inode); + if (!err) + goto out_opts; /* success */ + +out_root: + dput(root); + sb->s_root = NULL; +out_info: + kobject_put(&sbinfo->si_kobj); + sb->s_fs_info = NULL; +out_opts: + free_page((unsigned long)opts.opt); +out: + AuTraceErr(err); + err = cvt_err(err); + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static struct dentry *aufs_mount(struct file_system_type *fs_type, int flags, + const char *dev_name __maybe_unused, + void *raw_data) +{ + struct dentry *root; + + /* all timestamps always follow the ones on the branch */ + /* mnt->mnt_flags |= MNT_NOATIME | MNT_NODIRATIME; */ + root = mount_nodev(fs_type, flags, raw_data, aufs_fill_super); + if (IS_ERR(root)) + goto out; + + au_sbilist_add(root->d_sb); + +out: + return root; +} + +static void aufs_kill_sb(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + sbinfo = au_sbi(sb); + if (sbinfo) { + au_sbilist_del(sb); + aufs_write_lock(sb->s_root); + au_fhsm_fin(sb); + if (sbinfo->si_wbr_create_ops->fin) + sbinfo->si_wbr_create_ops->fin(sb); + if (au_opt_test(sbinfo->si_mntflags, UDBA_HNOTIFY)) { + au_opt_set_udba(sbinfo->si_mntflags, UDBA_NONE); + au_remount_refresh(sb, /*do_idop*/0); + } + if (au_opt_test(sbinfo->si_mntflags, PLINK)) + au_plink_put(sb, /*verbose*/1); + au_xino_clr(sb); + au_dr_opt_flush(sb); + sbinfo->si_sb = NULL; + aufs_write_unlock(sb->s_root); + au_nwt_flush(&sbinfo->si_nowait); + } + kill_anon_super(sb); +} + +struct file_system_type aufs_fs_type = { + .name = AUFS_FSTYPE, + /* a race between rename and others */ + .fs_flags = FS_RENAME_DOES_D_MOVE, + .mount = aufs_mount, + .kill_sb = aufs_kill_sb, + /* no need to __module_get() and module_put(). */ + .owner = THIS_MODULE, +}; diff --git a/fs/aufs/super.h b/fs/aufs/super.h new file mode 100644 index 0000000000000..eef21d205dbef --- /dev/null +++ b/fs/aufs/super.h @@ -0,0 +1,589 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * super_block operations + */ + +#ifndef __AUFS_SUPER_H__ +#define __AUFS_SUPER_H__ + +#ifdef __KERNEL__ + +#include +#include +#include "hbl.h" +#include "lcnt.h" +#include "rwsem.h" +#include "wkq.h" + +/* policies to select one among multiple writable branches */ +struct au_wbr_copyup_operations { + int (*copyup)(struct dentry *dentry); +}; + +#define AuWbr_DIR 1 /* target is a dir */ +#define AuWbr_PARENT (1 << 1) /* always require a parent */ + +#define au_ftest_wbr(flags, name) ((flags) & AuWbr_##name) +#define au_fset_wbr(flags, name) { (flags) |= AuWbr_##name; } +#define au_fclr_wbr(flags, name) { (flags) &= ~AuWbr_##name; } + +struct au_wbr_create_operations { + int (*create)(struct dentry *dentry, unsigned int flags); + int (*init)(struct super_block *sb); + int (*fin)(struct super_block *sb); +}; + +struct au_wbr_mfs { + struct mutex mfs_lock; /* protect this structure */ + unsigned long mfs_jiffy; + unsigned long mfs_expire; + aufs_bindex_t mfs_bindex; + + unsigned long long mfsrr_bytes; + unsigned long long mfsrr_watermark; +}; + +#define AuPlink_NHASH 100 +static inline int au_plink_hash(ino_t ino) +{ + return ino % AuPlink_NHASH; +} + +/* File-based Hierarchical Storage Management */ +struct au_fhsm { +#ifdef CONFIG_AUFS_FHSM + /* allow only one process who can receive the notification */ + spinlock_t fhsm_spin; + pid_t fhsm_pid; + wait_queue_head_t fhsm_wqh; + atomic_t fhsm_readable; + + /* these are protected by si_rwsem */ + unsigned long fhsm_expire; + aufs_bindex_t fhsm_bottom; +#endif +}; + +struct au_branch; +struct au_sbinfo { + /* nowait tasks in the system-wide workqueue */ + struct au_nowait_tasks si_nowait; + + /* + * tried sb->s_umount, but failed due to the dependency between i_mutex. + * rwsem for au_sbinfo is necessary. + */ + struct au_rwsem si_rwsem; + + /* + * dirty approach to protect sb->sb_inodes and ->s_files (gone) from + * remount. + */ + au_lcnt_t si_ninodes, si_nfiles; + + /* branch management */ + unsigned int si_generation; + + /* see AuSi_ flags */ + unsigned char au_si_status; + + aufs_bindex_t si_bbot; + + /* dirty trick to keep br_id plus */ + unsigned int si_last_br_id : + sizeof(aufs_bindex_t) * BITS_PER_BYTE - 1; + struct au_branch **si_branch; + + /* policy to select a writable branch */ + unsigned char si_wbr_copyup; + unsigned char si_wbr_create; + struct au_wbr_copyup_operations *si_wbr_copyup_ops; + struct au_wbr_create_operations *si_wbr_create_ops; + + /* round robin */ + atomic_t si_wbr_rr_next; + + /* most free space */ + struct au_wbr_mfs si_wbr_mfs; + + /* File-based Hierarchical Storage Management */ + struct au_fhsm si_fhsm; + + /* mount flags */ + /* include/asm-ia64/siginfo.h defines a macro named si_flags */ + unsigned int si_mntflags; + + /* external inode number (bitmap and translation table) */ + vfs_readf_t si_xread; + vfs_writef_t si_xwrite; + loff_t si_ximaxent; /* max entries in a xino */ + + struct file *si_xib; + struct mutex si_xib_mtx; /* protect xib members */ + unsigned long *si_xib_buf; + unsigned long si_xib_last_pindex; + int si_xib_next_bit; + + unsigned long si_xino_jiffy; + unsigned long si_xino_expire; + /* reserved for future use */ + /* unsigned long long si_xib_limit; */ /* Max xib file size */ + +#ifdef CONFIG_AUFS_EXPORT + /* i_generation */ + /* todo: make xigen file an array to support many inode numbers */ + struct file *si_xigen; + atomic_t si_xigen_next; +#endif + + /* dirty trick to support atomic_open */ + struct hlist_bl_head si_aopen; + + /* vdir parameters */ + unsigned long si_rdcache; /* max cache time in jiffies */ + unsigned int si_rdblk; /* deblk size */ + unsigned int si_rdhash; /* hash size */ + + /* + * If the number of whiteouts are larger than si_dirwh, leave all of + * them after au_whtmp_ren to reduce the cost of rmdir(2). + * future fsck.aufs or kernel thread will remove them later. + * Otherwise, remove all whiteouts and the dir in rmdir(2). + */ + unsigned int si_dirwh; + + /* pseudo_link list */ + struct hlist_bl_head si_plink[AuPlink_NHASH]; + wait_queue_head_t si_plink_wq; + spinlock_t si_plink_maint_lock; + pid_t si_plink_maint_pid; + + /* file list */ + struct hlist_bl_head si_files; + + /* with/without getattr, brother of sb->s_d_op */ + const struct inode_operations *si_iop_array; + + /* + * sysfs and lifetime management. + * this is not a small structure and it may be a waste of memory in case + * of sysfs is disabled, particularly when many aufs-es are mounted. + * but using sysfs is majority. + */ + struct kobject si_kobj; +#ifdef CONFIG_DEBUG_FS + struct dentry *si_dbgaufs; + struct dentry *si_dbgaufs_plink; + struct dentry *si_dbgaufs_xib; +#ifdef CONFIG_AUFS_EXPORT + struct dentry *si_dbgaufs_xigen; +#endif +#endif + +#ifdef CONFIG_AUFS_SBILIST + struct hlist_bl_node si_list; +#endif + + /* dirty, necessary for unmounting, sysfs and sysrq */ + struct super_block *si_sb; +}; + +/* sbinfo status flags */ +/* + * set true when refresh_dirs() failed at remount time. + * then try refreshing dirs at access time again. + * if it is false, refreshing dirs at access time is unnecessary + */ +#define AuSi_FAILED_REFRESH_DIR 1 +#define AuSi_FHSM (1 << 1) /* fhsm is active now */ +#define AuSi_NO_DREVAL (1 << 2) /* disable all d_revalidate */ + +#ifndef CONFIG_AUFS_FHSM +#undef AuSi_FHSM +#define AuSi_FHSM 0 +#endif + +static inline unsigned char au_do_ftest_si(struct au_sbinfo *sbi, + unsigned int flag) +{ + AuRwMustAnyLock(&sbi->si_rwsem); + return sbi->au_si_status & flag; +} +#define au_ftest_si(sbinfo, name) au_do_ftest_si(sbinfo, AuSi_##name) +#define au_fset_si(sbinfo, name) do { \ + AuRwMustWriteLock(&(sbinfo)->si_rwsem); \ + (sbinfo)->au_si_status |= AuSi_##name; \ +} while (0) +#define au_fclr_si(sbinfo, name) do { \ + AuRwMustWriteLock(&(sbinfo)->si_rwsem); \ + (sbinfo)->au_si_status &= ~AuSi_##name; \ +} while (0) + +/* ---------------------------------------------------------------------- */ + +/* policy to select one among writable branches */ +#define AuWbrCopyup(sbinfo, ...) \ + ((sbinfo)->si_wbr_copyup_ops->copyup(__VA_ARGS__)) +#define AuWbrCreate(sbinfo, ...) \ + ((sbinfo)->si_wbr_create_ops->create(__VA_ARGS__)) + +/* flags for si_read_lock()/aufs_read_lock()/di_read_lock() */ +#define AuLock_DW 1 /* write-lock dentry */ +#define AuLock_IR (1 << 1) /* read-lock inode */ +#define AuLock_IW (1 << 2) /* write-lock inode */ +#define AuLock_FLUSH (1 << 3) /* wait for 'nowait' tasks */ +#define AuLock_DIRS (1 << 4) /* target is a pair of dirs */ + /* except RENAME_EXCHANGE */ +#define AuLock_NOPLM (1 << 5) /* return err in plm mode */ +#define AuLock_NOPLMW (1 << 6) /* wait for plm mode ends */ +#define AuLock_GEN (1 << 7) /* test digen/iigen */ +#define au_ftest_lock(flags, name) ((flags) & AuLock_##name) +#define au_fset_lock(flags, name) \ + do { (flags) |= AuLock_##name; } while (0) +#define au_fclr_lock(flags, name) \ + do { (flags) &= ~AuLock_##name; } while (0) + +/* ---------------------------------------------------------------------- */ + +/* super.c */ +extern struct file_system_type aufs_fs_type; +struct inode *au_iget_locked(struct super_block *sb, ino_t ino); +typedef unsigned long long (*au_arraycb_t)(struct super_block *sb, void *array, + unsigned long long max, void *arg); +void *au_array_alloc(unsigned long long *hint, au_arraycb_t cb, + struct super_block *sb, void *arg); +struct inode **au_iarray_alloc(struct super_block *sb, unsigned long long *max); +void au_iarray_free(struct inode **a, unsigned long long max); + +/* sbinfo.c */ +void au_si_free(struct kobject *kobj); +int au_si_alloc(struct super_block *sb); +int au_sbr_realloc(struct au_sbinfo *sbinfo, int nbr, int may_shrink); + +unsigned int au_sigen_inc(struct super_block *sb); +aufs_bindex_t au_new_br_id(struct super_block *sb); + +int si_read_lock(struct super_block *sb, int flags); +int si_write_lock(struct super_block *sb, int flags); +int aufs_read_lock(struct dentry *dentry, int flags); +void aufs_read_unlock(struct dentry *dentry, int flags); +void aufs_write_lock(struct dentry *dentry); +void aufs_write_unlock(struct dentry *dentry); +int aufs_read_and_write_lock2(struct dentry *d1, struct dentry *d2, int flags); +void aufs_read_and_write_unlock2(struct dentry *d1, struct dentry *d2); + +/* wbr_policy.c */ +extern struct au_wbr_copyup_operations au_wbr_copyup_ops[]; +extern struct au_wbr_create_operations au_wbr_create_ops[]; +int au_cpdown_dirs(struct dentry *dentry, aufs_bindex_t bdst); +int au_wbr_nonopq(struct dentry *dentry, aufs_bindex_t bindex); +int au_wbr_do_copyup_bu(struct dentry *dentry, aufs_bindex_t btop); + +/* mvdown.c */ +int au_mvdown(struct dentry *dentry, struct aufs_mvdown __user *arg); + +#ifdef CONFIG_AUFS_FHSM +/* fhsm.c */ + +static inline pid_t au_fhsm_pid(struct au_fhsm *fhsm) +{ + pid_t pid; + + spin_lock(&fhsm->fhsm_spin); + pid = fhsm->fhsm_pid; + spin_unlock(&fhsm->fhsm_spin); + + return pid; +} + +void au_fhsm_wrote(struct super_block *sb, aufs_bindex_t bindex, int force); +void au_fhsm_wrote_all(struct super_block *sb, int force); +int au_fhsm_fd(struct super_block *sb, int oflags); +int au_fhsm_br_alloc(struct au_branch *br); +void au_fhsm_set_bottom(struct super_block *sb, aufs_bindex_t bindex); +void au_fhsm_fin(struct super_block *sb); +void au_fhsm_init(struct au_sbinfo *sbinfo); +void au_fhsm_set(struct au_sbinfo *sbinfo, unsigned int sec); +void au_fhsm_show(struct seq_file *seq, struct au_sbinfo *sbinfo); +#else +AuStubVoid(au_fhsm_wrote, struct super_block *sb, aufs_bindex_t bindex, + int force) +AuStubVoid(au_fhsm_wrote_all, struct super_block *sb, int force) +AuStub(int, au_fhsm_fd, return -EOPNOTSUPP, struct super_block *sb, int oflags) +AuStub(pid_t, au_fhsm_pid, return 0, struct au_fhsm *fhsm) +AuStubInt0(au_fhsm_br_alloc, struct au_branch *br) +AuStubVoid(au_fhsm_set_bottom, struct super_block *sb, aufs_bindex_t bindex) +AuStubVoid(au_fhsm_fin, struct super_block *sb) +AuStubVoid(au_fhsm_init, struct au_sbinfo *sbinfo) +AuStubVoid(au_fhsm_set, struct au_sbinfo *sbinfo, unsigned int sec) +AuStubVoid(au_fhsm_show, struct seq_file *seq, struct au_sbinfo *sbinfo) +#endif + +/* ---------------------------------------------------------------------- */ + +static inline struct au_sbinfo *au_sbi(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_EXPORT +int au_test_nfsd(void); +void au_export_init(struct super_block *sb); +void au_xigen_inc(struct inode *inode); +int au_xigen_new(struct inode *inode); +int au_xigen_set(struct super_block *sb, struct path *path); +void au_xigen_clr(struct super_block *sb); + +static inline int au_busy_or_stale(void) +{ + if (!au_test_nfsd()) + return -EBUSY; + return -ESTALE; +} +#else +AuStubInt0(au_test_nfsd, void) +AuStubVoid(au_export_init, struct super_block *sb) +AuStubVoid(au_xigen_inc, struct inode *inode) +AuStubInt0(au_xigen_new, struct inode *inode) +AuStubInt0(au_xigen_set, struct super_block *sb, struct path *path) +AuStubVoid(au_xigen_clr, struct super_block *sb) +AuStub(int, au_busy_or_stale, return -EBUSY, void) +#endif /* CONFIG_AUFS_EXPORT */ + +/* ---------------------------------------------------------------------- */ + +#ifdef CONFIG_AUFS_SBILIST +/* module.c */ +extern struct hlist_bl_head au_sbilist; + +static inline void au_sbilist_init(void) +{ + INIT_HLIST_BL_HEAD(&au_sbilist); +} + +static inline void au_sbilist_add(struct super_block *sb) +{ + au_hbl_add(&au_sbi(sb)->si_list, &au_sbilist); +} + +static inline void au_sbilist_del(struct super_block *sb) +{ + au_hbl_del(&au_sbi(sb)->si_list, &au_sbilist); +} + +#ifdef CONFIG_AUFS_MAGIC_SYSRQ +static inline void au_sbilist_lock(void) +{ + hlist_bl_lock(&au_sbilist); +} + +static inline void au_sbilist_unlock(void) +{ + hlist_bl_unlock(&au_sbilist); +} +#define AuGFP_SBILIST GFP_ATOMIC +#else +AuStubVoid(au_sbilist_lock, void) +AuStubVoid(au_sbilist_unlock, void) +#define AuGFP_SBILIST GFP_NOFS +#endif /* CONFIG_AUFS_MAGIC_SYSRQ */ +#else +AuStubVoid(au_sbilist_init, void) +AuStubVoid(au_sbilist_add, struct super_block *sb) +AuStubVoid(au_sbilist_del, struct super_block *sb) +AuStubVoid(au_sbilist_lock, void) +AuStubVoid(au_sbilist_unlock, void) +#define AuGFP_SBILIST GFP_NOFS +#endif + +/* ---------------------------------------------------------------------- */ + +static inline void dbgaufs_si_null(struct au_sbinfo *sbinfo) +{ + /* + * This function is a dynamic '__init' function actually, + * so the tiny check for si_rwsem is unnecessary. + */ + /* AuRwMustWriteLock(&sbinfo->si_rwsem); */ +#ifdef CONFIG_DEBUG_FS + sbinfo->si_dbgaufs = NULL; + sbinfo->si_dbgaufs_plink = NULL; + sbinfo->si_dbgaufs_xib = NULL; +#ifdef CONFIG_AUFS_EXPORT + sbinfo->si_dbgaufs_xigen = NULL; +#endif +#endif +} + +/* ---------------------------------------------------------------------- */ + +/* current->atomic_flags */ +/* this value should never corrupt the ones defined in linux/sched.h */ +#define PFA_AUFS 7 + +TASK_PFA_TEST(AUFS, test_aufs) /* task_test_aufs */ +TASK_PFA_SET(AUFS, aufs) /* task_set_aufs */ +TASK_PFA_CLEAR(AUFS, aufs) /* task_clear_aufs */ + +static inline int si_pid_test(struct super_block *sb) +{ + return !!task_test_aufs(current); +} + +static inline void si_pid_clr(struct super_block *sb) +{ + AuDebugOn(!task_test_aufs(current)); + task_clear_aufs(current); +} + +static inline void si_pid_set(struct super_block *sb) +{ + AuDebugOn(task_test_aufs(current)); + task_set_aufs(current); +} + +/* ---------------------------------------------------------------------- */ + +/* lock superblock. mainly for entry point functions */ +#define __si_read_lock(sb) au_rw_read_lock(&au_sbi(sb)->si_rwsem) +#define __si_write_lock(sb) au_rw_write_lock(&au_sbi(sb)->si_rwsem) +#define __si_read_trylock(sb) au_rw_read_trylock(&au_sbi(sb)->si_rwsem) +#define __si_write_trylock(sb) au_rw_write_trylock(&au_sbi(sb)->si_rwsem) +/* +#define __si_read_trylock_nested(sb) \ + au_rw_read_trylock_nested(&au_sbi(sb)->si_rwsem) +#define __si_write_trylock_nested(sb) \ + au_rw_write_trylock_nested(&au_sbi(sb)->si_rwsem) +*/ + +#define __si_read_unlock(sb) au_rw_read_unlock(&au_sbi(sb)->si_rwsem) +#define __si_write_unlock(sb) au_rw_write_unlock(&au_sbi(sb)->si_rwsem) +#define __si_downgrade_lock(sb) au_rw_dgrade_lock(&au_sbi(sb)->si_rwsem) + +#define SiMustNoWaiters(sb) AuRwMustNoWaiters(&au_sbi(sb)->si_rwsem) +#define SiMustAnyLock(sb) AuRwMustAnyLock(&au_sbi(sb)->si_rwsem) +#define SiMustWriteLock(sb) AuRwMustWriteLock(&au_sbi(sb)->si_rwsem) + +static inline void si_noflush_read_lock(struct super_block *sb) +{ + __si_read_lock(sb); + si_pid_set(sb); +} + +static inline int si_noflush_read_trylock(struct super_block *sb) +{ + int locked; + + locked = __si_read_trylock(sb); + if (locked) + si_pid_set(sb); + return locked; +} + +static inline void si_noflush_write_lock(struct super_block *sb) +{ + __si_write_lock(sb); + si_pid_set(sb); +} + +static inline int si_noflush_write_trylock(struct super_block *sb) +{ + int locked; + + locked = __si_write_trylock(sb); + if (locked) + si_pid_set(sb); + return locked; +} + +#if 0 /* reserved */ +static inline int si_read_trylock(struct super_block *sb, int flags) +{ + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&au_sbi(sb)->si_nowait); + return si_noflush_read_trylock(sb); +} +#endif + +static inline void si_read_unlock(struct super_block *sb) +{ + si_pid_clr(sb); + __si_read_unlock(sb); +} + +#if 0 /* reserved */ +static inline int si_write_trylock(struct super_block *sb, int flags) +{ + if (au_ftest_lock(flags, FLUSH)) + au_nwt_flush(&au_sbi(sb)->si_nowait); + return si_noflush_write_trylock(sb); +} +#endif + +static inline void si_write_unlock(struct super_block *sb) +{ + si_pid_clr(sb); + __si_write_unlock(sb); +} + +#if 0 /* reserved */ +static inline void si_downgrade_lock(struct super_block *sb) +{ + __si_downgrade_lock(sb); +} +#endif + +/* ---------------------------------------------------------------------- */ + +static inline aufs_bindex_t au_sbbot(struct super_block *sb) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_bbot; +} + +static inline unsigned int au_mntflags(struct super_block *sb) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_mntflags; +} + +static inline unsigned int au_sigen(struct super_block *sb) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_generation; +} + +static inline struct au_branch *au_sbr(struct super_block *sb, + aufs_bindex_t bindex) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_branch[0 + bindex]; +} + +static inline loff_t au_xi_maxent(struct super_block *sb) +{ + SiMustAnyLock(sb); + return au_sbi(sb)->si_ximaxent; +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_SUPER_H__ */ diff --git a/fs/aufs/sysaufs.c b/fs/aufs/sysaufs.c new file mode 100644 index 0000000000000..f615471231b5a --- /dev/null +++ b/fs/aufs/sysaufs.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sysfs interface and lifetime management + * they are necessary regardless sysfs is disabled. + */ + +#include +#include "aufs.h" + +unsigned long sysaufs_si_mask; +struct kset *sysaufs_kset; + +#define AuSiAttr(_name) { \ + .attr = { .name = __stringify(_name), .mode = 0444 }, \ + .show = sysaufs_si_##_name, \ +} + +static struct sysaufs_si_attr sysaufs_si_attr_xi_path = AuSiAttr(xi_path); +struct attribute *sysaufs_si_attrs[] = { + &sysaufs_si_attr_xi_path.attr, + NULL, +}; + +static const struct sysfs_ops au_sbi_ops = { + .show = sysaufs_si_show +}; + +static struct kobj_type au_sbi_ktype = { + .release = au_si_free, + .sysfs_ops = &au_sbi_ops, + .default_attrs = sysaufs_si_attrs +}; + +/* ---------------------------------------------------------------------- */ + +int sysaufs_si_init(struct au_sbinfo *sbinfo) +{ + int err; + + sbinfo->si_kobj.kset = sysaufs_kset; + /* cf. sysaufs_name() */ + err = kobject_init_and_add + (&sbinfo->si_kobj, &au_sbi_ktype, /*&sysaufs_kset->kobj*/NULL, + SysaufsSiNamePrefix "%lx", sysaufs_si_id(sbinfo)); + + return err; +} + +void sysaufs_fin(void) +{ + sysfs_remove_group(&sysaufs_kset->kobj, sysaufs_attr_group); + kset_unregister(sysaufs_kset); +} + +int __init sysaufs_init(void) +{ + int err; + + do { + get_random_bytes(&sysaufs_si_mask, sizeof(sysaufs_si_mask)); + } while (!sysaufs_si_mask); + + err = -EINVAL; + sysaufs_kset = kset_create_and_add(AUFS_NAME, NULL, fs_kobj); + if (unlikely(!sysaufs_kset)) + goto out; + err = PTR_ERR(sysaufs_kset); + if (IS_ERR(sysaufs_kset)) + goto out; + err = sysfs_create_group(&sysaufs_kset->kobj, sysaufs_attr_group); + if (unlikely(err)) + kset_unregister(sysaufs_kset); + +out: + return err; +} diff --git a/fs/aufs/sysaufs.h b/fs/aufs/sysaufs.h new file mode 100644 index 0000000000000..55c5c9868f3b0 --- /dev/null +++ b/fs/aufs/sysaufs.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sysfs interface and mount lifetime management + */ + +#ifndef __SYSAUFS_H__ +#define __SYSAUFS_H__ + +#ifdef __KERNEL__ + +#include +#include "module.h" + +struct super_block; +struct au_sbinfo; + +struct sysaufs_si_attr { + struct attribute attr; + int (*show)(struct seq_file *seq, struct super_block *sb); +}; + +/* ---------------------------------------------------------------------- */ + +/* sysaufs.c */ +extern unsigned long sysaufs_si_mask; +extern struct kset *sysaufs_kset; +extern struct attribute *sysaufs_si_attrs[]; +int sysaufs_si_init(struct au_sbinfo *sbinfo); +int __init sysaufs_init(void); +void sysaufs_fin(void); + +/* ---------------------------------------------------------------------- */ + +/* some people doesn't like to show a pointer in kernel */ +static inline unsigned long sysaufs_si_id(struct au_sbinfo *sbinfo) +{ + return sysaufs_si_mask ^ (unsigned long)sbinfo; +} + +#define SysaufsSiNamePrefix "si_" +#define SysaufsSiNameLen (sizeof(SysaufsSiNamePrefix) + 16) +static inline void sysaufs_name(struct au_sbinfo *sbinfo, char *name) +{ + snprintf(name, SysaufsSiNameLen, SysaufsSiNamePrefix "%lx", + sysaufs_si_id(sbinfo)); +} + +struct au_branch; +#ifdef CONFIG_SYSFS +/* sysfs.c */ +extern struct attribute_group *sysaufs_attr_group; + +int sysaufs_si_xi_path(struct seq_file *seq, struct super_block *sb); +ssize_t sysaufs_si_show(struct kobject *kobj, struct attribute *attr, + char *buf); +long au_brinfo_ioctl(struct file *file, unsigned long arg); +#ifdef CONFIG_COMPAT +long au_brinfo_compat_ioctl(struct file *file, unsigned long arg); +#endif + +void sysaufs_br_init(struct au_branch *br); +void sysaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex); +void sysaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex); + +#define sysaufs_brs_init() do {} while (0) + +#else +#define sysaufs_attr_group NULL + +AuStubInt0(sysaufs_si_xi_path, struct seq_file *seq, struct super_block *sb) +AuStub(ssize_t, sysaufs_si_show, return 0, struct kobject *kobj, + struct attribute *attr, char *buf) +AuStubVoid(sysaufs_br_init, struct au_branch *br) +AuStubVoid(sysaufs_brs_add, struct super_block *sb, aufs_bindex_t bindex) +AuStubVoid(sysaufs_brs_del, struct super_block *sb, aufs_bindex_t bindex) + +static inline void sysaufs_brs_init(void) +{ + sysaufs_brs = 0; +} + +#endif /* CONFIG_SYSFS */ + +#endif /* __KERNEL__ */ +#endif /* __SYSAUFS_H__ */ diff --git a/fs/aufs/sysfs.c b/fs/aufs/sysfs.c new file mode 100644 index 0000000000000..3f9340ae5e3e7 --- /dev/null +++ b/fs/aufs/sysfs.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sysfs interface + */ + +#include +#include +#include "aufs.h" + +#ifdef CONFIG_AUFS_FS_MODULE +/* this entry violates the "one line per file" policy of sysfs */ +static ssize_t config_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + ssize_t err; + static char *conf = +/* this file is generated at compiling */ +#include "conf.str" + ; + + err = snprintf(buf, PAGE_SIZE, conf); + if (unlikely(err >= PAGE_SIZE)) + err = -EFBIG; + return err; +} + +static struct kobj_attribute au_config_attr = __ATTR_RO(config); +#endif + +static struct attribute *au_attr[] = { +#ifdef CONFIG_AUFS_FS_MODULE + &au_config_attr.attr, +#endif + NULL, /* need to NULL terminate the list of attributes */ +}; + +static struct attribute_group sysaufs_attr_group_body = { + .attrs = au_attr +}; + +struct attribute_group *sysaufs_attr_group = &sysaufs_attr_group_body; + +/* ---------------------------------------------------------------------- */ + +int sysaufs_si_xi_path(struct seq_file *seq, struct super_block *sb) +{ + int err; + + SiMustAnyLock(sb); + + err = 0; + if (au_opt_test(au_mntflags(sb), XINO)) { + err = au_xino_path(seq, au_sbi(sb)->si_xib); + seq_putc(seq, '\n'); + } + return err; +} + +/* + * the lifetime of branch is independent from the entry under sysfs. + * sysfs handles the lifetime of the entry, and never call ->show() after it is + * unlinked. + */ +static int sysaufs_si_br(struct seq_file *seq, struct super_block *sb, + aufs_bindex_t bindex, int idx) +{ + int err; + struct path path; + struct dentry *root; + struct au_branch *br; + au_br_perm_str_t perm; + + AuDbg("b%d\n", bindex); + + err = 0; + root = sb->s_root; + di_read_lock_parent(root, !AuLock_IR); + br = au_sbr(sb, bindex); + + switch (idx) { + case AuBrSysfs_BR: + path.mnt = au_br_mnt(br); + path.dentry = au_h_dptr(root, bindex); + err = au_seq_path(seq, &path); + if (!err) { + au_optstr_br_perm(&perm, br->br_perm); + seq_printf(seq, "=%s\n", perm.a); + } + break; + case AuBrSysfs_BRID: + seq_printf(seq, "%d\n", br->br_id); + break; + } + di_read_unlock(root, !AuLock_IR); + if (unlikely(err || seq_has_overflowed(seq))) + err = -E2BIG; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +static struct seq_file *au_seq(char *p, ssize_t len) +{ + struct seq_file *seq; + + seq = kzalloc(sizeof(*seq), GFP_NOFS); + if (seq) { + /* mutex_init(&seq.lock); */ + seq->buf = p; + seq->size = len; + return seq; /* success */ + } + + seq = ERR_PTR(-ENOMEM); + return seq; +} + +#define SysaufsBr_PREFIX "br" +#define SysaufsBrid_PREFIX "brid" + +/* todo: file size may exceed PAGE_SIZE */ +ssize_t sysaufs_si_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + ssize_t err; + int idx; + long l; + aufs_bindex_t bbot; + struct au_sbinfo *sbinfo; + struct super_block *sb; + struct seq_file *seq; + char *name; + struct attribute **cattr; + + sbinfo = container_of(kobj, struct au_sbinfo, si_kobj); + sb = sbinfo->si_sb; + + /* + * prevent a race condition between sysfs and aufs. + * for instance, sysfs_file_read() calls sysfs_get_active_two() which + * prohibits maintaining the sysfs entries. + * hew we acquire read lock after sysfs_get_active_two(). + * on the other hand, the remount process may maintain the sysfs/aufs + * entries after acquiring write lock. + * it can cause a deadlock. + * simply we gave up processing read here. + */ + err = -EBUSY; + if (unlikely(!si_noflush_read_trylock(sb))) + goto out; + + seq = au_seq(buf, PAGE_SIZE); + err = PTR_ERR(seq); + if (IS_ERR(seq)) + goto out_unlock; + + name = (void *)attr->name; + cattr = sysaufs_si_attrs; + while (*cattr) { + if (!strcmp(name, (*cattr)->name)) { + err = container_of(*cattr, struct sysaufs_si_attr, attr) + ->show(seq, sb); + goto out_seq; + } + cattr++; + } + + if (!strncmp(name, SysaufsBrid_PREFIX, + sizeof(SysaufsBrid_PREFIX) - 1)) { + idx = AuBrSysfs_BRID; + name += sizeof(SysaufsBrid_PREFIX) - 1; + } else if (!strncmp(name, SysaufsBr_PREFIX, + sizeof(SysaufsBr_PREFIX) - 1)) { + idx = AuBrSysfs_BR; + name += sizeof(SysaufsBr_PREFIX) - 1; + } else + BUG(); + + err = kstrtol(name, 10, &l); + if (!err) { + bbot = au_sbbot(sb); + if (l <= bbot) + err = sysaufs_si_br(seq, sb, (aufs_bindex_t)l, idx); + else + err = -ENOENT; + } + +out_seq: + if (!err) { + err = seq->count; + /* sysfs limit */ + if (unlikely(err == PAGE_SIZE)) + err = -EFBIG; + } + au_kfree_rcu(seq); +out_unlock: + si_read_unlock(sb); +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_brinfo(struct super_block *sb, union aufs_brinfo __user *arg) +{ + int err; + int16_t brid; + aufs_bindex_t bindex, bbot; + size_t sz; + char *buf; + struct seq_file *seq; + struct au_branch *br; + + si_read_lock(sb, AuLock_FLUSH); + bbot = au_sbbot(sb); + err = bbot + 1; + if (!arg) + goto out; + + err = -ENOMEM; + buf = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!buf)) + goto out; + + seq = au_seq(buf, PAGE_SIZE); + err = PTR_ERR(seq); + if (IS_ERR(seq)) + goto out_buf; + + sz = sizeof(*arg) - offsetof(union aufs_brinfo, path); + for (bindex = 0; bindex <= bbot; bindex++, arg++) { + err = !access_ok(VERIFY_WRITE, arg, sizeof(*arg)); + if (unlikely(err)) + break; + + br = au_sbr(sb, bindex); + brid = br->br_id; + BUILD_BUG_ON(sizeof(brid) != sizeof(arg->id)); + err = __put_user(brid, &arg->id); + if (unlikely(err)) + break; + + BUILD_BUG_ON(sizeof(br->br_perm) != sizeof(arg->perm)); + err = __put_user(br->br_perm, &arg->perm); + if (unlikely(err)) + break; + + err = au_seq_path(seq, &br->br_path); + if (unlikely(err)) + break; + seq_putc(seq, '\0'); + if (!seq_has_overflowed(seq)) { + err = copy_to_user(arg->path, seq->buf, seq->count); + seq->count = 0; + if (unlikely(err)) + break; + } else { + err = -E2BIG; + goto out_seq; + } + } + if (unlikely(err)) + err = -EFAULT; + +out_seq: + au_kfree_rcu(seq); +out_buf: + free_page((unsigned long)buf); +out: + si_read_unlock(sb); + return err; +} + +long au_brinfo_ioctl(struct file *file, unsigned long arg) +{ + return au_brinfo(file->f_path.dentry->d_sb, (void __user *)arg); +} + +#ifdef CONFIG_COMPAT +long au_brinfo_compat_ioctl(struct file *file, unsigned long arg) +{ + return au_brinfo(file->f_path.dentry->d_sb, compat_ptr(arg)); +} +#endif + +/* ---------------------------------------------------------------------- */ + +void sysaufs_br_init(struct au_branch *br) +{ + int i; + struct au_brsysfs *br_sysfs; + struct attribute *attr; + + br_sysfs = br->br_sysfs; + for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) { + attr = &br_sysfs->attr; + sysfs_attr_init(attr); + attr->name = br_sysfs->name; + attr->mode = 0444; + br_sysfs++; + } +} + +void sysaufs_brs_del(struct super_block *sb, aufs_bindex_t bindex) +{ + struct au_branch *br; + struct kobject *kobj; + struct au_brsysfs *br_sysfs; + int i; + aufs_bindex_t bbot; + + if (!sysaufs_brs) + return; + + kobj = &au_sbi(sb)->si_kobj; + bbot = au_sbbot(sb); + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + br_sysfs = br->br_sysfs; + for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) { + sysfs_remove_file(kobj, &br_sysfs->attr); + br_sysfs++; + } + } +} + +void sysaufs_brs_add(struct super_block *sb, aufs_bindex_t bindex) +{ + int err, i; + aufs_bindex_t bbot; + struct kobject *kobj; + struct au_branch *br; + struct au_brsysfs *br_sysfs; + + if (!sysaufs_brs) + return; + + kobj = &au_sbi(sb)->si_kobj; + bbot = au_sbbot(sb); + for (; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + br_sysfs = br->br_sysfs; + snprintf(br_sysfs[AuBrSysfs_BR].name, sizeof(br_sysfs->name), + SysaufsBr_PREFIX "%d", bindex); + snprintf(br_sysfs[AuBrSysfs_BRID].name, sizeof(br_sysfs->name), + SysaufsBrid_PREFIX "%d", bindex); + for (i = 0; i < ARRAY_SIZE(br->br_sysfs); i++) { + err = sysfs_create_file(kobj, &br_sysfs->attr); + if (unlikely(err)) + pr_warn("failed %s under sysfs(%d)\n", + br_sysfs->name, err); + br_sysfs++; + } + } +} diff --git a/fs/aufs/sysrq.c b/fs/aufs/sysrq.c new file mode 100644 index 0000000000000..fc8aa4a2839f6 --- /dev/null +++ b/fs/aufs/sysrq.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * magic sysrq handler + */ + +/* #include */ +#include +#include "aufs.h" + +/* ---------------------------------------------------------------------- */ + +static void sysrq_sb(struct super_block *sb) +{ + char *plevel; + struct au_sbinfo *sbinfo; + struct file *file; + struct hlist_bl_head *files; + struct hlist_bl_node *pos; + struct au_finfo *finfo; + struct inode *i; + + plevel = au_plevel; + au_plevel = KERN_WARNING; + + /* since we define pr_fmt, call printk directly */ +#define pr(str) printk(KERN_WARNING AUFS_NAME ": " str) + + sbinfo = au_sbi(sb); + printk(KERN_WARNING "si=%lx\n", sysaufs_si_id(sbinfo)); + pr("superblock\n"); + au_dpri_sb(sb); + +#if 0 /* reserved */ + do { + int err, i, j, ndentry; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + + err = au_dpages_init(&dpages, GFP_ATOMIC); + if (unlikely(err)) + break; + err = au_dcsub_pages(&dpages, sb->s_root, NULL, NULL); + if (!err) + for (i = 0; i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + ndentry = dpage->ndentry; + for (j = 0; j < ndentry; j++) + au_dpri_dentry(dpage->dentries[j]); + } + au_dpages_free(&dpages); + } while (0); +#endif + + pr("isolated inode\n"); + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(i, &sb->s_inodes, i_sb_list) { + spin_lock(&i->i_lock); + if (hlist_empty(&i->i_dentry)) + au_dpri_inode(i); + spin_unlock(&i->i_lock); + } + spin_unlock(&sb->s_inode_list_lock); + + pr("files\n"); + files = &au_sbi(sb)->si_files; + hlist_bl_lock(files); + hlist_bl_for_each_entry(finfo, pos, files, fi_hlist) { + umode_t mode; + + file = finfo->fi_file; + mode = file_inode(file)->i_mode; + if (!special_file(mode)) + au_dpri_file(file); + } + hlist_bl_unlock(files); + pr("done\n"); + +#undef pr + au_plevel = plevel; +} + +/* ---------------------------------------------------------------------- */ + +/* module parameter */ +static char *aufs_sysrq_key = "a"; +module_param_named(sysrq, aufs_sysrq_key, charp, 0444); +MODULE_PARM_DESC(sysrq, "MagicSysRq key for " AUFS_NAME); + +static void au_sysrq(int key __maybe_unused) +{ + struct au_sbinfo *sbinfo; + struct hlist_bl_node *pos; + + lockdep_off(); + au_sbilist_lock(); + hlist_bl_for_each_entry(sbinfo, pos, &au_sbilist, si_list) + sysrq_sb(sbinfo->si_sb); + au_sbilist_unlock(); + lockdep_on(); +} + +static struct sysrq_key_op au_sysrq_op = { + .handler = au_sysrq, + .help_msg = "Aufs", + .action_msg = "Aufs", + .enable_mask = SYSRQ_ENABLE_DUMP +}; + +/* ---------------------------------------------------------------------- */ + +int __init au_sysrq_init(void) +{ + int err; + char key; + + err = -1; + key = *aufs_sysrq_key; + if ('a' <= key && key <= 'z') + err = register_sysrq_key(key, &au_sysrq_op); + if (unlikely(err)) + pr_err("err %d, sysrq=%c\n", err, key); + return err; +} + +void au_sysrq_fin(void) +{ + int err; + + err = unregister_sysrq_key(*aufs_sysrq_key, &au_sysrq_op); + if (unlikely(err)) + pr_err("err %d (ignored)\n", err); +} diff --git a/fs/aufs/vdir.c b/fs/aufs/vdir.c new file mode 100644 index 0000000000000..5ba006b80724f --- /dev/null +++ b/fs/aufs/vdir.c @@ -0,0 +1,896 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * virtual or vertical directory + */ + +#include +#include "aufs.h" + +static unsigned int calc_size(int nlen) +{ + return ALIGN(sizeof(struct au_vdir_de) + nlen, sizeof(ino_t)); +} + +static int set_deblk_end(union au_vdir_deblk_p *p, + union au_vdir_deblk_p *deblk_end) +{ + if (calc_size(0) <= deblk_end->deblk - p->deblk) { + p->de->de_str.len = 0; + /* smp_mb(); */ + return 0; + } + return -1; /* error */ +} + +/* returns true or false */ +static int is_deblk_end(union au_vdir_deblk_p *p, + union au_vdir_deblk_p *deblk_end) +{ + if (calc_size(0) <= deblk_end->deblk - p->deblk) + return !p->de->de_str.len; + return 1; +} + +static unsigned char *last_deblk(struct au_vdir *vdir) +{ + return vdir->vd_deblk[vdir->vd_nblk - 1]; +} + +/* ---------------------------------------------------------------------- */ + +/* estimate the appropriate size for name hash table */ +unsigned int au_rdhash_est(loff_t sz) +{ + unsigned int n; + + n = UINT_MAX; + sz >>= 10; + if (sz < n) + n = sz; + if (sz < AUFS_RDHASH_DEF) + n = AUFS_RDHASH_DEF; + /* pr_info("n %u\n", n); */ + return n; +} + +/* + * the allocated memory has to be freed by + * au_nhash_wh_free() or au_nhash_de_free(). + */ +int au_nhash_alloc(struct au_nhash *nhash, unsigned int num_hash, gfp_t gfp) +{ + struct hlist_head *head; + unsigned int u; + size_t sz; + + sz = sizeof(*nhash->nh_head) * num_hash; + head = kmalloc(sz, gfp); + if (head) { + nhash->nh_num = num_hash; + nhash->nh_head = head; + for (u = 0; u < num_hash; u++) + INIT_HLIST_HEAD(head++); + return 0; /* success */ + } + + return -ENOMEM; +} + +static void nhash_count(struct hlist_head *head) +{ +#if 0 /* debugging */ + unsigned long n; + struct hlist_node *pos; + + n = 0; + hlist_for_each(pos, head) + n++; + pr_info("%lu\n", n); +#endif +} + +static void au_nhash_wh_do_free(struct hlist_head *head) +{ + struct au_vdir_wh *pos; + struct hlist_node *node; + + hlist_for_each_entry_safe(pos, node, head, wh_hash) + au_kfree_rcu(pos); +} + +static void au_nhash_de_do_free(struct hlist_head *head) +{ + struct au_vdir_dehstr *pos; + struct hlist_node *node; + + hlist_for_each_entry_safe(pos, node, head, hash) + au_cache_free_vdir_dehstr(pos); +} + +static void au_nhash_do_free(struct au_nhash *nhash, + void (*free)(struct hlist_head *head)) +{ + unsigned int n; + struct hlist_head *head; + + n = nhash->nh_num; + if (!n) + return; + + head = nhash->nh_head; + while (n-- > 0) { + nhash_count(head); + free(head++); + } + au_kfree_try_rcu(nhash->nh_head); +} + +void au_nhash_wh_free(struct au_nhash *whlist) +{ + au_nhash_do_free(whlist, au_nhash_wh_do_free); +} + +static void au_nhash_de_free(struct au_nhash *delist) +{ + au_nhash_do_free(delist, au_nhash_de_do_free); +} + +/* ---------------------------------------------------------------------- */ + +int au_nhash_test_longer_wh(struct au_nhash *whlist, aufs_bindex_t btgt, + int limit) +{ + int num; + unsigned int u, n; + struct hlist_head *head; + struct au_vdir_wh *pos; + + num = 0; + n = whlist->nh_num; + head = whlist->nh_head; + for (u = 0; u < n; u++, head++) + hlist_for_each_entry(pos, head, wh_hash) + if (pos->wh_bindex == btgt && ++num > limit) + return 1; + return 0; +} + +static struct hlist_head *au_name_hash(struct au_nhash *nhash, + unsigned char *name, + unsigned int len) +{ + unsigned int v; + /* const unsigned int magic_bit = 12; */ + + AuDebugOn(!nhash->nh_num || !nhash->nh_head); + + v = 0; + if (len > 8) + len = 8; + while (len--) + v += *name++; + /* v = hash_long(v, magic_bit); */ + v %= nhash->nh_num; + return nhash->nh_head + v; +} + +static int au_nhash_test_name(struct au_vdir_destr *str, const char *name, + int nlen) +{ + return str->len == nlen && !memcmp(str->name, name, nlen); +} + +/* returns found or not */ +int au_nhash_test_known_wh(struct au_nhash *whlist, char *name, int nlen) +{ + struct hlist_head *head; + struct au_vdir_wh *pos; + struct au_vdir_destr *str; + + head = au_name_hash(whlist, name, nlen); + hlist_for_each_entry(pos, head, wh_hash) { + str = &pos->wh_str; + AuDbg("%.*s\n", str->len, str->name); + if (au_nhash_test_name(str, name, nlen)) + return 1; + } + return 0; +} + +/* returns found(true) or not */ +static int test_known(struct au_nhash *delist, char *name, int nlen) +{ + struct hlist_head *head; + struct au_vdir_dehstr *pos; + struct au_vdir_destr *str; + + head = au_name_hash(delist, name, nlen); + hlist_for_each_entry(pos, head, hash) { + str = pos->str; + AuDbg("%.*s\n", str->len, str->name); + if (au_nhash_test_name(str, name, nlen)) + return 1; + } + return 0; +} + +static void au_shwh_init_wh(struct au_vdir_wh *wh, ino_t ino, + unsigned char d_type) +{ +#ifdef CONFIG_AUFS_SHWH + wh->wh_ino = ino; + wh->wh_type = d_type; +#endif +} + +/* ---------------------------------------------------------------------- */ + +int au_nhash_append_wh(struct au_nhash *whlist, char *name, int nlen, ino_t ino, + unsigned int d_type, aufs_bindex_t bindex, + unsigned char shwh) +{ + int err; + struct au_vdir_destr *str; + struct au_vdir_wh *wh; + + AuDbg("%.*s\n", nlen, name); + AuDebugOn(!whlist->nh_num || !whlist->nh_head); + + err = -ENOMEM; + wh = kmalloc(sizeof(*wh) + nlen, GFP_NOFS); + if (unlikely(!wh)) + goto out; + + err = 0; + wh->wh_bindex = bindex; + if (shwh) + au_shwh_init_wh(wh, ino, d_type); + str = &wh->wh_str; + str->len = nlen; + memcpy(str->name, name, nlen); + hlist_add_head(&wh->wh_hash, au_name_hash(whlist, name, nlen)); + /* smp_mb(); */ + +out: + return err; +} + +static int append_deblk(struct au_vdir *vdir) +{ + int err; + unsigned long ul; + const unsigned int deblk_sz = vdir->vd_deblk_sz; + union au_vdir_deblk_p p, deblk_end; + unsigned char **o; + + err = -ENOMEM; + o = au_krealloc(vdir->vd_deblk, sizeof(*o) * (vdir->vd_nblk + 1), + GFP_NOFS, /*may_shrink*/0); + if (unlikely(!o)) + goto out; + + vdir->vd_deblk = o; + p.deblk = kmalloc(deblk_sz, GFP_NOFS); + if (p.deblk) { + ul = vdir->vd_nblk++; + vdir->vd_deblk[ul] = p.deblk; + vdir->vd_last.ul = ul; + vdir->vd_last.p.deblk = p.deblk; + deblk_end.deblk = p.deblk + deblk_sz; + err = set_deblk_end(&p, &deblk_end); + } + +out: + return err; +} + +static int append_de(struct au_vdir *vdir, char *name, int nlen, ino_t ino, + unsigned int d_type, struct au_nhash *delist) +{ + int err; + unsigned int sz; + const unsigned int deblk_sz = vdir->vd_deblk_sz; + union au_vdir_deblk_p p, *room, deblk_end; + struct au_vdir_dehstr *dehstr; + + p.deblk = last_deblk(vdir); + deblk_end.deblk = p.deblk + deblk_sz; + room = &vdir->vd_last.p; + AuDebugOn(room->deblk < p.deblk || deblk_end.deblk <= room->deblk + || !is_deblk_end(room, &deblk_end)); + + sz = calc_size(nlen); + if (unlikely(sz > deblk_end.deblk - room->deblk)) { + err = append_deblk(vdir); + if (unlikely(err)) + goto out; + + p.deblk = last_deblk(vdir); + deblk_end.deblk = p.deblk + deblk_sz; + /* smp_mb(); */ + AuDebugOn(room->deblk != p.deblk); + } + + err = -ENOMEM; + dehstr = au_cache_alloc_vdir_dehstr(); + if (unlikely(!dehstr)) + goto out; + + dehstr->str = &room->de->de_str; + hlist_add_head(&dehstr->hash, au_name_hash(delist, name, nlen)); + room->de->de_ino = ino; + room->de->de_type = d_type; + room->de->de_str.len = nlen; + memcpy(room->de->de_str.name, name, nlen); + + err = 0; + room->deblk += sz; + if (unlikely(set_deblk_end(room, &deblk_end))) + err = append_deblk(vdir); + /* smp_mb(); */ + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_vdir_free(struct au_vdir *vdir) +{ + unsigned char **deblk; + + deblk = vdir->vd_deblk; + while (vdir->vd_nblk--) + au_kfree_try_rcu(*deblk++); + au_kfree_try_rcu(vdir->vd_deblk); + au_cache_free_vdir(vdir); +} + +static struct au_vdir *alloc_vdir(struct file *file) +{ + struct au_vdir *vdir; + struct super_block *sb; + int err; + + sb = file->f_path.dentry->d_sb; + SiMustAnyLock(sb); + + err = -ENOMEM; + vdir = au_cache_alloc_vdir(); + if (unlikely(!vdir)) + goto out; + + vdir->vd_deblk = kzalloc(sizeof(*vdir->vd_deblk), GFP_NOFS); + if (unlikely(!vdir->vd_deblk)) + goto out_free; + + vdir->vd_deblk_sz = au_sbi(sb)->si_rdblk; + if (!vdir->vd_deblk_sz) { + /* estimate the appropriate size for deblk */ + vdir->vd_deblk_sz = au_dir_size(file, /*dentry*/NULL); + /* pr_info("vd_deblk_sz %u\n", vdir->vd_deblk_sz); */ + } + vdir->vd_nblk = 0; + vdir->vd_version = 0; + vdir->vd_jiffy = 0; + err = append_deblk(vdir); + if (!err) + return vdir; /* success */ + + au_kfree_try_rcu(vdir->vd_deblk); + +out_free: + au_cache_free_vdir(vdir); +out: + vdir = ERR_PTR(err); + return vdir; +} + +static int reinit_vdir(struct au_vdir *vdir) +{ + int err; + union au_vdir_deblk_p p, deblk_end; + + while (vdir->vd_nblk > 1) { + au_kfree_try_rcu(vdir->vd_deblk[vdir->vd_nblk - 1]); + /* vdir->vd_deblk[vdir->vd_nblk - 1] = NULL; */ + vdir->vd_nblk--; + } + p.deblk = vdir->vd_deblk[0]; + deblk_end.deblk = p.deblk + vdir->vd_deblk_sz; + err = set_deblk_end(&p, &deblk_end); + /* keep vd_dblk_sz */ + vdir->vd_last.ul = 0; + vdir->vd_last.p.deblk = vdir->vd_deblk[0]; + vdir->vd_version = 0; + vdir->vd_jiffy = 0; + /* smp_mb(); */ + return err; +} + +/* ---------------------------------------------------------------------- */ + +#define AuFillVdir_CALLED 1 +#define AuFillVdir_WHABLE (1 << 1) +#define AuFillVdir_SHWH (1 << 2) +#define au_ftest_fillvdir(flags, name) ((flags) & AuFillVdir_##name) +#define au_fset_fillvdir(flags, name) \ + do { (flags) |= AuFillVdir_##name; } while (0) +#define au_fclr_fillvdir(flags, name) \ + do { (flags) &= ~AuFillVdir_##name; } while (0) + +#ifndef CONFIG_AUFS_SHWH +#undef AuFillVdir_SHWH +#define AuFillVdir_SHWH 0 +#endif + +struct fillvdir_arg { + struct dir_context ctx; + struct file *file; + struct au_vdir *vdir; + struct au_nhash delist; + struct au_nhash whlist; + aufs_bindex_t bindex; + unsigned int flags; + int err; +}; + +static int fillvdir(struct dir_context *ctx, const char *__name, int nlen, + loff_t offset __maybe_unused, u64 h_ino, + unsigned int d_type) +{ + struct fillvdir_arg *arg = container_of(ctx, struct fillvdir_arg, ctx); + char *name = (void *)__name; + struct super_block *sb; + ino_t ino; + const unsigned char shwh = !!au_ftest_fillvdir(arg->flags, SHWH); + + arg->err = 0; + sb = arg->file->f_path.dentry->d_sb; + au_fset_fillvdir(arg->flags, CALLED); + /* smp_mb(); */ + if (nlen <= AUFS_WH_PFX_LEN + || memcmp(name, AUFS_WH_PFX, AUFS_WH_PFX_LEN)) { + if (test_known(&arg->delist, name, nlen) + || au_nhash_test_known_wh(&arg->whlist, name, nlen)) + goto out; /* already exists or whiteouted */ + + arg->err = au_ino(sb, arg->bindex, h_ino, d_type, &ino); + if (!arg->err) { + if (unlikely(nlen > AUFS_MAX_NAMELEN)) + d_type = DT_UNKNOWN; + arg->err = append_de(arg->vdir, name, nlen, ino, + d_type, &arg->delist); + } + } else if (au_ftest_fillvdir(arg->flags, WHABLE)) { + name += AUFS_WH_PFX_LEN; + nlen -= AUFS_WH_PFX_LEN; + if (au_nhash_test_known_wh(&arg->whlist, name, nlen)) + goto out; /* already whiteouted */ + + ino = 0; /* just to suppress a warning */ + if (shwh) + arg->err = au_wh_ino(sb, arg->bindex, h_ino, d_type, + &ino); + if (!arg->err) { + if (nlen <= AUFS_MAX_NAMELEN + AUFS_WH_PFX_LEN) + d_type = DT_UNKNOWN; + arg->err = au_nhash_append_wh + (&arg->whlist, name, nlen, ino, d_type, + arg->bindex, shwh); + } + } + +out: + if (!arg->err) + arg->vdir->vd_jiffy = jiffies; + /* smp_mb(); */ + AuTraceErr(arg->err); + return arg->err; +} + +static int au_handle_shwh(struct super_block *sb, struct au_vdir *vdir, + struct au_nhash *whlist, struct au_nhash *delist) +{ +#ifdef CONFIG_AUFS_SHWH + int err; + unsigned int nh, u; + struct hlist_head *head; + struct au_vdir_wh *pos; + struct hlist_node *n; + char *p, *o; + struct au_vdir_destr *destr; + + AuDebugOn(!au_opt_test(au_mntflags(sb), SHWH)); + + err = -ENOMEM; + o = p = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!p)) + goto out; + + err = 0; + nh = whlist->nh_num; + memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN); + p += AUFS_WH_PFX_LEN; + for (u = 0; u < nh; u++) { + head = whlist->nh_head + u; + hlist_for_each_entry_safe(pos, n, head, wh_hash) { + destr = &pos->wh_str; + memcpy(p, destr->name, destr->len); + err = append_de(vdir, o, destr->len + AUFS_WH_PFX_LEN, + pos->wh_ino, pos->wh_type, delist); + if (unlikely(err)) + break; + } + } + + free_page((unsigned long)o); + +out: + AuTraceErr(err); + return err; +#else + return 0; +#endif +} + +static int au_do_read_vdir(struct fillvdir_arg *arg) +{ + int err; + unsigned int rdhash; + loff_t offset; + aufs_bindex_t bbot, bindex, btop; + unsigned char shwh; + struct file *hf, *file; + struct super_block *sb; + + file = arg->file; + sb = file->f_path.dentry->d_sb; + SiMustAnyLock(sb); + + rdhash = au_sbi(sb)->si_rdhash; + if (!rdhash) + rdhash = au_rdhash_est(au_dir_size(file, /*dentry*/NULL)); + err = au_nhash_alloc(&arg->delist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out; + err = au_nhash_alloc(&arg->whlist, rdhash, GFP_NOFS); + if (unlikely(err)) + goto out_delist; + + err = 0; + arg->flags = 0; + shwh = 0; + if (au_opt_test(au_mntflags(sb), SHWH)) { + shwh = 1; + au_fset_fillvdir(arg->flags, SHWH); + } + btop = au_fbtop(file); + bbot = au_fbbot_dir(file); + for (bindex = btop; !err && bindex <= bbot; bindex++) { + hf = au_hf_dir(file, bindex); + if (!hf) + continue; + + offset = vfsub_llseek(hf, 0, SEEK_SET); + err = offset; + if (unlikely(offset)) + break; + + arg->bindex = bindex; + au_fclr_fillvdir(arg->flags, WHABLE); + if (shwh + || (bindex != bbot + && au_br_whable(au_sbr_perm(sb, bindex)))) + au_fset_fillvdir(arg->flags, WHABLE); + do { + arg->err = 0; + au_fclr_fillvdir(arg->flags, CALLED); + /* smp_mb(); */ + err = vfsub_iterate_dir(hf, &arg->ctx); + if (err >= 0) + err = arg->err; + } while (!err && au_ftest_fillvdir(arg->flags, CALLED)); + + /* + * dir_relax() may be good for concurrency, but aufs should not + * use it since it will cause a lockdep problem. + */ + } + + if (!err && shwh) + err = au_handle_shwh(sb, arg->vdir, &arg->whlist, &arg->delist); + + au_nhash_wh_free(&arg->whlist); + +out_delist: + au_nhash_de_free(&arg->delist); +out: + return err; +} + +static int read_vdir(struct file *file, int may_read) +{ + int err; + unsigned long expire; + unsigned char do_read; + struct fillvdir_arg arg = { + .ctx = { + .actor = fillvdir + } + }; + struct inode *inode; + struct au_vdir *vdir, *allocated; + + err = 0; + inode = file_inode(file); + IMustLock(inode); + IiMustWriteLock(inode); + SiMustAnyLock(inode->i_sb); + + allocated = NULL; + do_read = 0; + expire = au_sbi(inode->i_sb)->si_rdcache; + vdir = au_ivdir(inode); + if (!vdir) { + do_read = 1; + vdir = alloc_vdir(file); + err = PTR_ERR(vdir); + if (IS_ERR(vdir)) + goto out; + err = 0; + allocated = vdir; + } else if (may_read + && (!inode_eq_iversion(inode, vdir->vd_version) + || time_after(jiffies, vdir->vd_jiffy + expire))) { + do_read = 1; + err = reinit_vdir(vdir); + if (unlikely(err)) + goto out; + } + + if (!do_read) + return 0; /* success */ + + arg.file = file; + arg.vdir = vdir; + err = au_do_read_vdir(&arg); + if (!err) { + /* file->f_pos = 0; */ /* todo: ctx->pos? */ + vdir->vd_version = inode_query_iversion(inode); + vdir->vd_last.ul = 0; + vdir->vd_last.p.deblk = vdir->vd_deblk[0]; + if (allocated) + au_set_ivdir(inode, allocated); + } else if (allocated) + au_vdir_free(allocated); + +out: + return err; +} + +static int copy_vdir(struct au_vdir *tgt, struct au_vdir *src) +{ + int err, rerr; + unsigned long ul, n; + const unsigned int deblk_sz = src->vd_deblk_sz; + + AuDebugOn(tgt->vd_nblk != 1); + + err = -ENOMEM; + if (tgt->vd_nblk < src->vd_nblk) { + unsigned char **p; + + p = au_krealloc(tgt->vd_deblk, sizeof(*p) * src->vd_nblk, + GFP_NOFS, /*may_shrink*/0); + if (unlikely(!p)) + goto out; + tgt->vd_deblk = p; + } + + if (tgt->vd_deblk_sz != deblk_sz) { + unsigned char *p; + + tgt->vd_deblk_sz = deblk_sz; + p = au_krealloc(tgt->vd_deblk[0], deblk_sz, GFP_NOFS, + /*may_shrink*/1); + if (unlikely(!p)) + goto out; + tgt->vd_deblk[0] = p; + } + memcpy(tgt->vd_deblk[0], src->vd_deblk[0], deblk_sz); + tgt->vd_version = src->vd_version; + tgt->vd_jiffy = src->vd_jiffy; + + n = src->vd_nblk; + for (ul = 1; ul < n; ul++) { + tgt->vd_deblk[ul] = kmemdup(src->vd_deblk[ul], deblk_sz, + GFP_NOFS); + if (unlikely(!tgt->vd_deblk[ul])) + goto out; + tgt->vd_nblk++; + } + tgt->vd_nblk = n; + tgt->vd_last.ul = tgt->vd_last.ul; + tgt->vd_last.p.deblk = tgt->vd_deblk[tgt->vd_last.ul]; + tgt->vd_last.p.deblk += src->vd_last.p.deblk + - src->vd_deblk[src->vd_last.ul]; + /* smp_mb(); */ + return 0; /* success */ + +out: + rerr = reinit_vdir(tgt); + BUG_ON(rerr); + return err; +} + +int au_vdir_init(struct file *file) +{ + int err; + struct inode *inode; + struct au_vdir *vdir_cache, *allocated; + + /* test file->f_pos here instead of ctx->pos */ + err = read_vdir(file, !file->f_pos); + if (unlikely(err)) + goto out; + + allocated = NULL; + vdir_cache = au_fvdir_cache(file); + if (!vdir_cache) { + vdir_cache = alloc_vdir(file); + err = PTR_ERR(vdir_cache); + if (IS_ERR(vdir_cache)) + goto out; + allocated = vdir_cache; + } else if (!file->f_pos && vdir_cache->vd_version != file->f_version) { + /* test file->f_pos here instead of ctx->pos */ + err = reinit_vdir(vdir_cache); + if (unlikely(err)) + goto out; + } else + return 0; /* success */ + + inode = file_inode(file); + err = copy_vdir(vdir_cache, au_ivdir(inode)); + if (!err) { + file->f_version = inode_query_iversion(inode); + if (allocated) + au_set_fvdir_cache(file, allocated); + } else if (allocated) + au_vdir_free(allocated); + +out: + return err; +} + +static loff_t calc_offset(struct au_vdir *vdir) +{ + loff_t offset; + union au_vdir_deblk_p p; + + p.deblk = vdir->vd_deblk[vdir->vd_last.ul]; + offset = vdir->vd_last.p.deblk - p.deblk; + offset += vdir->vd_deblk_sz * vdir->vd_last.ul; + return offset; +} + +/* returns true or false */ +static int seek_vdir(struct file *file, struct dir_context *ctx) +{ + int valid; + unsigned int deblk_sz; + unsigned long ul, n; + loff_t offset; + union au_vdir_deblk_p p, deblk_end; + struct au_vdir *vdir_cache; + + valid = 1; + vdir_cache = au_fvdir_cache(file); + offset = calc_offset(vdir_cache); + AuDbg("offset %lld\n", offset); + if (ctx->pos == offset) + goto out; + + vdir_cache->vd_last.ul = 0; + vdir_cache->vd_last.p.deblk = vdir_cache->vd_deblk[0]; + if (!ctx->pos) + goto out; + + valid = 0; + deblk_sz = vdir_cache->vd_deblk_sz; + ul = div64_u64(ctx->pos, deblk_sz); + AuDbg("ul %lu\n", ul); + if (ul >= vdir_cache->vd_nblk) + goto out; + + n = vdir_cache->vd_nblk; + for (; ul < n; ul++) { + p.deblk = vdir_cache->vd_deblk[ul]; + deblk_end.deblk = p.deblk + deblk_sz; + offset = ul; + offset *= deblk_sz; + while (!is_deblk_end(&p, &deblk_end) && offset < ctx->pos) { + unsigned int l; + + l = calc_size(p.de->de_str.len); + offset += l; + p.deblk += l; + } + if (!is_deblk_end(&p, &deblk_end)) { + valid = 1; + vdir_cache->vd_last.ul = ul; + vdir_cache->vd_last.p = p; + break; + } + } + +out: + /* smp_mb(); */ + if (!valid) + AuDbg("valid %d\n", !valid); + return valid; +} + +int au_vdir_fill_de(struct file *file, struct dir_context *ctx) +{ + unsigned int l, deblk_sz; + union au_vdir_deblk_p deblk_end; + struct au_vdir *vdir_cache; + struct au_vdir_de *de; + + if (!seek_vdir(file, ctx)) + return 0; + + vdir_cache = au_fvdir_cache(file); + deblk_sz = vdir_cache->vd_deblk_sz; + while (1) { + deblk_end.deblk = vdir_cache->vd_deblk[vdir_cache->vd_last.ul]; + deblk_end.deblk += deblk_sz; + while (!is_deblk_end(&vdir_cache->vd_last.p, &deblk_end)) { + de = vdir_cache->vd_last.p.de; + AuDbg("%.*s, off%lld, i%lu, dt%d\n", + de->de_str.len, de->de_str.name, ctx->pos, + (unsigned long)de->de_ino, de->de_type); + if (unlikely(!dir_emit(ctx, de->de_str.name, + de->de_str.len, de->de_ino, + de->de_type))) { + /* todo: ignore the error caused by udba? */ + /* return err; */ + return 0; + } + + l = calc_size(de->de_str.len); + vdir_cache->vd_last.p.deblk += l; + ctx->pos += l; + } + if (vdir_cache->vd_last.ul < vdir_cache->vd_nblk - 1) { + vdir_cache->vd_last.ul++; + vdir_cache->vd_last.p.deblk + = vdir_cache->vd_deblk[vdir_cache->vd_last.ul]; + ctx->pos = deblk_sz * vdir_cache->vd_last.ul; + continue; + } + break; + } + + /* smp_mb(); */ + return 0; +} diff --git a/fs/aufs/vfsub.c b/fs/aufs/vfsub.c new file mode 100644 index 0000000000000..e954cd7d01101 --- /dev/null +++ b/fs/aufs/vfsub.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sub-routines for VFS + */ + +#include +#include +#include +#include +#include +#include "aufs.h" + +#ifdef CONFIG_AUFS_BR_FUSE +int vfsub_test_mntns(struct vfsmount *mnt, struct super_block *h_sb) +{ + if (!au_test_fuse(h_sb) || !au_userns) + return 0; + + return is_current_mnt_ns(mnt) ? 0 : -EACCES; +} +#endif + +int vfsub_sync_filesystem(struct super_block *h_sb, int wait) +{ + int err; + + lockdep_off(); + down_read(&h_sb->s_umount); + err = __sync_filesystem(h_sb, wait); + up_read(&h_sb->s_umount); + lockdep_on(); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +int vfsub_update_h_iattr(struct path *h_path, int *did) +{ + int err; + struct kstat st; + struct super_block *h_sb; + + /* for remote fs, leave work for its getattr or d_revalidate */ + /* for bad i_attr fs, handle them in aufs_getattr() */ + /* still some fs may acquire i_mutex. we need to skip them */ + err = 0; + if (!did) + did = &err; + h_sb = h_path->dentry->d_sb; + *did = (!au_test_fs_remote(h_sb) && au_test_fs_refresh_iattr(h_sb)); + if (*did) + err = vfsub_getattr(h_path, &st); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct file *vfsub_dentry_open(struct path *path, int flags) +{ + struct file *file; + + file = dentry_open(path, flags /* | __FMODE_NONOTIFY */, + current_cred()); + if (!IS_ERR_OR_NULL(file) + && (file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) + i_readcount_inc(d_inode(path->dentry)); + + return file; +} + +struct file *vfsub_filp_open(const char *path, int oflags, int mode) +{ + struct file *file; + + lockdep_off(); + file = filp_open(path, + oflags /* | __FMODE_NONOTIFY */, + mode); + lockdep_on(); + if (IS_ERR(file)) + goto out; + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/ + +out: + return file; +} + +/* + * Ideally this function should call VFS:do_last() in order to keep all its + * checkings. But it is very hard for aufs to regenerate several VFS internal + * structure such as nameidata. This is a second (or third) best approach. + * cf. linux/fs/namei.c:do_last(), lookup_open() and atomic_open(). + */ +int vfsub_atomic_open(struct inode *dir, struct dentry *dentry, + struct vfsub_aopen_args *args) +{ + int err; + struct au_branch *br = args->br; + struct file *file = args->file; + /* copied from linux/fs/namei.c:atomic_open() */ + struct dentry *const DENTRY_NOT_SET = (void *)-1UL; + + IMustLock(dir); + AuDebugOn(!dir->i_op->atomic_open); + + err = au_br_test_oflag(args->open_flag, br); + if (unlikely(err)) + goto out; + + au_lcnt_inc(&br->br_nfiles); + file->f_path.dentry = DENTRY_NOT_SET; + file->f_path.mnt = au_br_mnt(br); + AuDbg("%ps\n", dir->i_op->atomic_open); + err = dir->i_op->atomic_open(dir, dentry, file, args->open_flag, + args->create_mode); + if (unlikely(err < 0)) { + au_lcnt_dec(&br->br_nfiles); + goto out; + } + + /* temporary workaround for nfsv4 branch */ + if (au_test_nfs(dir->i_sb)) + nfs_mark_for_revalidate(dir); + + if (file->f_mode & FMODE_CREATED) + fsnotify_create(dir, dentry); + if (!(file->f_mode & FMODE_OPENED)) { + au_lcnt_dec(&br->br_nfiles); + goto out; + } + + /* todo: call VFS:may_open() here */ + /* todo: ima_file_check() too? */ + if (!err && (args->open_flag & __FMODE_EXEC)) + err = deny_write_access(file); + if (!err) + fsnotify_open(file); + else + au_lcnt_dec(&br->br_nfiles); + /* note that the file is created and still opened */ + +out: + return err; +} + +int vfsub_kern_path(const char *name, unsigned int flags, struct path *path) +{ + int err; + + err = kern_path(name, flags, path); + if (!err && d_is_positive(path->dentry)) + vfsub_update_h_iattr(path, /*did*/NULL); /*ignore*/ + return err; +} + +struct dentry *vfsub_lookup_one_len_unlocked(const char *name, + struct dentry *parent, int len) +{ + struct path path = { + .mnt = NULL + }; + + path.dentry = lookup_one_len_unlocked(name, parent, len); + if (IS_ERR(path.dentry)) + goto out; + if (d_is_positive(path.dentry)) + vfsub_update_h_iattr(&path, /*did*/NULL); /*ignore*/ + +out: + AuTraceErrPtr(path.dentry); + return path.dentry; +} + +struct dentry *vfsub_lookup_one_len(const char *name, struct dentry *parent, + int len) +{ + struct path path = { + .mnt = NULL + }; + + /* VFS checks it too, but by WARN_ON_ONCE() */ + IMustLock(d_inode(parent)); + + path.dentry = lookup_one_len(name, parent, len); + if (IS_ERR(path.dentry)) + goto out; + if (d_is_positive(path.dentry)) + vfsub_update_h_iattr(&path, /*did*/NULL); /*ignore*/ + +out: + AuTraceErrPtr(path.dentry); + return path.dentry; +} + +void vfsub_call_lkup_one(void *args) +{ + struct vfsub_lkup_one_args *a = args; + *a->errp = vfsub_lkup_one(a->name, a->parent); +} + +/* ---------------------------------------------------------------------- */ + +struct dentry *vfsub_lock_rename(struct dentry *d1, struct au_hinode *hdir1, + struct dentry *d2, struct au_hinode *hdir2) +{ + struct dentry *d; + + lockdep_off(); + d = lock_rename(d1, d2); + lockdep_on(); + au_hn_suspend(hdir1); + if (hdir1 != hdir2) + au_hn_suspend(hdir2); + + return d; +} + +void vfsub_unlock_rename(struct dentry *d1, struct au_hinode *hdir1, + struct dentry *d2, struct au_hinode *hdir2) +{ + au_hn_resume(hdir1); + if (hdir1 != hdir2) + au_hn_resume(hdir2); + lockdep_off(); + unlock_rename(d1, d2); + lockdep_on(); +} + +/* ---------------------------------------------------------------------- */ + +int vfsub_create(struct inode *dir, struct path *path, int mode, bool want_excl) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_mknod(path, d, mode, 0); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_create(dir, path->dentry, mode, want_excl); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_symlink(struct inode *dir, struct path *path, const char *symname) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_symlink(path, d, symname); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_symlink(dir, path->dentry, symname); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_mknod(struct inode *dir, struct path *path, int mode, dev_t dev) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_mknod(path, d, mode, new_encode_dev(dev)); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_mknod(dir, path->dentry, mode, dev); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +static int au_test_nlink(struct inode *inode) +{ + const unsigned int link_max = UINT_MAX >> 1; /* rough margin */ + + if (!au_test_fs_no_limit_nlink(inode->i_sb) + || inode->i_nlink < link_max) + return 0; + return -EMLINK; +} + +int vfsub_link(struct dentry *src_dentry, struct inode *dir, struct path *path, + struct inode **delegated_inode) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + err = au_test_nlink(d_inode(src_dentry)); + if (unlikely(err)) + return err; + + /* we don't call may_linkat() */ + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_link(src_dentry, path, d); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_link(src_dentry, dir, path->dentry, delegated_inode); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + /* fuse has different memory inode for the same inumber */ + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + tmp.dentry = src_dentry; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_rename(struct inode *src_dir, struct dentry *src_dentry, + struct inode *dir, struct path *path, + struct inode **delegated_inode, unsigned int flags) +{ + int err; + struct path tmp = { + .mnt = path->mnt + }; + struct dentry *d; + + IMustLock(dir); + IMustLock(src_dir); + + d = path->dentry; + path->dentry = d->d_parent; + tmp.dentry = src_dentry->d_parent; + err = security_path_rename(&tmp, src_dentry, path, d, /*flags*/0); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_rename(src_dir, src_dentry, dir, path->dentry, + delegated_inode, flags); + lockdep_on(); + if (!err) { + int did; + + tmp.dentry = d->d_parent; + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = src_dentry; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + tmp.dentry = src_dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_mkdir(struct inode *dir, struct path *path, int mode) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_mkdir(path, d, mode); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_mkdir(dir, path->dentry, mode); + lockdep_on(); + if (!err) { + struct path tmp = *path; + int did; + + vfsub_update_h_iattr(&tmp, &did); + if (did) { + tmp.dentry = path->dentry->d_parent; + vfsub_update_h_iattr(&tmp, /*did*/NULL); + } + /*ignore*/ + } + +out: + return err; +} + +int vfsub_rmdir(struct inode *dir, struct path *path) +{ + int err; + struct dentry *d; + + IMustLock(dir); + + d = path->dentry; + path->dentry = d->d_parent; + err = security_path_rmdir(path, d); + path->dentry = d; + if (unlikely(err)) + goto out; + + lockdep_off(); + err = vfs_rmdir(dir, path->dentry); + lockdep_on(); + if (!err) { + struct path tmp = { + .dentry = path->dentry->d_parent, + .mnt = path->mnt + }; + + vfsub_update_h_iattr(&tmp, /*did*/NULL); /*ignore*/ + } + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* todo: support mmap_sem? */ +ssize_t vfsub_read_u(struct file *file, char __user *ubuf, size_t count, + loff_t *ppos) +{ + ssize_t err; + + lockdep_off(); + err = vfs_read(file, ubuf, count, ppos); + lockdep_on(); + if (err >= 0) + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/ + return err; +} + +/* todo: kernel_read()? */ +ssize_t vfsub_read_k(struct file *file, void *kbuf, size_t count, + loff_t *ppos) +{ + ssize_t err; + mm_segment_t oldfs; + union { + void *k; + char __user *u; + } buf; + + buf.k = kbuf; + oldfs = get_fs(); + set_fs(KERNEL_DS); + err = vfsub_read_u(file, buf.u, count, ppos); + set_fs(oldfs); + return err; +} + +ssize_t vfsub_write_u(struct file *file, const char __user *ubuf, size_t count, + loff_t *ppos) +{ + ssize_t err; + + lockdep_off(); + err = vfs_write(file, ubuf, count, ppos); + lockdep_on(); + if (err >= 0) + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/ + return err; +} + +ssize_t vfsub_write_k(struct file *file, void *kbuf, size_t count, loff_t *ppos) +{ + ssize_t err; + mm_segment_t oldfs; + union { + void *k; + const char __user *u; + } buf; + + buf.k = kbuf; + oldfs = get_fs(); + set_fs(KERNEL_DS); + err = vfsub_write_u(file, buf.u, count, ppos); + set_fs(oldfs); + return err; +} + +int vfsub_flush(struct file *file, fl_owner_t id) +{ + int err; + + err = 0; + if (file->f_op->flush) { + if (!au_test_nfs(file->f_path.dentry->d_sb)) + err = file->f_op->flush(file, id); + else { + lockdep_off(); + err = file->f_op->flush(file, id); + lockdep_on(); + } + if (!err) + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); + /*ignore*/ + } + return err; +} + +int vfsub_iterate_dir(struct file *file, struct dir_context *ctx) +{ + int err; + + AuDbg("%pD, ctx{%ps, %llu}\n", file, ctx->actor, ctx->pos); + + lockdep_off(); + err = iterate_dir(file, ctx); + lockdep_on(); + if (err >= 0) + vfsub_update_h_iattr(&file->f_path, /*did*/NULL); /*ignore*/ + + return err; +} + +long vfsub_splice_to(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + long err; + + lockdep_off(); + err = do_splice_to(in, ppos, pipe, len, flags); + lockdep_on(); + file_accessed(in); + if (err >= 0) + vfsub_update_h_iattr(&in->f_path, /*did*/NULL); /*ignore*/ + return err; +} + +long vfsub_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) +{ + long err; + + lockdep_off(); + err = do_splice_from(pipe, out, ppos, len, flags); + lockdep_on(); + if (err >= 0) + vfsub_update_h_iattr(&out->f_path, /*did*/NULL); /*ignore*/ + return err; +} + +int vfsub_fsync(struct file *file, struct path *path, int datasync) +{ + int err; + + /* file can be NULL */ + lockdep_off(); + err = vfs_fsync(file, datasync); + lockdep_on(); + if (!err) { + if (!path) { + AuDebugOn(!file); + path = &file->f_path; + } + vfsub_update_h_iattr(path, /*did*/NULL); /*ignore*/ + } + return err; +} + +/* cf. open.c:do_sys_truncate() and do_sys_ftruncate() */ +int vfsub_trunc(struct path *h_path, loff_t length, unsigned int attr, + struct file *h_file) +{ + int err; + struct inode *h_inode; + struct super_block *h_sb; + + if (!h_file) { + err = vfsub_truncate(h_path, length); + goto out; + } + + h_inode = d_inode(h_path->dentry); + h_sb = h_inode->i_sb; + lockdep_off(); + sb_start_write(h_sb); + lockdep_on(); + err = locks_verify_truncate(h_inode, h_file, length); + if (!err) + err = security_path_truncate(h_path); + if (!err) { + lockdep_off(); + err = do_truncate(h_path->dentry, length, attr, h_file); + lockdep_on(); + } + lockdep_off(); + sb_end_write(h_sb); + lockdep_on(); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct au_vfsub_mkdir_args { + int *errp; + struct inode *dir; + struct path *path; + int mode; +}; + +static void au_call_vfsub_mkdir(void *args) +{ + struct au_vfsub_mkdir_args *a = args; + *a->errp = vfsub_mkdir(a->dir, a->path, a->mode); +} + +int vfsub_sio_mkdir(struct inode *dir, struct path *path, int mode) +{ + int err, do_sio, wkq_err; + + do_sio = au_test_h_perm_sio(dir, MAY_EXEC | MAY_WRITE); + if (!do_sio) { + lockdep_off(); + err = vfsub_mkdir(dir, path, mode); + lockdep_on(); + } else { + struct au_vfsub_mkdir_args args = { + .errp = &err, + .dir = dir, + .path = path, + .mode = mode + }; + wkq_err = au_wkq_wait(au_call_vfsub_mkdir, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + return err; +} + +struct au_vfsub_rmdir_args { + int *errp; + struct inode *dir; + struct path *path; +}; + +static void au_call_vfsub_rmdir(void *args) +{ + struct au_vfsub_rmdir_args *a = args; + *a->errp = vfsub_rmdir(a->dir, a->path); +} + +int vfsub_sio_rmdir(struct inode *dir, struct path *path) +{ + int err, do_sio, wkq_err; + + do_sio = au_test_h_perm_sio(dir, MAY_EXEC | MAY_WRITE); + if (!do_sio) { + lockdep_off(); + err = vfsub_rmdir(dir, path); + lockdep_on(); + } else { + struct au_vfsub_rmdir_args args = { + .errp = &err, + .dir = dir, + .path = path + }; + wkq_err = au_wkq_wait(au_call_vfsub_rmdir, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct notify_change_args { + int *errp; + struct path *path; + struct iattr *ia; + struct inode **delegated_inode; +}; + +static void call_notify_change(void *args) +{ + struct notify_change_args *a = args; + struct inode *h_inode; + + h_inode = d_inode(a->path->dentry); + IMustLock(h_inode); + + *a->errp = -EPERM; + if (!IS_IMMUTABLE(h_inode) && !IS_APPEND(h_inode)) { + lockdep_off(); + *a->errp = notify_change(a->path->dentry, a->ia, + a->delegated_inode); + lockdep_on(); + if (!*a->errp) + vfsub_update_h_iattr(a->path, /*did*/NULL); /*ignore*/ + } + AuTraceErr(*a->errp); +} + +int vfsub_notify_change(struct path *path, struct iattr *ia, + struct inode **delegated_inode) +{ + int err; + struct notify_change_args args = { + .errp = &err, + .path = path, + .ia = ia, + .delegated_inode = delegated_inode + }; + + call_notify_change(&args); + + return err; +} + +int vfsub_sio_notify_change(struct path *path, struct iattr *ia, + struct inode **delegated_inode) +{ + int err, wkq_err; + struct notify_change_args args = { + .errp = &err, + .path = path, + .ia = ia, + .delegated_inode = delegated_inode + }; + + wkq_err = au_wkq_wait(call_notify_change, &args); + if (unlikely(wkq_err)) + err = wkq_err; + + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct unlink_args { + int *errp; + struct inode *dir; + struct path *path; + struct inode **delegated_inode; +}; + +static void call_unlink(void *args) +{ + struct unlink_args *a = args; + struct dentry *d = a->path->dentry; + struct inode *h_inode; + const int stop_sillyrename = (au_test_nfs(d->d_sb) + && au_dcount(d) == 1); + + IMustLock(a->dir); + + a->path->dentry = d->d_parent; + *a->errp = security_path_unlink(a->path, d); + a->path->dentry = d; + if (unlikely(*a->errp)) + return; + + if (!stop_sillyrename) + dget(d); + h_inode = NULL; + if (d_is_positive(d)) { + h_inode = d_inode(d); + ihold(h_inode); + } + + lockdep_off(); + *a->errp = vfs_unlink(a->dir, d, a->delegated_inode); + lockdep_on(); + if (!*a->errp) { + struct path tmp = { + .dentry = d->d_parent, + .mnt = a->path->mnt + }; + vfsub_update_h_iattr(&tmp, /*did*/NULL); /*ignore*/ + } + + if (!stop_sillyrename) + dput(d); + if (h_inode) + iput(h_inode); + + AuTraceErr(*a->errp); +} + +/* + * @dir: must be locked. + * @dentry: target dentry. + */ +int vfsub_unlink(struct inode *dir, struct path *path, + struct inode **delegated_inode, int force) +{ + int err; + struct unlink_args args = { + .errp = &err, + .dir = dir, + .path = path, + .delegated_inode = delegated_inode + }; + + if (!force) + call_unlink(&args); + else { + int wkq_err; + + wkq_err = au_wkq_wait(call_unlink, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + + return err; +} diff --git a/fs/aufs/vfsub.h b/fs/aufs/vfsub.h new file mode 100644 index 0000000000000..f1e85dde0d5ea --- /dev/null +++ b/fs/aufs/vfsub.h @@ -0,0 +1,354 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * sub-routines for VFS + */ + +#ifndef __AUFS_VFSUB_H__ +#define __AUFS_VFSUB_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include "debug.h" + +/* copied from linux/fs/internal.h */ +/* todo: BAD approach!! */ +extern void __mnt_drop_write(struct vfsmount *); +extern struct file *alloc_empty_file(int, const struct cred *); + +/* ---------------------------------------------------------------------- */ + +/* lock subclass for lower inode */ +/* default MAX_LOCKDEP_SUBCLASSES(8) is not enough */ +/* reduce? gave up. */ +enum { + AuLsc_I_Begin = I_MUTEX_PARENT2, /* 5 */ + AuLsc_I_PARENT, /* lower inode, parent first */ + AuLsc_I_PARENT2, /* copyup dirs */ + AuLsc_I_PARENT3, /* copyup wh */ + AuLsc_I_CHILD, + AuLsc_I_CHILD2, + AuLsc_I_End +}; + +/* to debug easier, do not make them inlined functions */ +#define MtxMustLock(mtx) AuDebugOn(!mutex_is_locked(mtx)) +#define IMustLock(i) AuDebugOn(!inode_is_locked(i)) + +/* ---------------------------------------------------------------------- */ + +static inline void vfsub_drop_nlink(struct inode *inode) +{ + AuDebugOn(!inode->i_nlink); + drop_nlink(inode); +} + +static inline void vfsub_dead_dir(struct inode *inode) +{ + AuDebugOn(!S_ISDIR(inode->i_mode)); + inode->i_flags |= S_DEAD; + clear_nlink(inode); +} + +static inline int vfsub_native_ro(struct inode *inode) +{ + return sb_rdonly(inode->i_sb) + || IS_RDONLY(inode) + /* || IS_APPEND(inode) */ + || IS_IMMUTABLE(inode); +} + +#ifdef CONFIG_AUFS_BR_FUSE +int vfsub_test_mntns(struct vfsmount *mnt, struct super_block *h_sb); +#else +AuStubInt0(vfsub_test_mntns, struct vfsmount *mnt, struct super_block *h_sb); +#endif + +int vfsub_sync_filesystem(struct super_block *h_sb, int wait); + +/* ---------------------------------------------------------------------- */ + +int vfsub_update_h_iattr(struct path *h_path, int *did); +struct file *vfsub_dentry_open(struct path *path, int flags); +struct file *vfsub_filp_open(const char *path, int oflags, int mode); +struct au_branch; +struct vfsub_aopen_args { + struct file *file; + unsigned int open_flag; + umode_t create_mode; + struct au_branch *br; +}; +int vfsub_atomic_open(struct inode *dir, struct dentry *dentry, + struct vfsub_aopen_args *args); +int vfsub_kern_path(const char *name, unsigned int flags, struct path *path); + +struct dentry *vfsub_lookup_one_len_unlocked(const char *name, + struct dentry *parent, int len); +struct dentry *vfsub_lookup_one_len(const char *name, struct dentry *parent, + int len); + +struct vfsub_lkup_one_args { + struct dentry **errp; + struct qstr *name; + struct dentry *parent; +}; + +static inline struct dentry *vfsub_lkup_one(struct qstr *name, + struct dentry *parent) +{ + return vfsub_lookup_one_len(name->name, parent, name->len); +} + +void vfsub_call_lkup_one(void *args); + +/* ---------------------------------------------------------------------- */ + +static inline int vfsub_mnt_want_write(struct vfsmount *mnt) +{ + int err; + + lockdep_off(); + err = mnt_want_write(mnt); + lockdep_on(); + return err; +} + +static inline void vfsub_mnt_drop_write(struct vfsmount *mnt) +{ + lockdep_off(); + mnt_drop_write(mnt); + lockdep_on(); +} + +#if 0 /* reserved */ +static inline void vfsub_mnt_drop_write_file(struct file *file) +{ + lockdep_off(); + mnt_drop_write_file(file); + lockdep_on(); +} +#endif + +/* ---------------------------------------------------------------------- */ + +struct au_hinode; +struct dentry *vfsub_lock_rename(struct dentry *d1, struct au_hinode *hdir1, + struct dentry *d2, struct au_hinode *hdir2); +void vfsub_unlock_rename(struct dentry *d1, struct au_hinode *hdir1, + struct dentry *d2, struct au_hinode *hdir2); + +int vfsub_create(struct inode *dir, struct path *path, int mode, + bool want_excl); +int vfsub_symlink(struct inode *dir, struct path *path, + const char *symname); +int vfsub_mknod(struct inode *dir, struct path *path, int mode, dev_t dev); +int vfsub_link(struct dentry *src_dentry, struct inode *dir, + struct path *path, struct inode **delegated_inode); +int vfsub_rename(struct inode *src_hdir, struct dentry *src_dentry, + struct inode *hdir, struct path *path, + struct inode **delegated_inode, unsigned int flags); +int vfsub_mkdir(struct inode *dir, struct path *path, int mode); +int vfsub_rmdir(struct inode *dir, struct path *path); + +/* ---------------------------------------------------------------------- */ + +ssize_t vfsub_read_u(struct file *file, char __user *ubuf, size_t count, + loff_t *ppos); +ssize_t vfsub_read_k(struct file *file, void *kbuf, size_t count, + loff_t *ppos); +ssize_t vfsub_write_u(struct file *file, const char __user *ubuf, size_t count, + loff_t *ppos); +ssize_t vfsub_write_k(struct file *file, void *kbuf, size_t count, + loff_t *ppos); +int vfsub_flush(struct file *file, fl_owner_t id); +int vfsub_iterate_dir(struct file *file, struct dir_context *ctx); + +static inline loff_t vfsub_f_size_read(struct file *file) +{ + return i_size_read(file_inode(file)); +} + +static inline unsigned int vfsub_file_flags(struct file *file) +{ + unsigned int flags; + + spin_lock(&file->f_lock); + flags = file->f_flags; + spin_unlock(&file->f_lock); + + return flags; +} + +static inline int vfsub_file_execed(struct file *file) +{ + /* todo: direct access f_flags */ + return !!(vfsub_file_flags(file) & __FMODE_EXEC); +} + +#if 0 /* reserved */ +static inline void vfsub_file_accessed(struct file *h_file) +{ + file_accessed(h_file); + vfsub_update_h_iattr(&h_file->f_path, /*did*/NULL); /*ignore*/ +} +#endif + +#if 0 /* reserved */ +static inline void vfsub_touch_atime(struct vfsmount *h_mnt, + struct dentry *h_dentry) +{ + struct path h_path = { + .dentry = h_dentry, + .mnt = h_mnt + }; + touch_atime(&h_path); + vfsub_update_h_iattr(&h_path, /*did*/NULL); /*ignore*/ +} +#endif + +static inline int vfsub_update_time(struct inode *h_inode, + struct timespec64 *ts, int flags) +{ + return update_time(h_inode, ts, flags); + /* no vfsub_update_h_iattr() since we don't have struct path */ +} + +#ifdef CONFIG_FS_POSIX_ACL +static inline int vfsub_acl_chmod(struct inode *h_inode, umode_t h_mode) +{ + int err; + + err = posix_acl_chmod(h_inode, h_mode); + if (err == -EOPNOTSUPP) + err = 0; + return err; +} +#else +AuStubInt0(vfsub_acl_chmod, struct inode *h_inode, umode_t h_mode); +#endif + +long vfsub_splice_to(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags); +long vfsub_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags); + +static inline long vfsub_truncate(struct path *path, loff_t length) +{ + long err; + + lockdep_off(); + err = vfs_truncate(path, length); + lockdep_on(); + return err; +} + +int vfsub_trunc(struct path *h_path, loff_t length, unsigned int attr, + struct file *h_file); +int vfsub_fsync(struct file *file, struct path *path, int datasync); + +/* + * re-use branch fs's ioctl(FICLONE) while aufs itself doesn't support such + * ioctl. + */ +static inline int vfsub_clone_file_range(struct file *src, struct file *dst, + u64 len) +{ + int err; + + lockdep_off(); + err = vfs_clone_file_range(src, 0, dst, 0, len); + lockdep_on(); + + return err; +} + +/* copy_file_range(2) is a systemcall */ +static inline ssize_t vfsub_copy_file_range(struct file *src, loff_t src_pos, + struct file *dst, loff_t dst_pos, + size_t len, unsigned int flags) +{ + ssize_t ssz; + + lockdep_off(); + ssz = vfs_copy_file_range(src, src_pos, dst, dst_pos, len, flags); + lockdep_on(); + + return ssz; +} + +/* ---------------------------------------------------------------------- */ + +static inline loff_t vfsub_llseek(struct file *file, loff_t offset, int origin) +{ + loff_t err; + + lockdep_off(); + err = vfs_llseek(file, offset, origin); + lockdep_on(); + return err; +} + +/* ---------------------------------------------------------------------- */ + +int vfsub_sio_mkdir(struct inode *dir, struct path *path, int mode); +int vfsub_sio_rmdir(struct inode *dir, struct path *path); +int vfsub_sio_notify_change(struct path *path, struct iattr *ia, + struct inode **delegated_inode); +int vfsub_notify_change(struct path *path, struct iattr *ia, + struct inode **delegated_inode); +int vfsub_unlink(struct inode *dir, struct path *path, + struct inode **delegated_inode, int force); + +static inline int vfsub_getattr(const struct path *path, struct kstat *st) +{ + return vfs_getattr(path, st, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); +} + +/* ---------------------------------------------------------------------- */ + +static inline int vfsub_setxattr(struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) +{ + int err; + + lockdep_off(); + err = vfs_setxattr(dentry, name, value, size, flags); + lockdep_on(); + + return err; +} + +static inline int vfsub_removexattr(struct dentry *dentry, const char *name) +{ + int err; + + lockdep_off(); + err = vfs_removexattr(dentry, name); + lockdep_on(); + + return err; +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_VFSUB_H__ */ diff --git a/fs/aufs/wbr_policy.c b/fs/aufs/wbr_policy.c new file mode 100644 index 0000000000000..a7cc1109f69dd --- /dev/null +++ b/fs/aufs/wbr_policy.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * policies for selecting one among multiple writable branches + */ + +#include +#include "aufs.h" + +/* subset of cpup_attr() */ +static noinline_for_stack +int au_cpdown_attr(struct path *h_path, struct dentry *h_src) +{ + int err, sbits; + struct iattr ia; + struct inode *h_isrc; + + h_isrc = d_inode(h_src); + ia.ia_valid = ATTR_FORCE | ATTR_MODE | ATTR_UID | ATTR_GID; + ia.ia_mode = h_isrc->i_mode; + ia.ia_uid = h_isrc->i_uid; + ia.ia_gid = h_isrc->i_gid; + sbits = !!(ia.ia_mode & (S_ISUID | S_ISGID)); + au_cpup_attr_flags(d_inode(h_path->dentry), h_isrc->i_flags); + /* no delegation since it is just created */ + err = vfsub_sio_notify_change(h_path, &ia, /*delegated*/NULL); + + /* is this nfs only? */ + if (!err && sbits && au_test_nfs(h_path->dentry->d_sb)) { + ia.ia_valid = ATTR_FORCE | ATTR_MODE; + ia.ia_mode = h_isrc->i_mode; + err = vfsub_sio_notify_change(h_path, &ia, /*delegated*/NULL); + } + + return err; +} + +#define AuCpdown_PARENT_OPQ 1 +#define AuCpdown_WHED (1 << 1) +#define AuCpdown_MADE_DIR (1 << 2) +#define AuCpdown_DIROPQ (1 << 3) +#define au_ftest_cpdown(flags, name) ((flags) & AuCpdown_##name) +#define au_fset_cpdown(flags, name) \ + do { (flags) |= AuCpdown_##name; } while (0) +#define au_fclr_cpdown(flags, name) \ + do { (flags) &= ~AuCpdown_##name; } while (0) + +static int au_cpdown_dir_opq(struct dentry *dentry, aufs_bindex_t bdst, + unsigned int *flags) +{ + int err; + struct dentry *opq_dentry; + + opq_dentry = au_diropq_create(dentry, bdst); + err = PTR_ERR(opq_dentry); + if (IS_ERR(opq_dentry)) + goto out; + dput(opq_dentry); + au_fset_cpdown(*flags, DIROPQ); + +out: + return err; +} + +static int au_cpdown_dir_wh(struct dentry *dentry, struct dentry *h_parent, + struct inode *dir, aufs_bindex_t bdst) +{ + int err; + struct path h_path; + struct au_branch *br; + + br = au_sbr(dentry->d_sb, bdst); + h_path.dentry = au_wh_lkup(h_parent, &dentry->d_name, br); + err = PTR_ERR(h_path.dentry); + if (IS_ERR(h_path.dentry)) + goto out; + + err = 0; + if (d_is_positive(h_path.dentry)) { + h_path.mnt = au_br_mnt(br); + err = au_wh_unlink_dentry(au_h_iptr(dir, bdst), &h_path, + dentry); + } + dput(h_path.dentry); + +out: + return err; +} + +static int au_cpdown_dir(struct dentry *dentry, aufs_bindex_t bdst, + struct au_pin *pin, + struct dentry *h_parent, void *arg) +{ + int err, rerr; + aufs_bindex_t bopq, btop; + struct path h_path; + struct dentry *parent; + struct inode *h_dir, *h_inode, *inode, *dir; + unsigned int *flags = arg; + + btop = au_dbtop(dentry); + /* dentry is di-locked */ + parent = dget_parent(dentry); + dir = d_inode(parent); + h_dir = d_inode(h_parent); + AuDebugOn(h_dir != au_h_iptr(dir, bdst)); + IMustLock(h_dir); + + err = au_lkup_neg(dentry, bdst, /*wh*/0); + if (unlikely(err < 0)) + goto out; + h_path.dentry = au_h_dptr(dentry, bdst); + h_path.mnt = au_sbr_mnt(dentry->d_sb, bdst); + err = vfsub_sio_mkdir(au_h_iptr(dir, bdst), &h_path, 0755); + if (unlikely(err)) + goto out_put; + au_fset_cpdown(*flags, MADE_DIR); + + bopq = au_dbdiropq(dentry); + au_fclr_cpdown(*flags, WHED); + au_fclr_cpdown(*flags, DIROPQ); + if (au_dbwh(dentry) == bdst) + au_fset_cpdown(*flags, WHED); + if (!au_ftest_cpdown(*flags, PARENT_OPQ) && bopq <= bdst) + au_fset_cpdown(*flags, PARENT_OPQ); + h_inode = d_inode(h_path.dentry); + inode_lock_nested(h_inode, AuLsc_I_CHILD); + if (au_ftest_cpdown(*flags, WHED)) { + err = au_cpdown_dir_opq(dentry, bdst, flags); + if (unlikely(err)) { + inode_unlock(h_inode); + goto out_dir; + } + } + + err = au_cpdown_attr(&h_path, au_h_dptr(dentry, btop)); + inode_unlock(h_inode); + if (unlikely(err)) + goto out_opq; + + if (au_ftest_cpdown(*flags, WHED)) { + err = au_cpdown_dir_wh(dentry, h_parent, dir, bdst); + if (unlikely(err)) + goto out_opq; + } + + inode = d_inode(dentry); + if (au_ibbot(inode) < bdst) + au_set_ibbot(inode, bdst); + au_set_h_iptr(inode, bdst, au_igrab(h_inode), + au_hi_flags(inode, /*isdir*/1)); + au_fhsm_wrote(dentry->d_sb, bdst, /*force*/0); + goto out; /* success */ + + /* revert */ +out_opq: + if (au_ftest_cpdown(*flags, DIROPQ)) { + inode_lock_nested(h_inode, AuLsc_I_CHILD); + rerr = au_diropq_remove(dentry, bdst); + inode_unlock(h_inode); + if (unlikely(rerr)) { + AuIOErr("failed removing diropq for %pd b%d (%d)\n", + dentry, bdst, rerr); + err = -EIO; + goto out; + } + } +out_dir: + if (au_ftest_cpdown(*flags, MADE_DIR)) { + rerr = vfsub_sio_rmdir(au_h_iptr(dir, bdst), &h_path); + if (unlikely(rerr)) { + AuIOErr("failed removing %pd b%d (%d)\n", + dentry, bdst, rerr); + err = -EIO; + } + } +out_put: + au_set_h_dptr(dentry, bdst, NULL); + if (au_dbbot(dentry) == bdst) + au_update_dbbot(dentry); +out: + dput(parent); + return err; +} + +int au_cpdown_dirs(struct dentry *dentry, aufs_bindex_t bdst) +{ + int err; + unsigned int flags; + + flags = 0; + err = au_cp_dirs(dentry, bdst, au_cpdown_dir, &flags); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* policies for create */ + +int au_wbr_nonopq(struct dentry *dentry, aufs_bindex_t bindex) +{ + int err, i, j, ndentry; + aufs_bindex_t bopq; + struct au_dcsub_pages dpages; + struct au_dpage *dpage; + struct dentry **dentries, *parent, *d; + + err = au_dpages_init(&dpages, GFP_NOFS); + if (unlikely(err)) + goto out; + parent = dget_parent(dentry); + err = au_dcsub_pages_rev_aufs(&dpages, parent, /*do_include*/0); + if (unlikely(err)) + goto out_free; + + err = bindex; + for (i = 0; i < dpages.ndpage; i++) { + dpage = dpages.dpages + i; + dentries = dpage->dentries; + ndentry = dpage->ndentry; + for (j = 0; j < ndentry; j++) { + d = dentries[j]; + di_read_lock_parent2(d, !AuLock_IR); + bopq = au_dbdiropq(d); + di_read_unlock(d, !AuLock_IR); + if (bopq >= 0 && bopq < err) + err = bopq; + } + } + +out_free: + dput(parent); + au_dpages_free(&dpages); +out: + return err; +} + +static int au_wbr_bu(struct super_block *sb, aufs_bindex_t bindex) +{ + for (; bindex >= 0; bindex--) + if (!au_br_rdonly(au_sbr(sb, bindex))) + return bindex; + return -EROFS; +} + +/* top down parent */ +static int au_wbr_create_tdp(struct dentry *dentry, + unsigned int flags __maybe_unused) +{ + int err; + aufs_bindex_t btop, bindex; + struct super_block *sb; + struct dentry *parent, *h_parent; + + sb = dentry->d_sb; + btop = au_dbtop(dentry); + err = btop; + if (!au_br_rdonly(au_sbr(sb, btop))) + goto out; + + err = -EROFS; + parent = dget_parent(dentry); + for (bindex = au_dbtop(parent); bindex < btop; bindex++) { + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || d_is_negative(h_parent)) + continue; + + if (!au_br_rdonly(au_sbr(sb, bindex))) { + err = bindex; + break; + } + } + dput(parent); + + /* bottom up here */ + if (unlikely(err < 0)) { + err = au_wbr_bu(sb, btop - 1); + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + } + +out: + AuDbg("b%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* an exception for the policy other than tdp */ +static int au_wbr_create_exp(struct dentry *dentry) +{ + int err; + aufs_bindex_t bwh, bdiropq; + struct dentry *parent; + + err = -1; + bwh = au_dbwh(dentry); + parent = dget_parent(dentry); + bdiropq = au_dbdiropq(parent); + if (bwh >= 0) { + if (bdiropq >= 0) + err = min(bdiropq, bwh); + else + err = bwh; + AuDbg("%d\n", err); + } else if (bdiropq >= 0) { + err = bdiropq; + AuDbg("%d\n", err); + } + dput(parent); + + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + + if (err >= 0 && au_br_rdonly(au_sbr(dentry->d_sb, err))) + err = -1; + + AuDbg("%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* round robin */ +static int au_wbr_create_init_rr(struct super_block *sb) +{ + int err; + + err = au_wbr_bu(sb, au_sbbot(sb)); + atomic_set(&au_sbi(sb)->si_wbr_rr_next, -err); /* less important */ + /* smp_mb(); */ + + AuDbg("b%d\n", err); + return err; +} + +static int au_wbr_create_rr(struct dentry *dentry, unsigned int flags) +{ + int err, nbr; + unsigned int u; + aufs_bindex_t bindex, bbot; + struct super_block *sb; + atomic_t *next; + + err = au_wbr_create_exp(dentry); + if (err >= 0) + goto out; + + sb = dentry->d_sb; + next = &au_sbi(sb)->si_wbr_rr_next; + bbot = au_sbbot(sb); + nbr = bbot + 1; + for (bindex = 0; bindex <= bbot; bindex++) { + if (!au_ftest_wbr(flags, DIR)) { + err = atomic_dec_return(next) + 1; + /* modulo for 0 is meaningless */ + if (unlikely(!err)) + err = atomic_dec_return(next) + 1; + } else + err = atomic_read(next); + AuDbg("%d\n", err); + u = err; + err = u % nbr; + AuDbg("%d\n", err); + if (!au_br_rdonly(au_sbr(sb, err))) + break; + err = -EROFS; + } + + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + +out: + AuDbg("%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* most free space */ +static void au_mfs(struct dentry *dentry, struct dentry *parent) +{ + struct super_block *sb; + struct au_branch *br; + struct au_wbr_mfs *mfs; + struct dentry *h_parent; + aufs_bindex_t bindex, bbot; + int err; + unsigned long long b, bavail; + struct path h_path; + /* reduce the stack usage */ + struct kstatfs *st; + + st = kmalloc(sizeof(*st), GFP_NOFS); + if (unlikely(!st)) { + AuWarn1("failed updating mfs(%d), ignored\n", -ENOMEM); + return; + } + + bavail = 0; + sb = dentry->d_sb; + mfs = &au_sbi(sb)->si_wbr_mfs; + MtxMustLock(&mfs->mfs_lock); + mfs->mfs_bindex = -EROFS; + mfs->mfsrr_bytes = 0; + if (!parent) { + bindex = 0; + bbot = au_sbbot(sb); + } else { + bindex = au_dbtop(parent); + bbot = au_dbtaildir(parent); + } + + for (; bindex <= bbot; bindex++) { + if (parent) { + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || d_is_negative(h_parent)) + continue; + } + br = au_sbr(sb, bindex); + if (au_br_rdonly(br)) + continue; + + /* sb->s_root for NFS is unreliable */ + h_path.mnt = au_br_mnt(br); + h_path.dentry = h_path.mnt->mnt_root; + err = vfs_statfs(&h_path, st); + if (unlikely(err)) { + AuWarn1("failed statfs, b%d, %d\n", bindex, err); + continue; + } + + /* when the available size is equal, select the lower one */ + BUILD_BUG_ON(sizeof(b) < sizeof(st->f_bavail) + || sizeof(b) < sizeof(st->f_bsize)); + b = st->f_bavail * st->f_bsize; + br->br_wbr->wbr_bytes = b; + if (b >= bavail) { + bavail = b; + mfs->mfs_bindex = bindex; + mfs->mfs_jiffy = jiffies; + } + } + + mfs->mfsrr_bytes = bavail; + AuDbg("b%d\n", mfs->mfs_bindex); + au_kfree_rcu(st); +} + +static int au_wbr_create_mfs(struct dentry *dentry, unsigned int flags) +{ + int err; + struct dentry *parent; + struct super_block *sb; + struct au_wbr_mfs *mfs; + + err = au_wbr_create_exp(dentry); + if (err >= 0) + goto out; + + sb = dentry->d_sb; + parent = NULL; + if (au_ftest_wbr(flags, PARENT)) + parent = dget_parent(dentry); + mfs = &au_sbi(sb)->si_wbr_mfs; + mutex_lock(&mfs->mfs_lock); + if (time_after(jiffies, mfs->mfs_jiffy + mfs->mfs_expire) + || mfs->mfs_bindex < 0 + || au_br_rdonly(au_sbr(sb, mfs->mfs_bindex))) + au_mfs(dentry, parent); + mutex_unlock(&mfs->mfs_lock); + err = mfs->mfs_bindex; + dput(parent); + + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + +out: + AuDbg("b%d\n", err); + return err; +} + +static int au_wbr_create_init_mfs(struct super_block *sb) +{ + struct au_wbr_mfs *mfs; + + mfs = &au_sbi(sb)->si_wbr_mfs; + mutex_init(&mfs->mfs_lock); + mfs->mfs_jiffy = 0; + mfs->mfs_bindex = -EROFS; + + return 0; +} + +static int au_wbr_create_fin_mfs(struct super_block *sb __maybe_unused) +{ + mutex_destroy(&au_sbi(sb)->si_wbr_mfs.mfs_lock); + return 0; +} + +/* ---------------------------------------------------------------------- */ + +/* top down regardless parent, and then mfs */ +static int au_wbr_create_tdmfs(struct dentry *dentry, + unsigned int flags __maybe_unused) +{ + int err; + aufs_bindex_t bwh, btail, bindex, bfound, bmfs; + unsigned long long watermark; + struct super_block *sb; + struct au_wbr_mfs *mfs; + struct au_branch *br; + struct dentry *parent; + + sb = dentry->d_sb; + mfs = &au_sbi(sb)->si_wbr_mfs; + mutex_lock(&mfs->mfs_lock); + if (time_after(jiffies, mfs->mfs_jiffy + mfs->mfs_expire) + || mfs->mfs_bindex < 0) + au_mfs(dentry, /*parent*/NULL); + watermark = mfs->mfsrr_watermark; + bmfs = mfs->mfs_bindex; + mutex_unlock(&mfs->mfs_lock); + + /* another style of au_wbr_create_exp() */ + bwh = au_dbwh(dentry); + parent = dget_parent(dentry); + btail = au_dbtaildir(parent); + if (bwh >= 0 && bwh < btail) + btail = bwh; + + err = au_wbr_nonopq(dentry, btail); + if (unlikely(err < 0)) + goto out; + btail = err; + bfound = -1; + for (bindex = 0; bindex <= btail; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_rdonly(br)) + continue; + if (br->br_wbr->wbr_bytes > watermark) { + bfound = bindex; + break; + } + } + err = bfound; + if (err < 0) + err = bmfs; + +out: + dput(parent); + AuDbg("b%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* most free space and then round robin */ +static int au_wbr_create_mfsrr(struct dentry *dentry, unsigned int flags) +{ + int err; + struct au_wbr_mfs *mfs; + + err = au_wbr_create_mfs(dentry, flags); + if (err >= 0) { + mfs = &au_sbi(dentry->d_sb)->si_wbr_mfs; + mutex_lock(&mfs->mfs_lock); + if (mfs->mfsrr_bytes < mfs->mfsrr_watermark) + err = au_wbr_create_rr(dentry, flags); + mutex_unlock(&mfs->mfs_lock); + } + + AuDbg("b%d\n", err); + return err; +} + +static int au_wbr_create_init_mfsrr(struct super_block *sb) +{ + int err; + + au_wbr_create_init_mfs(sb); /* ignore */ + err = au_wbr_create_init_rr(sb); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* top down parent and most free space */ +static int au_wbr_create_pmfs(struct dentry *dentry, unsigned int flags) +{ + int err, e2; + unsigned long long b; + aufs_bindex_t bindex, btop, bbot; + struct super_block *sb; + struct dentry *parent, *h_parent; + struct au_branch *br; + + err = au_wbr_create_tdp(dentry, flags); + if (unlikely(err < 0)) + goto out; + parent = dget_parent(dentry); + btop = au_dbtop(parent); + bbot = au_dbtaildir(parent); + if (btop == bbot) + goto out_parent; /* success */ + + e2 = au_wbr_create_mfs(dentry, flags); + if (e2 < 0) + goto out_parent; /* success */ + + /* when the available size is equal, select upper one */ + sb = dentry->d_sb; + br = au_sbr(sb, err); + b = br->br_wbr->wbr_bytes; + AuDbg("b%d, %llu\n", err, b); + + for (bindex = btop; bindex <= bbot; bindex++) { + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || d_is_negative(h_parent)) + continue; + + br = au_sbr(sb, bindex); + if (!au_br_rdonly(br) && br->br_wbr->wbr_bytes > b) { + b = br->br_wbr->wbr_bytes; + err = bindex; + AuDbg("b%d, %llu\n", err, b); + } + } + + if (err >= 0) + err = au_wbr_nonopq(dentry, err); + +out_parent: + dput(parent); +out: + AuDbg("b%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * - top down parent + * - most free space with parent + * - most free space round-robin regardless parent + */ +static int au_wbr_create_pmfsrr(struct dentry *dentry, unsigned int flags) +{ + int err; + unsigned long long watermark; + struct super_block *sb; + struct au_branch *br; + struct au_wbr_mfs *mfs; + + err = au_wbr_create_pmfs(dentry, flags | AuWbr_PARENT); + if (unlikely(err < 0)) + goto out; + + sb = dentry->d_sb; + br = au_sbr(sb, err); + mfs = &au_sbi(sb)->si_wbr_mfs; + mutex_lock(&mfs->mfs_lock); + watermark = mfs->mfsrr_watermark; + mutex_unlock(&mfs->mfs_lock); + if (br->br_wbr->wbr_bytes < watermark) + /* regardless the parent dir */ + err = au_wbr_create_mfsrr(dentry, flags); + +out: + AuDbg("b%d\n", err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* policies for copyup */ + +/* top down parent */ +static int au_wbr_copyup_tdp(struct dentry *dentry) +{ + return au_wbr_create_tdp(dentry, /*flags, anything is ok*/0); +} + +/* bottom up parent */ +static int au_wbr_copyup_bup(struct dentry *dentry) +{ + int err; + aufs_bindex_t bindex, btop; + struct dentry *parent, *h_parent; + struct super_block *sb; + + err = -EROFS; + sb = dentry->d_sb; + parent = dget_parent(dentry); + btop = au_dbtop(parent); + for (bindex = au_dbtop(dentry); bindex >= btop; bindex--) { + h_parent = au_h_dptr(parent, bindex); + if (!h_parent || d_is_negative(h_parent)) + continue; + + if (!au_br_rdonly(au_sbr(sb, bindex))) { + err = bindex; + break; + } + } + dput(parent); + + /* bottom up here */ + if (unlikely(err < 0)) + err = au_wbr_bu(sb, btop - 1); + + AuDbg("b%d\n", err); + return err; +} + +/* bottom up */ +int au_wbr_do_copyup_bu(struct dentry *dentry, aufs_bindex_t btop) +{ + int err; + + err = au_wbr_bu(dentry->d_sb, btop); + AuDbg("b%d\n", err); + if (err > btop) + err = au_wbr_nonopq(dentry, err); + + AuDbg("b%d\n", err); + return err; +} + +static int au_wbr_copyup_bu(struct dentry *dentry) +{ + int err; + aufs_bindex_t btop; + + btop = au_dbtop(dentry); + err = au_wbr_do_copyup_bu(dentry, btop); + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct au_wbr_copyup_operations au_wbr_copyup_ops[] = { + [AuWbrCopyup_TDP] = { + .copyup = au_wbr_copyup_tdp + }, + [AuWbrCopyup_BUP] = { + .copyup = au_wbr_copyup_bup + }, + [AuWbrCopyup_BU] = { + .copyup = au_wbr_copyup_bu + } +}; + +struct au_wbr_create_operations au_wbr_create_ops[] = { + [AuWbrCreate_TDP] = { + .create = au_wbr_create_tdp + }, + [AuWbrCreate_RR] = { + .create = au_wbr_create_rr, + .init = au_wbr_create_init_rr + }, + [AuWbrCreate_MFS] = { + .create = au_wbr_create_mfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_MFSV] = { + .create = au_wbr_create_mfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_MFSRR] = { + .create = au_wbr_create_mfsrr, + .init = au_wbr_create_init_mfsrr, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_MFSRRV] = { + .create = au_wbr_create_mfsrr, + .init = au_wbr_create_init_mfsrr, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_TDMFS] = { + .create = au_wbr_create_tdmfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_TDMFSV] = { + .create = au_wbr_create_tdmfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_PMFS] = { + .create = au_wbr_create_pmfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_PMFSV] = { + .create = au_wbr_create_pmfs, + .init = au_wbr_create_init_mfs, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_PMFSRR] = { + .create = au_wbr_create_pmfsrr, + .init = au_wbr_create_init_mfsrr, + .fin = au_wbr_create_fin_mfs + }, + [AuWbrCreate_PMFSRRV] = { + .create = au_wbr_create_pmfsrr, + .init = au_wbr_create_init_mfsrr, + .fin = au_wbr_create_fin_mfs + } +}; diff --git a/fs/aufs/whout.c b/fs/aufs/whout.c new file mode 100644 index 0000000000000..efe0c16508dba --- /dev/null +++ b/fs/aufs/whout.c @@ -0,0 +1,1062 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * whiteout for logical deletion and opaque directory + */ + +#include "aufs.h" + +#define WH_MASK 0444 + +/* + * If a directory contains this file, then it is opaque. We start with the + * .wh. flag so that it is blocked by lookup. + */ +static struct qstr diropq_name = QSTR_INIT(AUFS_WH_DIROPQ, + sizeof(AUFS_WH_DIROPQ) - 1); + +/* + * generate whiteout name, which is NOT terminated by NULL. + * @name: original d_name.name + * @len: original d_name.len + * @wh: whiteout qstr + * returns zero when succeeds, otherwise error. + * succeeded value as wh->name should be freed by kfree(). + */ +int au_wh_name_alloc(struct qstr *wh, const struct qstr *name) +{ + char *p; + + if (unlikely(name->len > PATH_MAX - AUFS_WH_PFX_LEN)) + return -ENAMETOOLONG; + + wh->len = name->len + AUFS_WH_PFX_LEN; + p = kmalloc(wh->len, GFP_NOFS); + wh->name = p; + if (p) { + memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN); + memcpy(p + AUFS_WH_PFX_LEN, name->name, name->len); + /* smp_mb(); */ + return 0; + } + return -ENOMEM; +} + +/* ---------------------------------------------------------------------- */ + +/* + * test if the @wh_name exists under @h_parent. + * @try_sio specifies the necessary of super-io. + */ +int au_wh_test(struct dentry *h_parent, struct qstr *wh_name, int try_sio) +{ + int err; + struct dentry *wh_dentry; + + if (!try_sio) + wh_dentry = vfsub_lkup_one(wh_name, h_parent); + else + wh_dentry = au_sio_lkup_one(wh_name, h_parent); + err = PTR_ERR(wh_dentry); + if (IS_ERR(wh_dentry)) { + if (err == -ENAMETOOLONG) + err = 0; + goto out; + } + + err = 0; + if (d_is_negative(wh_dentry)) + goto out_wh; /* success */ + + err = 1; + if (d_is_reg(wh_dentry)) + goto out_wh; /* success */ + + err = -EIO; + AuIOErr("%pd Invalid whiteout entry type 0%o.\n", + wh_dentry, d_inode(wh_dentry)->i_mode); + +out_wh: + dput(wh_dentry); +out: + return err; +} + +/* + * test if the @h_dentry sets opaque or not. + */ +int au_diropq_test(struct dentry *h_dentry) +{ + int err; + struct inode *h_dir; + + h_dir = d_inode(h_dentry); + err = au_wh_test(h_dentry, &diropq_name, + au_test_h_perm_sio(h_dir, MAY_EXEC)); + return err; +} + +/* + * returns a negative dentry whose name is unique and temporary. + */ +struct dentry *au_whtmp_lkup(struct dentry *h_parent, struct au_branch *br, + struct qstr *prefix) +{ + struct dentry *dentry; + int i; + char defname[NAME_MAX - AUFS_MAX_NAMELEN + DNAME_INLINE_LEN + 1], + *name, *p; + /* strict atomic_t is unnecessary here */ + static unsigned short cnt; + struct qstr qs; + + BUILD_BUG_ON(sizeof(cnt) * 2 > AUFS_WH_TMP_LEN); + + name = defname; + qs.len = sizeof(defname) - DNAME_INLINE_LEN + prefix->len - 1; + if (unlikely(prefix->len > DNAME_INLINE_LEN)) { + dentry = ERR_PTR(-ENAMETOOLONG); + if (unlikely(qs.len > NAME_MAX)) + goto out; + dentry = ERR_PTR(-ENOMEM); + name = kmalloc(qs.len + 1, GFP_NOFS); + if (unlikely(!name)) + goto out; + } + + /* doubly whiteout-ed */ + memcpy(name, AUFS_WH_PFX AUFS_WH_PFX, AUFS_WH_PFX_LEN * 2); + p = name + AUFS_WH_PFX_LEN * 2; + memcpy(p, prefix->name, prefix->len); + p += prefix->len; + *p++ = '.'; + AuDebugOn(name + qs.len + 1 - p <= AUFS_WH_TMP_LEN); + + qs.name = name; + for (i = 0; i < 3; i++) { + sprintf(p, "%.*x", AUFS_WH_TMP_LEN, cnt++); + dentry = au_sio_lkup_one(&qs, h_parent); + if (IS_ERR(dentry) || d_is_negative(dentry)) + goto out_name; + dput(dentry); + } + /* pr_warn("could not get random name\n"); */ + dentry = ERR_PTR(-EEXIST); + AuDbg("%.*s\n", AuLNPair(&qs)); + BUG(); + +out_name: + if (name != defname) + au_kfree_try_rcu(name); +out: + AuTraceErrPtr(dentry); + return dentry; +} + +/* + * rename the @h_dentry on @br to the whiteouted temporary name. + */ +int au_whtmp_ren(struct dentry *h_dentry, struct au_branch *br) +{ + int err; + struct path h_path = { + .mnt = au_br_mnt(br) + }; + struct inode *h_dir, *delegated; + struct dentry *h_parent; + + h_parent = h_dentry->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + + h_path.dentry = au_whtmp_lkup(h_parent, br, &h_dentry->d_name); + err = PTR_ERR(h_path.dentry); + if (IS_ERR(h_path.dentry)) + goto out; + + /* under the same dir, no need to lock_rename() */ + delegated = NULL; + err = vfsub_rename(h_dir, h_dentry, h_dir, &h_path, &delegated, + /*flags*/0); + AuTraceErr(err); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal rename\n"); + iput(delegated); + } + dput(h_path.dentry); + +out: + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ +/* + * functions for removing a whiteout + */ + +static int do_unlink_wh(struct inode *h_dir, struct path *h_path) +{ + int err, force; + struct inode *delegated; + + /* + * forces superio when the dir has a sticky bit. + * this may be a violation of unix fs semantics. + */ + force = (h_dir->i_mode & S_ISVTX) + && !uid_eq(current_fsuid(), d_inode(h_path->dentry)->i_uid); + delegated = NULL; + err = vfsub_unlink(h_dir, h_path, &delegated, force); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + return err; +} + +int au_wh_unlink_dentry(struct inode *h_dir, struct path *h_path, + struct dentry *dentry) +{ + int err; + + err = do_unlink_wh(h_dir, h_path); + if (!err && dentry) + au_set_dbwh(dentry, -1); + + return err; +} + +static int unlink_wh_name(struct dentry *h_parent, struct qstr *wh, + struct au_branch *br) +{ + int err; + struct path h_path = { + .mnt = au_br_mnt(br) + }; + + err = 0; + h_path.dentry = vfsub_lkup_one(wh, h_parent); + if (IS_ERR(h_path.dentry)) + err = PTR_ERR(h_path.dentry); + else { + if (d_is_reg(h_path.dentry)) + err = do_unlink_wh(d_inode(h_parent), &h_path); + dput(h_path.dentry); + } + + return err; +} + +/* ---------------------------------------------------------------------- */ +/* + * initialize/clean whiteout for a branch + */ + +static void au_wh_clean(struct inode *h_dir, struct path *whpath, + const int isdir) +{ + int err; + struct inode *delegated; + + if (d_is_negative(whpath->dentry)) + return; + + if (isdir) + err = vfsub_rmdir(h_dir, whpath); + else { + delegated = NULL; + err = vfsub_unlink(h_dir, whpath, &delegated, /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + } + if (unlikely(err)) + pr_warn("failed removing %pd (%d), ignored.\n", + whpath->dentry, err); +} + +static int test_linkable(struct dentry *h_root) +{ + struct inode *h_dir = d_inode(h_root); + + if (h_dir->i_op->link) + return 0; + + pr_err("%pd (%s) doesn't support link(2), use noplink and rw+nolwh\n", + h_root, au_sbtype(h_root->d_sb)); + return -ENOSYS; /* the branch doesn't have its ->link() */ +} + +/* todo: should this mkdir be done in /sbin/mount.aufs helper? */ +static int au_whdir(struct inode *h_dir, struct path *path) +{ + int err; + + err = -EEXIST; + if (d_is_negative(path->dentry)) { + int mode = 0700; + + if (au_test_nfs(path->dentry->d_sb)) + mode |= 0111; + err = vfsub_mkdir(h_dir, path, mode); + } else if (d_is_dir(path->dentry)) + err = 0; + else + pr_err("unknown %pd exists\n", path->dentry); + + return err; +} + +struct au_wh_base { + const struct qstr *name; + struct dentry *dentry; +}; + +static void au_wh_init_ro(struct inode *h_dir, struct au_wh_base base[], + struct path *h_path) +{ + h_path->dentry = base[AuBrWh_BASE].dentry; + au_wh_clean(h_dir, h_path, /*isdir*/0); + h_path->dentry = base[AuBrWh_PLINK].dentry; + au_wh_clean(h_dir, h_path, /*isdir*/1); + h_path->dentry = base[AuBrWh_ORPH].dentry; + au_wh_clean(h_dir, h_path, /*isdir*/1); +} + +/* + * returns tri-state, + * minus: error, caller should print the message + * zero: success + * plus: error, caller should NOT print the message + */ +static int au_wh_init_rw_nolink(struct dentry *h_root, struct au_wbr *wbr, + int do_plink, struct au_wh_base base[], + struct path *h_path) +{ + int err; + struct inode *h_dir; + + h_dir = d_inode(h_root); + h_path->dentry = base[AuBrWh_BASE].dentry; + au_wh_clean(h_dir, h_path, /*isdir*/0); + h_path->dentry = base[AuBrWh_PLINK].dentry; + if (do_plink) { + err = test_linkable(h_root); + if (unlikely(err)) { + err = 1; + goto out; + } + + err = au_whdir(h_dir, h_path); + if (unlikely(err)) + goto out; + wbr->wbr_plink = dget(base[AuBrWh_PLINK].dentry); + } else + au_wh_clean(h_dir, h_path, /*isdir*/1); + h_path->dentry = base[AuBrWh_ORPH].dentry; + err = au_whdir(h_dir, h_path); + if (unlikely(err)) + goto out; + wbr->wbr_orph = dget(base[AuBrWh_ORPH].dentry); + +out: + return err; +} + +/* + * for the moment, aufs supports the branch filesystem which does not support + * link(2). testing on FAT which does not support i_op->setattr() fully either, + * copyup failed. finally, such filesystem will not be used as the writable + * branch. + * + * returns tri-state, see above. + */ +static int au_wh_init_rw(struct dentry *h_root, struct au_wbr *wbr, + int do_plink, struct au_wh_base base[], + struct path *h_path) +{ + int err; + struct inode *h_dir; + + WbrWhMustWriteLock(wbr); + + err = test_linkable(h_root); + if (unlikely(err)) { + err = 1; + goto out; + } + + /* + * todo: should this create be done in /sbin/mount.aufs helper? + */ + err = -EEXIST; + h_dir = d_inode(h_root); + if (d_is_negative(base[AuBrWh_BASE].dentry)) { + h_path->dentry = base[AuBrWh_BASE].dentry; + err = vfsub_create(h_dir, h_path, WH_MASK, /*want_excl*/true); + } else if (d_is_reg(base[AuBrWh_BASE].dentry)) + err = 0; + else + pr_err("unknown %pd2 exists\n", base[AuBrWh_BASE].dentry); + if (unlikely(err)) + goto out; + + h_path->dentry = base[AuBrWh_PLINK].dentry; + if (do_plink) { + err = au_whdir(h_dir, h_path); + if (unlikely(err)) + goto out; + wbr->wbr_plink = dget(base[AuBrWh_PLINK].dentry); + } else + au_wh_clean(h_dir, h_path, /*isdir*/1); + wbr->wbr_whbase = dget(base[AuBrWh_BASE].dentry); + + h_path->dentry = base[AuBrWh_ORPH].dentry; + err = au_whdir(h_dir, h_path); + if (unlikely(err)) + goto out; + wbr->wbr_orph = dget(base[AuBrWh_ORPH].dentry); + +out: + return err; +} + +/* + * initialize the whiteout base file/dir for @br. + */ +int au_wh_init(struct au_branch *br, struct super_block *sb) +{ + int err, i; + const unsigned char do_plink + = !!au_opt_test(au_mntflags(sb), PLINK); + struct inode *h_dir; + struct path path = br->br_path; + struct dentry *h_root = path.dentry; + struct au_wbr *wbr = br->br_wbr; + static const struct qstr base_name[] = { + [AuBrWh_BASE] = QSTR_INIT(AUFS_BASE_NAME, + sizeof(AUFS_BASE_NAME) - 1), + [AuBrWh_PLINK] = QSTR_INIT(AUFS_PLINKDIR_NAME, + sizeof(AUFS_PLINKDIR_NAME) - 1), + [AuBrWh_ORPH] = QSTR_INIT(AUFS_ORPHDIR_NAME, + sizeof(AUFS_ORPHDIR_NAME) - 1) + }; + struct au_wh_base base[] = { + [AuBrWh_BASE] = { + .name = base_name + AuBrWh_BASE, + .dentry = NULL + }, + [AuBrWh_PLINK] = { + .name = base_name + AuBrWh_PLINK, + .dentry = NULL + }, + [AuBrWh_ORPH] = { + .name = base_name + AuBrWh_ORPH, + .dentry = NULL + } + }; + + if (wbr) + WbrWhMustWriteLock(wbr); + + for (i = 0; i < AuBrWh_Last; i++) { + /* doubly whiteouted */ + struct dentry *d; + + d = au_wh_lkup(h_root, (void *)base[i].name, br); + err = PTR_ERR(d); + if (IS_ERR(d)) + goto out; + + base[i].dentry = d; + AuDebugOn(wbr + && wbr->wbr_wh[i] + && wbr->wbr_wh[i] != base[i].dentry); + } + + if (wbr) + for (i = 0; i < AuBrWh_Last; i++) { + dput(wbr->wbr_wh[i]); + wbr->wbr_wh[i] = NULL; + } + + err = 0; + if (!au_br_writable(br->br_perm)) { + h_dir = d_inode(h_root); + au_wh_init_ro(h_dir, base, &path); + } else if (!au_br_wh_linkable(br->br_perm)) { + err = au_wh_init_rw_nolink(h_root, wbr, do_plink, base, &path); + if (err > 0) + goto out; + else if (err) + goto out_err; + } else { + err = au_wh_init_rw(h_root, wbr, do_plink, base, &path); + if (err > 0) + goto out; + else if (err) + goto out_err; + } + goto out; /* success */ + +out_err: + pr_err("an error(%d) on the writable branch %pd(%s)\n", + err, h_root, au_sbtype(h_root->d_sb)); +out: + for (i = 0; i < AuBrWh_Last; i++) + dput(base[i].dentry); + return err; +} + +/* ---------------------------------------------------------------------- */ +/* + * whiteouts are all hard-linked usually. + * when its link count reaches a ceiling, we create a new whiteout base + * asynchronously. + */ + +struct reinit_br_wh { + struct super_block *sb; + struct au_branch *br; +}; + +static void reinit_br_wh(void *arg) +{ + int err; + aufs_bindex_t bindex; + struct path h_path; + struct reinit_br_wh *a = arg; + struct au_wbr *wbr; + struct inode *dir, *delegated; + struct dentry *h_root; + struct au_hinode *hdir; + + err = 0; + wbr = a->br->br_wbr; + /* big aufs lock */ + si_noflush_write_lock(a->sb); + if (!au_br_writable(a->br->br_perm)) + goto out; + bindex = au_br_index(a->sb, a->br->br_id); + if (unlikely(bindex < 0)) + goto out; + + di_read_lock_parent(a->sb->s_root, AuLock_IR); + dir = d_inode(a->sb->s_root); + hdir = au_hi(dir, bindex); + h_root = au_h_dptr(a->sb->s_root, bindex); + AuDebugOn(h_root != au_br_dentry(a->br)); + + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + wbr_wh_write_lock(wbr); + err = au_h_verify(wbr->wbr_whbase, au_opt_udba(a->sb), hdir->hi_inode, + h_root, a->br); + if (!err) { + h_path.dentry = wbr->wbr_whbase; + h_path.mnt = au_br_mnt(a->br); + delegated = NULL; + err = vfsub_unlink(hdir->hi_inode, &h_path, &delegated, + /*force*/0); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + } else { + pr_warn("%pd is moved, ignored\n", wbr->wbr_whbase); + err = 0; + } + dput(wbr->wbr_whbase); + wbr->wbr_whbase = NULL; + if (!err) + err = au_wh_init(a->br, a->sb); + wbr_wh_write_unlock(wbr); + au_hn_inode_unlock(hdir); + di_read_unlock(a->sb->s_root, AuLock_IR); + if (!err) + au_fhsm_wrote(a->sb, bindex, /*force*/0); + +out: + if (wbr) + atomic_dec(&wbr->wbr_wh_running); + au_lcnt_dec(&a->br->br_count); + si_write_unlock(a->sb); + au_nwt_done(&au_sbi(a->sb)->si_nowait); + au_kfree_rcu(a); + if (unlikely(err)) + AuIOErr("err %d\n", err); +} + +static void kick_reinit_br_wh(struct super_block *sb, struct au_branch *br) +{ + int do_dec, wkq_err; + struct reinit_br_wh *arg; + + do_dec = 1; + if (atomic_inc_return(&br->br_wbr->wbr_wh_running) != 1) + goto out; + + /* ignore ENOMEM */ + arg = kmalloc(sizeof(*arg), GFP_NOFS); + if (arg) { + /* + * dec(wh_running), kfree(arg) and dec(br_count) + * in reinit function + */ + arg->sb = sb; + arg->br = br; + au_lcnt_inc(&br->br_count); + wkq_err = au_wkq_nowait(reinit_br_wh, arg, sb, /*flags*/0); + if (unlikely(wkq_err)) { + atomic_dec(&br->br_wbr->wbr_wh_running); + au_lcnt_dec(&br->br_count); + au_kfree_rcu(arg); + } + do_dec = 0; + } + +out: + if (do_dec) + atomic_dec(&br->br_wbr->wbr_wh_running); +} + +/* ---------------------------------------------------------------------- */ + +/* + * create the whiteout @wh. + */ +static int link_or_create_wh(struct super_block *sb, aufs_bindex_t bindex, + struct dentry *wh) +{ + int err; + struct path h_path = { + .dentry = wh + }; + struct au_branch *br; + struct au_wbr *wbr; + struct dentry *h_parent; + struct inode *h_dir, *delegated; + + h_parent = wh->d_parent; /* dir inode is locked */ + h_dir = d_inode(h_parent); + IMustLock(h_dir); + + br = au_sbr(sb, bindex); + h_path.mnt = au_br_mnt(br); + wbr = br->br_wbr; + wbr_wh_read_lock(wbr); + if (wbr->wbr_whbase) { + delegated = NULL; + err = vfsub_link(wbr->wbr_whbase, h_dir, &h_path, &delegated); + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal link\n"); + iput(delegated); + } + if (!err || err != -EMLINK) + goto out; + + /* link count full. re-initialize br_whbase. */ + kick_reinit_br_wh(sb, br); + } + + /* return this error in this context */ + err = vfsub_create(h_dir, &h_path, WH_MASK, /*want_excl*/true); + if (!err) + au_fhsm_wrote(sb, bindex, /*force*/0); + +out: + wbr_wh_read_unlock(wbr); + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * create or remove the diropq. + */ +static struct dentry *do_diropq(struct dentry *dentry, aufs_bindex_t bindex, + unsigned int flags) +{ + struct dentry *opq_dentry, *h_dentry; + struct super_block *sb; + struct au_branch *br; + int err; + + sb = dentry->d_sb; + br = au_sbr(sb, bindex); + h_dentry = au_h_dptr(dentry, bindex); + opq_dentry = vfsub_lkup_one(&diropq_name, h_dentry); + if (IS_ERR(opq_dentry)) + goto out; + + if (au_ftest_diropq(flags, CREATE)) { + err = link_or_create_wh(sb, bindex, opq_dentry); + if (!err) { + au_set_dbdiropq(dentry, bindex); + goto out; /* success */ + } + } else { + struct path tmp = { + .dentry = opq_dentry, + .mnt = au_br_mnt(br) + }; + err = do_unlink_wh(au_h_iptr(d_inode(dentry), bindex), &tmp); + if (!err) + au_set_dbdiropq(dentry, -1); + } + dput(opq_dentry); + opq_dentry = ERR_PTR(err); + +out: + return opq_dentry; +} + +struct do_diropq_args { + struct dentry **errp; + struct dentry *dentry; + aufs_bindex_t bindex; + unsigned int flags; +}; + +static void call_do_diropq(void *args) +{ + struct do_diropq_args *a = args; + *a->errp = do_diropq(a->dentry, a->bindex, a->flags); +} + +struct dentry *au_diropq_sio(struct dentry *dentry, aufs_bindex_t bindex, + unsigned int flags) +{ + struct dentry *diropq, *h_dentry; + + h_dentry = au_h_dptr(dentry, bindex); + if (!au_test_h_perm_sio(d_inode(h_dentry), MAY_EXEC | MAY_WRITE)) + diropq = do_diropq(dentry, bindex, flags); + else { + int wkq_err; + struct do_diropq_args args = { + .errp = &diropq, + .dentry = dentry, + .bindex = bindex, + .flags = flags + }; + + wkq_err = au_wkq_wait(call_do_diropq, &args); + if (unlikely(wkq_err)) + diropq = ERR_PTR(wkq_err); + } + + return diropq; +} + +/* ---------------------------------------------------------------------- */ + +/* + * lookup whiteout dentry. + * @h_parent: lower parent dentry which must exist and be locked + * @base_name: name of dentry which will be whiteouted + * returns dentry for whiteout. + */ +struct dentry *au_wh_lkup(struct dentry *h_parent, struct qstr *base_name, + struct au_branch *br) +{ + int err; + struct qstr wh_name; + struct dentry *wh_dentry; + + err = au_wh_name_alloc(&wh_name, base_name); + wh_dentry = ERR_PTR(err); + if (!err) { + wh_dentry = vfsub_lkup_one(&wh_name, h_parent); + au_kfree_try_rcu(wh_name.name); + } + return wh_dentry; +} + +/* + * link/create a whiteout for @dentry on @bindex. + */ +struct dentry *au_wh_create(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent) +{ + struct dentry *wh_dentry; + struct super_block *sb; + int err; + + sb = dentry->d_sb; + wh_dentry = au_wh_lkup(h_parent, &dentry->d_name, au_sbr(sb, bindex)); + if (!IS_ERR(wh_dentry) && d_is_negative(wh_dentry)) { + err = link_or_create_wh(sb, bindex, wh_dentry); + if (!err) { + au_set_dbwh(dentry, bindex); + au_fhsm_wrote(sb, bindex, /*force*/0); + } else { + dput(wh_dentry); + wh_dentry = ERR_PTR(err); + } + } + + return wh_dentry; +} + +/* ---------------------------------------------------------------------- */ + +/* Delete all whiteouts in this directory on branch bindex. */ +static int del_wh_children(struct dentry *h_dentry, struct au_nhash *whlist, + aufs_bindex_t bindex, struct au_branch *br) +{ + int err; + unsigned long ul, n; + struct qstr wh_name; + char *p; + struct hlist_head *head; + struct au_vdir_wh *pos; + struct au_vdir_destr *str; + + err = -ENOMEM; + p = (void *)__get_free_page(GFP_NOFS); + wh_name.name = p; + if (unlikely(!wh_name.name)) + goto out; + + err = 0; + memcpy(p, AUFS_WH_PFX, AUFS_WH_PFX_LEN); + p += AUFS_WH_PFX_LEN; + n = whlist->nh_num; + head = whlist->nh_head; + for (ul = 0; !err && ul < n; ul++, head++) { + hlist_for_each_entry(pos, head, wh_hash) { + if (pos->wh_bindex != bindex) + continue; + + str = &pos->wh_str; + if (str->len + AUFS_WH_PFX_LEN <= PATH_MAX) { + memcpy(p, str->name, str->len); + wh_name.len = AUFS_WH_PFX_LEN + str->len; + err = unlink_wh_name(h_dentry, &wh_name, br); + if (!err) + continue; + break; + } + AuIOErr("whiteout name too long %.*s\n", + str->len, str->name); + err = -EIO; + break; + } + } + free_page((unsigned long)wh_name.name); + +out: + return err; +} + +struct del_wh_children_args { + int *errp; + struct dentry *h_dentry; + struct au_nhash *whlist; + aufs_bindex_t bindex; + struct au_branch *br; +}; + +static void call_del_wh_children(void *args) +{ + struct del_wh_children_args *a = args; + *a->errp = del_wh_children(a->h_dentry, a->whlist, a->bindex, a->br); +} + +/* ---------------------------------------------------------------------- */ + +struct au_whtmp_rmdir *au_whtmp_rmdir_alloc(struct super_block *sb, gfp_t gfp) +{ + struct au_whtmp_rmdir *whtmp; + int err; + unsigned int rdhash; + + SiMustAnyLock(sb); + + whtmp = kzalloc(sizeof(*whtmp), gfp); + if (unlikely(!whtmp)) { + whtmp = ERR_PTR(-ENOMEM); + goto out; + } + + /* no estimation for dir size */ + rdhash = au_sbi(sb)->si_rdhash; + if (!rdhash) + rdhash = AUFS_RDHASH_DEF; + err = au_nhash_alloc(&whtmp->whlist, rdhash, gfp); + if (unlikely(err)) { + au_kfree_rcu(whtmp); + whtmp = ERR_PTR(err); + } + +out: + return whtmp; +} + +void au_whtmp_rmdir_free(struct au_whtmp_rmdir *whtmp) +{ + if (whtmp->br) + au_lcnt_dec(&whtmp->br->br_count); + dput(whtmp->wh_dentry); + iput(whtmp->dir); + au_nhash_wh_free(&whtmp->whlist); + au_kfree_rcu(whtmp); +} + +/* + * rmdir the whiteouted temporary named dir @h_dentry. + * @whlist: whiteouted children. + */ +int au_whtmp_rmdir(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct au_nhash *whlist) +{ + int err; + unsigned int h_nlink; + struct path h_tmp; + struct inode *wh_inode, *h_dir; + struct au_branch *br; + + h_dir = d_inode(wh_dentry->d_parent); /* dir inode is locked */ + IMustLock(h_dir); + + br = au_sbr(dir->i_sb, bindex); + wh_inode = d_inode(wh_dentry); + inode_lock_nested(wh_inode, AuLsc_I_CHILD); + + /* + * someone else might change some whiteouts while we were sleeping. + * it means this whlist may have an obsoleted entry. + */ + if (!au_test_h_perm_sio(wh_inode, MAY_EXEC | MAY_WRITE)) + err = del_wh_children(wh_dentry, whlist, bindex, br); + else { + int wkq_err; + struct del_wh_children_args args = { + .errp = &err, + .h_dentry = wh_dentry, + .whlist = whlist, + .bindex = bindex, + .br = br + }; + + wkq_err = au_wkq_wait(call_del_wh_children, &args); + if (unlikely(wkq_err)) + err = wkq_err; + } + inode_unlock(wh_inode); + + if (!err) { + h_tmp.dentry = wh_dentry; + h_tmp.mnt = au_br_mnt(br); + h_nlink = h_dir->i_nlink; + err = vfsub_rmdir(h_dir, &h_tmp); + /* some fs doesn't change the parent nlink in some cases */ + h_nlink -= h_dir->i_nlink; + } + + if (!err) { + if (au_ibtop(dir) == bindex) { + /* todo: dir->i_mutex is necessary */ + au_cpup_attr_timesizes(dir); + if (h_nlink) + vfsub_drop_nlink(dir); + } + return 0; /* success */ + } + + pr_warn("failed removing %pd(%d), ignored\n", wh_dentry, err); + return err; +} + +static void call_rmdir_whtmp(void *args) +{ + int err; + aufs_bindex_t bindex; + struct au_whtmp_rmdir *a = args; + struct super_block *sb; + struct dentry *h_parent; + struct inode *h_dir; + struct au_hinode *hdir; + + /* rmdir by nfsd may cause deadlock with this i_mutex */ + /* inode_lock(a->dir); */ + err = -EROFS; + sb = a->dir->i_sb; + si_read_lock(sb, !AuLock_FLUSH); + if (!au_br_writable(a->br->br_perm)) + goto out; + bindex = au_br_index(sb, a->br->br_id); + if (unlikely(bindex < 0)) + goto out; + + err = -EIO; + ii_write_lock_parent(a->dir); + h_parent = dget_parent(a->wh_dentry); + h_dir = d_inode(h_parent); + hdir = au_hi(a->dir, bindex); + err = vfsub_mnt_want_write(au_br_mnt(a->br)); + if (unlikely(err)) + goto out_mnt; + au_hn_inode_lock_nested(hdir, AuLsc_I_PARENT); + err = au_h_verify(a->wh_dentry, au_opt_udba(sb), h_dir, h_parent, + a->br); + if (!err) + err = au_whtmp_rmdir(a->dir, bindex, a->wh_dentry, &a->whlist); + au_hn_inode_unlock(hdir); + vfsub_mnt_drop_write(au_br_mnt(a->br)); + +out_mnt: + dput(h_parent); + ii_write_unlock(a->dir); +out: + /* inode_unlock(a->dir); */ + au_whtmp_rmdir_free(a); + si_read_unlock(sb); + au_nwt_done(&au_sbi(sb)->si_nowait); + if (unlikely(err)) + AuIOErr("err %d\n", err); +} + +void au_whtmp_kick_rmdir(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct au_whtmp_rmdir *args) +{ + int wkq_err; + struct super_block *sb; + + IMustLock(dir); + + /* all post-process will be done in do_rmdir_whtmp(). */ + sb = dir->i_sb; + args->dir = au_igrab(dir); + args->br = au_sbr(sb, bindex); + au_lcnt_inc(&args->br->br_count); + args->wh_dentry = dget(wh_dentry); + wkq_err = au_wkq_nowait(call_rmdir_whtmp, args, sb, /*flags*/0); + if (unlikely(wkq_err)) { + pr_warn("rmdir error %pd (%d), ignored\n", wh_dentry, wkq_err); + au_whtmp_rmdir_free(args); + } +} diff --git a/fs/aufs/whout.h b/fs/aufs/whout.h new file mode 100644 index 0000000000000..65459ba0e7907 --- /dev/null +++ b/fs/aufs/whout.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * whiteout for logical deletion and opaque directory + */ + +#ifndef __AUFS_WHOUT_H__ +#define __AUFS_WHOUT_H__ + +#ifdef __KERNEL__ + +#include "dir.h" + +/* whout.c */ +int au_wh_name_alloc(struct qstr *wh, const struct qstr *name); +int au_wh_test(struct dentry *h_parent, struct qstr *wh_name, int try_sio); +int au_diropq_test(struct dentry *h_dentry); +struct au_branch; +struct dentry *au_whtmp_lkup(struct dentry *h_parent, struct au_branch *br, + struct qstr *prefix); +int au_whtmp_ren(struct dentry *h_dentry, struct au_branch *br); +int au_wh_unlink_dentry(struct inode *h_dir, struct path *h_path, + struct dentry *dentry); +int au_wh_init(struct au_branch *br, struct super_block *sb); + +/* diropq flags */ +#define AuDiropq_CREATE 1 +#define au_ftest_diropq(flags, name) ((flags) & AuDiropq_##name) +#define au_fset_diropq(flags, name) \ + do { (flags) |= AuDiropq_##name; } while (0) +#define au_fclr_diropq(flags, name) \ + do { (flags) &= ~AuDiropq_##name; } while (0) + +struct dentry *au_diropq_sio(struct dentry *dentry, aufs_bindex_t bindex, + unsigned int flags); +struct dentry *au_wh_lkup(struct dentry *h_parent, struct qstr *base_name, + struct au_branch *br); +struct dentry *au_wh_create(struct dentry *dentry, aufs_bindex_t bindex, + struct dentry *h_parent); + +/* real rmdir for the whiteout-ed dir */ +struct au_whtmp_rmdir { + struct inode *dir; + struct au_branch *br; + struct dentry *wh_dentry; + struct au_nhash whlist; +}; + +struct au_whtmp_rmdir *au_whtmp_rmdir_alloc(struct super_block *sb, gfp_t gfp); +void au_whtmp_rmdir_free(struct au_whtmp_rmdir *whtmp); +int au_whtmp_rmdir(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct au_nhash *whlist); +void au_whtmp_kick_rmdir(struct inode *dir, aufs_bindex_t bindex, + struct dentry *wh_dentry, struct au_whtmp_rmdir *args); + +/* ---------------------------------------------------------------------- */ + +static inline struct dentry *au_diropq_create(struct dentry *dentry, + aufs_bindex_t bindex) +{ + return au_diropq_sio(dentry, bindex, AuDiropq_CREATE); +} + +static inline int au_diropq_remove(struct dentry *dentry, aufs_bindex_t bindex) +{ + return PTR_ERR(au_diropq_sio(dentry, bindex, !AuDiropq_CREATE)); +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_WHOUT_H__ */ diff --git a/fs/aufs/wkq.c b/fs/aufs/wkq.c new file mode 100644 index 0000000000000..9d92e1fd2e4bf --- /dev/null +++ b/fs/aufs/wkq.c @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * workqueue for asynchronous/super-io operations + * todo: try new credential scheme + */ + +#include +#include "aufs.h" + +/* internal workqueue named AUFS_WKQ_NAME */ + +static struct workqueue_struct *au_wkq; + +struct au_wkinfo { + struct work_struct wk; + struct kobject *kobj; + + unsigned int flags; /* see wkq.h */ + + au_wkq_func_t func; + void *args; + +#ifdef CONFIG_LOCKDEP + int dont_check; + struct held_lock **hlock; +#endif + + struct completion *comp; +}; + +/* ---------------------------------------------------------------------- */ +/* + * Aufs passes some operations to the workqueue such as the internal copyup. + * This scheme looks rather unnatural for LOCKDEP debugging feature, since the + * job run by workqueue depends upon the locks acquired in the other task. + * Delegating a small operation to the workqueue, aufs passes its lockdep + * information too. And the job in the workqueue restores the info in order to + * pretend as if it acquired those locks. This is just to make LOCKDEP work + * correctly and expectedly. + */ + +#ifndef CONFIG_LOCKDEP +AuStubInt0(au_wkq_lockdep_alloc, struct au_wkinfo *wkinfo); +AuStubVoid(au_wkq_lockdep_free, struct au_wkinfo *wkinfo); +AuStubVoid(au_wkq_lockdep_pre, struct au_wkinfo *wkinfo); +AuStubVoid(au_wkq_lockdep_post, struct au_wkinfo *wkinfo); +AuStubVoid(au_wkq_lockdep_init, struct au_wkinfo *wkinfo); +#else +static void au_wkq_lockdep_init(struct au_wkinfo *wkinfo) +{ + wkinfo->hlock = NULL; + wkinfo->dont_check = 0; +} + +/* + * 1: matched + * 0: unmatched + */ +static int au_wkq_lockdep_test(struct lock_class_key *key, const char *name) +{ + static DEFINE_SPINLOCK(spin); + static struct { + char *name; + struct lock_class_key *key; + } a[] = { + { .name = "&sbinfo->si_rwsem" }, + { .name = "&finfo->fi_rwsem" }, + { .name = "&dinfo->di_rwsem" }, + { .name = "&iinfo->ii_rwsem" } + }; + static int set; + int i; + + /* lockless read from 'set.' see below */ + if (set == ARRAY_SIZE(a)) { + for (i = 0; i < ARRAY_SIZE(a); i++) + if (a[i].key == key) + goto match; + goto unmatch; + } + + spin_lock(&spin); + if (set) + for (i = 0; i < ARRAY_SIZE(a); i++) + if (a[i].key == key) { + spin_unlock(&spin); + goto match; + } + for (i = 0; i < ARRAY_SIZE(a); i++) { + if (a[i].key) { + if (unlikely(a[i].key == key)) { /* rare but possible */ + spin_unlock(&spin); + goto match; + } else + continue; + } + if (strstr(a[i].name, name)) { + /* + * the order of these three lines is important for the + * lockless read above. + */ + a[i].key = key; + spin_unlock(&spin); + set++; + /* AuDbg("%d, %s\n", set, name); */ + goto match; + } + } + spin_unlock(&spin); + goto unmatch; + +match: + return 1; +unmatch: + return 0; +} + +static int au_wkq_lockdep_alloc(struct au_wkinfo *wkinfo) +{ + int err, n; + struct task_struct *curr; + struct held_lock **hl, *held_locks, *p; + + err = 0; + curr = current; + wkinfo->dont_check = lockdep_recursing(curr); + if (wkinfo->dont_check) + goto out; + n = curr->lockdep_depth; + if (!n) + goto out; + + err = -ENOMEM; + wkinfo->hlock = kmalloc_array(n + 1, sizeof(*wkinfo->hlock), GFP_NOFS); + if (unlikely(!wkinfo->hlock)) + goto out; + + err = 0; +#if 0 /* left for debugging */ + if (0 && au_debug_test()) + lockdep_print_held_locks(curr); +#endif + held_locks = curr->held_locks; + hl = wkinfo->hlock; + while (n--) { + p = held_locks++; + if (au_wkq_lockdep_test(p->instance->key, p->instance->name)) + *hl++ = p; + } + *hl = NULL; + +out: + return err; +} + +static void au_wkq_lockdep_free(struct au_wkinfo *wkinfo) +{ + au_kfree_try_rcu(wkinfo->hlock); +} + +static void au_wkq_lockdep_pre(struct au_wkinfo *wkinfo) +{ + struct held_lock *p, **hl = wkinfo->hlock; + int subclass; + + if (wkinfo->dont_check) + lockdep_off(); + if (!hl) + return; + while ((p = *hl++)) { /* assignment */ + subclass = lockdep_hlock_class(p)->subclass; + /* AuDbg("%s, %d\n", p->instance->name, subclass); */ + if (p->read) + rwsem_acquire_read(p->instance, subclass, 0, + /*p->acquire_ip*/_RET_IP_); + else + rwsem_acquire(p->instance, subclass, 0, + /*p->acquire_ip*/_RET_IP_); + } +} + +static void au_wkq_lockdep_post(struct au_wkinfo *wkinfo) +{ + struct held_lock *p, **hl = wkinfo->hlock; + + if (wkinfo->dont_check) + lockdep_on(); + if (!hl) + return; + while ((p = *hl++)) /* assignment */ + rwsem_release(p->instance, 0, /*p->acquire_ip*/_RET_IP_); +} +#endif + +static void wkq_func(struct work_struct *wk) +{ + struct au_wkinfo *wkinfo = container_of(wk, struct au_wkinfo, wk); + + AuDebugOn(!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)); + AuDebugOn(rlimit(RLIMIT_FSIZE) != RLIM_INFINITY); + + au_wkq_lockdep_pre(wkinfo); + wkinfo->func(wkinfo->args); + au_wkq_lockdep_post(wkinfo); + if (au_ftest_wkq(wkinfo->flags, WAIT)) + complete(wkinfo->comp); + else { + kobject_put(wkinfo->kobj); + module_put(THIS_MODULE); /* todo: ?? */ + au_kfree_rcu(wkinfo); + } +} + +/* + * Since struct completion is large, try allocating it dynamically. + */ +#define AuWkqCompDeclare(name) struct completion *comp = NULL + +static int au_wkq_comp_alloc(struct au_wkinfo *wkinfo, struct completion **comp) +{ + *comp = kmalloc(sizeof(**comp), GFP_NOFS); + if (*comp) { + init_completion(*comp); + wkinfo->comp = *comp; + return 0; + } + return -ENOMEM; +} + +static void au_wkq_comp_free(struct completion *comp) +{ + au_kfree_rcu(comp); +} + +static void au_wkq_run(struct au_wkinfo *wkinfo) +{ + if (au_ftest_wkq(wkinfo->flags, NEST)) { + if (au_wkq_test()) { + AuWarn1("wkq from wkq, unless silly-rename on NFS," + " due to a dead dir by UDBA," + " or async xino write?\n"); + AuDebugOn(au_ftest_wkq(wkinfo->flags, WAIT)); + } + } else + au_dbg_verify_kthread(); + + if (au_ftest_wkq(wkinfo->flags, WAIT)) { + INIT_WORK_ONSTACK(&wkinfo->wk, wkq_func); + queue_work(au_wkq, &wkinfo->wk); + } else { + INIT_WORK(&wkinfo->wk, wkq_func); + schedule_work(&wkinfo->wk); + } +} + +/* + * Be careful. It is easy to make deadlock happen. + * processA: lock, wkq and wait + * processB: wkq and wait, lock in wkq + * --> deadlock + */ +int au_wkq_do_wait(unsigned int flags, au_wkq_func_t func, void *args) +{ + int err; + AuWkqCompDeclare(comp); + struct au_wkinfo wkinfo = { + .flags = flags, + .func = func, + .args = args + }; + + err = au_wkq_comp_alloc(&wkinfo, &comp); + if (unlikely(err)) + goto out; + err = au_wkq_lockdep_alloc(&wkinfo); + if (unlikely(err)) + goto out_comp; + if (!err) { + au_wkq_run(&wkinfo); + /* no timeout, no interrupt */ + wait_for_completion(wkinfo.comp); + } + au_wkq_lockdep_free(&wkinfo); + +out_comp: + au_wkq_comp_free(comp); +out: + destroy_work_on_stack(&wkinfo.wk); + return err; +} + +/* + * Note: dget/dput() in func for aufs dentries are not supported. It will be a + * problem in a concurrent umounting. + */ +int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, + unsigned int flags) +{ + int err; + struct au_wkinfo *wkinfo; + + atomic_inc(&au_sbi(sb)->si_nowait.nw_len); + + /* + * wkq_func() must free this wkinfo. + * it highly depends upon the implementation of workqueue. + */ + err = 0; + wkinfo = kmalloc(sizeof(*wkinfo), GFP_NOFS); + if (wkinfo) { + wkinfo->kobj = &au_sbi(sb)->si_kobj; + wkinfo->flags = flags & ~AuWkq_WAIT; + wkinfo->func = func; + wkinfo->args = args; + wkinfo->comp = NULL; + au_wkq_lockdep_init(wkinfo); + kobject_get(wkinfo->kobj); + __module_get(THIS_MODULE); /* todo: ?? */ + + au_wkq_run(wkinfo); + } else { + err = -ENOMEM; + au_nwt_done(&au_sbi(sb)->si_nowait); + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +void au_nwt_init(struct au_nowait_tasks *nwt) +{ + atomic_set(&nwt->nw_len, 0); + /* smp_mb(); */ /* atomic_set */ + init_waitqueue_head(&nwt->nw_wq); +} + +void au_wkq_fin(void) +{ + destroy_workqueue(au_wkq); +} + +int __init au_wkq_init(void) +{ + int err; + + err = 0; + au_wkq = alloc_workqueue(AUFS_WKQ_NAME, 0, WQ_DFL_ACTIVE); + if (IS_ERR(au_wkq)) + err = PTR_ERR(au_wkq); + else if (!au_wkq) + err = -ENOMEM; + + return err; +} diff --git a/fs/aufs/wkq.h b/fs/aufs/wkq.h new file mode 100644 index 0000000000000..5be76b69d8ffd --- /dev/null +++ b/fs/aufs/wkq.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * workqueue for asynchronous/super-io operations + * todo: try new credentials management scheme + */ + +#ifndef __AUFS_WKQ_H__ +#define __AUFS_WKQ_H__ + +#ifdef __KERNEL__ + +#include + +struct super_block; + +/* ---------------------------------------------------------------------- */ + +/* + * in the next operation, wait for the 'nowait' tasks in system-wide workqueue + */ +struct au_nowait_tasks { + atomic_t nw_len; + wait_queue_head_t nw_wq; +}; + +/* ---------------------------------------------------------------------- */ + +typedef void (*au_wkq_func_t)(void *args); + +/* wkq flags */ +#define AuWkq_WAIT 1 +#define AuWkq_NEST (1 << 1) +#define au_ftest_wkq(flags, name) ((flags) & AuWkq_##name) +#define au_fset_wkq(flags, name) \ + do { (flags) |= AuWkq_##name; } while (0) +#define au_fclr_wkq(flags, name) \ + do { (flags) &= ~AuWkq_##name; } while (0) + +/* wkq.c */ +int au_wkq_do_wait(unsigned int flags, au_wkq_func_t func, void *args); +int au_wkq_nowait(au_wkq_func_t func, void *args, struct super_block *sb, + unsigned int flags); +void au_nwt_init(struct au_nowait_tasks *nwt); +int __init au_wkq_init(void); +void au_wkq_fin(void); + +/* ---------------------------------------------------------------------- */ + +static inline int au_wkq_test(void) +{ + return current->flags & PF_WQ_WORKER; +} + +static inline int au_wkq_wait(au_wkq_func_t func, void *args) +{ + return au_wkq_do_wait(AuWkq_WAIT, func, args); +} + +static inline void au_nwt_done(struct au_nowait_tasks *nwt) +{ + if (atomic_dec_and_test(&nwt->nw_len)) + wake_up_all(&nwt->nw_wq); +} + +static inline int au_nwt_flush(struct au_nowait_tasks *nwt) +{ + wait_event(nwt->nw_wq, !atomic_read(&nwt->nw_len)); + return 0; +} + +#endif /* __KERNEL__ */ +#endif /* __AUFS_WKQ_H__ */ diff --git a/fs/aufs/xattr.c b/fs/aufs/xattr.c new file mode 100644 index 0000000000000..103c4275e5fcb --- /dev/null +++ b/fs/aufs/xattr.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2014-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * handling xattr functions + */ + +#include +#include +#include +#include "aufs.h" + +static int au_xattr_ignore(int err, char *name, unsigned int ignore_flags) +{ + if (!ignore_flags) + goto out; + switch (err) { + case -ENOMEM: + case -EDQUOT: + goto out; + } + + if ((ignore_flags & AuBrAttr_ICEX) == AuBrAttr_ICEX) { + err = 0; + goto out; + } + +#define cmp(brattr, prefix) do { \ + if (!strncmp(name, XATTR_##prefix##_PREFIX, \ + XATTR_##prefix##_PREFIX_LEN)) { \ + if (ignore_flags & AuBrAttr_ICEX_##brattr) \ + err = 0; \ + goto out; \ + } \ + } while (0) + + cmp(SEC, SECURITY); + cmp(SYS, SYSTEM); + cmp(TR, TRUSTED); + cmp(USR, USER); +#undef cmp + + if (ignore_flags & AuBrAttr_ICEX_OTH) + err = 0; + +out: + return err; +} + +static const int au_xattr_out_of_list = AuBrAttr_ICEX_OTH << 1; + +static int au_do_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, + char *name, char **buf, unsigned int ignore_flags, + unsigned int verbose) +{ + int err; + ssize_t ssz; + struct inode *h_idst; + + ssz = vfs_getxattr_alloc(h_src, name, buf, 0, GFP_NOFS); + err = ssz; + if (unlikely(err <= 0)) { + if (err == -ENODATA + || (err == -EOPNOTSUPP + && ((ignore_flags & au_xattr_out_of_list) + || (au_test_nfs_noacl(d_inode(h_src)) + && (!strcmp(name, XATTR_NAME_POSIX_ACL_ACCESS) + || !strcmp(name, + XATTR_NAME_POSIX_ACL_DEFAULT)))) + )) + err = 0; + if (err && (verbose || au_debug_test())) + pr_err("%s, err %d\n", name, err); + goto out; + } + + /* unlock it temporary */ + h_idst = d_inode(h_dst); + inode_unlock(h_idst); + err = vfsub_setxattr(h_dst, name, *buf, ssz, /*flags*/0); + inode_lock_nested(h_idst, AuLsc_I_CHILD2); + if (unlikely(err)) { + if (verbose || au_debug_test()) + pr_err("%s, err %d\n", name, err); + err = au_xattr_ignore(err, name, ignore_flags); + } + +out: + return err; +} + +int au_cpup_xattr(struct dentry *h_dst, struct dentry *h_src, int ignore_flags, + unsigned int verbose) +{ + int err, unlocked, acl_access, acl_default; + ssize_t ssz; + struct inode *h_isrc, *h_idst; + char *value, *p, *o, *e; + + /* try stopping to update the source inode while we are referencing */ + /* there should not be the parent-child relationship between them */ + h_isrc = d_inode(h_src); + h_idst = d_inode(h_dst); + inode_unlock(h_idst); + inode_lock_shared_nested(h_isrc, AuLsc_I_CHILD); + inode_lock_nested(h_idst, AuLsc_I_CHILD2); + unlocked = 0; + + /* some filesystems don't list POSIX ACL, for example tmpfs */ + ssz = vfs_listxattr(h_src, NULL, 0); + err = ssz; + if (unlikely(err < 0)) { + AuTraceErr(err); + if (err == -ENODATA + || err == -EOPNOTSUPP) + err = 0; /* ignore */ + goto out; + } + + err = 0; + p = NULL; + o = NULL; + if (ssz) { + err = -ENOMEM; + p = kmalloc(ssz, GFP_NOFS); + o = p; + if (unlikely(!p)) + goto out; + err = vfs_listxattr(h_src, p, ssz); + } + inode_unlock_shared(h_isrc); + unlocked = 1; + AuDbg("err %d, ssz %zd\n", err, ssz); + if (unlikely(err < 0)) + goto out_free; + + err = 0; + e = p + ssz; + value = NULL; + acl_access = 0; + acl_default = 0; + while (!err && p < e) { + acl_access |= !strncmp(p, XATTR_NAME_POSIX_ACL_ACCESS, + sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1); + acl_default |= !strncmp(p, XATTR_NAME_POSIX_ACL_DEFAULT, + sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) + - 1); + err = au_do_cpup_xattr(h_dst, h_src, p, &value, ignore_flags, + verbose); + p += strlen(p) + 1; + } + AuTraceErr(err); + ignore_flags |= au_xattr_out_of_list; + if (!err && !acl_access) { + err = au_do_cpup_xattr(h_dst, h_src, + XATTR_NAME_POSIX_ACL_ACCESS, &value, + ignore_flags, verbose); + AuTraceErr(err); + } + if (!err && !acl_default) { + err = au_do_cpup_xattr(h_dst, h_src, + XATTR_NAME_POSIX_ACL_DEFAULT, &value, + ignore_flags, verbose); + AuTraceErr(err); + } + + au_kfree_try_rcu(value); + +out_free: + au_kfree_try_rcu(o); +out: + if (!unlocked) + inode_unlock_shared(h_isrc); + AuTraceErr(err); + return err; +} + +/* ---------------------------------------------------------------------- */ + +static int au_smack_reentering(struct super_block *sb) +{ +#if IS_ENABLED(CONFIG_SECURITY_SMACK) || IS_ENABLED(CONFIG_SECURITY_SELINUX) + /* + * as a part of lookup, smack_d_instantiate() is called, and it calls + * i_op->getxattr(). ouch. + */ + return si_pid_test(sb); +#else + return 0; +#endif +} + +enum { + AU_XATTR_LIST, + AU_XATTR_GET +}; + +struct au_lgxattr { + int type; + union { + struct { + char *list; + size_t size; + } list; + struct { + const char *name; + void *value; + size_t size; + } get; + } u; +}; + +static ssize_t au_lgxattr(struct dentry *dentry, struct inode *inode, + struct au_lgxattr *arg) +{ + ssize_t err; + int reenter; + struct path h_path; + struct super_block *sb; + + sb = dentry->d_sb; + reenter = au_smack_reentering(sb); + if (!reenter) { + err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM); + if (unlikely(err)) + goto out; + } + err = au_h_path_getattr(dentry, inode, /*force*/1, &h_path, reenter); + if (unlikely(err)) + goto out_si; + if (unlikely(!h_path.dentry)) + /* illegally overlapped or something */ + goto out_di; /* pretending success */ + + /* always topmost entry only */ + switch (arg->type) { + case AU_XATTR_LIST: + err = vfs_listxattr(h_path.dentry, + arg->u.list.list, arg->u.list.size); + break; + case AU_XATTR_GET: + AuDebugOn(d_is_negative(h_path.dentry)); + err = vfs_getxattr(h_path.dentry, + arg->u.get.name, arg->u.get.value, + arg->u.get.size); + break; + } + +out_di: + if (!reenter) + di_read_unlock(dentry, AuLock_IR); +out_si: + if (!reenter) + si_read_unlock(sb); +out: + AuTraceErr(err); + return err; +} + +ssize_t aufs_listxattr(struct dentry *dentry, char *list, size_t size) +{ + struct au_lgxattr arg = { + .type = AU_XATTR_LIST, + .u.list = { + .list = list, + .size = size + }, + }; + + return au_lgxattr(dentry, /*inode*/NULL, &arg); +} + +static ssize_t au_getxattr(struct dentry *dentry, struct inode *inode, + const char *name, void *value, size_t size) +{ + struct au_lgxattr arg = { + .type = AU_XATTR_GET, + .u.get = { + .name = name, + .value = value, + .size = size + }, + }; + + return au_lgxattr(dentry, inode, &arg); +} + +static int au_setxattr(struct dentry *dentry, struct inode *inode, + const char *name, const void *value, size_t size, + int flags) +{ + struct au_sxattr arg = { + .type = AU_XATTR_SET, + .u.set = { + .name = name, + .value = value, + .size = size, + .flags = flags + }, + }; + + return au_sxattr(dentry, inode, &arg); +} + +/* ---------------------------------------------------------------------- */ + +static int au_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return au_getxattr(dentry, inode, name, buffer, size); +} + +static int au_xattr_set(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, size_t size, + int flags) +{ + return au_setxattr(dentry, inode, name, value, size, flags); +} + +static const struct xattr_handler au_xattr_handler = { + .name = "", + .prefix = "", + .get = au_xattr_get, + .set = au_xattr_set +}; + +static const struct xattr_handler *au_xattr_handlers[] = { +#ifdef CONFIG_FS_POSIX_ACL + &posix_acl_access_xattr_handler, + &posix_acl_default_xattr_handler, +#endif + &au_xattr_handler, /* must be last */ + NULL +}; + +void au_xattr_init(struct super_block *sb) +{ + sb->s_xattr = au_xattr_handlers; +} diff --git a/fs/aufs/xino.c b/fs/aufs/xino.c new file mode 100644 index 0000000000000..b3152c0ce0b53 --- /dev/null +++ b/fs/aufs/xino.c @@ -0,0 +1,1966 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * external inode number translation table and bitmap + * + * things to consider + * - the lifetime + * + au_xino object + * + XINO files (xino, xib, xigen) + * + dynamic debugfs entries (xiN) + * + static debugfs entries (xib, xigen) + * + static sysfs entry (xi_path) + * - several entry points to handle them. + * + mount(2) without xino option (default) + * + mount(2) with xino option + * + mount(2) with noxino option + * + umount(2) + * + remount with add/del branches + * + remount with xino/noxino options + */ + +#include +#include +#include "aufs.h" + +static aufs_bindex_t sbr_find_shared(struct super_block *sb, aufs_bindex_t btop, + aufs_bindex_t bbot, + struct super_block *h_sb) +{ + /* todo: try binary-search if the branches are many */ + for (; btop <= bbot; btop++) + if (h_sb == au_sbr_sb(sb, btop)) + return btop; + return -1; +} + +/* + * find another branch who is on the same filesystem of the specified + * branch{@btgt}. search until @bbot. + */ +static aufs_bindex_t is_sb_shared(struct super_block *sb, aufs_bindex_t btgt, + aufs_bindex_t bbot) +{ + aufs_bindex_t bindex; + struct super_block *tgt_sb; + + tgt_sb = au_sbr_sb(sb, btgt); + bindex = sbr_find_shared(sb, /*btop*/0, btgt - 1, tgt_sb); + if (bindex < 0) + bindex = sbr_find_shared(sb, btgt + 1, bbot, tgt_sb); + + return bindex; +} + +/* ---------------------------------------------------------------------- */ + +/* + * stop unnecessary notify events at creating xino files + */ + +aufs_bindex_t au_xi_root(struct super_block *sb, struct dentry *dentry) +{ + aufs_bindex_t bfound, bindex, bbot; + struct dentry *parent; + struct au_branch *br; + + bfound = -1; + parent = dentry->d_parent; /* safe d_parent access */ + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_dentry(br) == parent) { + bfound = bindex; + break; + } + } + + AuDbg("bfound b%d\n", bfound); + return bfound; +} + +struct au_xino_lock_dir { + struct au_hinode *hdir; + struct dentry *parent; + struct inode *dir; +}; + +static struct dentry *au_dget_parent_lock(struct dentry *dentry, + unsigned int lsc) +{ + struct dentry *parent; + struct inode *dir; + + parent = dget_parent(dentry); + dir = d_inode(parent); + inode_lock_nested(dir, lsc); +#if 0 /* it should not happen */ + spin_lock(&dentry->d_lock); + if (unlikely(dentry->d_parent != parent)) { + spin_unlock(&dentry->d_lock); + inode_unlock(dir); + dput(parent); + parent = NULL; + goto out; + } + spin_unlock(&dentry->d_lock); + +out: +#endif + return parent; +} + +static void au_xino_lock_dir(struct super_block *sb, struct path *xipath, + struct au_xino_lock_dir *ldir) +{ + aufs_bindex_t bindex; + + ldir->hdir = NULL; + bindex = au_xi_root(sb, xipath->dentry); + if (bindex >= 0) { + /* rw branch root */ + ldir->hdir = au_hi(d_inode(sb->s_root), bindex); + au_hn_inode_lock_nested(ldir->hdir, AuLsc_I_PARENT); + } else { + /* other */ + ldir->parent = au_dget_parent_lock(xipath->dentry, + AuLsc_I_PARENT); + ldir->dir = d_inode(ldir->parent); + } +} + +static void au_xino_unlock_dir(struct au_xino_lock_dir *ldir) +{ + if (ldir->hdir) + au_hn_inode_unlock(ldir->hdir); + else { + inode_unlock(ldir->dir); + dput(ldir->parent); + } +} + +/* ---------------------------------------------------------------------- */ + +/* + * create and set a new xino file + */ +struct file *au_xino_create(struct super_block *sb, char *fpath, int silent, + int wbrtop) +{ + struct file *file; + struct dentry *h_parent, *d; + struct inode *h_dir, *inode; + int err; + static DEFINE_MUTEX(mtx); + + /* + * at mount-time, and the xino file is the default path, + * hnotify is disabled so we have no notify events to ignore. + * when a user specified the xino, we cannot get au_hdir to be ignored. + */ + if (!wbrtop) + mutex_lock(&mtx); + file = vfsub_filp_open(fpath, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE + /* | __FMODE_NONOTIFY */, + 0666); + if (IS_ERR(file)) { + if (!wbrtop) + mutex_unlock(&mtx); + if (!silent) + pr_err("open %s(%ld)\n", fpath, PTR_ERR(file)); + return file; + } + + /* keep file count */ + err = 0; + d = file->f_path.dentry; + h_parent = au_dget_parent_lock(d, AuLsc_I_PARENT); + if (!wbrtop) + mutex_unlock(&mtx); + /* mnt_want_write() is unnecessary here */ + h_dir = d_inode(h_parent); + inode = file_inode(file); + /* no delegation since it is just created */ + if (inode->i_nlink) + err = vfsub_unlink(h_dir, &file->f_path, /*delegated*/NULL, + /*force*/0); + inode_unlock(h_dir); + dput(h_parent); + if (unlikely(err)) { + if (!silent) + pr_err("unlink %s(%d)\n", fpath, err); + goto out; + } + + err = -EINVAL; + if (unlikely(sb == d->d_sb)) { + if (!silent) + pr_err("%s must be outside\n", fpath); + goto out; + } + if (unlikely(au_test_fs_bad_xino(d->d_sb))) { + if (!silent) + pr_err("xino doesn't support %s(%s)\n", + fpath, au_sbtype(d->d_sb)); + goto out; + } + return file; /* success */ + +out: + fput(file); + file = ERR_PTR(err); + return file; +} + +/* + * create a new xinofile at the same place/path as @base. + */ +struct file *au_xino_create2(struct super_block *sb, struct path *base, + struct file *copy_src) +{ + struct file *file; + struct dentry *dentry, *parent; + struct inode *dir, *delegated; + struct qstr *name; + struct path path; + int err, do_unlock; + struct au_xino_lock_dir ldir; + + do_unlock = 1; + au_xino_lock_dir(sb, base, &ldir); + dentry = base->dentry; + parent = dentry->d_parent; /* dir inode is locked */ + dir = d_inode(parent); + IMustLock(dir); + + name = &dentry->d_name; + path.dentry = vfsub_lookup_one_len(name->name, parent, name->len); + if (IS_ERR(path.dentry)) { + file = (void *)path.dentry; + pr_err("%pd lookup err %ld\n", dentry, PTR_ERR(path.dentry)); + goto out; + } + + /* no need to mnt_want_write() since we call dentry_open() later */ + err = vfs_create(dir, path.dentry, 0666, NULL); + if (unlikely(err)) { + file = ERR_PTR(err); + pr_err("%pd create err %d\n", dentry, err); + goto out_dput; + } + + path.mnt = base->mnt; + file = vfsub_dentry_open(&path, + O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE + /* | __FMODE_NONOTIFY */); + if (IS_ERR(file)) { + pr_err("%pd open err %ld\n", dentry, PTR_ERR(file)); + goto out_dput; + } + + delegated = NULL; + err = vfsub_unlink(dir, &file->f_path, &delegated, /*force*/0); + au_xino_unlock_dir(&ldir); + do_unlock = 0; + if (unlikely(err == -EWOULDBLOCK)) { + pr_warn("cannot retry for NFSv4 delegation" + " for an internal unlink\n"); + iput(delegated); + } + if (unlikely(err)) { + pr_err("%pd unlink err %d\n", dentry, err); + goto out_fput; + } + + if (copy_src) { + /* no one can touch copy_src xino */ + err = au_copy_file(file, copy_src, vfsub_f_size_read(copy_src)); + if (unlikely(err)) { + pr_err("%pd copy err %d\n", dentry, err); + goto out_fput; + } + } + goto out_dput; /* success */ + +out_fput: + fput(file); + file = ERR_PTR(err); +out_dput: + dput(path.dentry); +out: + if (do_unlock) + au_xino_unlock_dir(&ldir); + return file; +} + +struct file *au_xino_file1(struct au_xino *xi) +{ + struct file *file; + unsigned int u, nfile; + + file = NULL; + nfile = xi->xi_nfile; + for (u = 0; u < nfile; u++) { + file = xi->xi_file[u]; + if (file) + break; + } + + return file; +} + +static int au_xino_file_set(struct au_xino *xi, int idx, struct file *file) +{ + int err; + struct file *f; + void *p; + + if (file) + get_file(file); + + err = 0; + f = NULL; + if (idx < xi->xi_nfile) { + f = xi->xi_file[idx]; + if (f) + fput(f); + } else { + p = au_kzrealloc(xi->xi_file, + sizeof(*xi->xi_file) * xi->xi_nfile, + sizeof(*xi->xi_file) * (idx + 1), + GFP_NOFS, /*may_shrink*/0); + if (p) { + MtxMustLock(&xi->xi_mtx); + xi->xi_file = p; + xi->xi_nfile = idx + 1; + } else { + err = -ENOMEM; + if (file) + fput(file); + goto out; + } + } + xi->xi_file[idx] = file; + +out: + return err; +} + +/* + * if @xinew->xi is not set, then create new xigen file. + */ +struct file *au_xi_new(struct super_block *sb, struct au_xi_new *xinew) +{ + struct file *file; + int err; + + SiMustAnyLock(sb); + + file = au_xino_create2(sb, xinew->base, xinew->copy_src); + if (IS_ERR(file)) { + err = PTR_ERR(file); + pr_err("%s[%d], err %d\n", + xinew->xi ? "xino" : "xigen", + xinew->idx, err); + goto out; + } + + if (xinew->xi) + err = au_xino_file_set(xinew->xi, xinew->idx, file); + else { + BUG(); + /* todo: make xigen file an array */ + /* err = au_xigen_file_set(sb, xinew->idx, file); */ + } + fput(file); + if (unlikely(err)) + file = ERR_PTR(err); + +out: + return file; +} + +/* ---------------------------------------------------------------------- */ + +/* + * truncate xino files + */ +static int au_xino_do_trunc(struct super_block *sb, aufs_bindex_t bindex, + int idx, struct kstatfs *st) +{ + int err; + blkcnt_t blocks; + struct file *file, *new_xino; + struct au_xi_new xinew = { + .idx = idx + }; + + err = 0; + xinew.xi = au_sbr(sb, bindex)->br_xino; + file = au_xino_file(xinew.xi, idx); + if (!file) + goto out; + + xinew.base = &file->f_path; + err = vfs_statfs(xinew.base, st); + if (unlikely(err)) { + AuErr1("statfs err %d, ignored\n", err); + err = 0; + goto out; + } + + blocks = file_inode(file)->i_blocks; + pr_info("begin truncating xino(b%d-%d), ib%llu, %llu/%llu free blks\n", + bindex, idx, (u64)blocks, st->f_bfree, st->f_blocks); + + xinew.copy_src = file; + new_xino = au_xi_new(sb, &xinew); + if (IS_ERR(new_xino)) { + err = PTR_ERR(new_xino); + pr_err("xino(b%d-%d), err %d, ignored\n", bindex, idx, err); + goto out; + } + + err = vfs_statfs(&new_xino->f_path, st); + if (!err) + pr_info("end truncating xino(b%d-%d), ib%llu, %llu/%llu free blks\n", + bindex, idx, (u64)file_inode(new_xino)->i_blocks, + st->f_bfree, st->f_blocks); + else { + AuErr1("statfs err %d, ignored\n", err); + err = 0; + } + +out: + return err; +} + +int au_xino_trunc(struct super_block *sb, aufs_bindex_t bindex, int idx_begin) +{ + int err, i; + unsigned long jiffy; + aufs_bindex_t bbot; + struct kstatfs *st; + struct au_branch *br; + struct au_xino *xi; + + err = -ENOMEM; + st = kmalloc(sizeof(*st), GFP_NOFS); + if (unlikely(!st)) + goto out; + + err = -EINVAL; + bbot = au_sbbot(sb); + if (unlikely(bindex < 0 || bbot < bindex)) + goto out_st; + + err = 0; + jiffy = jiffies; + br = au_sbr(sb, bindex); + xi = br->br_xino; + for (i = idx_begin; !err && i < xi->xi_nfile; i++) + err = au_xino_do_trunc(sb, bindex, i, st); + if (!err) + au_sbi(sb)->si_xino_jiffy = jiffy; + +out_st: + au_kfree_rcu(st); +out: + return err; +} + +struct xino_do_trunc_args { + struct super_block *sb; + struct au_branch *br; + int idx; +}; + +static void xino_do_trunc(void *_args) +{ + struct xino_do_trunc_args *args = _args; + struct super_block *sb; + struct au_branch *br; + struct inode *dir; + int err, idx; + aufs_bindex_t bindex; + + err = 0; + sb = args->sb; + dir = d_inode(sb->s_root); + br = args->br; + idx = args->idx; + + si_noflush_write_lock(sb); + ii_read_lock_parent(dir); + bindex = au_br_index(sb, br->br_id); + err = au_xino_trunc(sb, bindex, idx); + ii_read_unlock(dir); + if (unlikely(err)) + pr_warn("err b%d, (%d)\n", bindex, err); + atomic_dec(&br->br_xino->xi_truncating); + au_lcnt_dec(&br->br_count); + si_write_unlock(sb); + au_nwt_done(&au_sbi(sb)->si_nowait); + au_kfree_rcu(args); +} + +/* + * returns the index in the xi_file array whose corresponding file is necessary + * to truncate, or -1 which means no need to truncate. + */ +static int xino_trunc_test(struct super_block *sb, struct au_branch *br) +{ + int err; + unsigned int u; + struct kstatfs st; + struct au_sbinfo *sbinfo; + struct au_xino *xi; + struct file *file; + + /* todo: si_xino_expire and the ratio should be customizable */ + sbinfo = au_sbi(sb); + if (time_before(jiffies, + sbinfo->si_xino_jiffy + sbinfo->si_xino_expire)) + return -1; + + /* truncation border */ + xi = br->br_xino; + for (u = 0; u < xi->xi_nfile; u++) { + file = au_xino_file(xi, u); + if (!file) + continue; + + err = vfs_statfs(&file->f_path, &st); + if (unlikely(err)) { + AuErr1("statfs err %d, ignored\n", err); + return -1; + } + if (div64_u64(st.f_bfree * 100, st.f_blocks) + >= AUFS_XINO_DEF_TRUNC) + return u; + } + + return -1; +} + +static void xino_try_trunc(struct super_block *sb, struct au_branch *br) +{ + int idx; + struct xino_do_trunc_args *args; + int wkq_err; + + idx = xino_trunc_test(sb, br); + if (idx < 0) + return; + + if (atomic_inc_return(&br->br_xino->xi_truncating) > 1) + goto out; + + /* lock and kfree() will be called in trunc_xino() */ + args = kmalloc(sizeof(*args), GFP_NOFS); + if (unlikely(!args)) { + AuErr1("no memory\n"); + goto out; + } + + au_lcnt_inc(&br->br_count); + args->sb = sb; + args->br = br; + args->idx = idx; + wkq_err = au_wkq_nowait(xino_do_trunc, args, sb, /*flags*/0); + if (!wkq_err) + return; /* success */ + + pr_err("wkq %d\n", wkq_err); + au_lcnt_dec(&br->br_count); + au_kfree_rcu(args); + +out: + atomic_dec(&br->br_xino->xi_truncating); +} + +/* ---------------------------------------------------------------------- */ + +struct au_xi_calc { + int idx; + loff_t pos; +}; + +static void au_xi_calc(struct super_block *sb, ino_t h_ino, + struct au_xi_calc *calc) +{ + loff_t maxent; + + maxent = au_xi_maxent(sb); + calc->idx = div64_u64_rem(h_ino, maxent, &calc->pos); + calc->pos *= sizeof(ino_t); +} + +static int au_xino_do_new_async(struct super_block *sb, struct au_branch *br, + struct au_xi_calc *calc) +{ + int err; + struct file *file; + struct au_xino *xi = br->br_xino; + struct au_xi_new xinew = { + .xi = xi + }; + + SiMustAnyLock(sb); + + err = 0; + if (!xi) + goto out; + + mutex_lock(&xi->xi_mtx); + file = au_xino_file(xi, calc->idx); + if (file) + goto out_mtx; + + file = au_xino_file(xi, /*idx*/-1); + AuDebugOn(!file); + xinew.idx = calc->idx; + xinew.base = &file->f_path; + /* xinew.copy_src = NULL; */ + file = au_xi_new(sb, &xinew); + if (IS_ERR(file)) + err = PTR_ERR(file); + +out_mtx: + mutex_unlock(&xi->xi_mtx); +out: + return err; +} + +struct au_xino_do_new_async_args { + struct super_block *sb; + struct au_branch *br; + struct au_xi_calc calc; + ino_t ino; +}; + +struct au_xi_writing { + struct hlist_bl_node node; + ino_t h_ino, ino; +}; + +static int au_xino_do_write(vfs_writef_t write, struct file *file, + struct au_xi_calc *calc, ino_t ino); + +static void au_xino_call_do_new_async(void *args) +{ + struct au_xino_do_new_async_args *a = args; + struct au_branch *br; + struct super_block *sb; + struct au_sbinfo *sbi; + struct inode *root; + struct file *file; + struct au_xi_writing *del, *p; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + int err; + + br = a->br; + sb = a->sb; + sbi = au_sbi(sb); + si_noflush_read_lock(sb); + root = d_inode(sb->s_root); + ii_read_lock_child(root); + err = au_xino_do_new_async(sb, br, &a->calc); + if (unlikely(err)) { + AuIOErr("err %d\n", err); + goto out; + } + + file = au_xino_file(br->br_xino, a->calc.idx); + AuDebugOn(!file); + err = au_xino_do_write(sbi->si_xwrite, file, &a->calc, a->ino); + if (unlikely(err)) { + AuIOErr("err %d\n", err); + goto out; + } + + del = NULL; + hbl = &br->br_xino->xi_writing; + hlist_bl_lock(hbl); + au_hbl_for_each(pos, hbl) { + p = container_of(pos, struct au_xi_writing, node); + if (p->ino == a->ino) { + del = p; + hlist_bl_del(&p->node); + break; + } + } + hlist_bl_unlock(hbl); + au_kfree_rcu(del); + +out: + au_lcnt_dec(&br->br_count); + ii_read_unlock(root); + si_read_unlock(sb); + au_nwt_done(&sbi->si_nowait); + au_kfree_rcu(a); +} + +/* + * create a new xino file asynchronously + */ +static int au_xino_new_async(struct super_block *sb, struct au_branch *br, + struct au_xi_calc *calc, ino_t ino) +{ + int err; + struct au_xino_do_new_async_args *arg; + + err = -ENOMEM; + arg = kmalloc(sizeof(*arg), GFP_NOFS); + if (unlikely(!arg)) + goto out; + + arg->sb = sb; + arg->br = br; + arg->calc = *calc; + arg->ino = ino; + au_lcnt_inc(&br->br_count); + err = au_wkq_nowait(au_xino_call_do_new_async, arg, sb, AuWkq_NEST); + if (unlikely(err)) { + pr_err("wkq %d\n", err); + au_lcnt_dec(&br->br_count); + au_kfree_rcu(arg); + } + +out: + return err; +} + +/* + * read @ino from xinofile for the specified branch{@sb, @bindex} + * at the position of @h_ino. + */ +int au_xino_read(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + ino_t *ino) +{ + int err; + ssize_t sz; + struct au_xi_calc calc; + struct au_sbinfo *sbinfo; + struct file *file; + struct au_xino *xi; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos; + struct au_xi_writing *p; + + *ino = 0; + if (!au_opt_test(au_mntflags(sb), XINO)) + return 0; /* no xino */ + + err = 0; + au_xi_calc(sb, h_ino, &calc); + xi = au_sbr(sb, bindex)->br_xino; + file = au_xino_file(xi, calc.idx); + if (!file) { + hbl = &xi->xi_writing; + hlist_bl_lock(hbl); + au_hbl_for_each(pos, hbl) { + p = container_of(pos, struct au_xi_writing, node); + if (p->h_ino == h_ino) { + AuDbg("hi%llu, i%llu, found\n", + (u64)p->h_ino, (u64)p->ino); + *ino = p->ino; + break; + } + } + hlist_bl_unlock(hbl); + return 0; + } else if (vfsub_f_size_read(file) < calc.pos + sizeof(*ino)) + return 0; /* no xino */ + + sbinfo = au_sbi(sb); + sz = xino_fread(sbinfo->si_xread, file, ino, sizeof(*ino), &calc.pos); + if (sz == sizeof(*ino)) + return 0; /* success */ + + err = sz; + if (unlikely(sz >= 0)) { + err = -EIO; + AuIOErr("xino read error (%zd)\n", sz); + } + return err; +} + +static int au_xino_do_write(vfs_writef_t write, struct file *file, + struct au_xi_calc *calc, ino_t ino) +{ + ssize_t sz; + + sz = xino_fwrite(write, file, &ino, sizeof(ino), &calc->pos); + if (sz == sizeof(ino)) + return 0; /* success */ + + AuIOErr("write failed (%zd)\n", sz); + return -EIO; +} + +/* + * write @ino to the xinofile for the specified branch{@sb, @bindex} + * at the position of @h_ino. + * even if @ino is zero, it is written to the xinofile and means no entry. + * if the size of the xino file on a specific filesystem exceeds the watermark, + * try truncating it. + */ +int au_xino_write(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + ino_t ino) +{ + int err; + unsigned int mnt_flags; + struct au_xi_calc calc; + struct file *file; + struct au_branch *br; + struct au_xino *xi; + struct au_xi_writing *p; + + SiMustAnyLock(sb); + + mnt_flags = au_mntflags(sb); + if (!au_opt_test(mnt_flags, XINO)) + return 0; + + au_xi_calc(sb, h_ino, &calc); + br = au_sbr(sb, bindex); + xi = br->br_xino; + file = au_xino_file(xi, calc.idx); + if (!file) { + /* store the inum pair into the list */ + p = kmalloc(sizeof(*p), GFP_NOFS | __GFP_NOFAIL); + p->h_ino = h_ino; + p->ino = ino; + au_hbl_add(&p->node, &xi->xi_writing); + + /* create and write a new xino file asynchronously */ + err = au_xino_new_async(sb, br, &calc, ino); + if (!err) + return 0; /* success */ + goto out; + } + + err = au_xino_do_write(au_sbi(sb)->si_xwrite, file, &calc, ino); + if (!err) { + br = au_sbr(sb, bindex); + if (au_opt_test(mnt_flags, TRUNC_XINO) + && au_test_fs_trunc_xino(au_br_sb(br))) + xino_try_trunc(sb, br); + return 0; /* success */ + } + +out: + AuIOErr("write failed (%d)\n", err); + return -EIO; +} + +static ssize_t xino_fread_wkq(vfs_readf_t func, struct file *file, void *buf, + size_t size, loff_t *pos); + +/* todo: unnecessary to support mmap_sem since kernel-space? */ +ssize_t xino_fread(vfs_readf_t func, struct file *file, void *kbuf, size_t size, + loff_t *pos) +{ + ssize_t err; + mm_segment_t oldfs; + union { + void *k; + char __user *u; + } buf; + int i; + const int prevent_endless = 10; + + i = 0; + buf.k = kbuf; + oldfs = get_fs(); + set_fs(KERNEL_DS); + do { + err = func(file, buf.u, size, pos); + if (err == -EINTR + && !au_wkq_test() + && fatal_signal_pending(current)) { + set_fs(oldfs); + err = xino_fread_wkq(func, file, kbuf, size, pos); + BUG_ON(err == -EINTR); + oldfs = get_fs(); + set_fs(KERNEL_DS); + } + } while (i++ < prevent_endless + && (err == -EAGAIN || err == -EINTR)); + set_fs(oldfs); + +#if 0 /* reserved for future use */ + if (err > 0) + fsnotify_access(file->f_path.dentry); +#endif + + return err; +} + +struct xino_fread_args { + ssize_t *errp; + vfs_readf_t func; + struct file *file; + void *buf; + size_t size; + loff_t *pos; +}; + +static void call_xino_fread(void *args) +{ + struct xino_fread_args *a = args; + *a->errp = xino_fread(a->func, a->file, a->buf, a->size, a->pos); +} + +static ssize_t xino_fread_wkq(vfs_readf_t func, struct file *file, void *buf, + size_t size, loff_t *pos) +{ + ssize_t err; + int wkq_err; + struct xino_fread_args args = { + .errp = &err, + .func = func, + .file = file, + .buf = buf, + .size = size, + .pos = pos + }; + + wkq_err = au_wkq_wait(call_xino_fread, &args); + if (unlikely(wkq_err)) + err = wkq_err; + + return err; +} + +static ssize_t xino_fwrite_wkq(vfs_writef_t func, struct file *file, void *buf, + size_t size, loff_t *pos); + +static ssize_t do_xino_fwrite(vfs_writef_t func, struct file *file, void *kbuf, + size_t size, loff_t *pos) +{ + ssize_t err; + mm_segment_t oldfs; + union { + void *k; + const char __user *u; + } buf; + int i; + const int prevent_endless = 10; + + i = 0; + buf.k = kbuf; + oldfs = get_fs(); + set_fs(KERNEL_DS); + do { + err = func(file, buf.u, size, pos); + if (err == -EINTR + && !au_wkq_test() + && fatal_signal_pending(current)) { + set_fs(oldfs); + err = xino_fwrite_wkq(func, file, kbuf, size, pos); + BUG_ON(err == -EINTR); + oldfs = get_fs(); + set_fs(KERNEL_DS); + } + } while (i++ < prevent_endless + && (err == -EAGAIN || err == -EINTR)); + set_fs(oldfs); + +#if 0 /* reserved for future use */ + if (err > 0) + fsnotify_modify(file->f_path.dentry); +#endif + + return err; +} + +struct do_xino_fwrite_args { + ssize_t *errp; + vfs_writef_t func; + struct file *file; + void *buf; + size_t size; + loff_t *pos; +}; + +static void call_do_xino_fwrite(void *args) +{ + struct do_xino_fwrite_args *a = args; + *a->errp = do_xino_fwrite(a->func, a->file, a->buf, a->size, a->pos); +} + +static ssize_t xino_fwrite_wkq(vfs_writef_t func, struct file *file, void *buf, + size_t size, loff_t *pos) +{ + ssize_t err; + int wkq_err; + struct do_xino_fwrite_args args = { + .errp = &err, + .func = func, + .file = file, + .buf = buf, + .size = size, + .pos = pos + }; + + /* + * it breaks RLIMIT_FSIZE and normal user's limit, + * users should care about quota and real 'filesystem full.' + */ + wkq_err = au_wkq_wait(call_do_xino_fwrite, &args); + if (unlikely(wkq_err)) + err = wkq_err; + + return err; +} + +ssize_t xino_fwrite(vfs_writef_t func, struct file *file, void *buf, + size_t size, loff_t *pos) +{ + ssize_t err; + + if (rlimit(RLIMIT_FSIZE) == RLIM_INFINITY) { + lockdep_off(); + err = do_xino_fwrite(func, file, buf, size, pos); + lockdep_on(); + } else { + lockdep_off(); + err = xino_fwrite_wkq(func, file, buf, size, pos); + lockdep_on(); + } + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * inode number bitmap + */ +static const int page_bits = (int)PAGE_SIZE * BITS_PER_BYTE; +static ino_t xib_calc_ino(unsigned long pindex, int bit) +{ + ino_t ino; + + AuDebugOn(bit < 0 || page_bits <= bit); + ino = AUFS_FIRST_INO + pindex * page_bits + bit; + return ino; +} + +static void xib_calc_bit(ino_t ino, unsigned long *pindex, int *bit) +{ + AuDebugOn(ino < AUFS_FIRST_INO); + ino -= AUFS_FIRST_INO; + *pindex = ino / page_bits; + *bit = ino % page_bits; +} + +static int xib_pindex(struct super_block *sb, unsigned long pindex) +{ + int err; + loff_t pos; + ssize_t sz; + struct au_sbinfo *sbinfo; + struct file *xib; + unsigned long *p; + + sbinfo = au_sbi(sb); + MtxMustLock(&sbinfo->si_xib_mtx); + AuDebugOn(pindex > ULONG_MAX / PAGE_SIZE + || !au_opt_test(sbinfo->si_mntflags, XINO)); + + if (pindex == sbinfo->si_xib_last_pindex) + return 0; + + xib = sbinfo->si_xib; + p = sbinfo->si_xib_buf; + pos = sbinfo->si_xib_last_pindex; + pos *= PAGE_SIZE; + sz = xino_fwrite(sbinfo->si_xwrite, xib, p, PAGE_SIZE, &pos); + if (unlikely(sz != PAGE_SIZE)) + goto out; + + pos = pindex; + pos *= PAGE_SIZE; + if (vfsub_f_size_read(xib) >= pos + PAGE_SIZE) + sz = xino_fread(sbinfo->si_xread, xib, p, PAGE_SIZE, &pos); + else { + memset(p, 0, PAGE_SIZE); + sz = xino_fwrite(sbinfo->si_xwrite, xib, p, PAGE_SIZE, &pos); + } + if (sz == PAGE_SIZE) { + sbinfo->si_xib_last_pindex = pindex; + return 0; /* success */ + } + +out: + AuIOErr1("write failed (%zd)\n", sz); + err = sz; + if (sz >= 0) + err = -EIO; + return err; +} + +static void au_xib_clear_bit(struct inode *inode) +{ + int err, bit; + unsigned long pindex; + struct super_block *sb; + struct au_sbinfo *sbinfo; + + AuDebugOn(inode->i_nlink); + + sb = inode->i_sb; + xib_calc_bit(inode->i_ino, &pindex, &bit); + AuDebugOn(page_bits <= bit); + sbinfo = au_sbi(sb); + mutex_lock(&sbinfo->si_xib_mtx); + err = xib_pindex(sb, pindex); + if (!err) { + clear_bit(bit, sbinfo->si_xib_buf); + sbinfo->si_xib_next_bit = bit; + } + mutex_unlock(&sbinfo->si_xib_mtx); +} + +/* ---------------------------------------------------------------------- */ + +/* + * truncate a xino bitmap file + */ + +/* todo: slow */ +static int do_xib_restore(struct super_block *sb, struct file *file, void *page) +{ + int err, bit; + ssize_t sz; + unsigned long pindex; + loff_t pos, pend; + struct au_sbinfo *sbinfo; + vfs_readf_t func; + ino_t *ino; + unsigned long *p; + + err = 0; + sbinfo = au_sbi(sb); + MtxMustLock(&sbinfo->si_xib_mtx); + p = sbinfo->si_xib_buf; + func = sbinfo->si_xread; + pend = vfsub_f_size_read(file); + pos = 0; + while (pos < pend) { + sz = xino_fread(func, file, page, PAGE_SIZE, &pos); + err = sz; + if (unlikely(sz <= 0)) + goto out; + + err = 0; + for (ino = page; sz > 0; ino++, sz -= sizeof(ino)) { + if (unlikely(*ino < AUFS_FIRST_INO)) + continue; + + xib_calc_bit(*ino, &pindex, &bit); + AuDebugOn(page_bits <= bit); + err = xib_pindex(sb, pindex); + if (!err) + set_bit(bit, p); + else + goto out; + } + } + +out: + return err; +} + +static int xib_restore(struct super_block *sb) +{ + int err, i; + unsigned int nfile; + aufs_bindex_t bindex, bbot; + void *page; + struct au_branch *br; + struct au_xino *xi; + struct file *file; + + err = -ENOMEM; + page = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!page)) + goto out; + + err = 0; + bbot = au_sbbot(sb); + for (bindex = 0; !err && bindex <= bbot; bindex++) + if (!bindex || is_sb_shared(sb, bindex, bindex - 1) < 0) { + br = au_sbr(sb, bindex); + xi = br->br_xino; + nfile = xi->xi_nfile; + for (i = 0; i < nfile; i++) { + file = au_xino_file(xi, i); + if (file) + err = do_xib_restore(sb, file, page); + } + } else + AuDbg("skip shared b%d\n", bindex); + free_page((unsigned long)page); + +out: + return err; +} + +int au_xib_trunc(struct super_block *sb) +{ + int err; + ssize_t sz; + loff_t pos; + struct au_sbinfo *sbinfo; + unsigned long *p; + struct file *file; + + SiMustWriteLock(sb); + + err = 0; + sbinfo = au_sbi(sb); + if (!au_opt_test(sbinfo->si_mntflags, XINO)) + goto out; + + file = sbinfo->si_xib; + if (vfsub_f_size_read(file) <= PAGE_SIZE) + goto out; + + file = au_xino_create2(sb, &sbinfo->si_xib->f_path, NULL); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + fput(sbinfo->si_xib); + sbinfo->si_xib = file; + + p = sbinfo->si_xib_buf; + memset(p, 0, PAGE_SIZE); + pos = 0; + sz = xino_fwrite(sbinfo->si_xwrite, sbinfo->si_xib, p, PAGE_SIZE, &pos); + if (unlikely(sz != PAGE_SIZE)) { + err = sz; + AuIOErr("err %d\n", err); + if (sz >= 0) + err = -EIO; + goto out; + } + + mutex_lock(&sbinfo->si_xib_mtx); + /* mnt_want_write() is unnecessary here */ + err = xib_restore(sb); + mutex_unlock(&sbinfo->si_xib_mtx); + +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +struct au_xino *au_xino_alloc(unsigned int nfile) +{ + struct au_xino *xi; + + xi = kzalloc(sizeof(*xi), GFP_NOFS); + if (unlikely(!xi)) + goto out; + xi->xi_nfile = nfile; + xi->xi_file = kcalloc(nfile, sizeof(*xi->xi_file), GFP_NOFS); + if (unlikely(!xi->xi_file)) + goto out_free; + + xi->xi_nondir.total = 8; /* initial size */ + xi->xi_nondir.array = kcalloc(xi->xi_nondir.total, sizeof(ino_t), + GFP_NOFS); + if (unlikely(!xi->xi_nondir.array)) + goto out_file; + + spin_lock_init(&xi->xi_nondir.spin); + init_waitqueue_head(&xi->xi_nondir.wqh); + mutex_init(&xi->xi_mtx); + INIT_HLIST_BL_HEAD(&xi->xi_writing); + atomic_set(&xi->xi_truncating, 0); + kref_init(&xi->xi_kref); + goto out; /* success */ + +out_file: + au_kfree_try_rcu(xi->xi_file); +out_free: + au_kfree_rcu(xi); + xi = NULL; +out: + return xi; +} + +static int au_xino_init(struct au_branch *br, int idx, struct file *file) +{ + int err; + struct au_xino *xi; + + err = 0; + xi = au_xino_alloc(idx + 1); + if (unlikely(!xi)) { + err = -ENOMEM; + goto out; + } + + if (file) + get_file(file); + xi->xi_file[idx] = file; + AuDebugOn(br->br_xino); + br->br_xino = xi; + +out: + return err; +} + +static void au_xino_release(struct kref *kref) +{ + struct au_xino *xi; + int i; + unsigned long ul; + struct hlist_bl_head *hbl; + struct hlist_bl_node *pos, *n; + struct au_xi_writing *p; + + xi = container_of(kref, struct au_xino, xi_kref); + for (i = 0; i < xi->xi_nfile; i++) + if (xi->xi_file[i]) + fput(xi->xi_file[i]); + for (i = xi->xi_nondir.total - 1; i >= 0; i--) + AuDebugOn(xi->xi_nondir.array[i]); + mutex_destroy(&xi->xi_mtx); + hbl = &xi->xi_writing; + ul = au_hbl_count(hbl); + if (unlikely(ul)) { + pr_warn("xi_writing %lu\n", ul); + hlist_bl_lock(hbl); + hlist_bl_for_each_entry_safe(p, pos, n, hbl, node) { + hlist_bl_del(&p->node); + /* kmemleak reported au_kfree_rcu() doesn't free it */ + kfree(p); + } + hlist_bl_unlock(hbl); + } + au_kfree_try_rcu(xi->xi_file); + au_kfree_try_rcu(xi->xi_nondir.array); + au_kfree_rcu(xi); +} + +int au_xino_put(struct au_branch *br) +{ + int ret; + struct au_xino *xi; + + ret = 0; + xi = br->br_xino; + if (xi) { + br->br_xino = NULL; + ret = kref_put(&xi->xi_kref, au_xino_release); + } + + return ret; +} + +/* ---------------------------------------------------------------------- */ + +/* + * xino mount option handlers + */ + +/* xino bitmap */ +static void xino_clear_xib(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + /* unnecessary to clear sbinfo->si_xread and ->si_xwrite */ + if (sbinfo->si_xib) + fput(sbinfo->si_xib); + sbinfo->si_xib = NULL; + if (sbinfo->si_xib_buf) + free_page((unsigned long)sbinfo->si_xib_buf); + sbinfo->si_xib_buf = NULL; +} + +static int au_xino_set_xib(struct super_block *sb, struct path *path) +{ + int err; + loff_t pos; + struct au_sbinfo *sbinfo; + struct file *file; + struct super_block *xi_sb; + + SiMustWriteLock(sb); + + sbinfo = au_sbi(sb); + file = au_xino_create2(sb, path, sbinfo->si_xib); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + if (sbinfo->si_xib) + fput(sbinfo->si_xib); + sbinfo->si_xib = file; + sbinfo->si_xread = vfs_readf(file); + sbinfo->si_xwrite = vfs_writef(file); + xi_sb = file_inode(file)->i_sb; + sbinfo->si_ximaxent = xi_sb->s_maxbytes; + if (unlikely(sbinfo->si_ximaxent < PAGE_SIZE)) { + err = -EIO; + pr_err("s_maxbytes(%llu) on %s is too small\n", + (u64)sbinfo->si_ximaxent, au_sbtype(xi_sb)); + goto out_unset; + } + sbinfo->si_ximaxent /= sizeof(ino_t); + + err = -ENOMEM; + if (!sbinfo->si_xib_buf) + sbinfo->si_xib_buf = (void *)get_zeroed_page(GFP_NOFS); + if (unlikely(!sbinfo->si_xib_buf)) + goto out_unset; + + sbinfo->si_xib_last_pindex = 0; + sbinfo->si_xib_next_bit = 0; + if (vfsub_f_size_read(file) < PAGE_SIZE) { + pos = 0; + err = xino_fwrite(sbinfo->si_xwrite, file, sbinfo->si_xib_buf, + PAGE_SIZE, &pos); + if (unlikely(err != PAGE_SIZE)) + goto out_free; + } + err = 0; + goto out; /* success */ + +out_free: + if (sbinfo->si_xib_buf) + free_page((unsigned long)sbinfo->si_xib_buf); + sbinfo->si_xib_buf = NULL; + if (err >= 0) + err = -EIO; +out_unset: + fput(sbinfo->si_xib); + sbinfo->si_xib = NULL; +out: + AuTraceErr(err); + return err; +} + +/* xino for each branch */ +static void xino_clear_br(struct super_block *sb) +{ + aufs_bindex_t bindex, bbot; + struct au_branch *br; + + bbot = au_sbbot(sb); + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + AuDebugOn(!br); + au_xino_put(br); + } +} + +static void au_xino_set_br_shared(struct super_block *sb, struct au_branch *br, + aufs_bindex_t bshared) +{ + struct au_branch *brshared; + + brshared = au_sbr(sb, bshared); + AuDebugOn(!brshared->br_xino); + AuDebugOn(!brshared->br_xino->xi_file); + if (br->br_xino != brshared->br_xino) { + au_xino_get(brshared); + au_xino_put(br); + br->br_xino = brshared->br_xino; + } +} + +struct au_xino_do_set_br { + vfs_writef_t writef; + struct au_branch *br; + ino_t h_ino; + aufs_bindex_t bshared; +}; + +static int au_xino_do_set_br(struct super_block *sb, struct path *path, + struct au_xino_do_set_br *args) +{ + int err; + struct au_xi_calc calc; + struct file *file; + struct au_branch *br; + struct au_xi_new xinew = { + .base = path + }; + + br = args->br; + xinew.xi = br->br_xino; + au_xi_calc(sb, args->h_ino, &calc); + xinew.copy_src = au_xino_file(xinew.xi, calc.idx); + if (args->bshared >= 0) + /* shared xino */ + au_xino_set_br_shared(sb, br, args->bshared); + else if (!xinew.xi) { + /* new xino */ + err = au_xino_init(br, calc.idx, xinew.copy_src); + if (unlikely(err)) + goto out; + } + + /* force re-creating */ + xinew.xi = br->br_xino; + xinew.idx = calc.idx; + mutex_lock(&xinew.xi->xi_mtx); + file = au_xi_new(sb, &xinew); + mutex_unlock(&xinew.xi->xi_mtx); + err = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + AuDebugOn(!file); + + err = au_xino_do_write(args->writef, file, &calc, AUFS_ROOT_INO); + if (unlikely(err)) + au_xino_put(br); + +out: + AuTraceErr(err); + return err; +} + +static int au_xino_set_br(struct super_block *sb, struct path *path) +{ + int err; + aufs_bindex_t bindex, bbot; + struct au_xino_do_set_br args; + struct inode *inode; + + SiMustWriteLock(sb); + + bbot = au_sbbot(sb); + inode = d_inode(sb->s_root); + args.writef = au_sbi(sb)->si_xwrite; + for (bindex = 0; bindex <= bbot; bindex++) { + args.h_ino = au_h_iptr(inode, bindex)->i_ino; + args.br = au_sbr(sb, bindex); + args.bshared = is_sb_shared(sb, bindex, bindex - 1); + err = au_xino_do_set_br(sb, path, &args); + if (unlikely(err)) + break; + } + + AuTraceErr(err); + return err; +} + +void au_xino_clr(struct super_block *sb) +{ + struct au_sbinfo *sbinfo; + + au_xigen_clr(sb); + xino_clear_xib(sb); + xino_clear_br(sb); + dbgaufs_brs_del(sb, 0); + sbinfo = au_sbi(sb); + /* lvalue, do not call au_mntflags() */ + au_opt_clr(sbinfo->si_mntflags, XINO); +} + +int au_xino_set(struct super_block *sb, struct au_opt_xino *xiopt, int remount) +{ + int err, skip; + struct dentry *dentry, *parent, *cur_dentry, *cur_parent; + struct qstr *dname, *cur_name; + struct file *cur_xino; + struct au_sbinfo *sbinfo; + struct path *path, *cur_path; + + SiMustWriteLock(sb); + + err = 0; + sbinfo = au_sbi(sb); + path = &xiopt->file->f_path; + dentry = path->dentry; + parent = dget_parent(dentry); + if (remount) { + skip = 0; + cur_xino = sbinfo->si_xib; + if (cur_xino) { + cur_path = &cur_xino->f_path; + cur_dentry = cur_path->dentry; + cur_parent = dget_parent(cur_dentry); + cur_name = &cur_dentry->d_name; + dname = &dentry->d_name; + skip = (cur_parent == parent + && au_qstreq(dname, cur_name)); + dput(cur_parent); + } + if (skip) + goto out; + } + + au_opt_set(sbinfo->si_mntflags, XINO); + err = au_xino_set_xib(sb, path); + /* si_x{read,write} are set */ + if (!err) + err = au_xigen_set(sb, path); + if (!err) + err = au_xino_set_br(sb, path); + if (!err) { + dbgaufs_brs_add(sb, 0, /*topdown*/1); + goto out; /* success */ + } + + /* reset all */ + AuIOErr("failed setting xino(%d).\n", err); + au_xino_clr(sb); + +out: + dput(parent); + return err; +} + +/* + * create a xinofile at the default place/path. + */ +struct file *au_xino_def(struct super_block *sb) +{ + struct file *file; + char *page, *p; + struct au_branch *br; + struct super_block *h_sb; + struct path path; + aufs_bindex_t bbot, bindex, bwr; + + br = NULL; + bbot = au_sbbot(sb); + bwr = -1; + for (bindex = 0; bindex <= bbot; bindex++) { + br = au_sbr(sb, bindex); + if (au_br_writable(br->br_perm) + && !au_test_fs_bad_xino(au_br_sb(br))) { + bwr = bindex; + break; + } + } + + if (bwr >= 0) { + file = ERR_PTR(-ENOMEM); + page = (void *)__get_free_page(GFP_NOFS); + if (unlikely(!page)) + goto out; + path.mnt = au_br_mnt(br); + path.dentry = au_h_dptr(sb->s_root, bwr); + p = d_path(&path, page, PATH_MAX - sizeof(AUFS_XINO_FNAME)); + file = (void *)p; + if (!IS_ERR(p)) { + strcat(p, "/" AUFS_XINO_FNAME); + AuDbg("%s\n", p); + file = au_xino_create(sb, p, /*silent*/0, /*wbrtop*/1); + } + free_page((unsigned long)page); + } else { + file = au_xino_create(sb, AUFS_XINO_DEFPATH, /*silent*/0, + /*wbrtop*/0); + if (IS_ERR(file)) + goto out; + h_sb = file->f_path.dentry->d_sb; + if (unlikely(au_test_fs_bad_xino(h_sb))) { + pr_err("xino doesn't support %s(%s)\n", + AUFS_XINO_DEFPATH, au_sbtype(h_sb)); + fput(file); + file = ERR_PTR(-EINVAL); + } + } + +out: + return file; +} + +/* ---------------------------------------------------------------------- */ + +/* + * initialize the xinofile for the specified branch @br + * at the place/path where @base_file indicates. + * test whether another branch is on the same filesystem or not, + * if found then share the xinofile with another branch. + */ +int au_xino_init_br(struct super_block *sb, struct au_branch *br, ino_t h_ino, + struct path *base) +{ + int err; + struct au_xino_do_set_br args = { + .h_ino = h_ino, + .br = br + }; + + args.writef = au_sbi(sb)->si_xwrite; + args.bshared = sbr_find_shared(sb, /*btop*/0, au_sbbot(sb), + au_br_sb(br)); + err = au_xino_do_set_br(sb, base, &args); + if (unlikely(err)) + au_xino_put(br); + + return err; +} + +/* ---------------------------------------------------------------------- */ + +/* + * get an unused inode number from bitmap + */ +ino_t au_xino_new_ino(struct super_block *sb) +{ + ino_t ino; + unsigned long *p, pindex, ul, pend; + struct au_sbinfo *sbinfo; + struct file *file; + int free_bit, err; + + if (!au_opt_test(au_mntflags(sb), XINO)) + return iunique(sb, AUFS_FIRST_INO); + + sbinfo = au_sbi(sb); + mutex_lock(&sbinfo->si_xib_mtx); + p = sbinfo->si_xib_buf; + free_bit = sbinfo->si_xib_next_bit; + if (free_bit < page_bits && !test_bit(free_bit, p)) + goto out; /* success */ + free_bit = find_first_zero_bit(p, page_bits); + if (free_bit < page_bits) + goto out; /* success */ + + pindex = sbinfo->si_xib_last_pindex; + for (ul = pindex - 1; ul < ULONG_MAX; ul--) { + err = xib_pindex(sb, ul); + if (unlikely(err)) + goto out_err; + free_bit = find_first_zero_bit(p, page_bits); + if (free_bit < page_bits) + goto out; /* success */ + } + + file = sbinfo->si_xib; + pend = vfsub_f_size_read(file) / PAGE_SIZE; + for (ul = pindex + 1; ul <= pend; ul++) { + err = xib_pindex(sb, ul); + if (unlikely(err)) + goto out_err; + free_bit = find_first_zero_bit(p, page_bits); + if (free_bit < page_bits) + goto out; /* success */ + } + BUG(); + +out: + set_bit(free_bit, p); + sbinfo->si_xib_next_bit = free_bit + 1; + pindex = sbinfo->si_xib_last_pindex; + mutex_unlock(&sbinfo->si_xib_mtx); + ino = xib_calc_ino(pindex, free_bit); + AuDbg("i%lu\n", (unsigned long)ino); + return ino; +out_err: + mutex_unlock(&sbinfo->si_xib_mtx); + AuDbg("i0\n"); + return 0; +} + +/* for s_op->delete_inode() */ +void au_xino_delete_inode(struct inode *inode, const int unlinked) +{ + int err; + unsigned int mnt_flags; + aufs_bindex_t bindex, bbot, bi; + unsigned char try_trunc; + struct au_iinfo *iinfo; + struct super_block *sb; + struct au_hinode *hi; + struct inode *h_inode; + struct au_branch *br; + vfs_writef_t xwrite; + struct au_xi_calc calc; + struct file *file; + + AuDebugOn(au_is_bad_inode(inode)); + + sb = inode->i_sb; + mnt_flags = au_mntflags(sb); + if (!au_opt_test(mnt_flags, XINO) + || inode->i_ino == AUFS_ROOT_INO) + return; + + if (unlinked) { + au_xigen_inc(inode); + au_xib_clear_bit(inode); + } + + iinfo = au_ii(inode); + bindex = iinfo->ii_btop; + if (bindex < 0) + return; + + xwrite = au_sbi(sb)->si_xwrite; + try_trunc = !!au_opt_test(mnt_flags, TRUNC_XINO); + hi = au_hinode(iinfo, bindex); + bbot = iinfo->ii_bbot; + for (; bindex <= bbot; bindex++, hi++) { + h_inode = hi->hi_inode; + if (!h_inode + || (!unlinked && h_inode->i_nlink)) + continue; + + /* inode may not be revalidated */ + bi = au_br_index(sb, hi->hi_id); + if (bi < 0) + continue; + + br = au_sbr(sb, bi); + au_xi_calc(sb, h_inode->i_ino, &calc); + file = au_xino_file(br->br_xino, calc.idx); + if (IS_ERR_OR_NULL(file)) + continue; + + err = au_xino_do_write(xwrite, file, &calc, /*ino*/0); + if (!err && try_trunc + && au_test_fs_trunc_xino(au_br_sb(br))) + xino_try_trunc(sb, br); + } +} + +/* ---------------------------------------------------------------------- */ + +static int au_xinondir_find(struct au_xino *xi, ino_t h_ino) +{ + int found, total, i; + + found = -1; + total = xi->xi_nondir.total; + for (i = 0; i < total; i++) { + if (xi->xi_nondir.array[i] != h_ino) + continue; + found = i; + break; + } + + return found; +} + +static int au_xinondir_expand(struct au_xino *xi) +{ + int err, sz; + ino_t *p; + + BUILD_BUG_ON(KMALLOC_MAX_SIZE > INT_MAX); + + err = -ENOMEM; + sz = xi->xi_nondir.total * sizeof(ino_t); + if (unlikely(sz > KMALLOC_MAX_SIZE / 2)) + goto out; + p = au_kzrealloc(xi->xi_nondir.array, sz, sz << 1, GFP_ATOMIC, + /*may_shrink*/0); + if (p) { + xi->xi_nondir.array = p; + xi->xi_nondir.total <<= 1; + AuDbg("xi_nondir.total %d\n", xi->xi_nondir.total); + err = 0; + } + +out: + return err; +} + +void au_xinondir_leave(struct super_block *sb, aufs_bindex_t bindex, + ino_t h_ino, int idx) +{ + struct au_xino *xi; + + AuDebugOn(!au_opt_test(au_mntflags(sb), XINO)); + xi = au_sbr(sb, bindex)->br_xino; + AuDebugOn(idx < 0 || xi->xi_nondir.total <= idx); + + spin_lock(&xi->xi_nondir.spin); + AuDebugOn(xi->xi_nondir.array[idx] != h_ino); + xi->xi_nondir.array[idx] = 0; + spin_unlock(&xi->xi_nondir.spin); + wake_up_all(&xi->xi_nondir.wqh); +} + +int au_xinondir_enter(struct super_block *sb, aufs_bindex_t bindex, ino_t h_ino, + int *idx) +{ + int err, found, empty; + struct au_xino *xi; + + err = 0; + *idx = -1; + if (!au_opt_test(au_mntflags(sb), XINO)) + goto out; /* no xino */ + + xi = au_sbr(sb, bindex)->br_xino; + +again: + spin_lock(&xi->xi_nondir.spin); + found = au_xinondir_find(xi, h_ino); + if (found == -1) { + empty = au_xinondir_find(xi, /*h_ino*/0); + if (empty == -1) { + empty = xi->xi_nondir.total; + err = au_xinondir_expand(xi); + if (unlikely(err)) + goto out_unlock; + } + xi->xi_nondir.array[empty] = h_ino; + *idx = empty; + } else { + spin_unlock(&xi->xi_nondir.spin); + wait_event(xi->xi_nondir.wqh, + xi->xi_nondir.array[found] != h_ino); + goto again; + } + +out_unlock: + spin_unlock(&xi->xi_nondir.spin); +out: + return err; +} + +/* ---------------------------------------------------------------------- */ + +int au_xino_path(struct seq_file *seq, struct file *file) +{ + int err; + + err = au_seq_path(seq, &file->f_path); + if (unlikely(err)) + goto out; + +#define Deleted "\\040(deleted)" + seq->count -= sizeof(Deleted) - 1; + AuDebugOn(memcmp(seq->buf + seq->count, Deleted, + sizeof(Deleted) - 1)); +#undef Deleted + +out: + return err; +} diff --git a/include/uapi/linux/aufs_type.h b/include/uapi/linux/aufs_type.h new file mode 100644 index 0000000000000..34fadaf9b4886 --- /dev/null +++ b/include/uapi/linux/aufs_type.h @@ -0,0 +1,452 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2005-2020 Junjiro R. Okajima + * + * This program, aufs is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __AUFS_TYPE_H__ +#define __AUFS_TYPE_H__ + +#define AUFS_NAME "aufs" + +#ifdef __KERNEL__ +/* + * define it before including all other headers. + * sched.h may use pr_* macros before defining "current", so define the + * no-current version first, and re-define later. + */ +#define pr_fmt(fmt) AUFS_NAME " %s:%d: " fmt, __func__, __LINE__ +#include +#undef pr_fmt +#define pr_fmt(fmt) \ + AUFS_NAME " %s:%d:%.*s[%d]: " fmt, __func__, __LINE__, \ + (int)sizeof(current->comm), current->comm, current->pid +#include +#else +#include +#include +#include +#endif /* __KERNEL__ */ + +#define AUFS_VERSION "4.19.63+-20200518" + +/* todo? move this to linux-2.6.19/include/magic.h */ +#define AUFS_SUPER_MAGIC ('a' << 24 | 'u' << 16 | 'f' << 8 | 's') + +/* ---------------------------------------------------------------------- */ + +#ifdef __KERNEL__ +#ifdef CONFIG_AUFS_BRANCH_MAX_127 +typedef int8_t aufs_bindex_t; +#define AUFS_BRANCH_MAX 127 +#else +typedef int16_t aufs_bindex_t; +#ifdef CONFIG_AUFS_BRANCH_MAX_511 +#define AUFS_BRANCH_MAX 511 +#elif defined(CONFIG_AUFS_BRANCH_MAX_1023) +#define AUFS_BRANCH_MAX 1023 +#elif defined(CONFIG_AUFS_BRANCH_MAX_32767) +#define AUFS_BRANCH_MAX 32767 +#endif +#endif + +#ifndef AUFS_BRANCH_MAX +#error unknown CONFIG_AUFS_BRANCH_MAX value +#endif +#endif /* __KERNEL__ */ + +/* ---------------------------------------------------------------------- */ + +#define AUFS_FSTYPE AUFS_NAME + +#define AUFS_ROOT_INO 2 +#define AUFS_FIRST_INO 11 + +#define AUFS_WH_PFX ".wh." +#define AUFS_WH_PFX_LEN ((int)sizeof(AUFS_WH_PFX) - 1) +#define AUFS_WH_TMP_LEN 4 +/* a limit for rmdir/rename a dir and copyup */ +#define AUFS_MAX_NAMELEN (NAME_MAX \ + - AUFS_WH_PFX_LEN * 2 /* doubly whiteouted */\ + - 1 /* dot */\ + - AUFS_WH_TMP_LEN) /* hex */ +#define AUFS_XINO_FNAME "." AUFS_NAME ".xino" +#define AUFS_XINO_DEFPATH "/tmp/" AUFS_XINO_FNAME +#define AUFS_XINO_DEF_SEC 30 /* seconds */ +#define AUFS_XINO_DEF_TRUNC 45 /* percentage */ +#define AUFS_DIRWH_DEF 3 +#define AUFS_RDCACHE_DEF 10 /* seconds */ +#define AUFS_RDCACHE_MAX 3600 /* seconds */ +#define AUFS_RDBLK_DEF 512 /* bytes */ +#define AUFS_RDHASH_DEF 32 +#define AUFS_WKQ_NAME AUFS_NAME "d" +#define AUFS_MFS_DEF_SEC 30 /* seconds */ +#define AUFS_MFS_MAX_SEC 3600 /* seconds */ +#define AUFS_FHSM_CACHE_DEF_SEC 30 /* seconds */ +#define AUFS_PLINK_WARN 50 /* number of plinks in a single bucket */ + +/* pseudo-link maintenace under /proc */ +#define AUFS_PLINK_MAINT_NAME "plink_maint" +#define AUFS_PLINK_MAINT_DIR "fs/" AUFS_NAME +#define AUFS_PLINK_MAINT_PATH AUFS_PLINK_MAINT_DIR "/" AUFS_PLINK_MAINT_NAME + +/* dirren, renamed dir */ +#define AUFS_DR_INFO_PFX AUFS_WH_PFX ".dr." +#define AUFS_DR_BRHINO_NAME AUFS_WH_PFX "hino" +/* whiteouted doubly */ +#define AUFS_WH_DR_INFO_PFX AUFS_WH_PFX AUFS_DR_INFO_PFX +#define AUFS_WH_DR_BRHINO AUFS_WH_PFX AUFS_DR_BRHINO_NAME + +#define AUFS_DIROPQ_NAME AUFS_WH_PFX ".opq" /* whiteouted doubly */ +#define AUFS_WH_DIROPQ AUFS_WH_PFX AUFS_DIROPQ_NAME + +#define AUFS_BASE_NAME AUFS_WH_PFX AUFS_NAME +#define AUFS_PLINKDIR_NAME AUFS_WH_PFX "plnk" +#define AUFS_ORPHDIR_NAME AUFS_WH_PFX "orph" + +/* doubly whiteouted */ +#define AUFS_WH_BASE AUFS_WH_PFX AUFS_BASE_NAME +#define AUFS_WH_PLINKDIR AUFS_WH_PFX AUFS_PLINKDIR_NAME +#define AUFS_WH_ORPHDIR AUFS_WH_PFX AUFS_ORPHDIR_NAME + +/* branch permissions and attributes */ +#define AUFS_BRPERM_RW "rw" +#define AUFS_BRPERM_RO "ro" +#define AUFS_BRPERM_RR "rr" +#define AUFS_BRATTR_COO_REG "coo_reg" +#define AUFS_BRATTR_COO_ALL "coo_all" +#define AUFS_BRATTR_FHSM "fhsm" +#define AUFS_BRATTR_UNPIN "unpin" +#define AUFS_BRATTR_ICEX "icex" +#define AUFS_BRATTR_ICEX_SEC "icexsec" +#define AUFS_BRATTR_ICEX_SYS "icexsys" +#define AUFS_BRATTR_ICEX_TR "icextr" +#define AUFS_BRATTR_ICEX_USR "icexusr" +#define AUFS_BRATTR_ICEX_OTH "icexoth" +#define AUFS_BRRATTR_WH "wh" +#define AUFS_BRWATTR_NLWH "nolwh" +#define AUFS_BRWATTR_MOO "moo" + +#define AuBrPerm_RW 1 /* writable, hardlinkable wh */ +#define AuBrPerm_RO (1 << 1) /* readonly */ +#define AuBrPerm_RR (1 << 2) /* natively readonly */ +#define AuBrPerm_Mask (AuBrPerm_RW | AuBrPerm_RO | AuBrPerm_RR) + +#define AuBrAttr_COO_REG (1 << 3) /* copy-up on open */ +#define AuBrAttr_COO_ALL (1 << 4) +#define AuBrAttr_COO_Mask (AuBrAttr_COO_REG | AuBrAttr_COO_ALL) + +#define AuBrAttr_FHSM (1 << 5) /* file-based hsm */ +#define AuBrAttr_UNPIN (1 << 6) /* rename-able top dir of + branch. meaningless since + linux-3.18-rc1 */ + +/* ignore error in copying XATTR */ +#define AuBrAttr_ICEX_SEC (1 << 7) +#define AuBrAttr_ICEX_SYS (1 << 8) +#define AuBrAttr_ICEX_TR (1 << 9) +#define AuBrAttr_ICEX_USR (1 << 10) +#define AuBrAttr_ICEX_OTH (1 << 11) +#define AuBrAttr_ICEX (AuBrAttr_ICEX_SEC \ + | AuBrAttr_ICEX_SYS \ + | AuBrAttr_ICEX_TR \ + | AuBrAttr_ICEX_USR \ + | AuBrAttr_ICEX_OTH) + +#define AuBrRAttr_WH (1 << 12) /* whiteout-able */ +#define AuBrRAttr_Mask AuBrRAttr_WH + +#define AuBrWAttr_NoLinkWH (1 << 13) /* un-hardlinkable whiteouts */ +#define AuBrWAttr_MOO (1 << 14) /* move-up on open */ +#define AuBrWAttr_Mask (AuBrWAttr_NoLinkWH | AuBrWAttr_MOO) + +#define AuBrAttr_CMOO_Mask (AuBrAttr_COO_Mask | AuBrWAttr_MOO) + +/* #warning test userspace */ +#ifdef __KERNEL__ +#ifndef CONFIG_AUFS_FHSM +#undef AuBrAttr_FHSM +#define AuBrAttr_FHSM 0 +#endif +#ifndef CONFIG_AUFS_XATTR +#undef AuBrAttr_ICEX +#define AuBrAttr_ICEX 0 +#undef AuBrAttr_ICEX_SEC +#define AuBrAttr_ICEX_SEC 0 +#undef AuBrAttr_ICEX_SYS +#define AuBrAttr_ICEX_SYS 0 +#undef AuBrAttr_ICEX_TR +#define AuBrAttr_ICEX_TR 0 +#undef AuBrAttr_ICEX_USR +#define AuBrAttr_ICEX_USR 0 +#undef AuBrAttr_ICEX_OTH +#define AuBrAttr_ICEX_OTH 0 +#endif +#endif + +/* the longest combination */ +/* AUFS_BRATTR_ICEX and AUFS_BRATTR_ICEX_TR don't affect here */ +#define AuBrPermStrSz sizeof(AUFS_BRPERM_RW \ + "+" AUFS_BRATTR_COO_REG \ + "+" AUFS_BRATTR_FHSM \ + "+" AUFS_BRATTR_UNPIN \ + "+" AUFS_BRATTR_ICEX_SEC \ + "+" AUFS_BRATTR_ICEX_SYS \ + "+" AUFS_BRATTR_ICEX_USR \ + "+" AUFS_BRATTR_ICEX_OTH \ + "+" AUFS_BRWATTR_NLWH) + +typedef struct { + char a[AuBrPermStrSz]; +} au_br_perm_str_t; + +static inline int au_br_writable(int brperm) +{ + return brperm & AuBrPerm_RW; +} + +static inline int au_br_whable(int brperm) +{ + return brperm & (AuBrPerm_RW | AuBrRAttr_WH); +} + +static inline int au_br_wh_linkable(int brperm) +{ + return !(brperm & AuBrWAttr_NoLinkWH); +} + +static inline int au_br_cmoo(int brperm) +{ + return brperm & AuBrAttr_CMOO_Mask; +} + +static inline int au_br_fhsm(int brperm) +{ + return brperm & AuBrAttr_FHSM; +} + +/* ---------------------------------------------------------------------- */ + +/* ioctl */ +enum { + /* readdir in userspace */ + AuCtl_RDU, + AuCtl_RDU_INO, + + AuCtl_WBR_FD, /* pathconf wrapper */ + AuCtl_IBUSY, /* busy inode */ + AuCtl_MVDOWN, /* move-down */ + AuCtl_BR, /* info about branches */ + AuCtl_FHSM_FD /* connection for fhsm */ +}; + +/* borrowed from linux/include/linux/kernel.h */ +#ifndef ALIGN +#ifdef _GNU_SOURCE +#define ALIGN(x, a) __ALIGN_MASK(x, (typeof(x))(a)-1) +#else +#define ALIGN(x, a) (((x) + (a) - 1) & ~((a) - 1)) +#endif +#define __ALIGN_MASK(x, mask) (((x)+(mask))&~(mask)) +#endif + +/* borrowed from linux/include/linux/compiler-gcc3.h */ +#ifndef __aligned +#define __aligned(x) __attribute__((aligned(x))) +#endif + +#ifdef __KERNEL__ +#ifndef __packed +#define __packed __attribute__((packed)) +#endif +#endif + +struct au_rdu_cookie { + uint64_t h_pos; + int16_t bindex; + uint8_t flags; + uint8_t pad; + uint32_t generation; +} __aligned(8); + +struct au_rdu_ent { + uint64_t ino; + int16_t bindex; + uint8_t type; + uint8_t nlen; + uint8_t wh; + char name[]; +} __aligned(8); + +static inline int au_rdu_len(int nlen) +{ + /* include the terminating NULL */ + return ALIGN(sizeof(struct au_rdu_ent) + nlen + 1, + sizeof(uint64_t)); +} + +union au_rdu_ent_ul { + struct au_rdu_ent __user *e; + uint64_t ul; +}; + +enum { + AufsCtlRduV_SZ, + AufsCtlRduV_End +}; + +struct aufs_rdu { + /* input */ + union { + uint64_t sz; /* AuCtl_RDU */ + uint64_t nent; /* AuCtl_RDU_INO */ + }; + union au_rdu_ent_ul ent; + uint16_t verify[AufsCtlRduV_End]; + + /* input/output */ + uint32_t blk; + + /* output */ + union au_rdu_ent_ul tail; + /* number of entries which were added in a single call */ + uint64_t rent; + uint8_t full; + uint8_t shwh; + + struct au_rdu_cookie cookie; +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +/* dirren. the branch is identified by the filename who contains this */ +struct au_drinfo { + uint64_t ino; + union { + uint8_t oldnamelen; + uint64_t _padding; + }; + uint8_t oldname[]; +} __aligned(8); + +struct au_drinfo_fdata { + uint32_t magic; + struct au_drinfo drinfo; +} __aligned(8); + +#define AUFS_DRINFO_MAGIC_V1 ('a' << 24 | 'd' << 16 | 'r' << 8 | 0x01) +/* future */ +#define AUFS_DRINFO_MAGIC_V2 ('a' << 24 | 'd' << 16 | 'r' << 8 | 0x02) + +/* ---------------------------------------------------------------------- */ + +struct aufs_wbr_fd { + uint32_t oflags; + int16_t brid; +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +struct aufs_ibusy { + uint64_t ino, h_ino; + int16_t bindex; +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +/* error code for move-down */ +/* the actual message strings are implemented in aufs-util.git */ +enum { + EAU_MVDOWN_OPAQUE = 1, + EAU_MVDOWN_WHITEOUT, + EAU_MVDOWN_UPPER, + EAU_MVDOWN_BOTTOM, + EAU_MVDOWN_NOUPPER, + EAU_MVDOWN_NOLOWERBR, + EAU_Last +}; + +/* flags for move-down */ +#define AUFS_MVDOWN_DMSG 1 +#define AUFS_MVDOWN_OWLOWER (1 << 1) /* overwrite lower */ +#define AUFS_MVDOWN_KUPPER (1 << 2) /* keep upper */ +#define AUFS_MVDOWN_ROLOWER (1 << 3) /* do even if lower is RO */ +#define AUFS_MVDOWN_ROLOWER_R (1 << 4) /* did on lower RO */ +#define AUFS_MVDOWN_ROUPPER (1 << 5) /* do even if upper is RO */ +#define AUFS_MVDOWN_ROUPPER_R (1 << 6) /* did on upper RO */ +#define AUFS_MVDOWN_BRID_UPPER (1 << 7) /* upper brid */ +#define AUFS_MVDOWN_BRID_LOWER (1 << 8) /* lower brid */ +#define AUFS_MVDOWN_FHSM_LOWER (1 << 9) /* find fhsm attr for lower */ +#define AUFS_MVDOWN_STFS (1 << 10) /* req. stfs */ +#define AUFS_MVDOWN_STFS_FAILED (1 << 11) /* output: stfs is unusable */ +#define AUFS_MVDOWN_BOTTOM (1 << 12) /* output: no more lowers */ + +/* index for move-down */ +enum { + AUFS_MVDOWN_UPPER, + AUFS_MVDOWN_LOWER, + AUFS_MVDOWN_NARRAY +}; + +/* + * additional info of move-down + * number of free blocks and inodes. + * subset of struct kstatfs, but smaller and always 64bit. + */ +struct aufs_stfs { + uint64_t f_blocks; + uint64_t f_bavail; + uint64_t f_files; + uint64_t f_ffree; +}; + +struct aufs_stbr { + int16_t brid; /* optional input */ + int16_t bindex; /* output */ + struct aufs_stfs stfs; /* output when AUFS_MVDOWN_STFS set */ +} __aligned(8); + +struct aufs_mvdown { + uint32_t flags; /* input/output */ + struct aufs_stbr stbr[AUFS_MVDOWN_NARRAY]; /* input/output */ + int8_t au_errno; /* output */ +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +union aufs_brinfo { + /* PATH_MAX may differ between kernel-space and user-space */ + char _spacer[4096]; + struct { + int16_t id; + int perm; + char path[]; + }; +} __aligned(8); + +/* ---------------------------------------------------------------------- */ + +#define AuCtlType 'A' +#define AUFS_CTL_RDU _IOWR(AuCtlType, AuCtl_RDU, struct aufs_rdu) +#define AUFS_CTL_RDU_INO _IOWR(AuCtlType, AuCtl_RDU_INO, struct aufs_rdu) +#define AUFS_CTL_WBR_FD _IOW(AuCtlType, AuCtl_WBR_FD, \ + struct aufs_wbr_fd) +#define AUFS_CTL_IBUSY _IOWR(AuCtlType, AuCtl_IBUSY, struct aufs_ibusy) +#define AUFS_CTL_MVDOWN _IOWR(AuCtlType, AuCtl_MVDOWN, \ + struct aufs_mvdown) +#define AUFS_CTL_BRINFO _IOW(AuCtlType, AuCtl_BR, union aufs_brinfo) +#define AUFS_CTL_FHSM_FD _IOW(AuCtlType, AuCtl_FHSM_FD, int) + +#endif /* __AUFS_TYPE_H__ */ From e13f4748307f4f7613ca36c7a38fb558106f7505 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:41:20 -0500 Subject: [PATCH 006/213] merge: can-isotp: https://github.com/hartkopp/can-isotp https://github.com/hartkopp/can-isotp/commit/ced84cab120a1ddde2bd4561a18514707c8a6184 Signed-off-by: Robert Nelson --- include/uapi/linux/can/isotp.h | 135 +++ net/can/isotp.c | 1537 ++++++++++++++++++++++++++++++++ 2 files changed, 1672 insertions(+) create mode 100644 include/uapi/linux/can/isotp.h create mode 100644 net/can/isotp.c diff --git a/include/uapi/linux/can/isotp.h b/include/uapi/linux/can/isotp.h new file mode 100644 index 0000000000000..548e732f33a8e --- /dev/null +++ b/include/uapi/linux/can/isotp.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ +/* + * socketcan/can/isotp.h + * + * Definitions for isotp CAN sockets + * + * Author: Oliver Hartkopp + * Copyright (c) 2008 Volkswagen Group Electronic Research + * All rights reserved. + * + * Send feedback to + * + */ + +#ifndef CAN_ISOTP_H +#define CAN_ISOTP_H + +#include + +#define SOL_CAN_ISOTP (SOL_CAN_BASE + CAN_ISOTP) + +/* for socket options affecting the socket (not the global system) */ + +#define CAN_ISOTP_OPTS 1 /* pass struct can_isotp_options */ + +#define CAN_ISOTP_RECV_FC 2 /* pass struct can_isotp_fc_options */ + +/* sockopts to force stmin timer values for protocol regression tests */ + +#define CAN_ISOTP_TX_STMIN 3 /* pass __u32 value in nano secs */ + /* use this time instead of value */ + /* provided in FC from the receiver */ + +#define CAN_ISOTP_RX_STMIN 4 /* pass __u32 value in nano secs */ + /* ignore received CF frames which */ + /* timestamps differ less than val */ + +#define CAN_ISOTP_LL_OPTS 5 /* pass struct can_isotp_ll_options */ + +struct can_isotp_options { + + __u32 flags; /* set flags for isotp behaviour. */ + /* __u32 value : flags see below */ + + __u32 frame_txtime; /* frame transmission time (N_As/N_Ar) */ + /* __u32 value : time in nano secs */ + + __u8 ext_address; /* set address for extended addressing */ + /* __u8 value : extended address */ + + __u8 txpad_content; /* set content of padding byte (tx) */ + /* __u8 value : content on tx path */ + + __u8 rxpad_content; /* set content of padding byte (rx) */ + /* __u8 value : content on rx path */ + + __u8 rx_ext_address; /* set address for extended addressing */ + /* __u8 value : extended address (rx) */ +}; + +struct can_isotp_fc_options { + + __u8 bs; /* blocksize provided in FC frame */ + /* __u8 value : blocksize. 0 = off */ + + __u8 stmin; /* separation time provided in FC frame */ + /* __u8 value : */ + /* 0x00 - 0x7F : 0 - 127 ms */ + /* 0x80 - 0xF0 : reserved */ + /* 0xF1 - 0xF9 : 100 us - 900 us */ + /* 0xFA - 0xFF : reserved */ + + __u8 wftmax; /* max. number of wait frame transmiss. */ + /* __u8 value : 0 = omit FC N_PDU WT */ +}; + +struct can_isotp_ll_options { + + __u8 mtu; /* generated & accepted CAN frame type */ + /* __u8 value : */ + /* CAN_MTU (16) -> standard CAN 2.0 */ + /* CANFD_MTU (72) -> CAN FD frame */ + + __u8 tx_dl; /* tx link layer data length in bytes */ + /* (configured maximum payload length) */ + /* __u8 value : 8,12,16,20,24,32,48,64 */ + /* => rx path supports all LL_DL values */ + + __u8 tx_flags; /* set into struct canfd_frame.flags */ + /* at frame creation: e.g. CANFD_BRS */ + /* Obsolete when the BRS flag is fixed */ + /* by the CAN netdriver configuration */ +}; + +/* flags for isotp behaviour */ + +#define CAN_ISOTP_LISTEN_MODE 0x001 /* listen only (do not send FC) */ +#define CAN_ISOTP_EXTEND_ADDR 0x002 /* enable extended addressing */ +#define CAN_ISOTP_TX_PADDING 0x004 /* enable CAN frame padding tx path */ +#define CAN_ISOTP_RX_PADDING 0x008 /* enable CAN frame padding rx path */ +#define CAN_ISOTP_CHK_PAD_LEN 0x010 /* check received CAN frame padding */ +#define CAN_ISOTP_CHK_PAD_DATA 0x020 /* check received CAN frame padding */ +#define CAN_ISOTP_HALF_DUPLEX 0x040 /* half duplex error state handling */ +#define CAN_ISOTP_FORCE_TXSTMIN 0x080 /* ignore stmin from received FC */ +#define CAN_ISOTP_FORCE_RXSTMIN 0x100 /* ignore CFs depending on rx stmin */ +#define CAN_ISOTP_RX_EXT_ADDR 0x200 /* different rx extended addressing */ + + +/* default values */ + +#define CAN_ISOTP_DEFAULT_FLAGS 0 +#define CAN_ISOTP_DEFAULT_EXT_ADDRESS 0x00 +#define CAN_ISOTP_DEFAULT_PAD_CONTENT 0xCC /* prevent bit-stuffing */ +#define CAN_ISOTP_DEFAULT_FRAME_TXTIME 0 +#define CAN_ISOTP_DEFAULT_RECV_BS 0 +#define CAN_ISOTP_DEFAULT_RECV_STMIN 0x00 +#define CAN_ISOTP_DEFAULT_RECV_WFTMAX 0 + +#define CAN_ISOTP_DEFAULT_LL_MTU CAN_MTU +#define CAN_ISOTP_DEFAULT_LL_TX_DL CAN_MAX_DLEN +#define CAN_ISOTP_DEFAULT_LL_TX_FLAGS 0 + +/* + * Remark on CAN_ISOTP_DEFAULT_RECV_* values: + * + * We can strongly assume, that the Linux Kernel implementation of + * CAN_ISOTP is capable to run with BS=0, STmin=0 and WFTmax=0. + * But as we like to be able to behave as a commonly available ECU, + * these default settings can be changed via sockopts. + * For that reason the STmin value is intentionally _not_ checked for + * consistency and copied directly into the flow control (FC) frame. + * + */ + +#endif diff --git a/net/can/isotp.c b/net/can/isotp.c new file mode 100644 index 0000000000000..a0d20d9443bf5 --- /dev/null +++ b/net/can/isotp.c @@ -0,0 +1,1537 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ +/* + * isotp.c - ISO 15765-2 CAN transport protocol for protocol family CAN + * + * WARNING: This is ALPHA code for discussions and first tests that should + * not be used in production environments. + * + * In the discussion the Socket-API to the userspace or the ISO-TP socket + * options or the return values we may change! Current behaviour: + * + * - no ISO-TP specific return values are provided to the userspace + * - when a transfer (tx) is on the run the next write() blocks until it's done + * - no support for sending wait frames to the data source in the rx path + * + * Copyright (c) 2017 Volkswagen Group Electronic Research + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of Volkswagen nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * Alternatively, provided that this notice is retained in full, this + * software may be distributed under the terms of the GNU General + * Public License ("GPL") version 2, in which case the provisions of the + * GPL apply INSTEAD OF those given above. + * + * The provided data structures and external interfaces from this code + * are not restricted to be used by modules with a GPL compatible license. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + * + * Send feedback to + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0) +#include +#define CAN_SKBRES sizeof(struct can_skb_priv) +#else +#define CAN_SKBRES 0 +#endif +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) +#include +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25) +#include "compat.h" +#endif + +#define CAN_ISOTP_VERSION "20181217" +static __initdata const char banner[] = + KERN_INFO "can: isotp protocol (rev " CAN_ISOTP_VERSION " alpha)\n"; + +MODULE_DESCRIPTION("PF_CAN isotp 15765-2:2016 protocol"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Oliver Hartkopp "); +MODULE_ALIAS("can-proto-6"); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,22) +#error This modules needs hrtimers (available since Kernel 2.6.22) +#endif + +#define DBG(fmt, args...) (printk( KERN_DEBUG "can-isotp: %s: " fmt, \ + __func__, ##args)) +#undef DBG +#define DBG(fmt, args...) + +#define SINGLE_MASK(id) ((id & CAN_EFF_FLAG) ? \ + (CAN_EFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG) : \ + (CAN_SFF_MASK | CAN_EFF_FLAG | CAN_RTR_FLAG)) + +/* + ISO 15765-2:2016 supports more than 4095 byte per ISO PDU as the FF_DL can + take full 32 bit values (4 Gbyte). We would need some good concept to handle + this between user space and kernel space. For now increase the static buffer + to something about 8 kbyte to be able to test this new functionality. +*/ +#define MAX_MSG_LENGTH 8200 + +/* N_PCI type values in bits 7-4 of N_PCI bytes */ +#define N_PCI_SF 0x00 /* single frame */ +#define N_PCI_FF 0x10 /* first frame */ +#define N_PCI_CF 0x20 /* consecutive frame */ +#define N_PCI_FC 0x30 /* flow control */ + +#define N_PCI_SZ 1 /* size of the PCI byte #1 */ +#define SF_PCI_SZ4 1 /* size of SingleFrame PCI including 4 bit SF_DL */ +#define SF_PCI_SZ8 2 /* size of SingleFrame PCI including 8 bit SF_DL */ +#define FF_PCI_SZ12 2 /* size of FirstFrame PCI including 12 bit FF_DL */ +#define FF_PCI_SZ32 6 /* size of FirstFrame PCI including 32 bit FF_DL */ +#define FC_CONTENT_SZ 3 /* flow control content size in byte (FS/BS/STmin) */ + +#define ISOTP_CHECK_PADDING (CAN_ISOTP_CHK_PAD_LEN | CAN_ISOTP_CHK_PAD_DATA) + +/* Flow Status given in FC frame */ +#define ISOTP_FC_CTS 0 /* clear to send */ +#define ISOTP_FC_WT 1 /* wait */ +#define ISOTP_FC_OVFLW 2 /* overflow */ + +enum { + ISOTP_IDLE = 0, + ISOTP_WAIT_FIRST_FC, + ISOTP_WAIT_FC, + ISOTP_WAIT_DATA, + ISOTP_SENDING +}; + +struct tpcon { + int idx; + int len; + u8 state; + u8 bs; + u8 sn; + u8 ll_dl; + u8 buf[MAX_MSG_LENGTH+1]; +}; + +struct isotp_sock { + struct sock sk; + int bound; + int ifindex; + canid_t txid; + canid_t rxid; + ktime_t tx_gap; + ktime_t lastrxcf_tstamp; + struct hrtimer rxtimer, txtimer; + struct tasklet_struct txtsklet; + struct can_isotp_options opt; + struct can_isotp_fc_options rxfc, txfc; + struct can_isotp_ll_options ll; + __u32 force_tx_stmin; + __u32 force_rx_stmin; + struct tpcon rx, tx; + struct notifier_block notifier; + wait_queue_head_t wait; +}; + +static inline struct isotp_sock *isotp_sk(const struct sock *sk) +{ + return (struct isotp_sock *)sk; +} + +static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + rxtimer); + if (so->rx.state == ISOTP_WAIT_DATA) { + DBG("we did not get new data frames in time.\n"); + + /* reset tx state */ + so->rx.state = ISOTP_IDLE; + } + + return HRTIMER_NORESTART; +} + +static void isotp_skb_reserve(struct sk_buff *skb, struct net_device *dev) +{ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,9,0) + can_skb_reserve(skb); + can_skb_prv(skb)->ifindex = dev->ifindex; +#endif +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,5) + can_skb_prv(skb)->skbcnt = 0; +#endif +} + +static void isotp_skb_destructor(struct sk_buff *skb) +{ + sock_put(skb->sk); +} + +static inline void isotp_skb_set_owner(struct sk_buff *skb, struct sock *sk) +{ + if (sk) { + sock_hold(sk); + skb->destructor = isotp_skb_destructor; + skb->sk = sk; + } +} + +static int isotp_send_fc(struct sock *sk, int ae, u8 flowstatus) +{ + struct net_device *dev; + struct sk_buff *nskb; + struct canfd_frame *ncf; + struct isotp_sock *so = isotp_sk(sk); + + nskb = alloc_skb(so->ll.mtu + CAN_SKBRES, gfp_any()); + if (!nskb) + return 1; + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) { + kfree_skb(nskb); + return 1; + } + isotp_skb_reserve(nskb, dev); + nskb->dev = dev; + isotp_skb_set_owner(nskb, sk); + ncf = (struct canfd_frame *) nskb->data; + skb_put(nskb, so->ll.mtu); + + /* create & send flow control reply */ + ncf->can_id = so->txid; + + if (so->opt.flags & CAN_ISOTP_TX_PADDING) { + memset(ncf->data, so->opt.txpad_content, CAN_MAX_DLEN); + ncf->len = CAN_MAX_DLEN; + } else + ncf->len = ae + FC_CONTENT_SZ; + + ncf->data[ae] = N_PCI_FC | flowstatus; + ncf->data[ae + 1] = so->rxfc.bs; + ncf->data[ae + 2] = so->rxfc.stmin; + + if (ae) + ncf->data[0] = so->opt.ext_address; + + if (so->ll.mtu == CANFD_MTU) + ncf->flags = so->ll.tx_flags; + + can_send(nskb, 1); + dev_put(dev); + + /* reset blocksize counter */ + so->rx.bs = 0; + + /* reset last CF frame rx timestamp for rx stmin enforcement */ + so->lastrxcf_tstamp = ktime_set(0,0); + + /* start rx timeout watchdog */ + hrtimer_start(&so->rxtimer, ktime_set(1,0), HRTIMER_MODE_REL); + return 0; +} + +static void isotp_rcv_skb(struct sk_buff *skb, struct sock *sk) +{ + struct sockaddr_can *addr = (struct sockaddr_can *)skb->cb; + + BUILD_BUG_ON(sizeof(skb->cb) < sizeof(struct sockaddr_can)); + + memset(addr, 0, sizeof(*addr)); + addr->can_family = AF_CAN; + addr->can_ifindex = skb->dev->ifindex; + + if (sock_queue_rcv_skb(sk, skb) < 0) + kfree_skb(skb); +} + +static u8 padlen(u8 datalen) +{ + const u8 plen[] = {8, 8, 8, 8, 8, 8, 8, 8, 8, /* 0 - 8 */ + 12, 12, 12, 12, /* 9 - 12 */ + 16, 16, 16, 16, /* 13 - 16 */ + 20, 20, 20, 20, /* 17 - 20 */ + 24, 24, 24, 24, /* 21 - 24 */ + 32, 32, 32, 32, 32, 32, 32, 32, /* 25 - 32 */ + 48, 48, 48, 48, 48, 48, 48, 48, /* 33 - 40 */ + 48, 48, 48, 48, 48, 48, 48, 48}; /* 41 - 48 */ + + if (datalen > 48) + return 64; + + return plen[datalen]; +} + +/* check for length optimization and return 1/true when the check fails */ +static int check_optimized(struct canfd_frame *cf, int start_index) +{ + /* + * for CAN_DL <= 8 the start_index is equal to the CAN_DL as the + * padding would start at this point. E.g. if the padding would + * start at cf.data[7] cf->len has to be 7 to be optimal. + * Note: The data[] index starts with zero. + */ + if (cf->len <= CAN_MAX_DLEN) + return (cf->len != start_index); + + /* + * This relation is also valid in the non-linear DLC range, where + * we need to take care of the minimal next possible CAN_DL. + * The correct check would be (padlen(cf->len) != padlen(start_index)). + * But as cf->len can only take discrete values from 12, .., 64 at this + * point the padlen(cf->len) is always equal to cf->len. + */ + return (cf->len != padlen(start_index)); +} + +/* check padding and return 1/true when the check fails */ +static int check_pad(struct isotp_sock *so, struct canfd_frame *cf, + int start_index, __u8 content) +{ + int i; + + /* no RX_PADDING value => check length of optimized frame length */ + if (!(so->opt.flags & CAN_ISOTP_RX_PADDING)) { + + if (so->opt.flags & CAN_ISOTP_CHK_PAD_LEN) + return check_optimized(cf, start_index); + + /* no valid test against empty value => ignore frame */ + return 1; + } + + /* check datalength of correctly padded CAN frame */ + if ((so->opt.flags & CAN_ISOTP_CHK_PAD_LEN) && + cf->len != padlen(cf->len)) + return 1; + + /* check padding content */ + if (so->opt.flags & CAN_ISOTP_CHK_PAD_DATA) { + for (i = start_index; i < cf->len; i++) + if (cf->data[i] != content) + return 1; + } + return 0; +} + +static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) +{ + if (so->tx.state != ISOTP_WAIT_FC && + so->tx.state != ISOTP_WAIT_FIRST_FC) + return 0; + + hrtimer_cancel(&so->txtimer); + + if ((cf->len < ae + FC_CONTENT_SZ) || + ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, ae + FC_CONTENT_SZ, so->opt.rxpad_content))) { + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + return 1; + } + + /* get communication parameters only from the first FC frame */ + if (so->tx.state == ISOTP_WAIT_FIRST_FC) { + + so->txfc.bs = cf->data[ae + 1]; + so->txfc.stmin = cf->data[ae + 2]; + + /* fix wrong STmin values according spec */ + if ((so->txfc.stmin > 0x7F) && + ((so->txfc.stmin < 0xF1) || (so->txfc.stmin > 0xF9))) + so->txfc.stmin = 0x7F; + + so->tx_gap = ktime_set(0,0); + /* add transmission time for CAN frame N_As */ + so->tx_gap = ktime_add_ns(so->tx_gap, so->opt.frame_txtime); + /* add waiting time for consecutive frames N_Cs */ + if (so->opt.flags & CAN_ISOTP_FORCE_TXSTMIN) + so->tx_gap = ktime_add_ns(so->tx_gap, + so->force_tx_stmin); + else if (so->txfc.stmin < 0x80) + so->tx_gap = ktime_add_ns(so->tx_gap, + so->txfc.stmin * 1000000); + else + so->tx_gap = ktime_add_ns(so->tx_gap, + (so->txfc.stmin - 0xF0) + * 100000); + so->tx.state = ISOTP_WAIT_FC; + } + + DBG("FC frame: FS %d, BS %d, STmin 0x%02X, tx_gap %lld\n", + cf->data[ae] & 0x0F & 0x0F, so->txfc.bs, so->txfc.stmin, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + (long long)so->tx_gap); +#else + (long long)so->tx_gap.tv64); +#endif + + switch (cf->data[ae] & 0x0F) { + + case ISOTP_FC_CTS: + so->tx.bs = 0; + so->tx.state = ISOTP_SENDING; + DBG("starting txtimer for sending\n"); + /* start cyclic timer for sending CF frame */ + hrtimer_start(&so->txtimer, so->tx_gap, + HRTIMER_MODE_REL); + break; + + case ISOTP_FC_WT: + DBG("starting waiting for next FC\n"); + /* start timer to wait for next FC frame */ + hrtimer_start(&so->txtimer, ktime_set(1,0), + HRTIMER_MODE_REL); + break; + + case ISOTP_FC_OVFLW: + DBG("overflow in receiver side\n"); + + default: + /* stop this tx job. TODO: error reporting? */ + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + } + return 0; +} + +static int isotp_rcv_sf(struct sock *sk, struct canfd_frame *cf, int pcilen, + struct sk_buff *skb, int len) +{ + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *nskb; + + hrtimer_cancel(&so->rxtimer); + so->rx.state = ISOTP_IDLE; + + if (!len || len > cf->len - pcilen) + return 1; + + if ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, pcilen + len, so->opt.rxpad_content)) + return 1; + + nskb = alloc_skb(len, gfp_any()); + if (!nskb) + return 1; + + memcpy(skb_put(nskb, len), &cf->data[pcilen], len); + + nskb->tstamp = skb->tstamp; + nskb->dev = skb->dev; + isotp_rcv_skb(nskb, sk); + return 0; +} + +static int isotp_rcv_ff(struct sock *sk, struct canfd_frame *cf, int ae) +{ + struct isotp_sock *so = isotp_sk(sk); + int i; + int off; + int ff_pci_sz; + + hrtimer_cancel(&so->rxtimer); + so->rx.state = ISOTP_IDLE; + + /* get the used sender LL_DL from the (first) CAN frame data length */ + so->rx.ll_dl = padlen(cf->len); + + /* the first frame has to use the entire frame up to LL_DL length */ + if (cf->len != so->rx.ll_dl) + return 1; + + /* get the FF_DL */ + so->rx.len = (cf->data[ae] & 0x0F) << 8; + so->rx.len += cf->data[ae + 1]; + + /* Check for FF_DL escape sequence supporting 32 bit PDU length */ + if (so->rx.len) + ff_pci_sz = FF_PCI_SZ12; + else { + /* FF_DL = 0 => get real length from next 4 bytes */ + so->rx.len = cf->data[ae + 2] << 24; + so->rx.len += cf->data[ae + 3] << 16; + so->rx.len += cf->data[ae + 4] << 8; + so->rx.len += cf->data[ae + 5]; + ff_pci_sz = FF_PCI_SZ32; + } + + /* take care of a potential SF_DL ESC offset for TX_DL > 8 */ + off = (so->rx.ll_dl > CAN_MAX_DLEN)? 1:0; + + if (so->rx.len + ae + off + ff_pci_sz < so->rx.ll_dl) + return 1; + + if (so->rx.len > MAX_MSG_LENGTH) { + /* send FC frame with overflow status */ + isotp_send_fc(sk, ae, ISOTP_FC_OVFLW); + return 1; + } + + /* copy the first received data bytes */ + so->rx.idx = 0; + for (i = ae + ff_pci_sz; i < so->rx.ll_dl; i++) + so->rx.buf[so->rx.idx++] = cf->data[i]; + + /* initial setup for this pdu receiption */ + so->rx.sn = 1; + so->rx.state = ISOTP_WAIT_DATA; + + /* no creation of flow control frames */ + if (so->opt.flags & CAN_ISOTP_LISTEN_MODE) + return 0; + + /* send our first FC frame */ + isotp_send_fc(sk, ae, ISOTP_FC_CTS); + return 0; +} + +static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, + struct sk_buff *skb) +{ + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *nskb; + int i; + + if (so->rx.state != ISOTP_WAIT_DATA) + return 0; + + /* drop if timestamp gap is less than force_rx_stmin nano secs */ + if (so->opt.flags & CAN_ISOTP_FORCE_RXSTMIN) { + + if (ktime_to_ns(ktime_sub(skb->tstamp, so->lastrxcf_tstamp)) < + so->force_rx_stmin) + return 0; + + so->lastrxcf_tstamp = skb->tstamp; + } + + hrtimer_cancel(&so->rxtimer); + + /* CFs are never longer than the FF */ + if (cf->len > so->rx.ll_dl) + return 1; + + /* CFs have usually the LL_DL length */ + if (cf->len < so->rx.ll_dl) { + /* this is only allowed for the last CF */ + if (so->rx.len - so->rx.idx > so->rx.ll_dl - ae - N_PCI_SZ) + return 1; + } + + if ((cf->data[ae] & 0x0F) != so->rx.sn) { + DBG("wrong sn %d. expected %d.\n", + cf->data[ae] & 0x0F, so->rx.sn); + /* some error reporting? */ + so->rx.state = ISOTP_IDLE; + return 1; + } + so->rx.sn++; + so->rx.sn %= 16; + + for (i = ae + N_PCI_SZ; i < cf->len; i++) { + so->rx.buf[so->rx.idx++] = cf->data[i]; + if (so->rx.idx >= so->rx.len) + break; + } + + if (so->rx.idx >= so->rx.len) { + + /* we are done */ + so->rx.state = ISOTP_IDLE; + + if ((so->opt.flags & ISOTP_CHECK_PADDING) && + check_pad(so, cf, i+1, so->opt.rxpad_content)) + return 1; + + nskb = alloc_skb(so->rx.len, gfp_any()); + if (!nskb) + return 1; + + memcpy(skb_put(nskb, so->rx.len), so->rx.buf, + so->rx.len); + + nskb->tstamp = skb->tstamp; + nskb->dev = skb->dev; + isotp_rcv_skb(nskb, sk); + return 0; + } + + /* no creation of flow control frames */ + if (so->opt.flags & CAN_ISOTP_LISTEN_MODE) + return 0; + + /* perform blocksize handling, if enabled */ + if (!so->rxfc.bs || ++so->rx.bs < so->rxfc.bs) { + + /* start rx timeout watchdog */ + hrtimer_start(&so->rxtimer, ktime_set(1,0), + HRTIMER_MODE_REL); + return 0; + } + + /* we reached the specified blocksize so->rxfc.bs */ + isotp_send_fc(sk, ae, ISOTP_FC_CTS); + return 0; +} + +static void isotp_rcv(struct sk_buff *skb, void *data) +{ + struct sock *sk = (struct sock *)data; + struct isotp_sock *so = isotp_sk(sk); + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0; + u8 n_pci_type, sf_dl; + + BUG_ON(skb->len != CAN_MTU && skb->len != CANFD_MTU); + + /* + * Strictly receive only frames with the configured MTU size + * => clear separation of CAN2.0 / CAN FD transport channels + */ + if (skb->len != so->ll.mtu) + return; + + cf = (struct canfd_frame *) skb->data; + + /* if enabled: check receiption of my configured extended address */ + if (ae && cf->data[0] != so->opt.rx_ext_address) + return; + + n_pci_type = cf->data[ae] & 0xF0; + + if (so->opt.flags & CAN_ISOTP_HALF_DUPLEX) { + /* check rx/tx path half duplex expectations */ + if ((so->tx.state != ISOTP_IDLE && n_pci_type != N_PCI_FC) || + (so->rx.state != ISOTP_IDLE && n_pci_type == N_PCI_FC)) + return; + } + + switch (n_pci_type) { + case N_PCI_FC: + /* tx path: flow control frame containing the FC parameters */ + isotp_rcv_fc(so, cf, ae); + break; + + case N_PCI_SF: + /* + * rx path: single frame + * + * As we do not have a rx.ll_dl configuration, we can only test + * if the CAN frames payload length matches the LL_DL == 8 + * requirements - no matter if it's CAN 2.0 or CAN FD + */ + + /* get the SF_DL from the N_PCI byte */ + sf_dl = cf->data[ae] & 0x0F; + + if (cf->len <= CAN_MAX_DLEN) + isotp_rcv_sf(sk, cf, SF_PCI_SZ4 + ae, skb, sf_dl); + else if (skb->len == CANFD_MTU) { + /* + * We have a CAN FD frame and CAN_DL is greater than 8: + * Only frames with the SF_DL == 0 ESC value are valid. + * + * If so take care of the increased SF PCI size + * (SF_PCI_SZ8) to point to the message content behind + * the extended SF PCI info and get the real SF_DL + * length value from the formerly first data byte. + */ + if (sf_dl == 0) + isotp_rcv_sf(sk, cf, SF_PCI_SZ8 + ae, skb, + cf->data[SF_PCI_SZ4 + ae]); + } + break; + + case N_PCI_FF: + /* rx path: first frame */ + isotp_rcv_ff(sk, cf, ae); + break; + + case N_PCI_CF: + /* rx path: consecutive frame */ + isotp_rcv_cf(sk, cf, ae, skb); + break; + + } +} + +static void isotp_fill_dataframe(struct canfd_frame *cf, struct isotp_sock *so, + int ae, int off) +{ + int pcilen = N_PCI_SZ + ae + off; + int space = so->tx.ll_dl - pcilen; + int num = min_t(int, so->tx.len - so->tx.idx, space); + int i; + + cf->can_id = so->txid; + cf->len = num + pcilen; + + if (num < space) { + if (so->opt.flags & CAN_ISOTP_TX_PADDING) { + /* user requested padding */ + cf->len = padlen(cf->len); + memset(cf->data, so->opt.txpad_content, cf->len); + } else if (cf->len > CAN_MAX_DLEN) { + /* mandatory padding for CAN FD frames */ + cf->len = padlen(cf->len); + memset(cf->data, CAN_ISOTP_DEFAULT_PAD_CONTENT, + cf->len); + } + } + + for (i = 0; i < num; i++) + cf->data[pcilen + i] = so->tx.buf[so->tx.idx++]; + + if (ae) + cf->data[0] = so->opt.ext_address; +} + +static void isotp_create_fframe(struct canfd_frame *cf, struct isotp_sock *so, + int ae) +{ + int i; + int ff_pci_sz; + + cf->can_id = so->txid; + cf->len = so->tx.ll_dl; + if (ae) + cf->data[0] = so->opt.ext_address; + + /* create N_PCI bytes with 12/32 bit FF_DL data length */ + if (so->tx.len > 4095) { + /* use 32 bit FF_DL notation */ + cf->data[ae] = N_PCI_FF; + cf->data[ae + 1] = 0; + cf->data[ae + 2] = (u8) (so->tx.len >> 24) & 0xFFU; + cf->data[ae + 3] = (u8) (so->tx.len >> 16) & 0xFFU; + cf->data[ae + 4] = (u8) (so->tx.len >> 8) & 0xFFU; + cf->data[ae + 5] = (u8) so->tx.len & 0xFFU; + ff_pci_sz = FF_PCI_SZ32; + } else { + /* use 12 bit FF_DL notation */ + cf->data[ae] = (u8) (so->tx.len>>8) | N_PCI_FF; + cf->data[ae + 1] = (u8) so->tx.len & 0xFFU; + ff_pci_sz = FF_PCI_SZ12; + } + + /* add first data bytes depending on ae */ + for (i = ae + ff_pci_sz; i < so->tx.ll_dl; i++) + cf->data[i] = so->tx.buf[so->tx.idx++]; + + so->tx.sn = 1; + so->tx.state = ISOTP_WAIT_FIRST_FC; +} + +static void isotp_tx_timer_tsklet(unsigned long data) +{ + struct isotp_sock *so = (struct isotp_sock *)data; + struct sock *sk = &so->sk; + struct sk_buff *skb; + struct net_device *dev; + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0; + + switch (so->tx.state) { + + case ISOTP_WAIT_FC: + case ISOTP_WAIT_FIRST_FC: + + /* we did not get any flow control frame in time */ + + DBG("we did not get FC frame in time.\n"); + + /* reset tx state */ + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + break; + + case ISOTP_SENDING: + + /* push out the next segmented pdu */ + + DBG("next pdu to send.\n"); + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) + break; + +isotp_tx_burst: + skb = alloc_skb(so->ll.mtu + CAN_SKBRES, gfp_any()); + if (!skb) { + dev_put(dev); + break; + } + + isotp_skb_reserve(skb, dev); + cf = (struct canfd_frame *)skb->data; + skb_put(skb, so->ll.mtu); + + /* create consecutive frame */ + isotp_fill_dataframe(cf, so, ae, 0); + + /* place consecutive frame N_PCI in appropriate index */ + cf->data[ae] = N_PCI_CF | so->tx.sn++; + so->tx.sn %= 16; + so->tx.bs++; + + if (so->ll.mtu == CANFD_MTU) + cf->flags = so->ll.tx_flags; + + skb->dev = dev; + isotp_skb_set_owner(skb, sk); + can_send(skb, 1); + + if (so->tx.idx >= so->tx.len) { + /* we are done */ + DBG("we are done\n"); + so->tx.state = ISOTP_IDLE; + dev_put(dev); + wake_up_interruptible(&so->wait); + break; + } + + if (so->txfc.bs && so->tx.bs >= so->txfc.bs) { + /* stop and wait for FC */ + DBG("BS stop and wait for FC\n"); + so->tx.state = ISOTP_WAIT_FC; + dev_put(dev); + hrtimer_start(&so->txtimer, + ktime_add(ktime_get(), ktime_set(1,0)), + HRTIMER_MODE_ABS); + break; + } + + /* no gap between data frames needed => use burst mode */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) + if (!so->tx_gap) + goto isotp_tx_burst; +#else + if (!so->tx_gap.tv64) + goto isotp_tx_burst; +#endif + + /* start timer to send next data frame with correct delay */ + dev_put(dev); + hrtimer_start(&so->txtimer, + ktime_add(ktime_get(), so->tx_gap), + HRTIMER_MODE_ABS); + break; + + default: + BUG_ON(1); + } +} + +static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) +{ + struct isotp_sock *so = container_of(hrtimer, struct isotp_sock, + txtimer); + tasklet_schedule(&so->txtsklet); + + return HRTIMER_NORESTART; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) +static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) +#else +static int isotp_sendmsg(struct kiocb *iocb, struct socket *sock, + struct msghdr *msg, size_t size) +#endif +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + struct sk_buff *skb; + struct net_device *dev; + struct canfd_frame *cf; + int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR)? 1:0; + int off; + int err; + + if (!so->bound) + return -EADDRNOTAVAIL; + + /* we do not support multiple buffers - for now */ + if (so->tx.state != ISOTP_IDLE) { + if (msg->msg_flags & MSG_DONTWAIT) + return -EAGAIN; + + /* wait for complete transmission of current pdu */ + wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + } + + if (!size || size > MAX_MSG_LENGTH) + return -EINVAL; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) + err = memcpy_from_msg(so->tx.buf, msg, size); +#else + err = memcpy_fromiovec(so->tx.buf, msg->msg_iov, size); +#endif + if (err < 0) + return err; + + dev = dev_get_by_index(sock_net(sk), so->ifindex); + if (!dev) + return -ENXIO; + + skb = sock_alloc_send_skb(sk, so->ll.mtu + CAN_SKBRES, + msg->msg_flags & MSG_DONTWAIT, &err); + if (!skb) { + dev_put(dev); + return err; + } + + isotp_skb_reserve(skb, dev); + + so->tx.state = ISOTP_SENDING; + so->tx.len = size; + so->tx.idx = 0; + + cf = (struct canfd_frame *)skb->data; + skb_put(skb, so->ll.mtu); + + /* take care of a potential SF_DL ESC offset for TX_DL > 8 */ + off = (so->tx.ll_dl > CAN_MAX_DLEN)? 1:0; + + /* check for single frame transmission depending on TX_DL */ + if (size <= so->tx.ll_dl - SF_PCI_SZ4 - ae - off) { + + /* + * The message size generally fits into a SingleFrame - good. + * + * SF_DL ESC offset optimization: + * + * When TX_DL is greater 8 but the message would still fit + * into a 8 byte CAN frame, we can omit the offset. + * This prevents a protocol caused length extension from + * CAN_DL = 8 to CAN_DL = 12 due to the SF_SL ESC handling. + */ + if (size <= CAN_MAX_DLEN - SF_PCI_SZ4 - ae) + off = 0; + + isotp_fill_dataframe(cf, so, ae, off); + + /* place single frame N_PCI w/o length in appropriate index */ + cf->data[ae] = N_PCI_SF; + + /* place SF_DL size value depending on the SF_DL ESC offset */ + if (off) + cf->data[SF_PCI_SZ4 + ae] = size; + else + cf->data[ae] |= size; + + so->tx.state = ISOTP_IDLE; + wake_up_interruptible(&so->wait); + } else { + /* send first frame and wait for FC */ + + isotp_create_fframe(cf, so, ae); + + DBG("starting txtimer for fc\n"); + /* start timeout for FC */ + hrtimer_start(&so->txtimer, ktime_set(1,0), HRTIMER_MODE_REL); + } + + /* send the first or only CAN frame */ + if (so->ll.mtu == CANFD_MTU) + cf->flags = so->ll.tx_flags; + + skb->dev = dev; + skb->sk = sk; + err = can_send(skb, 1); + dev_put(dev); + if (err) + return err; + + return size; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,1,0) +static int isotp_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, + int flags) +#else +static int isotp_recvmsg(struct kiocb *iocb, struct socket *sock, + struct msghdr *msg, size_t size, int flags) +#endif +{ + struct sock *sk = sock->sk; + struct sk_buff *skb; + int err = 0; + int noblock; + + noblock = flags & MSG_DONTWAIT; + flags &= ~MSG_DONTWAIT; + + skb = skb_recv_datagram(sk, flags, noblock, &err); + if (!skb) + return err; + + if (size < skb->len) + msg->msg_flags |= MSG_TRUNC; + else + size = skb->len; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,19,0) + err = memcpy_to_msg(msg, skb->data, size); +#else + err = memcpy_toiovec(msg->msg_iov, skb->data, size); +#endif + if (err < 0) { + skb_free_datagram(sk, skb); + return err; + } + + sock_recv_timestamp(msg, sk, skb); + + if (msg->msg_name) { + msg->msg_namelen = sizeof(struct sockaddr_can); + memcpy(msg->msg_name, skb->cb, msg->msg_namelen); + } + + skb_free_datagram(sk, skb); + + return size; +} + +static int isotp_release(struct socket *sock) +{ + struct sock *sk = sock->sk; + struct isotp_sock *so; + struct net *net; + + if (!sk) + return 0; + + so = isotp_sk(sk); + net = sock_net(sk); + + /* wait for complete transmission of current pdu */ + wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + + unregister_netdevice_notifier(&so->notifier); + + lock_sock(sk); + + hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->rxtimer); + tasklet_kill(&so->txtsklet); + + /* remove current filters & unregister */ + if (so->bound) { + if (so->ifindex) { + struct net_device *dev; + + dev = dev_get_by_index(net, so->ifindex); + if (dev) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(net, dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + dev_put(dev); + } + } + } + + so->ifindex = 0; + so->bound = 0; + + sock_orphan(sk); + sock->sk = NULL; + + release_sock(sk); + sock_put(sk); + + return 0; +} + +static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len) +{ + struct sockaddr_can *addr = (struct sockaddr_can *)uaddr; + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + struct net *net = sock_net(sk); + int ifindex; + struct net_device *dev; + int err = 0; + int notify_enetdown = 0; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,4,0) + if (len < CAN_REQUIRED_SIZE(struct sockaddr_can, can_addr.tp)) +#else + if (len < sizeof(*addr)) +#endif + return -EINVAL; + + if (addr->can_addr.tp.rx_id == addr->can_addr.tp.tx_id) + return -EADDRNOTAVAIL; + + if ((addr->can_addr.tp.rx_id | addr->can_addr.tp.tx_id) & + (CAN_ERR_FLAG | CAN_RTR_FLAG)) + return -EADDRNOTAVAIL; + + if (!addr->can_ifindex) + return -ENODEV; + + lock_sock(sk); + + if (so->bound && addr->can_ifindex == so->ifindex && + addr->can_addr.tp.rx_id == so->rxid && + addr->can_addr.tp.tx_id == so->txid) + goto out; + + dev = dev_get_by_index(net, addr->can_ifindex); + if (!dev) { + err = -ENODEV; + goto out; + } + if (dev->type != ARPHRD_CAN) { + dev_put(dev); + err = -ENODEV; + goto out; + } + if (dev->mtu < so->ll.mtu) { + dev_put(dev); + err = -EINVAL; + goto out; + } + if (!(dev->flags & IFF_UP)) + notify_enetdown = 1; + + ifindex = dev->ifindex; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_register(net, dev, addr->can_addr.tp.rx_id, +#else + can_rx_register(dev, addr->can_addr.tp.rx_id, +#endif + SINGLE_MASK(addr->can_addr.tp.rx_id), isotp_rcv, sk, +#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4,9,11)) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(4,4,50)) && (LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0))) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(3,18,49)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0))) || \ + ((LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,42)) && (LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0))) + "isotp", sk); +#else + "isotp"); +#endif + dev_put(dev); + + if (so->bound) { + /* unregister old filter */ + if (so->ifindex) { + dev = dev_get_by_index(net, so->ifindex); + if (dev) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(net, dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + dev_put(dev); + } + } + } + + /* switch to new settings */ + so->ifindex = ifindex; + so->rxid = addr->can_addr.tp.rx_id; + so->txid = addr->can_addr.tp.tx_id; + so->bound = 1; + +out: + release_sock(sk); + + if (notify_enetdown) { + sk->sk_err = ENETDOWN; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + } + + return err; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0) +static int isotp_getname(struct socket *sock, struct sockaddr *uaddr, int peer) +#else +static int isotp_getname(struct socket *sock, struct sockaddr *uaddr, + int *len, int peer) +#endif +{ + struct sockaddr_can *addr = (struct sockaddr_can *)uaddr; + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + + if (peer) + return -EOPNOTSUPP; + + addr->can_family = AF_CAN; + addr->can_ifindex = so->ifindex; + addr->can_addr.tp.rx_id = so->rxid; + addr->can_addr.tp.tx_id = so->txid; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,17,0) + return sizeof(*addr); +#else + *len = sizeof(*addr); + + return 0; +#endif +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,32) +static int isotp_setsockopt(struct socket *sock, int level, int optname, + char __user *optval, unsigned int optlen) +#else +static int isotp_setsockopt(struct socket *sock, int level, int optname, + char __user *optval, int optlen) +#endif +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + int ret = 0; + + if (level != SOL_CAN_ISOTP) + return -EINVAL; + if (optlen < 0) + return -EINVAL; + + switch (optname) { + + case CAN_ISOTP_OPTS: + if (optlen != sizeof(struct can_isotp_options)) + return -EINVAL; + + if (copy_from_user(&so->opt, optval, optlen)) + return -EFAULT; + + /* no separate rx_ext_address is given => use ext_address */ + if (!(so->opt.flags & CAN_ISOTP_RX_EXT_ADDR)) + so->opt.rx_ext_address = so->opt.ext_address; + break; + + case CAN_ISOTP_RECV_FC: + if (optlen != sizeof(struct can_isotp_fc_options)) + return -EINVAL; + + if (copy_from_user(&so->rxfc, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_TX_STMIN: + if (optlen != sizeof(__u32)) + return -EINVAL; + + if (copy_from_user(&so->force_tx_stmin, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_RX_STMIN: + if (optlen != sizeof(__u32)) + return -EINVAL; + + if (copy_from_user(&so->force_rx_stmin, optval, optlen)) + return -EFAULT; + break; + + case CAN_ISOTP_LL_OPTS: + if (optlen != sizeof(struct can_isotp_ll_options)) + return -EINVAL; + else { + struct can_isotp_ll_options ll; + + if (copy_from_user(&ll, optval, optlen)) + return -EFAULT; + + /* check for correct ISO 11898-1 DLC data lentgh */ + if (ll.tx_dl != padlen(ll.tx_dl)) + return -EINVAL; + + if (ll.mtu != CAN_MTU && ll.mtu != CANFD_MTU) + return -EINVAL; + + if (ll.mtu == CAN_MTU && ll.tx_dl > CAN_MAX_DLEN) + return -EINVAL; + + memcpy(&so->ll, &ll, sizeof(ll)); + + /* set ll_dl for tx path to similar place as for rx */ + so->tx.ll_dl = ll.tx_dl; + } + break; + + default: + ret = -ENOPROTOOPT; + } + + return ret; +} + +static int isotp_getsockopt(struct socket *sock, int level, int optname, + char __user *optval, int __user *optlen) +{ + struct sock *sk = sock->sk; + struct isotp_sock *so = isotp_sk(sk); + int len; + void *val; + + if (level != SOL_CAN_ISOTP) + return -EINVAL; + if (get_user(len, optlen)) + return -EFAULT; + if (len < 0) + return -EINVAL; + + switch (optname) { + + case CAN_ISOTP_OPTS: + len = min_t(int, len, sizeof(struct can_isotp_options)); + val = &so->opt; + break; + + case CAN_ISOTP_RECV_FC: + len = min_t(int, len, sizeof(struct can_isotp_fc_options)); + val = &so->rxfc; + break; + + case CAN_ISOTP_TX_STMIN: + len = min_t(int, len, sizeof(__u32)); + val = &so->force_tx_stmin; + break; + + case CAN_ISOTP_RX_STMIN: + len = min_t(int, len, sizeof(__u32)); + val = &so->force_rx_stmin; + break; + + case CAN_ISOTP_LL_OPTS: + len = min_t(int, len, sizeof(struct can_isotp_ll_options)); + val = &so->ll; + break; + + default: + return -ENOPROTOOPT; + } + + if (put_user(len, optlen)) + return -EFAULT; + if (copy_to_user(optval, val, len)) + return -EFAULT; + return 0; +} + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3,11,0) +static int isotp_notifier(struct notifier_block *nb, unsigned long msg, + void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); +#else +static int isotp_notifier(struct notifier_block *nb, + unsigned long msg, void *data) +{ + struct net_device *dev = (struct net_device *)data; +#endif + struct isotp_sock *so = container_of(nb, struct isotp_sock, notifier); + struct sock *sk = &so->sk; + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + if (!net_eq(dev_net(dev), sock_net(sk))) + return NOTIFY_DONE; +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26) + if (dev_net(dev) != &init_net) + return NOTIFY_DONE; +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,24) + if (dev->nd_net != &init_net) + return NOTIFY_DONE; +#endif + + if (dev->type != ARPHRD_CAN) + return NOTIFY_DONE; + + if (so->ifindex != dev->ifindex) + return NOTIFY_DONE; + + switch (msg) { + + case NETDEV_UNREGISTER: + lock_sock(sk); + /* remove current filters & unregister */ + if (so->bound) +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,12,0) + can_rx_unregister(dev_net(dev), dev, so->rxid, +#else + can_rx_unregister(dev, so->rxid, +#endif + SINGLE_MASK(so->rxid), + isotp_rcv, sk); + + so->ifindex = 0; + so->bound = 0; + release_sock(sk); + + sk->sk_err = ENODEV; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + break; + + case NETDEV_DOWN: + sk->sk_err = ENETDOWN; + if (!sock_flag(sk, SOCK_DEAD)) + sk->sk_error_report(sk); + break; + } + + return NOTIFY_DONE; +} + +static int isotp_init(struct sock *sk) +{ + struct isotp_sock *so = isotp_sk(sk); + + so->ifindex = 0; + so->bound = 0; + + so->opt.flags = CAN_ISOTP_DEFAULT_FLAGS; + so->opt.ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS; + so->opt.rx_ext_address = CAN_ISOTP_DEFAULT_EXT_ADDRESS; + so->opt.rxpad_content = CAN_ISOTP_DEFAULT_PAD_CONTENT; + so->opt.txpad_content = CAN_ISOTP_DEFAULT_PAD_CONTENT; + so->opt.frame_txtime = CAN_ISOTP_DEFAULT_FRAME_TXTIME; + so->rxfc.bs = CAN_ISOTP_DEFAULT_RECV_BS; + so->rxfc.stmin = CAN_ISOTP_DEFAULT_RECV_STMIN; + so->rxfc.wftmax = CAN_ISOTP_DEFAULT_RECV_WFTMAX; + so->ll.mtu = CAN_ISOTP_DEFAULT_LL_MTU; + so->ll.tx_dl = CAN_ISOTP_DEFAULT_LL_TX_DL; + so->ll.tx_flags = CAN_ISOTP_DEFAULT_LL_TX_FLAGS; + + /* set ll_dl for tx path to similar place as for rx */ + so->tx.ll_dl = so->ll.tx_dl; + + so->rx.state = ISOTP_IDLE; + so->tx.state = ISOTP_IDLE; + + hrtimer_init(&so->rxtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + so->rxtimer.function = isotp_rx_timer_handler; + hrtimer_init(&so->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + so->txtimer.function = isotp_tx_timer_handler; + + tasklet_init(&so->txtsklet, isotp_tx_timer_tsklet, (unsigned long)so); + + init_waitqueue_head(&so->wait); + + so->notifier.notifier_call = isotp_notifier; + register_netdevice_notifier(&so->notifier); + + return 0; +} + +int isotp_sock_no_ioctlcmd(struct socket *sock, unsigned int cmd, + unsigned long arg) +{ + /* no ioctls for socket layer -> hand it down to NIC layer */ + return -ENOIOCTLCMD; +} + +static const struct proto_ops isotp_ops = { + .family = PF_CAN, + .release = isotp_release, + .bind = isotp_bind, + .connect = sock_no_connect, + .socketpair = sock_no_socketpair, + .accept = sock_no_accept, + .getname = isotp_getname, + .poll = datagram_poll, + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,2,0) + .ioctl = isotp_sock_no_ioctlcmd, + .gettstamp = sock_gettstamp, +#else + .ioctl = can_ioctl, /* use can_ioctl() from af_can.c */ +#endif + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .setsockopt = isotp_setsockopt, + .getsockopt = isotp_getsockopt, + .sendmsg = isotp_sendmsg, + .recvmsg = isotp_recvmsg, + .mmap = sock_no_mmap, + .sendpage = sock_no_sendpage, +}; + +static struct proto isotp_proto __read_mostly = { + .name = "CAN_ISOTP", + .owner = THIS_MODULE, + .obj_size = sizeof(struct isotp_sock), + .init = isotp_init, +}; + +static const struct can_proto isotp_can_proto = { + .type = SOCK_DGRAM, + .protocol = CAN_ISOTP, +#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,33) + .capability = -1, +#endif + .ops = &isotp_ops, + .prot = &isotp_proto, +}; + +static __init int isotp_module_init(void) +{ + int err; + + printk(banner); + + err = can_proto_register(&isotp_can_proto); + if (err < 0) + printk(KERN_ERR "can: registration of isotp protocol failed\n"); + + return err; +} + +static __exit void isotp_module_exit(void) +{ + can_proto_unregister(&isotp_can_proto); +} + +module_init(isotp_module_init); +module_exit(isotp_module_exit); From bfcd59dbf6374ac46fbebdb4cbb643fb0ede3f03 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 29 Jul 2019 11:58:09 -0500 Subject: [PATCH 007/213] CAN_ISOTP: wire up to kconfig/makefile Signed-off-by: Robert Nelson --- net/can/Kconfig | 10 ++++++++++ net/can/Makefile | 3 +++ 2 files changed, 13 insertions(+) diff --git a/net/can/Kconfig b/net/can/Kconfig index a4399be54ff4c..da3781da68e3c 100644 --- a/net/can/Kconfig +++ b/net/can/Kconfig @@ -51,6 +51,16 @@ config CAN_GW They can be modified with AND/OR/XOR/SET operations as configured by the netlink configuration interface known e.g. from iptables. +config CAN_ISOTP + tristate "ISO 15765-2:2016 CAN transport protocol" + default y + ---help--- + ISO 15765-2 CAN transport protocol for protocol family CAN + + This implementation is already widely used in automotive use-cases, e.g. + for UDS based OBD diagnosis. Although some small adaptions may be applied + to make it ready for Linux Mainline. Feedback is welcome. + source "drivers/net/can/Kconfig" endif diff --git a/net/can/Makefile b/net/can/Makefile index 1242bbbfe57f5..fde676965946a 100644 --- a/net/can/Makefile +++ b/net/can/Makefile @@ -15,3 +15,6 @@ can-bcm-y := bcm.o obj-$(CONFIG_CAN_GW) += can-gw.o can-gw-y := gw.o + +obj-$(CONFIG_CAN_ISOTP) += can-isotp.o +can-isotp-y := isotp.o From 4609c894a1ac7057ea6a17e4966867380be39022 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:46:24 -0500 Subject: [PATCH 008/213] merge: WireGuard https://git.zx2c4.com/WireGuard/commit/edad0d6e99e5133b1e8e865d727a25fff6399cb4 Signed-off-by: Robert Nelson --- net/Kconfig | 1 + net/Makefile | 1 + net/wireguard/Kconfig | 33 + net/wireguard/Makefile | 15 + net/wireguard/allowedips.c | 381 + net/wireguard/allowedips.h | 59 + net/wireguard/compat/Makefile.include | 98 + .../compat/checksum/checksum_partial_compat.h | 208 + net/wireguard/compat/compat-asm.h | 48 + net/wireguard/compat/compat.h | 945 ++ net/wireguard/compat/dst_cache/dst_cache.c | 175 + .../compat/dst_cache/include/net/dst_cache.h | 97 + .../compat/fpu-x86/include/asm/fpu/api.h | 1 + .../include/asm/intel-family.h | 73 + net/wireguard/compat/memneq/include.h | 5 + net/wireguard/compat/memneq/memneq.c | 170 + .../compat/neon-arm/include/asm/neon.h | 7 + .../compat/ptr_ring/include/linux/ptr_ring.h | 640 ++ .../compat/simd-asm/include/asm/simd.h | 21 + .../compat/simd/include/linux/simd.h | 70 + .../compat/siphash/include/linux/siphash.h | 140 + net/wireguard/compat/siphash/siphash.c | 539 + .../udp_tunnel/include/net/udp_tunnel.h | 94 + net/wireguard/compat/udp_tunnel/udp_tunnel.c | 387 + .../udp_tunnel/udp_tunnel_partial_compat.h | 226 + net/wireguard/cookie.c | 236 + net/wireguard/cookie.h | 59 + net/wireguard/crypto/Makefile.include | 57 + net/wireguard/crypto/include/zinc/blake2s.h | 56 + net/wireguard/crypto/include/zinc/chacha20.h | 70 + .../crypto/include/zinc/chacha20poly1305.h | 50 + .../crypto/include/zinc/curve25519.h | 28 + net/wireguard/crypto/include/zinc/poly1305.h | 31 + net/wireguard/crypto/zinc.h | 15 + .../crypto/zinc/blake2s/blake2s-x86_64-glue.c | 72 + .../crypto/zinc/blake2s/blake2s-x86_64.S | 258 + net/wireguard/crypto/zinc/blake2s/blake2s.c | 276 + .../crypto/zinc/chacha20/chacha20-arm-glue.c | 98 + .../crypto/zinc/chacha20/chacha20-arm.pl | 1227 +++ .../crypto/zinc/chacha20/chacha20-arm64.pl | 1163 +++ .../crypto/zinc/chacha20/chacha20-mips-glue.c | 27 + .../crypto/zinc/chacha20/chacha20-mips.S | 424 + .../zinc/chacha20/chacha20-unrolled-arm.S | 461 + .../zinc/chacha20/chacha20-x86_64-glue.c | 105 + .../crypto/zinc/chacha20/chacha20-x86_64.pl | 4106 ++++++++ net/wireguard/crypto/zinc/chacha20/chacha20.c | 193 + net/wireguard/crypto/zinc/chacha20poly1305.c | 402 + .../zinc/curve25519/curve25519-arm-glue.c | 43 + .../crypto/zinc/curve25519/curve25519-arm.S | 2064 ++++ .../zinc/curve25519/curve25519-fiat32.c | 860 ++ .../zinc/curve25519/curve25519-hacl64.c | 784 ++ .../zinc/curve25519/curve25519-x86_64-glue.c | 50 + .../zinc/curve25519/curve25519-x86_64.c | 2366 +++++ .../crypto/zinc/curve25519/curve25519.c | 113 + .../crypto/zinc/poly1305/poly1305-arm-glue.c | 140 + .../crypto/zinc/poly1305/poly1305-arm.pl | 1276 +++ .../crypto/zinc/poly1305/poly1305-arm64.pl | 974 ++ .../crypto/zinc/poly1305/poly1305-donna32.c | 205 + .../crypto/zinc/poly1305/poly1305-donna64.c | 182 + .../crypto/zinc/poly1305/poly1305-mips-glue.c | 37 + .../crypto/zinc/poly1305/poly1305-mips.S | 407 + .../crypto/zinc/poly1305/poly1305-mips64.pl | 466 + .../zinc/poly1305/poly1305-x86_64-glue.c | 156 + .../crypto/zinc/poly1305/poly1305-x86_64.pl | 4266 ++++++++ net/wireguard/crypto/zinc/poly1305/poly1305.c | 168 + net/wireguard/crypto/zinc/selftest/blake2s.c | 2090 ++++ net/wireguard/crypto/zinc/selftest/chacha20.c | 2698 +++++ .../crypto/zinc/selftest/chacha20poly1305.c | 9076 +++++++++++++++++ .../crypto/zinc/selftest/curve25519.c | 1315 +++ net/wireguard/crypto/zinc/selftest/poly1305.c | 1107 ++ net/wireguard/crypto/zinc/selftest/run.h | 48 + net/wireguard/device.c | 470 + net/wireguard/device.h | 73 + net/wireguard/main.c | 68 + net/wireguard/messages.h | 128 + net/wireguard/netlink.c | 660 ++ net/wireguard/netlink.h | 12 + net/wireguard/noise.c | 830 ++ net/wireguard/noise.h | 137 + net/wireguard/peer.c | 240 + net/wireguard/peer.h | 83 + net/wireguard/peerlookup.c | 221 + net/wireguard/peerlookup.h | 64 + net/wireguard/queueing.c | 53 + net/wireguard/queueing.h | 197 + net/wireguard/ratelimiter.c | 235 + net/wireguard/ratelimiter.h | 19 + net/wireguard/receive.c | 605 ++ net/wireguard/selftest/allowedips.c | 683 ++ net/wireguard/selftest/counter.c | 104 + net/wireguard/selftest/ratelimiter.c | 226 + net/wireguard/send.c | 421 + net/wireguard/socket.c | 437 + net/wireguard/socket.h | 44 + net/wireguard/timers.c | 243 + net/wireguard/timers.h | 31 + net/wireguard/uapi/wireguard.h | 196 + net/wireguard/version.h | 1 + 98 files changed, 51220 insertions(+) create mode 100644 net/wireguard/Kconfig create mode 100644 net/wireguard/Makefile create mode 100644 net/wireguard/allowedips.c create mode 100644 net/wireguard/allowedips.h create mode 100644 net/wireguard/compat/Makefile.include create mode 100644 net/wireguard/compat/checksum/checksum_partial_compat.h create mode 100644 net/wireguard/compat/compat-asm.h create mode 100644 net/wireguard/compat/compat.h create mode 100644 net/wireguard/compat/dst_cache/dst_cache.c create mode 100644 net/wireguard/compat/dst_cache/include/net/dst_cache.h create mode 100644 net/wireguard/compat/fpu-x86/include/asm/fpu/api.h create mode 100644 net/wireguard/compat/intel-family-x86/include/asm/intel-family.h create mode 100644 net/wireguard/compat/memneq/include.h create mode 100644 net/wireguard/compat/memneq/memneq.c create mode 100644 net/wireguard/compat/neon-arm/include/asm/neon.h create mode 100644 net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h create mode 100644 net/wireguard/compat/simd-asm/include/asm/simd.h create mode 100644 net/wireguard/compat/simd/include/linux/simd.h create mode 100644 net/wireguard/compat/siphash/include/linux/siphash.h create mode 100644 net/wireguard/compat/siphash/siphash.c create mode 100644 net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h create mode 100644 net/wireguard/compat/udp_tunnel/udp_tunnel.c create mode 100644 net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h create mode 100644 net/wireguard/cookie.c create mode 100644 net/wireguard/cookie.h create mode 100644 net/wireguard/crypto/Makefile.include create mode 100644 net/wireguard/crypto/include/zinc/blake2s.h create mode 100644 net/wireguard/crypto/include/zinc/chacha20.h create mode 100644 net/wireguard/crypto/include/zinc/chacha20poly1305.h create mode 100644 net/wireguard/crypto/include/zinc/curve25519.h create mode 100644 net/wireguard/crypto/include/zinc/poly1305.h create mode 100644 net/wireguard/crypto/zinc.h create mode 100644 net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c create mode 100644 net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S create mode 100644 net/wireguard/crypto/zinc/blake2s/blake2s.c create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-mips.S create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl create mode 100644 net/wireguard/crypto/zinc/chacha20/chacha20.c create mode 100644 net/wireguard/crypto/zinc/chacha20poly1305.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-arm.S create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c create mode 100644 net/wireguard/crypto/zinc/curve25519/curve25519.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-mips.S create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl create mode 100644 net/wireguard/crypto/zinc/poly1305/poly1305.c create mode 100644 net/wireguard/crypto/zinc/selftest/blake2s.c create mode 100644 net/wireguard/crypto/zinc/selftest/chacha20.c create mode 100644 net/wireguard/crypto/zinc/selftest/chacha20poly1305.c create mode 100644 net/wireguard/crypto/zinc/selftest/curve25519.c create mode 100644 net/wireguard/crypto/zinc/selftest/poly1305.c create mode 100644 net/wireguard/crypto/zinc/selftest/run.h create mode 100644 net/wireguard/device.c create mode 100644 net/wireguard/device.h create mode 100644 net/wireguard/main.c create mode 100644 net/wireguard/messages.h create mode 100644 net/wireguard/netlink.c create mode 100644 net/wireguard/netlink.h create mode 100644 net/wireguard/noise.c create mode 100644 net/wireguard/noise.h create mode 100644 net/wireguard/peer.c create mode 100644 net/wireguard/peer.h create mode 100644 net/wireguard/peerlookup.c create mode 100644 net/wireguard/peerlookup.h create mode 100644 net/wireguard/queueing.c create mode 100644 net/wireguard/queueing.h create mode 100644 net/wireguard/ratelimiter.c create mode 100644 net/wireguard/ratelimiter.h create mode 100644 net/wireguard/receive.c create mode 100644 net/wireguard/selftest/allowedips.c create mode 100644 net/wireguard/selftest/counter.c create mode 100644 net/wireguard/selftest/ratelimiter.c create mode 100644 net/wireguard/send.c create mode 100644 net/wireguard/socket.c create mode 100644 net/wireguard/socket.h create mode 100644 net/wireguard/timers.c create mode 100644 net/wireguard/timers.h create mode 100644 net/wireguard/uapi/wireguard.h create mode 100644 net/wireguard/version.h diff --git a/net/Kconfig b/net/Kconfig index e1f1846835f98..5ecc6ecccc0ec 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -86,6 +86,7 @@ config INET Short answer: say Y. if INET +source "net/wireguard/Kconfig" source "net/ipv4/Kconfig" source "net/ipv6/Kconfig" source "net/netlabel/Kconfig" diff --git a/net/Makefile b/net/Makefile index bb22792979c83..6454146e3d96e 100644 --- a/net/Makefile +++ b/net/Makefile @@ -15,6 +15,7 @@ obj-$(CONFIG_NET) += $(tmp-y) obj-$(CONFIG_LLC) += llc/ obj-$(CONFIG_NET) += ethernet/ 802/ sched/ netlink/ bpf/ obj-$(CONFIG_NETFILTER) += netfilter/ +obj-$(CONFIG_WIREGUARD) += wireguard/ obj-$(CONFIG_INET) += ipv4/ obj-$(CONFIG_TLS) += tls/ obj-$(CONFIG_XFRM) += xfrm/ diff --git a/net/wireguard/Kconfig b/net/wireguard/Kconfig new file mode 100644 index 0000000000000..156e9dbfc0516 --- /dev/null +++ b/net/wireguard/Kconfig @@ -0,0 +1,33 @@ +config WIREGUARD + tristate "IP: WireGuard secure network tunnel" + depends on NET && INET + depends on IPV6 || !IPV6 + select NET_UDP_TUNNEL + select DST_CACHE + select CRYPTO + select CRYPTO_ALGAPI + select VFP + select VFPv3 if CPU_V7 + select NEON if CPU_V7 + select KERNEL_MODE_NEON if CPU_V7 + default m + help + WireGuard is a secure, fast, and easy to use replacement for IPsec + that uses modern cryptography and clever networking tricks. It's + designed to be fairly general purpose and abstract enough to fit most + use cases, while at the same time remaining extremely simple to + configure. See www.wireguard.com for more info. + + It's safe to say Y or M here, as the driver is very lightweight and + is only in use when an administrator chooses to add an interface. + +config WIREGUARD_DEBUG + bool "Debugging checks and verbose messages" + depends on WIREGUARD + help + This will write log messages for handshake and other events + that occur for a WireGuard interface. It will also perform some + extra validation checks and unit tests at various points. This is + only useful for debugging. + + Say N here unless you know what you're doing. diff --git a/net/wireguard/Makefile b/net/wireguard/Makefile new file mode 100644 index 0000000000000..364deb7053766 --- /dev/null +++ b/net/wireguard/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + +ccflags-y := -O3 -fvisibility=hidden +ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DDEBUG -g +ccflags-y += -D'pr_fmt(fmt)=KBUILD_MODNAME ": " fmt' +ccflags-y += -Wframe-larger-than=2048 + +wireguard-y := main.o noise.o device.o peer.o timers.o queueing.o send.o receive.o socket.o peerlookup.o allowedips.o ratelimiter.o cookie.o netlink.o + +include $(src)/crypto/Makefile.include +include $(src)/compat/Makefile.include + +obj-$(if $(KBUILD_EXTMOD),m,$(CONFIG_WIREGUARD)) := wireguard.o diff --git a/net/wireguard/allowedips.c b/net/wireguard/allowedips.c new file mode 100644 index 0000000000000..72667d5399c34 --- /dev/null +++ b/net/wireguard/allowedips.c @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "allowedips.h" +#include "peer.h" + +static void swap_endian(u8 *dst, const u8 *src, u8 bits) +{ + if (bits == 32) { + *(u32 *)dst = be32_to_cpu(*(const __be32 *)src); + } else if (bits == 128) { + ((u64 *)dst)[0] = be64_to_cpu(((const __be64 *)src)[0]); + ((u64 *)dst)[1] = be64_to_cpu(((const __be64 *)src)[1]); + } +} + +static void copy_and_assign_cidr(struct allowedips_node *node, const u8 *src, + u8 cidr, u8 bits) +{ + node->cidr = cidr; + node->bit_at_a = cidr / 8U; +#ifdef __LITTLE_ENDIAN + node->bit_at_a ^= (bits / 8U - 1U) % 8U; +#endif + node->bit_at_b = 7U - (cidr % 8U); + node->bitlen = bits; + memcpy(node->bits, src, bits / 8U); +} +#define CHOOSE_NODE(parent, key) \ + parent->bit[(key[parent->bit_at_a] >> parent->bit_at_b) & 1] + +static void node_free_rcu(struct rcu_head *rcu) +{ + kfree(container_of(rcu, struct allowedips_node, rcu)); +} + +static void push_rcu(struct allowedips_node **stack, + struct allowedips_node __rcu *p, unsigned int *len) +{ + if (rcu_access_pointer(p)) { + WARN_ON(IS_ENABLED(DEBUG) && *len >= 128); + stack[(*len)++] = rcu_dereference_raw(p); + } +} + +static void root_free_rcu(struct rcu_head *rcu) +{ + struct allowedips_node *node, *stack[128] = { + container_of(rcu, struct allowedips_node, rcu) }; + unsigned int len = 1; + + while (len > 0 && (node = stack[--len])) { + push_rcu(stack, node->bit[0], &len); + push_rcu(stack, node->bit[1], &len); + kfree(node); + } +} + +static void root_remove_peer_lists(struct allowedips_node *root) +{ + struct allowedips_node *node, *stack[128] = { root }; + unsigned int len = 1; + + while (len > 0 && (node = stack[--len])) { + push_rcu(stack, node->bit[0], &len); + push_rcu(stack, node->bit[1], &len); + if (rcu_access_pointer(node->peer)) + list_del(&node->peer_list); + } +} + +static void walk_remove_by_peer(struct allowedips_node __rcu **top, + struct wg_peer *peer, struct mutex *lock) +{ +#define REF(p) rcu_access_pointer(p) +#define DEREF(p) rcu_dereference_protected(*(p), lockdep_is_held(lock)) +#define PUSH(p) ({ \ + WARN_ON(IS_ENABLED(DEBUG) && len >= 128); \ + stack[len++] = p; \ + }) + + struct allowedips_node __rcu **stack[128], **nptr; + struct allowedips_node *node, *prev; + unsigned int len; + + if (unlikely(!peer || !REF(*top))) + return; + + for (prev = NULL, len = 0, PUSH(top); len > 0; prev = node) { + nptr = stack[len - 1]; + node = DEREF(nptr); + if (!node) { + --len; + continue; + } + if (!prev || REF(prev->bit[0]) == node || + REF(prev->bit[1]) == node) { + if (REF(node->bit[0])) + PUSH(&node->bit[0]); + else if (REF(node->bit[1])) + PUSH(&node->bit[1]); + } else if (REF(node->bit[0]) == prev) { + if (REF(node->bit[1])) + PUSH(&node->bit[1]); + } else { + if (rcu_dereference_protected(node->peer, + lockdep_is_held(lock)) == peer) { + RCU_INIT_POINTER(node->peer, NULL); + list_del_init(&node->peer_list); + if (!node->bit[0] || !node->bit[1]) { + rcu_assign_pointer(*nptr, DEREF( + &node->bit[!REF(node->bit[0])])); + call_rcu(&node->rcu, node_free_rcu); + node = DEREF(nptr); + } + } + --len; + } + } + +#undef REF +#undef DEREF +#undef PUSH +} + +static unsigned int fls128(u64 a, u64 b) +{ + return a ? fls64(a) + 64U : fls64(b); +} + +static u8 common_bits(const struct allowedips_node *node, const u8 *key, + u8 bits) +{ + if (bits == 32) + return 32U - fls(*(const u32 *)node->bits ^ *(const u32 *)key); + else if (bits == 128) + return 128U - fls128( + *(const u64 *)&node->bits[0] ^ *(const u64 *)&key[0], + *(const u64 *)&node->bits[8] ^ *(const u64 *)&key[8]); + return 0; +} + +static bool prefix_matches(const struct allowedips_node *node, const u8 *key, + u8 bits) +{ + /* This could be much faster if it actually just compared the common + * bits properly, by precomputing a mask bswap(~0 << (32 - cidr)), and + * the rest, but it turns out that common_bits is already super fast on + * modern processors, even taking into account the unfortunate bswap. + * So, we just inline it like this instead. + */ + return common_bits(node, key, bits) >= node->cidr; +} + +static struct allowedips_node *find_node(struct allowedips_node *trie, u8 bits, + const u8 *key) +{ + struct allowedips_node *node = trie, *found = NULL; + + while (node && prefix_matches(node, key, bits)) { + if (rcu_access_pointer(node->peer)) + found = node; + if (node->cidr == bits) + break; + node = rcu_dereference_bh(CHOOSE_NODE(node, key)); + } + return found; +} + +/* Returns a strong reference to a peer */ +static struct wg_peer *lookup(struct allowedips_node __rcu *root, u8 bits, + const void *be_ip) +{ + /* Aligned so it can be passed to fls/fls64 */ + u8 ip[16] __aligned(__alignof(u64)); + struct allowedips_node *node; + struct wg_peer *peer = NULL; + + swap_endian(ip, be_ip, bits); + + rcu_read_lock_bh(); +retry: + node = find_node(rcu_dereference_bh(root), bits, ip); + if (node) { + peer = wg_peer_get_maybe_zero(rcu_dereference_bh(node->peer)); + if (!peer) + goto retry; + } + rcu_read_unlock_bh(); + return peer; +} + +static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key, + u8 cidr, u8 bits, struct allowedips_node **rnode, + struct mutex *lock) +{ + struct allowedips_node *node = rcu_dereference_protected(trie, + lockdep_is_held(lock)); + struct allowedips_node *parent = NULL; + bool exact = false; + + while (node && node->cidr <= cidr && prefix_matches(node, key, bits)) { + parent = node; + if (parent->cidr == cidr) { + exact = true; + break; + } + node = rcu_dereference_protected(CHOOSE_NODE(parent, key), + lockdep_is_held(lock)); + } + *rnode = parent; + return exact; +} + +static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + struct allowedips_node *node, *parent, *down, *newnode; + + if (unlikely(cidr > bits || !peer)) + return -EINVAL; + + if (!rcu_access_pointer(*trie)) { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) + return -ENOMEM; + RCU_INIT_POINTER(node->peer, peer); + list_add_tail(&node->peer_list, &peer->allowedips_list); + copy_and_assign_cidr(node, key, cidr, bits); + rcu_assign_pointer(*trie, node); + return 0; + } + if (node_placement(*trie, key, cidr, bits, &node, lock)) { + rcu_assign_pointer(node->peer, peer); + list_move_tail(&node->peer_list, &peer->allowedips_list); + return 0; + } + + newnode = kzalloc(sizeof(*newnode), GFP_KERNEL); + if (unlikely(!newnode)) + return -ENOMEM; + RCU_INIT_POINTER(newnode->peer, peer); + list_add_tail(&newnode->peer_list, &peer->allowedips_list); + copy_and_assign_cidr(newnode, key, cidr, bits); + + if (!node) { + down = rcu_dereference_protected(*trie, lockdep_is_held(lock)); + } else { + down = rcu_dereference_protected(CHOOSE_NODE(node, key), + lockdep_is_held(lock)); + if (!down) { + rcu_assign_pointer(CHOOSE_NODE(node, key), newnode); + return 0; + } + } + cidr = min(cidr, common_bits(down, key, bits)); + parent = node; + + if (newnode->cidr == cidr) { + rcu_assign_pointer(CHOOSE_NODE(newnode, down->bits), down); + if (!parent) + rcu_assign_pointer(*trie, newnode); + else + rcu_assign_pointer(CHOOSE_NODE(parent, newnode->bits), + newnode); + } else { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) { + kfree(newnode); + return -ENOMEM; + } + INIT_LIST_HEAD(&node->peer_list); + copy_and_assign_cidr(node, newnode->bits, cidr, bits); + + rcu_assign_pointer(CHOOSE_NODE(node, down->bits), down); + rcu_assign_pointer(CHOOSE_NODE(node, newnode->bits), newnode); + if (!parent) + rcu_assign_pointer(*trie, node); + else + rcu_assign_pointer(CHOOSE_NODE(parent, node->bits), + node); + } + return 0; +} + +void wg_allowedips_init(struct allowedips *table) +{ + table->root4 = table->root6 = NULL; + table->seq = 1; +} + +void wg_allowedips_free(struct allowedips *table, struct mutex *lock) +{ + struct allowedips_node __rcu *old4 = table->root4, *old6 = table->root6; + + ++table->seq; + RCU_INIT_POINTER(table->root4, NULL); + RCU_INIT_POINTER(table->root6, NULL); + if (rcu_access_pointer(old4)) { + struct allowedips_node *node = rcu_dereference_protected(old4, + lockdep_is_held(lock)); + + root_remove_peer_lists(node); + call_rcu(&node->rcu, root_free_rcu); + } + if (rcu_access_pointer(old6)) { + struct allowedips_node *node = rcu_dereference_protected(old6, + lockdep_is_held(lock)); + + root_remove_peer_lists(node); + call_rcu(&node->rcu, root_free_rcu); + } +} + +int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + /* Aligned so it can be passed to fls */ + u8 key[4] __aligned(__alignof(u32)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 32); + return add(&table->root4, 32, key, cidr, peer, lock); +} + +int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + /* Aligned so it can be passed to fls64 */ + u8 key[16] __aligned(__alignof(u64)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 128); + return add(&table->root6, 128, key, cidr, peer, lock); +} + +void wg_allowedips_remove_by_peer(struct allowedips *table, + struct wg_peer *peer, struct mutex *lock) +{ + ++table->seq; + walk_remove_by_peer(&table->root4, peer, lock); + walk_remove_by_peer(&table->root6, peer, lock); +} + +int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr) +{ + const unsigned int cidr_bytes = DIV_ROUND_UP(node->cidr, 8U); + swap_endian(ip, node->bits, node->bitlen); + memset(ip + cidr_bytes, 0, node->bitlen / 8U - cidr_bytes); + if (node->cidr) + ip[cidr_bytes - 1U] &= ~0U << (-node->cidr % 8U); + + *cidr = node->cidr; + return node->bitlen == 32 ? AF_INET : AF_INET6; +} + +/* Returns a strong reference to a peer */ +struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, + struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return lookup(table->root4, 32, &ip_hdr(skb)->daddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + return lookup(table->root6, 128, &ipv6_hdr(skb)->daddr); + return NULL; +} + +/* Returns a strong reference to a peer */ +struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, + struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return lookup(table->root4, 32, &ip_hdr(skb)->saddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + return lookup(table->root6, 128, &ipv6_hdr(skb)->saddr); + return NULL; +} + +#include "selftest/allowedips.c" diff --git a/net/wireguard/allowedips.h b/net/wireguard/allowedips.h new file mode 100644 index 0000000000000..e5c83cafcef4c --- /dev/null +++ b/net/wireguard/allowedips.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_ALLOWEDIPS_H +#define _WG_ALLOWEDIPS_H + +#include +#include +#include + +struct wg_peer; + +struct allowedips_node { + struct wg_peer __rcu *peer; + struct allowedips_node __rcu *bit[2]; + /* While it may seem scandalous that we waste space for v4, + * we're alloc'ing to the nearest power of 2 anyway, so this + * doesn't actually make a difference. + */ + u8 bits[16] __aligned(__alignof(u64)); + u8 cidr, bit_at_a, bit_at_b, bitlen; + + /* Keep rarely used list at bottom to be beyond cache line. */ + union { + struct list_head peer_list; + struct rcu_head rcu; + }; +}; + +struct allowedips { + struct allowedips_node __rcu *root4; + struct allowedips_node __rcu *root6; + u64 seq; +}; + +void wg_allowedips_init(struct allowedips *table); +void wg_allowedips_free(struct allowedips *table, struct mutex *mutex); +int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +void wg_allowedips_remove_by_peer(struct allowedips *table, + struct wg_peer *peer, struct mutex *lock); +/* The ip input pointer should be __aligned(__alignof(u64))) */ +int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr); + +/* These return a strong reference to a peer: */ +struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, + struct sk_buff *skb); +struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, + struct sk_buff *skb); + +#ifdef DEBUG +bool wg_allowedips_selftest(void); +#endif + +#endif /* _WG_ALLOWEDIPS_H */ diff --git a/net/wireguard/compat/Makefile.include b/net/wireguard/compat/Makefile.include new file mode 100644 index 0000000000000..7bf345b5db524 --- /dev/null +++ b/net/wireguard/compat/Makefile.include @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + +kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) + +ccflags-y += -include $(kbuild-dir)/compat/compat.h +asflags-y += -include $(kbuild-dir)/compat/compat-asm.h + +ifeq ($(wildcard $(srctree)/include/linux/ptr_ring.h),) +ccflags-y += -I$(kbuild-dir)/compat/ptr_ring/include +endif + +ifeq ($(wildcard $(srctree)/include/linux/siphash.h),) +ccflags-y += -I$(kbuild-dir)/compat/siphash/include +wireguard-y += compat/siphash/siphash.o +endif + +ifeq ($(wildcard $(srctree)/include/net/dst_cache.h),) +ccflags-y += -I$(kbuild-dir)/compat/dst_cache/include +wireguard-y += compat/dst_cache/dst_cache.o +endif + +ifeq ($(wildcard $(srctree)/arch/x86/include/asm/intel-family.h)$(CONFIG_X86),y) +ccflags-y += -I$(kbuild-dir)/compat/intel-family-x86/include +endif + +ifeq ($(wildcard $(srctree)/arch/x86/include/asm/fpu/api.h)$(CONFIG_X86),y) +ccflags-y += -I$(kbuild-dir)/compat/fpu-x86/include +endif + +ifeq ($(wildcard $(srctree)/arch/$(SRCARCH)/include/asm/simd.h)$(shell grep -s -F "generic-y += simd.h" "$(srctree)/arch/$(SRCARCH)/Makefile" "$(srctree)/arch/$(SRCARCH)/Makefile"),) +ccflags-y += -I$(kbuild-dir)/compat/simd-asm/include +endif + +ifeq ($(wildcard $(srctree)/include/linux/simd.h),) +ccflags-y += -I$(kbuild-dir)/compat/simd/include +endif + +ifeq ($(wildcard $(srctree)/include/net/udp_tunnel.h),) +ccflags-y += -I$(kbuild-dir)/compat/udp_tunnel/include +wireguard-y += compat/udp_tunnel/udp_tunnel.o +endif + +ifeq ($(shell grep -s -F "int crypto_memneq" "$(srctree)/include/crypto/algapi.h"),) +ccflags-y += -include $(kbuild-dir)/compat/memneq/include.h +wireguard-y += compat/memneq/memneq.o +endif + +ifeq ($(shell grep -s -F "addr_gen_mode" "$(srctree)/include/linux/ipv6.h"),) +ccflags-y += -DCOMPAT_CANNOT_USE_DEV_CNF +endif + +ifdef CONFIG_HZ +ifeq ($(wildcard $(srctree)/include/generated/timeconst.h),) +ccflags-y += $(shell echo 'define gcd(a,b){auto t;while(b){t=b;b=a%b;a=t;};return a;};hz=$(CONFIG_HZ);cd=gcd(hz,1000000);print "-DHZ_TO_USEC_NUM=",1000000/cd," -DHZ_TO_USEC_DEN=",hz/cd;halt;' | bc -q) +endif +endif + +ifeq ($(wildcard $(srctree)/arch/arm/include/asm/neon.h)$(CONFIG_ARM),y) +ccflags-y += -I$(kbuild-dir)/compat/neon-arm/include +endif +ifeq ($(wildcard $(srctree)/arch/arm64/include/asm/neon.h)$(CONFIG_ARM64),y) +ccflags-y += -I$(kbuild-dir)/compat/neon-arm/include +endif + +ifeq ($(CONFIG_X86_64),y) + ifeq ($(ssse3_instr),) + ssse3_instr := $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1) + ccflags-y += $(ssse3_instr) + asflags-y += $(ssse3_instr) + endif + ifeq ($(avx_instr),) + avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1) + ccflags-y += $(avx_instr) + asflags-y += $(avx_instr) + endif + ifeq ($(avx2_instr),) + avx2_instr := $(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1) + ccflags-y += $(avx2_instr) + asflags-y += $(avx2_instr) + endif + ifeq ($(avx512_instr),) + avx512_instr := $(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1) + ccflags-y += $(avx512_instr) + asflags-y += $(avx512_instr) + endif + ifeq ($(bmi2_instr),) + bmi2_instr :=$(call as-instr,mulx %rax$(comma)%rax$(comma)%rax,-DCONFIG_AS_BMI2=1) + ccflags-y += $(bmi2_instr) + asflags-y += $(bmi2_instr) + endif + ifeq ($(adx_instr),) + adx_instr :=$(call as-instr,adcx %rax$(comma)%rax,-DCONFIG_AS_ADX=1) + ccflags-y += $(adx_instr) + asflags-y += $(adx_instr) + endif +endif diff --git a/net/wireguard/compat/checksum/checksum_partial_compat.h b/net/wireguard/compat/checksum/checksum_partial_compat.h new file mode 100644 index 0000000000000..3dfe5397a94f5 --- /dev/null +++ b/net/wireguard/compat/checksum/checksum_partial_compat.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include +#include + +#define IP6_MF 0x0001 +#define IP6_OFFSET 0xFFF8 +static inline int skb_maybe_pull_tail(struct sk_buff *skb, unsigned int len, unsigned int max) +{ + if (skb_headlen(skb) >= len) + return 0; + if (max > skb->len) + max = skb->len; + if (__pskb_pull_tail(skb, max - skb_headlen(skb)) == NULL) + return -ENOMEM; + if (skb_headlen(skb) < len) + return -EPROTO; + return 0; +} +#define MAX_IP_HDR_LEN 128 +static inline int skb_checksum_setup_ip(struct sk_buff *skb, bool recalculate) +{ + unsigned int off; + bool fragment; + int err; + fragment = false; + err = skb_maybe_pull_tail(skb, sizeof(struct iphdr), MAX_IP_HDR_LEN); + if (err < 0) + goto out; + if (ip_hdr(skb)->frag_off & htons(IP_OFFSET | IP_MF)) + fragment = true; + off = ip_hdrlen(skb); + err = -EPROTO; + if (fragment) + goto out; + switch (ip_hdr(skb)->protocol) { + case IPPROTO_TCP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct tcphdr), + MAX_IP_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct tcphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + tcp_hdr(skb)->check = + ~csum_tcpudp_magic(ip_hdr(skb)->saddr, + ip_hdr(skb)->daddr, + skb->len - off, + IPPROTO_TCP, 0); + break; + case IPPROTO_UDP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct udphdr), + MAX_IP_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct udphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + udp_hdr(skb)->check = + ~csum_tcpudp_magic(ip_hdr(skb)->saddr, + ip_hdr(skb)->daddr, + skb->len - off, + IPPROTO_UDP, 0); + break; + default: + goto out; + } + err = 0; +out: + return err; +} +#define MAX_IPV6_HDR_LEN 256 +#define OPT_HDR(type, skb, off) \ + (type *)(skb_network_header(skb) + (off)) +static inline int skb_checksum_setup_ipv6(struct sk_buff *skb, bool recalculate) +{ + int err; + u8 nexthdr; + unsigned int off; + unsigned int len; + bool fragment; + bool done; + fragment = false; + done = false; + off = sizeof(struct ipv6hdr); + err = skb_maybe_pull_tail(skb, off, MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + nexthdr = ipv6_hdr(skb)->nexthdr; + len = sizeof(struct ipv6hdr) + ntohs(ipv6_hdr(skb)->payload_len); + while (off <= len && !done) { + switch (nexthdr) { + case IPPROTO_DSTOPTS: + case IPPROTO_HOPOPTS: + case IPPROTO_ROUTING: { + struct ipv6_opt_hdr *hp; + + err = skb_maybe_pull_tail(skb, off + sizeof(struct ipv6_opt_hdr), MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + hp = OPT_HDR(struct ipv6_opt_hdr, skb, off); + nexthdr = hp->nexthdr; + off += ipv6_optlen(hp); + break; + } + case IPPROTO_FRAGMENT: { + struct frag_hdr *hp; + err = skb_maybe_pull_tail(skb, off + sizeof(struct frag_hdr), MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + hp = OPT_HDR(struct frag_hdr, skb, off); + if (hp->frag_off & htons(IP6_OFFSET | IP6_MF)) + fragment = true; + nexthdr = hp->nexthdr; + off += sizeof(struct frag_hdr); + break; + } + default: + done = true; + break; + } + } + err = -EPROTO; + if (!done || fragment) + goto out; + switch (nexthdr) { + case IPPROTO_TCP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct tcphdr), + MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct tcphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + tcp_hdr(skb)->check = + ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr, + skb->len - off, + IPPROTO_TCP, 0); + break; + case IPPROTO_UDP: + err = skb_maybe_pull_tail(skb, + off + sizeof(struct udphdr), + MAX_IPV6_HDR_LEN); + if (err < 0) + goto out; + + if (!skb_partial_csum_set(skb, off, + offsetof(struct udphdr, check))) { + err = -EPROTO; + goto out; + } + + if (recalculate) + udp_hdr(skb)->check = + ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr, + skb->len - off, + IPPROTO_UDP, 0); + break; + default: + goto out; + } + err = 0; +out: + return err; +} +static inline int skb_checksum_setup(struct sk_buff *skb, bool recalculate) +{ + int err; + switch (skb->protocol) { + case htons(ETH_P_IP): + err = skb_checksum_setup_ip(skb, recalculate); + break; + + case htons(ETH_P_IPV6): + err = skb_checksum_setup_ipv6(skb, recalculate); + break; + default: + err = -EPROTO; + break; + } + return err; +} diff --git a/net/wireguard/compat/compat-asm.h b/net/wireguard/compat/compat-asm.h new file mode 100644 index 0000000000000..bafd70beb68f4 --- /dev/null +++ b/net/wireguard/compat/compat-asm.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COMPATASM_H +#define _WG_COMPATASM_H + +#include +#include +#include + +/* PaX compatibility */ +#if defined(RAP_PLUGIN) +#undef ENTRY +#define ENTRY RAP_ENTRY +#endif + +#if defined(__LINUX_ARM_ARCH__) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo + .macro ret\c, reg +#if __LINUX_ARM_ARCH__ < 6 + mov\c pc, \reg +#else + .ifeqs "\reg", "lr" + bx\c \reg + .else + mov\c pc, \reg + .endif +#endif + .endm + .endr +#endif + +#if defined(__LINUX_ARM_ARCH__) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) +#include +#define lspush push +#define lspull pull +#undef push +#undef pull +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) +#define SYM_FUNC_START ENTRY +#define SYM_FUNC_END ENDPROC +#endif + +#endif /* _WG_COMPATASM_H */ diff --git a/net/wireguard/compat/compat.h b/net/wireguard/compat/compat.h new file mode 100644 index 0000000000000..824f57cad01f2 --- /dev/null +++ b/net/wireguard/compat/compat.h @@ -0,0 +1,945 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COMPAT_H +#define _WG_COMPAT_H + +#include +#include +#include +#include + +#ifdef RHEL_MAJOR +#if RHEL_MAJOR == 7 +#define ISRHEL7 +#elif RHEL_MAJOR == 8 +#define ISRHEL8 +#endif +#endif +#ifdef UTS_UBUNTU_RELEASE_ABI +#if LINUX_VERSION_CODE == KERNEL_VERSION(3, 13, 11) +#define ISUBUNTU1404 +#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) +#define ISUBUNTU1604 +#endif +#endif +#ifdef CONFIG_SUSE_KERNEL +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0) +#define ISOPENSUSE42 +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) +#define ISOPENSUSE15 +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 0) +#error "WireGuard requires Linux >= 3.10" +#endif + +#if defined(ISRHEL7) +#include +#define headers_end headers_start +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) +#define headers_start data +#define headers_end data +#endif + +#include +#include +#ifndef __ro_after_init +#define __ro_after_init __read_mostly +#endif + +#include +#ifndef READ_ONCE +#define READ_ONCE ACCESS_ONCE +#endif +#ifndef WRITE_ONCE +#ifdef ACCESS_ONCE_RW +#define WRITE_ONCE(p, v) (ACCESS_ONCE_RW(p) = (v)) +#else +#define WRITE_ONCE(p, v) (ACCESS_ONCE(p) = (v)) +#endif +#endif + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include "udp_tunnel/udp_tunnel_partial_compat.h" +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && !defined(DEBUG) && defined(net_dbg_ratelimited) +#undef net_dbg_ratelimited +#define net_dbg_ratelimited(fmt, ...) do { if (0) no_printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#include +#ifndef RCU_LOCKDEP_WARN +#define RCU_LOCKDEP_WARN(cond, message) rcu_lockdep_assert(!(cond), message) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && !defined(ISRHEL7) +#define ipv6_dst_lookup(a, b, c, d) ipv6_dst_lookup(b, c, d) +#endif + +#if (LINUX_VERSION_CODE == KERNEL_VERSION(4, 4, 0) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 5) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 17) && LINUX_VERSION_CODE > KERNEL_VERSION(3, 19, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 27) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 8) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 40) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) || \ + (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 54))) && !defined(ISUBUNTU1404) && (!defined(ISRHEL7) || RHEL_MINOR < 7) /* TODO: remove < 7 workaround once CentOS 7.7 comes out. */ +#include +#include +#define IP6_ECN_set_ce(a, b) IP6_ECN_set_ce(b) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && IS_ENABLED(CONFIG_IPV6) && !defined(ISRHEL7) +#include +struct ipv6_stub_type { + void *udpv6_encap_enable; + int (*ipv6_dst_lookup)(struct sock *sk, struct dst_entry **dst, struct flowi6 *fl6); +}; +static const struct ipv6_stub_type ipv6_stub_impl = { + .udpv6_encap_enable = (void *)1, + .ipv6_dst_lookup = ip6_dst_lookup +}; +static const struct ipv6_stub_type *ipv6_stub = &ipv6_stub_impl; +#endif + + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0) && IS_ENABLED(CONFIG_IPV6) && !defined(ISOPENSUSE42) && !defined(ISRHEL7) +#include +static inline bool ipv6_mod_enabled(void) +{ + return ipv6_stub != NULL && ipv6_stub->udpv6_encap_enable != NULL; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) && !defined(ISRHEL7) +#include +static inline void skb_reset_tc(struct sk_buff *skb) +{ +#ifdef CONFIG_NET_CLS_ACT + skb->tc_verd = 0; +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +#include +#include +static inline u32 __compat_get_random_u32(void) +{ + static siphash_key_t key; + static u32 counter = 0; +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) + static bool has_seeded = false; + if (unlikely(!has_seeded)) { + get_random_bytes(&key, sizeof(key)); + has_seeded = true; + } +#else + get_random_once(&key, sizeof(key)); +#endif + return siphash_2u32(counter++, get_random_int(), &key); +} +#define get_random_u32 __compat_get_random_u32 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) && !defined(ISRHEL7) +static inline void netif_keep_dst(struct net_device *dev) +{ + dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; +} +#define COMPAT_CANNOT_USE_CSUM_LEVEL +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) && !defined(ISRHEL7) +#include +#ifndef netdev_alloc_pcpu_stats +#define pcpu_sw_netstats pcpu_tstats +#endif +#ifndef netdev_alloc_pcpu_stats +#define netdev_alloc_pcpu_stats alloc_percpu +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) && !defined(ISRHEL7) +#include +#ifndef netdev_alloc_pcpu_stats +#define netdev_alloc_pcpu_stats(type) \ +({ \ + typeof(type) __percpu *pcpu_stats = alloc_percpu(type); \ + if (pcpu_stats) { \ + int __cpu; \ + for_each_possible_cpu(__cpu) { \ + typeof(type) *stat; \ + stat = per_cpu_ptr(pcpu_stats, __cpu); \ + u64_stats_init(&stat->syncp); \ + } \ + } \ + pcpu_stats; \ +}) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) && !defined(ISRHEL7) +#include "checksum/checksum_partial_compat.h" +static inline void *__compat_pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) +{ + if (tail != skb) { + skb->data_len += len; + skb->len += len; + } + return skb_put(tail, len); +} +#define pskb_put __compat_pskb_put +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) && !defined(ISRHEL7) +#include +static inline void skb_scrub_packet(struct sk_buff *skb, bool xnet) +{ +#ifdef CONFIG_CAVIUM_OCTEON_IPFWD_OFFLOAD + memset(&skb->cvm_info, 0, sizeof(skb->cvm_info)); + skb->cvm_reserved = 0; +#endif + skb->tstamp.tv64 = 0; + skb->pkt_type = PACKET_HOST; + skb->skb_iif = 0; + skb_dst_drop(skb); + secpath_reset(skb); + nf_reset(skb); + nf_reset_trace(skb); + if (!xnet) + return; + skb_orphan(skb); + skb->mark = 0; +} +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) || defined(ISUBUNTU1404)) && !defined(ISRHEL7) +#include +static inline u32 __compat_prandom_u32_max(u32 ep_ro) +{ + return (u32)(((u64)prandom_u32() * ep_ro) >> 32); +} +#define prandom_u32_max __compat_prandom_u32_max +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 75) && !defined(ISRHEL7) +#ifndef U8_MAX +#define U8_MAX ((u8)~0U) +#endif +#ifndef S8_MAX +#define S8_MAX ((s8)(U8_MAX >> 1)) +#endif +#ifndef S8_MIN +#define S8_MIN ((s8)(-S8_MAX - 1)) +#endif +#ifndef U16_MAX +#define U16_MAX ((u16)~0U) +#endif +#ifndef S16_MAX +#define S16_MAX ((s16)(U16_MAX >> 1)) +#endif +#ifndef S16_MIN +#define S16_MIN ((s16)(-S16_MAX - 1)) +#endif +#ifndef U32_MAX +#define U32_MAX ((u32)~0U) +#endif +#ifndef S32_MAX +#define S32_MAX ((s32)(U32_MAX >> 1)) +#endif +#ifndef S32_MIN +#define S32_MIN ((s32)(-S32_MAX - 1)) +#endif +#ifndef U64_MAX +#define U64_MAX ((u64)~0ULL) +#endif +#ifndef S64_MAX +#define S64_MAX ((s64)(U64_MAX >> 1)) +#endif +#ifndef S64_MIN +#define S64_MIN ((s64)(-S64_MAX - 1)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 60) && !defined(ISRHEL7) +/* Making this static may very well invalidate its usefulness, + * but so it goes with compat code. */ +static inline void memzero_explicit(void *s, size_t count) +{ + memset(s, 0, count); + barrier(); +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && !defined(ISRHEL7) +static const struct in6_addr __compat_in6addr_any = IN6ADDR_ANY_INIT; +#define in6addr_any __compat_in6addr_any +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) && !defined(ISOPENSUSE15) +#include +#include +#include +struct rng_initializer { + struct completion done; + struct random_ready_callback cb; +}; +static inline void rng_initialized_callback(struct random_ready_callback *cb) +{ + complete(&container_of(cb, struct rng_initializer, cb)->done); +} +static inline int wait_for_random_bytes(void) +{ + static bool rng_is_initialized = false; + int ret; + if (unlikely(!rng_is_initialized)) { + struct rng_initializer rng = { + .done = COMPLETION_INITIALIZER(rng.done), + .cb = { .owner = THIS_MODULE, .func = rng_initialized_callback } + }; + ret = add_random_ready_callback(&rng.cb); + if (!ret) { + ret = wait_for_completion_interruptible(&rng.done); + if (ret) { + del_random_ready_callback(&rng.cb); + return ret; + } + } else if (ret != -EALREADY) + return ret; + rng_is_initialized = true; + } + return 0; +} +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +/* This is a disaster. Without this API, we really have no way of + * knowing if it's initialized. We just return that it has and hope + * for the best... */ +static inline int wait_for_random_bytes(void) +{ + return 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0) && !defined(ISRHEL8) +#include +#include +struct rng_is_initialized_callback { + struct random_ready_callback cb; + atomic_t *rng_state; +}; +static inline void rng_is_initialized_callback(struct random_ready_callback *cb) +{ + struct rng_is_initialized_callback *rdy = container_of(cb, struct rng_is_initialized_callback, cb); + atomic_set(rdy->rng_state, 2); + kfree(rdy); +} +static inline bool rng_is_initialized(void) +{ + static atomic_t rng_state = ATOMIC_INIT(0); + + if (atomic_read(&rng_state) == 2) + return true; + + if (atomic_cmpxchg(&rng_state, 0, 1) == 0) { + int ret; + struct rng_is_initialized_callback *rdy = kmalloc(sizeof(*rdy), GFP_ATOMIC); + if (!rdy) { + atomic_set(&rng_state, 0); + return false; + } + rdy->cb.owner = THIS_MODULE; + rdy->cb.func = rng_is_initialized_callback; + rdy->rng_state = &rng_state; + ret = add_random_ready_callback(&rdy->cb); + if (ret) + kfree(rdy); + if (ret == -EALREADY) { + atomic_set(&rng_state, 2); + return true; + } else if (ret) + atomic_set(&rng_state, 0); + return false; + } + return false; +} +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +/* This is a disaster. Without this API, we really have no way of + * knowing if it's initialized. We just return that it has and hope + * for the best... */ +static inline bool rng_is_initialized(void) +{ + return true; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISOPENSUSE15) +static inline int get_random_bytes_wait(void *buf, int nbytes) +{ + int ret = wait_for_random_bytes(); + if (unlikely(ret)) + return ret; + get_random_bytes(buf, nbytes); + return 0; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) && !defined(ISRHEL7) +#define system_power_efficient_wq system_unbound_wq +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 0) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#include +#ifndef ktime_get_real_ts64 +#define timespec64 timespec +#define ktime_get_real_ts64 ktime_get_real_ts +#endif +#else +#include +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) +static inline u64 __compat_jiffies64_to_nsecs(u64 j) +{ +#if !(NSEC_PER_SEC % HZ) + return (NSEC_PER_SEC / HZ) * j; +#else + return div_u64(j * HZ_TO_USEC_NUM, HZ_TO_USEC_DEN) * 1000; +#endif +} +#define jiffies64_to_nsecs __compat_jiffies64_to_nsecs +#endif +static inline u64 ktime_get_coarse_boottime_ns(void) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) + return ktime_to_ns(ktime_get_boottime()); +#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 12) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 20, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 53) + return ktime_to_ns(ktime_mono_to_any(ns_to_ktime(jiffies64_to_nsecs(get_jiffies_64())), TK_OFFS_BOOT)); +#else + return ktime_to_ns(ktime_get_coarse_boottime()); +#endif +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#include +static inline __be32 __compat_confirm_addr_indev(struct in_device *in_dev, __be32 dst, __be32 local, int scope) +{ + int same = 0; + __be32 addr = 0; + for_ifa(in_dev) { + if (!addr && (local == ifa->ifa_local || !local) && ifa->ifa_scope <= scope) { + addr = ifa->ifa_local; + if (same) + break; + } + if (!same) { + same = (!local || inet_ifa_match(local, ifa)) && (!dst || inet_ifa_match(dst, ifa)); + if (same && addr) { + if (local || !dst) + break; + if (inet_ifa_match(addr, ifa)) + break; + if (ifa->ifa_scope <= scope) { + addr = ifa->ifa_local; + break; + } + same = 0; + } + } + } endfor_ifa(in_dev); + return same ? addr : 0; +} +static inline __be32 __compat_inet_confirm_addr(struct net *net, struct in_device *in_dev, __be32 dst, __be32 local, int scope) +{ + __be32 addr = 0; + struct net_device *dev; + if (in_dev) + return __compat_confirm_addr_indev(in_dev, dst, local, scope); + rcu_read_lock(); + for_each_netdev_rcu(net, dev) { + in_dev = __in_dev_get_rcu(dev); + if (in_dev) { + addr = __compat_confirm_addr_indev(in_dev, dst, local, scope); + if (addr) + break; + } + } + rcu_read_unlock(); + return addr; +} +#define inet_confirm_addr __compat_inet_confirm_addr +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#include +#include +static inline void *__compat_kvmalloc(size_t size, gfp_t flags) +{ + gfp_t kmalloc_flags = flags; + void *ret; + if (size > PAGE_SIZE) { + kmalloc_flags |= __GFP_NOWARN; + if (!(kmalloc_flags & __GFP_REPEAT) || (size <= PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) + kmalloc_flags |= __GFP_NORETRY; + } + ret = kmalloc(size, kmalloc_flags); + if (ret || size <= PAGE_SIZE) + return ret; + return __vmalloc(size, flags, PAGE_KERNEL); +} +static inline void *__compat_kvzalloc(size_t size, gfp_t flags) +{ + return __compat_kvmalloc(size, flags | __GFP_ZERO); +} +#define kvmalloc __compat_kvmalloc +#define kvzalloc __compat_kvzalloc +#endif + +#if ((LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 41)) && !defined(ISUBUNTU1404) +#include +#include +static inline void __compat_kvfree(const void *addr) +{ + if (is_vmalloc_addr(addr)) + vfree(addr); + else + kfree(addr); +} +#define kvfree __compat_kvfree +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 9) +#include +#define priv_destructor destructor +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISOPENSUSE15) +#define wg_newlink(a,b,c,d,e) wg_newlink(a,b,c,d) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#include +#define nlmsg_parse(a, b, c, d, e, f) nlmsg_parse(a, b, c, d, e) +#define nla_parse_nested(a, b, c, d, e) nla_parse_nested(a, b, c, d) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && !defined(ISRHEL7) +static inline struct nlattr **genl_family_attrbuf(const struct genl_family *family) +{ + return family->attrbuf; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +#define PTR_ERR_OR_ZERO(p) PTR_RET(p) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 7, 0) +#include +#define nla_put_u64_64bit(a, b, c, d) nla_put_u64(a, b, c) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) +#include +#ifndef GENL_UNS_ADMIN_PERM +#define GENL_UNS_ADMIN_PERM GENL_ADMIN_PERM +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) && !defined(ISRHEL7) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0) +#define genl_register_family(a) genl_register_family_with_ops(a, genl_ops, ARRAY_SIZE(genl_ops)) +#define COMPAT_CANNOT_USE_CONST_GENL_OPS +#else +#define genl_register_family(a) genl_register_family_with_ops(a, genl_ops) +#endif +#define COMPAT_CANNOT_USE_GENL_NOPS +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 2) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 16) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 65) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 101) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 84) +#define __COMPAT_NETLINK_DUMP_BLOCK { \ + int ret; \ + skb->end -= nlmsg_total_size(sizeof(int)); \ + ret = wg_get_device_dump_real(skb, cb); \ + skb->end += nlmsg_total_size(sizeof(int)); \ + return ret; \ +} +#define __COMPAT_NETLINK_DUMP_OVERRIDE +#else +#define __COMPAT_NETLINK_DUMP_BLOCK return wg_get_device_dump_real(skb, cb); +#endif +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 8) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 15, 0)) || (LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 25) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 87) +#define wg_get_device_dump(a, b) wg_get_device_dump_real(a, b); \ +static int wg_get_device_dump(a, b) { \ + struct wg_device *wg = (struct wg_device *)cb->args[0]; \ + if (!wg) { \ + int ret = wg_get_device_start(cb); \ + if (ret) \ + return ret; \ + } \ + __COMPAT_NETLINK_DUMP_BLOCK \ +} \ +static int wg_get_device_dump_real(a, b) +#define COMPAT_CANNOT_USE_NETLINK_START +#elif defined(__COMPAT_NETLINK_DUMP_OVERRIDE) +#define wg_get_device_dump(a, b) wg_get_device_dump_real(a, b); \ +static int wg_get_device_dump(a, b) { \ + __COMPAT_NETLINK_DUMP_BLOCK \ +} \ +static int wg_get_device_dump_real(a, b) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#define COMPAT_CANNOT_USE_IN6_DEV_GET +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#define COMPAT_CANNOT_USE_IFF_NO_QUEUE +#endif + +#if defined(CONFIG_X86_64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) +#include +#include +static inline int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) +{ + return boot_cpu_has(X86_FEATURE_XSAVE) && xgetbv(XCR_XFEATURE_ENABLED_MASK) & xfeatures_needed; +} +#endif +#ifndef XFEATURE_MASK_YMM +#define XFEATURE_MASK_YMM XSTATE_YMM +#endif +#ifndef XFEATURE_MASK_SSE +#define XFEATURE_MASK_SSE XSTATE_SSE +#endif +#ifndef XSTATE_AVX512 +#define XSTATE_AVX512 (XSTATE_OPMASK | XSTATE_ZMM_Hi256 | XSTATE_Hi16_ZMM) +#endif +#ifndef XFEATURE_MASK_AVX512 +#define XFEATURE_MASK_AVX512 XSTATE_AVX512 +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && defined(CONFIG_X86_64) +/* This is incredibly dumb and reckless, but as it turns out, there's + * not really hardware Linux runs properly on that supports F but not BW + * and VL, so in practice this isn't so bad. Plus, this is compat layer, + * so the bar remains fairly low. + */ +#include +#ifndef X86_FEATURE_AVX512BW +#define X86_FEATURE_AVX512BW X86_FEATURE_AVX512F +#endif +#ifndef X86_FEATURE_AVX512VL +#define X86_FEATURE_AVX512VL X86_FEATURE_AVX512F +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0) +struct __compat_dummy_container { char dev; }; +#define netdev_notifier_info net_device *)data); __attribute((unused)) char __compat_dummy_variable = ((struct __compat_dummy_container +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +#define timer_setup(a, b, c) setup_timer(a, ((void (*)(unsigned long))b), ((unsigned long)a)) +#define from_timer(var, callback_timer, timer_fieldname) container_of(callback_timer, typeof(*var), timer_fieldname) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 3) +#define COMPAT_CANNOT_USE_AVX512 +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +#include +#define genl_dump_check_consistent(a, b) genl_dump_check_consistent(a, b, &genl_family) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0) && !defined(ISRHEL7) && !defined(ISOPENSUSE15) +static inline void *skb_put_data(struct sk_buff *skb, const void *data, unsigned int len) +{ + void *tmp = skb_put(skb, len); + memcpy(tmp, data, len); + return tmp; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) && !defined(ISRHEL7) +#define napi_complete_done(n, work_done) napi_complete(n) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) +#include +/* NAPI_STATE_SCHED gets set by netif_napi_add anyway, so this is safe. + * Also, kernels without NAPI_STATE_NO_BUSY_POLL don't have a call to + * napi_hash_add inside of netif_napi_add. + */ +#define NAPI_STATE_NO_BUSY_POLL NAPI_STATE_SCHED +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#include +#ifndef atomic_read_acquire +#define atomic_read_acquire(v) ({ int __compat_p1 = atomic_read(v); smp_rmb(); __compat_p1; }) +#endif +#ifndef atomic_set_release +#define atomic_set_release(v, i) ({ smp_wmb(); atomic_set(v, i); }) +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) +#include +#ifndef atomic_read_acquire +#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) +#endif +#ifndef atomic_set_release +#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 17, 0) +static inline void le32_to_cpu_array(u32 *buf, unsigned int words) +{ + while (words--) { + __le32_to_cpus(buf); + buf++; + } +} +static inline void cpu_to_le32_array(u32 *buf, unsigned int words) +{ + while (words--) { + __cpu_to_le32s(buf); + buf++; + } +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +#include +static inline void crypto_xor_cpy(u8 *dst, const u8 *src1, const u8 *src2, + unsigned int size) +{ + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + __builtin_constant_p(size) && + (size % sizeof(unsigned long)) == 0) { + unsigned long *d = (unsigned long *)dst; + unsigned long *s1 = (unsigned long *)src1; + unsigned long *s2 = (unsigned long *)src2; + + while (size > 0) { + *d++ = *s1++ ^ *s2++; + size -= sizeof(unsigned long); + } + } else { + if (unlikely(dst != src1)) + memmove(dst, src1, size); +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) + crypto_xor(dst, src2, size); +#else + __crypto_xor(dst, src2, size); +#endif + } +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) +#define read_cpuid_part() read_cpuid_part_number() +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) && !defined(ISRHEL7) +#define hlist_add_behind(a, b) hlist_add_after(b, a) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0) +#define totalram_pages() totalram_pages +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 18, 0) +struct __kernel_timespec { + int64_t tv_sec, tv_nsec; +}; +#elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#ifdef __kernel_timespec +#undef __kernel_timespec +struct __kernel_timespec { + int64_t tv_sec, tv_nsec; +}; +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) +#include +#ifndef ALIGN_DOWN +#define ALIGN_DOWN(x, a) __ALIGN_KERNEL((x) - ((a) - 1), (a)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#define skb_probe_transport_header(a) skb_probe_transport_header(a, 0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) && !defined(ISRHEL7) +#define ignore_df local_df +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +/* Note that all intentional uses of the non-_bh variety need to explicitly + * undef these, conditionalized on COMPAT_CANNOT_DEPRECIATE_BH_RCU. + */ +#include +static __always_inline void old_synchronize_rcu(void) +{ + synchronize_rcu(); +} +static __always_inline void old_call_rcu(void *a, void *b) +{ + call_rcu(a, b); +} +static __always_inline void old_rcu_barrier(void) +{ + rcu_barrier(); +} +#ifdef synchronize_rcu +#undef synchronize_rcu +#endif +#ifdef call_rcu +#undef call_rcu +#endif +#ifdef rcu_barrier +#undef rcu_barrier +#endif +#define synchronize_rcu synchronize_rcu_bh +#define call_rcu call_rcu_bh +#define rcu_barrier rcu_barrier_bh +#define COMPAT_CANNOT_DEPRECIATE_BH_RCU +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 10) && !defined(ISRHEL8) +static inline void skb_mark_not_on_list(struct sk_buff *skb) +{ + skb->next = NULL; +} +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0) && !defined(ISRHEL8) +#define NLA_EXACT_LEN NLA_UNSPEC +#endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0) +#define NLA_MIN_LEN NLA_UNSPEC +#define COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 2, 0) && defined(__aarch64__) +#define cpu_have_named_feature(name) (elf_hwcap & (HWCAP_ ## name)) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0) +#include +#ifndef offsetofend +#define offsetofend(TYPE, MEMBER) (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER)) +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0) +#define genl_dumpit_info(cb) ({ \ + struct { struct nlattr **attrs; } *a = (void *)((u8 *)cb->args + offsetofend(struct dump_ctx, next_allowedip)); \ + BUILD_BUG_ON(sizeof(cb->args) < offsetofend(struct dump_ctx, next_allowedip) + sizeof(*a)); \ + a->attrs = genl_family_attrbuf(&genl_family); \ + if (nlmsg_parse(cb->nlh, GENL_HDRLEN + genl_family.hdrsize, a->attrs, genl_family.maxattr, device_policy, NULL) < 0) \ + memset(a->attrs, 0, (genl_family.maxattr + 1) * sizeof(struct nlattr *)); \ + a; \ +}) +#endif + +#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 5) && LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(5, 3, 18) +#define ipv6_dst_lookup_flow(a, b, c, d) ipv6_dst_lookup(a, b, &dst, c) + (void *)0 ?: dst +#endif + +#if defined(ISUBUNTU1604) +#include +#ifndef _WG_LINUX_SIPHASH_H +#define hsiphash_2u32 siphash_2u32 +#define hsiphash_3u32 siphash_3u32 +#define hsiphash_key_t siphash_key_t +#endif +#endif + +#ifdef CONFIG_VE +#include +#ifdef NETIF_F_VIRTUAL +#undef NETIF_F_LLTX +#define NETIF_F_LLTX (__NETIF_F(LLTX) | __NETIF_F(VIRTUAL)) +#endif +#endif + +/* https://github.com/ClangBuiltLinux/linux/issues/7 */ +#if defined( __clang__) && (!defined(CONFIG_CLANG_VERSION) || CONFIG_CLANG_VERSION < 80000) +#include +#undef BUILD_BUG_ON +#define BUILD_BUG_ON(x) +#endif + +/* https://lkml.kernel.org/r/20170624021727.17835-1-Jason@zx2c4.com */ +#if IS_ENABLED(CONFIG_NF_CONNTRACK) +#include +#include +#include +#include +#include +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0) +#include +#endif +static inline void new_icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info) +{ + enum ip_conntrack_info ctinfo; + struct nf_conn *ct = nf_ct_get(skb_in, &ctinfo); + if (skb_network_header(skb_in) < skb_in->head || (skb_network_header(skb_in) + sizeof(struct iphdr)) > skb_tail_pointer(skb_in)) + return; + if (ct) + ip_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.ip; + icmp_send(skb_in, type, code, info); +} +static inline void new_icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) +{ + enum ip_conntrack_info ctinfo; + struct nf_conn *ct = nf_ct_get(skb, &ctinfo); + if (skb_network_header(skb) < skb->head || (skb_network_header(skb) + sizeof(struct ipv6hdr)) > skb_tail_pointer(skb)) + return; + if (ct) + ipv6_hdr(skb)->saddr = ct->tuplehash[0].tuple.src.u3.in6; + icmpv6_send(skb, type, code, info); +} +#define icmp_send(a,b,c,d) new_icmp_send(a,b,c,d) +#define icmpv6_send(a,b,c,d) new_icmpv6_send(a,b,c,d) +#endif + +/* PaX compatibility */ +#ifdef CONSTIFY_PLUGIN +#include +#undef __read_mostly +#define __read_mostly +#endif +#if (defined(RAP_PLUGIN) || defined(CONFIG_CFI_CLANG)) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0) +#include +#define wg_expired_retransmit_handshake(a) wg_expired_retransmit_handshake(unsigned long timer) +#define wg_expired_send_keepalive(a) wg_expired_send_keepalive(unsigned long timer) +#define wg_expired_new_handshake(a) wg_expired_new_handshake(unsigned long timer) +#define wg_expired_zero_key_material(a) wg_expired_zero_key_material(unsigned long timer) +#define wg_expired_send_persistent_keepalive(a) wg_expired_send_persistent_keepalive(unsigned long timer) +#undef timer_setup +#define timer_setup(a, b, c) setup_timer(a, ((void (*)(unsigned long))b), ((unsigned long)a)) +#undef from_timer +#define from_timer(var, callback_timer, timer_fieldname) container_of((struct timer_list *)callback_timer, typeof(*var), timer_fieldname) +#endif + +#endif /* _WG_COMPAT_H */ diff --git a/net/wireguard/compat/dst_cache/dst_cache.c b/net/wireguard/compat/dst_cache/dst_cache.c new file mode 100644 index 0000000000000..7ec22f768a8fb --- /dev/null +++ b/net/wireguard/compat/dst_cache/dst_cache.c @@ -0,0 +1,175 @@ +/* + * net/core/dst_cache.c - dst entry cache + * + * Copyright (c) 2016 Paolo Abeni + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) || LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 50) +static inline u32 rt6_get_cookie(const struct rt6_info *rt) +{ + if ((unlikely(rt->dst.flags & DST_NOCACHE) && rt->dst.from)) + rt = (struct rt6_info *)(rt->dst.from); + + return rt->rt6i_node ? rt->rt6i_node->fn_sernum : 0; +} +#endif +#endif +#include + +struct dst_cache_pcpu { + unsigned long refresh_ts; + struct dst_entry *dst; + u32 cookie; + union { + struct in_addr in_saddr; + struct in6_addr in6_saddr; + }; +}; + +static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache, + struct dst_entry *dst, u32 cookie) +{ + dst_release(dst_cache->dst); + if (dst) + dst_hold(dst); + + dst_cache->cookie = cookie; + dst_cache->dst = dst; +} + +static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, + struct dst_cache_pcpu *idst) +{ + struct dst_entry *dst; + + dst = idst->dst; + if (!dst) + goto fail; + + /* the cache already hold a dst reference; it can't go away */ + dst_hold(dst); + + if (unlikely(!time_after(idst->refresh_ts, dst_cache->reset_ts) || + (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) { + dst_cache_per_cpu_dst_set(idst, NULL, 0); + dst_release(dst); + goto fail; + } + return dst; + +fail: + idst->refresh_ts = jiffies; + return NULL; +} + +struct dst_entry *dst_cache_get(struct dst_cache *dst_cache) +{ + if (!dst_cache->cache) + return NULL; + + return dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache)); +} + +struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr) +{ + struct dst_cache_pcpu *idst; + struct dst_entry *dst; + + if (!dst_cache->cache) + return NULL; + + idst = this_cpu_ptr(dst_cache->cache); + dst = dst_cache_per_cpu_get(dst_cache, idst); + if (!dst) + return NULL; + + *saddr = idst->in_saddr.s_addr; + return container_of(dst, struct rtable, dst); +} + +void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, + __be32 saddr) +{ + struct dst_cache_pcpu *idst; + + if (!dst_cache->cache) + return; + + idst = this_cpu_ptr(dst_cache->cache); + dst_cache_per_cpu_dst_set(idst, dst, 0); + idst->in_saddr.s_addr = saddr; +} + +#if IS_ENABLED(CONFIG_IPV6) +void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, + const struct in6_addr *addr) +{ + struct dst_cache_pcpu *idst; + + if (!dst_cache->cache) + return; + + idst = this_cpu_ptr(dst_cache->cache); + dst_cache_per_cpu_dst_set(this_cpu_ptr(dst_cache->cache), dst, + rt6_get_cookie((struct rt6_info *)dst)); + idst->in6_saddr = *addr; +} + +struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, + struct in6_addr *saddr) +{ + struct dst_cache_pcpu *idst; + struct dst_entry *dst; + + if (!dst_cache->cache) + return NULL; + + idst = this_cpu_ptr(dst_cache->cache); + dst = dst_cache_per_cpu_get(dst_cache, idst); + if (!dst) + return NULL; + + *saddr = idst->in6_saddr; + return dst; +} +#endif + +int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) + BUG_ON(gfp & GFP_ATOMIC); + dst_cache->cache = alloc_percpu(struct dst_cache_pcpu); +#else + dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu, + gfp | __GFP_ZERO); +#endif + if (!dst_cache->cache) + return -ENOMEM; + + dst_cache_reset(dst_cache); + return 0; +} + +void dst_cache_destroy(struct dst_cache *dst_cache) +{ + int i; + + if (!dst_cache->cache) + return; + + for_each_possible_cpu(i) + dst_release(per_cpu_ptr(dst_cache->cache, i)->dst); + + free_percpu(dst_cache->cache); +} diff --git a/net/wireguard/compat/dst_cache/include/net/dst_cache.h b/net/wireguard/compat/dst_cache/include/net/dst_cache.h new file mode 100644 index 0000000000000..48021c0d6be14 --- /dev/null +++ b/net/wireguard/compat/dst_cache/include/net/dst_cache.h @@ -0,0 +1,97 @@ +#ifndef _WG_NET_DST_CACHE_H +#define _WG_NET_DST_CACHE_H + +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#endif + +struct dst_cache { + struct dst_cache_pcpu __percpu *cache; + unsigned long reset_ts; +}; + +/** + * dst_cache_get - perform cache lookup + * @dst_cache: the cache + * + * The caller should use dst_cache_get_ip4() if it need to retrieve the + * source address to be used when xmitting to the cached dst. + * local BH must be disabled. + */ +struct dst_entry *dst_cache_get(struct dst_cache *dst_cache); + +/** + * dst_cache_get_ip4 - perform cache lookup and fetch ipv4 source address + * @dst_cache: the cache + * @saddr: return value for the retrieved source address + * + * local BH must be disabled. + */ +struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr); + +/** + * dst_cache_set_ip4 - store the ipv4 dst into the cache + * @dst_cache: the cache + * @dst: the entry to be cached + * @saddr: the source address to be stored inside the cache + * + * local BH must be disabled. + */ +void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, + __be32 saddr); + +#if IS_ENABLED(CONFIG_IPV6) + +/** + * dst_cache_set_ip6 - store the ipv6 dst into the cache + * @dst_cache: the cache + * @dst: the entry to be cached + * @saddr: the source address to be stored inside the cache + * + * local BH must be disabled. + */ +void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, + const struct in6_addr *addr); + +/** + * dst_cache_get_ip6 - perform cache lookup and fetch ipv6 source address + * @dst_cache: the cache + * @saddr: return value for the retrieved source address + * + * local BH must be disabled. + */ +struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, + struct in6_addr *saddr); +#endif + +/** + * dst_cache_reset - invalidate the cache contents + * @dst_cache: the cache + * + * This do not free the cached dst to avoid races and contentions. + * the dst will be freed on later cache lookup. + */ +static inline void dst_cache_reset(struct dst_cache *dst_cache) +{ + dst_cache->reset_ts = jiffies; +} + +/** + * dst_cache_init - initialize the cache, allocating the required storage + * @dst_cache: the cache + * @gfp: allocation flags + */ +int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp); + +/** + * dst_cache_destroy - empty the cache and free the allocated storage + * @dst_cache: the cache + * + * No synchronization is enforced: it must be called only when the cache + * is unused. + */ +void dst_cache_destroy(struct dst_cache *dst_cache); + +#endif /* _WG_NET_DST_CACHE_H */ diff --git a/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h b/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h new file mode 100644 index 0000000000000..f3f9117bcb885 --- /dev/null +++ b/net/wireguard/compat/fpu-x86/include/asm/fpu/api.h @@ -0,0 +1 @@ +#include diff --git a/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h b/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h new file mode 100644 index 0000000000000..35a6bc4da8adf --- /dev/null +++ b/net/wireguard/compat/intel-family-x86/include/asm/intel-family.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_INTEL_FAMILY_H +#define _ASM_X86_INTEL_FAMILY_H + +/* + * "Big Core" Processors (Branded as Core, Xeon, etc...) + * + * The "_X" parts are generally the EP and EX Xeons, or the + * "Extreme" ones, like Broadwell-E. + * + * Things ending in "2" are usually because we have no better + * name for them. There's no processor called "SILVERMONT2". + */ + +#define INTEL_FAM6_CORE_YONAH 0x0E + +#define INTEL_FAM6_CORE2_MEROM 0x0F +#define INTEL_FAM6_CORE2_MEROM_L 0x16 +#define INTEL_FAM6_CORE2_PENRYN 0x17 +#define INTEL_FAM6_CORE2_DUNNINGTON 0x1D + +#define INTEL_FAM6_NEHALEM 0x1E +#define INTEL_FAM6_NEHALEM_G 0x1F /* Auburndale / Havendale */ +#define INTEL_FAM6_NEHALEM_EP 0x1A +#define INTEL_FAM6_NEHALEM_EX 0x2E + +#define INTEL_FAM6_WESTMERE 0x25 +#define INTEL_FAM6_WESTMERE_EP 0x2C +#define INTEL_FAM6_WESTMERE_EX 0x2F + +#define INTEL_FAM6_SANDYBRIDGE 0x2A +#define INTEL_FAM6_SANDYBRIDGE_X 0x2D +#define INTEL_FAM6_IVYBRIDGE 0x3A +#define INTEL_FAM6_IVYBRIDGE_X 0x3E + +#define INTEL_FAM6_HASWELL_CORE 0x3C +#define INTEL_FAM6_HASWELL_X 0x3F +#define INTEL_FAM6_HASWELL_ULT 0x45 +#define INTEL_FAM6_HASWELL_GT3E 0x46 + +#define INTEL_FAM6_BROADWELL_CORE 0x3D +#define INTEL_FAM6_BROADWELL_GT3E 0x47 +#define INTEL_FAM6_BROADWELL_X 0x4F +#define INTEL_FAM6_BROADWELL_XEON_D 0x56 + +#define INTEL_FAM6_SKYLAKE_MOBILE 0x4E +#define INTEL_FAM6_SKYLAKE_DESKTOP 0x5E +#define INTEL_FAM6_SKYLAKE_X 0x55 +#define INTEL_FAM6_KABYLAKE_MOBILE 0x8E +#define INTEL_FAM6_KABYLAKE_DESKTOP 0x9E + +/* "Small Core" Processors (Atom) */ + +#define INTEL_FAM6_ATOM_PINEVIEW 0x1C +#define INTEL_FAM6_ATOM_LINCROFT 0x26 +#define INTEL_FAM6_ATOM_PENWELL 0x27 +#define INTEL_FAM6_ATOM_CLOVERVIEW 0x35 +#define INTEL_FAM6_ATOM_CEDARVIEW 0x36 +#define INTEL_FAM6_ATOM_SILVERMONT1 0x37 /* BayTrail/BYT / Valleyview */ +#define INTEL_FAM6_ATOM_SILVERMONT2 0x4D /* Avaton/Rangely */ +#define INTEL_FAM6_ATOM_AIRMONT 0x4C /* CherryTrail / Braswell */ +#define INTEL_FAM6_ATOM_MERRIFIELD 0x4A /* Tangier */ +#define INTEL_FAM6_ATOM_MOOREFIELD 0x5A /* Anniedale */ +#define INTEL_FAM6_ATOM_GOLDMONT 0x5C +#define INTEL_FAM6_ATOM_DENVERTON 0x5F /* Goldmont Microserver */ +#define INTEL_FAM6_ATOM_GEMINI_LAKE 0x7A + +/* Xeon Phi */ + +#define INTEL_FAM6_XEON_PHI_KNL 0x57 /* Knights Landing */ +#define INTEL_FAM6_XEON_PHI_KNM 0x85 /* Knights Mill */ + +#endif /* _ASM_X86_INTEL_FAMILY_H */ diff --git a/net/wireguard/compat/memneq/include.h b/net/wireguard/compat/memneq/include.h new file mode 100644 index 0000000000000..2d18acd9b6c87 --- /dev/null +++ b/net/wireguard/compat/memneq/include.h @@ -0,0 +1,5 @@ +extern noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size); +static inline int crypto_memneq(const void *a, const void *b, size_t size) +{ + return __crypto_memneq(a, b, size) != 0UL ? 1 : 0; +} diff --git a/net/wireguard/compat/memneq/memneq.c b/net/wireguard/compat/memneq/memneq.c new file mode 100644 index 0000000000000..1c427d4055379 --- /dev/null +++ b/net/wireguard/compat/memneq/memneq.c @@ -0,0 +1,170 @@ +/* + * Constant-time equality testing of memory regions. + * + * Authors: + * + * James Yonan + * Daniel Borkmann + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * The full GNU General Public License is included in this distribution + * in the file called LICENSE.GPL. + * + * BSD LICENSE + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of OpenVPN Technologies nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +/* Make the optimizer believe the variable can be manipulated arbitrarily. */ +#define COMPILER_OPTIMIZER_HIDE_VAR(var) asm("" : "=r" (var) : "0" (var)) + +#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ + +/* Generic path for arbitrary size */ +static inline unsigned long +__crypto_memneq_generic(const void *a, const void *b, size_t size) +{ + unsigned long neq = 0; + +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) + while (size >= sizeof(unsigned long)) { + neq |= *(unsigned long *)a ^ *(unsigned long *)b; + COMPILER_OPTIMIZER_HIDE_VAR(neq); + a += sizeof(unsigned long); + b += sizeof(unsigned long); + size -= sizeof(unsigned long); + } +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + while (size > 0) { + neq |= *(unsigned char *)a ^ *(unsigned char *)b; + COMPILER_OPTIMIZER_HIDE_VAR(neq); + a += 1; + b += 1; + size -= 1; + } + return neq; +} + +/* Loop-free fast-path for frequently used 16-byte size */ +static inline unsigned long __crypto_memneq_16(const void *a, const void *b) +{ + unsigned long neq = 0; + +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (sizeof(unsigned long) == 8) { + neq |= *(unsigned long *)(a) ^ *(unsigned long *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned long *)(a+8) ^ *(unsigned long *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } else if (sizeof(unsigned int) == 4) { + neq |= *(unsigned int *)(a) ^ *(unsigned int *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+4) ^ *(unsigned int *)(b+4); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+8) ^ *(unsigned int *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned int *)(a+12) ^ *(unsigned int *)(b+12); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } else +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + { + neq |= *(unsigned char *)(a) ^ *(unsigned char *)(b); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+1) ^ *(unsigned char *)(b+1); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+2) ^ *(unsigned char *)(b+2); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+3) ^ *(unsigned char *)(b+3); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+4) ^ *(unsigned char *)(b+4); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+5) ^ *(unsigned char *)(b+5); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+6) ^ *(unsigned char *)(b+6); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+7) ^ *(unsigned char *)(b+7); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+8) ^ *(unsigned char *)(b+8); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+9) ^ *(unsigned char *)(b+9); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+10) ^ *(unsigned char *)(b+10); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+11) ^ *(unsigned char *)(b+11); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+12) ^ *(unsigned char *)(b+12); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+13) ^ *(unsigned char *)(b+13); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+14) ^ *(unsigned char *)(b+14); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+15) ^ *(unsigned char *)(b+15); + COMPILER_OPTIMIZER_HIDE_VAR(neq); + } + + return neq; +} + +/* Compare two areas of memory without leaking timing information, + * and with special optimizations for common sizes. Users should + * not call this function directly, but should instead use + * crypto_memneq defined in crypto/algapi.h. + */ +noinline unsigned long __crypto_memneq(const void *a, const void *b, + size_t size) +{ + switch (size) { + case 16: + return __crypto_memneq_16(a, b); + default: + return __crypto_memneq_generic(a, b, size); + } +} + +#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ diff --git a/net/wireguard/compat/neon-arm/include/asm/neon.h b/net/wireguard/compat/neon-arm/include/asm/neon.h new file mode 100644 index 0000000000000..980d831e201a4 --- /dev/null +++ b/net/wireguard/compat/neon-arm/include/asm/neon.h @@ -0,0 +1,7 @@ +#ifndef _ARCH_ARM_ASM_NEON +#define _ARCH_ARM_ASM_NEON +#define kernel_neon_begin() \ + BUILD_BUG_ON_MSG(1, "This kernel does not support ARM NEON") +#define kernel_neon_end() \ + BUILD_BUG_ON_MSG(1, "This kernel does not support ARM NEON") +#endif diff --git a/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h b/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h new file mode 100644 index 0000000000000..37b4bb2545b32 --- /dev/null +++ b/net/wireguard/compat/ptr_ring/include/linux/ptr_ring.h @@ -0,0 +1,640 @@ +/* + * Definitions for the 'struct ptr_ring' datastructure. + * + * Author: + * Michael S. Tsirkin + * + * Copyright (C) 2016 Red Hat, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * This is a limited-size FIFO maintaining pointers in FIFO order, with + * one CPU producing entries and another consuming entries from a FIFO. + * + * This implementation tries to minimize cache-contention when there is a + * single producer and a single consumer CPU. + */ + +#ifndef _LINUX_PTR_RING_H +#define _LINUX_PTR_RING_H 1 + +#ifdef __KERNEL__ +#include +#include +#include +#include +#include +#include +#include +#endif + +struct ptr_ring { + int producer ____cacheline_aligned_in_smp; + spinlock_t producer_lock; + int consumer_head ____cacheline_aligned_in_smp; /* next valid entry */ + int consumer_tail; /* next entry to invalidate */ + spinlock_t consumer_lock; + /* Shared consumer/producer data */ + /* Read-only by both the producer and the consumer */ + int size ____cacheline_aligned_in_smp; /* max entries in queue */ + int batch; /* number of entries to consume in a batch */ + void **queue; +}; + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). If ring is ever resized, callers must hold + * producer_lock - see e.g. ptr_ring_full. Otherwise, if callers don't hold + * producer_lock, the next call to __ptr_ring_produce may fail. + */ +static inline bool __ptr_ring_full(struct ptr_ring *r) +{ + return r->queue[r->producer]; +} + +static inline bool ptr_ring_full(struct ptr_ring *r) +{ + bool ret; + + spin_lock(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock(&r->producer_lock); + + return ret; +} + +static inline bool ptr_ring_full_irq(struct ptr_ring *r) +{ + bool ret; + + spin_lock_irq(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock_irq(&r->producer_lock); + + return ret; +} + +static inline bool ptr_ring_full_any(struct ptr_ring *r) +{ + unsigned long flags; + bool ret; + + spin_lock_irqsave(&r->producer_lock, flags); + ret = __ptr_ring_full(r); + spin_unlock_irqrestore(&r->producer_lock, flags); + + return ret; +} + +static inline bool ptr_ring_full_bh(struct ptr_ring *r) +{ + bool ret; + + spin_lock_bh(&r->producer_lock); + ret = __ptr_ring_full(r); + spin_unlock_bh(&r->producer_lock); + + return ret; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must hold producer_lock. + */ +static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr) +{ + if (unlikely(!r->size) || r->queue[r->producer]) + return -ENOSPC; + + r->queue[r->producer++] = ptr; + if (unlikely(r->producer >= r->size)) + r->producer = 0; + return 0; +} + +/* + * Note: resize (below) nests producer lock within consumer lock, so if you + * consume in interrupt or BH context, you must disable interrupts/BH when + * calling this. + */ +static inline int ptr_ring_produce(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock(&r->producer_lock); + + return ret; +} + +static inline int ptr_ring_produce_irq(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock_irq(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_irq(&r->producer_lock); + + return ret; +} + +static inline int ptr_ring_produce_any(struct ptr_ring *r, void *ptr) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&r->producer_lock, flags); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_irqrestore(&r->producer_lock, flags); + + return ret; +} + +static inline int ptr_ring_produce_bh(struct ptr_ring *r, void *ptr) +{ + int ret; + + spin_lock_bh(&r->producer_lock); + ret = __ptr_ring_produce(r, ptr); + spin_unlock_bh(&r->producer_lock); + + return ret; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must take consumer_lock + * if they dereference the pointer - see e.g. PTR_RING_PEEK_CALL. + * If ring is never resized, and if the pointer is merely + * tested, there's no need to take the lock - see e.g. __ptr_ring_empty. + */ +static inline void *__ptr_ring_peek(struct ptr_ring *r) +{ + if (likely(r->size)) + return r->queue[r->consumer_head]; + return NULL; +} + +/* Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must take consumer_lock + * if the ring is ever resized - see e.g. ptr_ring_empty. + */ +static inline bool __ptr_ring_empty(struct ptr_ring *r) +{ + return !__ptr_ring_peek(r); +} + +static inline bool ptr_ring_empty(struct ptr_ring *r) +{ + bool ret; + + spin_lock(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock(&r->consumer_lock); + + return ret; +} + +static inline bool ptr_ring_empty_irq(struct ptr_ring *r) +{ + bool ret; + + spin_lock_irq(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock_irq(&r->consumer_lock); + + return ret; +} + +static inline bool ptr_ring_empty_any(struct ptr_ring *r) +{ + unsigned long flags; + bool ret; + + spin_lock_irqsave(&r->consumer_lock, flags); + ret = __ptr_ring_empty(r); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ret; +} + +static inline bool ptr_ring_empty_bh(struct ptr_ring *r) +{ + bool ret; + + spin_lock_bh(&r->consumer_lock); + ret = __ptr_ring_empty(r); + spin_unlock_bh(&r->consumer_lock); + + return ret; +} + +/* Must only be called after __ptr_ring_peek returned !NULL */ +static inline void __ptr_ring_discard_one(struct ptr_ring *r) +{ + /* Fundamentally, what we want to do is update consumer + * index and zero out the entry so producer can reuse it. + * Doing it naively at each consume would be as simple as: + * r->queue[r->consumer++] = NULL; + * if (unlikely(r->consumer >= r->size)) + * r->consumer = 0; + * but that is suboptimal when the ring is full as producer is writing + * out new entries in the same cache line. Defer these updates until a + * batch of entries has been consumed. + */ + int head = r->consumer_head++; + + /* Once we have processed enough entries invalidate them in + * the ring all at once so producer can reuse their space in the ring. + * We also do this when we reach end of the ring - not mandatory + * but helps keep the implementation simple. + */ + if (unlikely(r->consumer_head - r->consumer_tail >= r->batch || + r->consumer_head >= r->size)) { + /* Zero out entries in the reverse order: this way we touch the + * cache line that producer might currently be reading the last; + * producer won't make progress and touch other cache lines + * besides the first one until we write out all entries. + */ + while (likely(head >= r->consumer_tail)) + r->queue[head--] = NULL; + r->consumer_tail = r->consumer_head; + } + if (unlikely(r->consumer_head >= r->size)) { + r->consumer_head = 0; + r->consumer_tail = 0; + } +} + +static inline void *__ptr_ring_consume(struct ptr_ring *r) +{ + void *ptr; + + ptr = __ptr_ring_peek(r); + if (ptr) + __ptr_ring_discard_one(r); + + return ptr; +} + +static inline int __ptr_ring_consume_batched(struct ptr_ring *r, + void **array, int n) +{ + void *ptr; + int i; + + for (i = 0; i < n; i++) { + ptr = __ptr_ring_consume(r); + if (!ptr) + break; + array[i] = ptr; + } + + return i; +} + +/* + * Note: resize (below) nests producer lock within consumer lock, so if you + * call this in interrupt or BH context, you must disable interrupts/BH when + * producing. + */ +static inline void *ptr_ring_consume(struct ptr_ring *r) +{ + void *ptr; + + spin_lock(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock(&r->consumer_lock); + + return ptr; +} + +static inline void *ptr_ring_consume_irq(struct ptr_ring *r) +{ + void *ptr; + + spin_lock_irq(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock_irq(&r->consumer_lock); + + return ptr; +} + +static inline void *ptr_ring_consume_any(struct ptr_ring *r) +{ + unsigned long flags; + void *ptr; + + spin_lock_irqsave(&r->consumer_lock, flags); + ptr = __ptr_ring_consume(r); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ptr; +} + +static inline void *ptr_ring_consume_bh(struct ptr_ring *r) +{ + void *ptr; + + spin_lock_bh(&r->consumer_lock); + ptr = __ptr_ring_consume(r); + spin_unlock_bh(&r->consumer_lock); + + return ptr; +} + +static inline int ptr_ring_consume_batched(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock(&r->consumer_lock); + + return ret; +} + +static inline int ptr_ring_consume_batched_irq(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock_irq(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_irq(&r->consumer_lock); + + return ret; +} + +static inline int ptr_ring_consume_batched_any(struct ptr_ring *r, + void **array, int n) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&r->consumer_lock, flags); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_irqrestore(&r->consumer_lock, flags); + + return ret; +} + +static inline int ptr_ring_consume_batched_bh(struct ptr_ring *r, + void **array, int n) +{ + int ret; + + spin_lock_bh(&r->consumer_lock); + ret = __ptr_ring_consume_batched(r, array, n); + spin_unlock_bh(&r->consumer_lock); + + return ret; +} + +/* Cast to structure type and call a function without discarding from FIFO. + * Function must return a value. + * Callers must take consumer_lock. + */ +#define __PTR_RING_PEEK_CALL(r, f) ((f)(__ptr_ring_peek(r))) + +#define PTR_RING_PEEK_CALL(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_IRQ(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock_irq(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_irq(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_BH(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + \ + spin_lock_bh(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_bh(&(r)->consumer_lock); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +#define PTR_RING_PEEK_CALL_ANY(r, f) ({ \ + typeof((f)(NULL)) __PTR_RING_PEEK_CALL_v; \ + unsigned long __PTR_RING_PEEK_CALL_f;\ + \ + spin_lock_irqsave(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ + __PTR_RING_PEEK_CALL_v = __PTR_RING_PEEK_CALL(r, f); \ + spin_unlock_irqrestore(&(r)->consumer_lock, __PTR_RING_PEEK_CALL_f); \ + __PTR_RING_PEEK_CALL_v; \ +}) + +static inline void **__ptr_ring_init_queue_alloc(unsigned int size, gfp_t gfp) +{ + return kcalloc(size, sizeof(void *), gfp); +} + +static inline void __ptr_ring_set_size(struct ptr_ring *r, int size) +{ + r->size = size; + r->batch = SMP_CACHE_BYTES * 2 / sizeof(*(r->queue)); + /* We need to set batch at least to 1 to make logic + * in __ptr_ring_discard_one work correctly. + * Batching too much (because ring is small) would cause a lot of + * burstiness. Needs tuning, for now disable batching. + */ + if (r->batch > r->size / 2 || !r->batch) + r->batch = 1; +} + +static inline int ptr_ring_init(struct ptr_ring *r, int size, gfp_t gfp) +{ + r->queue = __ptr_ring_init_queue_alloc(size, gfp); + if (!r->queue) + return -ENOMEM; + + __ptr_ring_set_size(r, size); + r->producer = r->consumer_head = r->consumer_tail = 0; + spin_lock_init(&r->producer_lock); + spin_lock_init(&r->consumer_lock); + + return 0; +} + +/* + * Return entries into ring. Destroy entries that don't fit. + * + * Note: this is expected to be a rare slow path operation. + * + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline void ptr_ring_unconsume(struct ptr_ring *r, void **batch, int n, + void (*destroy)(void *)) +{ + unsigned long flags; + int head; + + spin_lock_irqsave(&r->consumer_lock, flags); + spin_lock(&r->producer_lock); + + if (!r->size) + goto done; + + /* + * Clean out buffered entries (for simplicity). This way following code + * can test entries for NULL and if not assume they are valid. + */ + head = r->consumer_head - 1; + while (likely(head >= r->consumer_tail)) + r->queue[head--] = NULL; + r->consumer_tail = r->consumer_head; + + /* + * Go over entries in batch, start moving head back and copy entries. + * Stop when we run into previously unconsumed entries. + */ + while (n) { + head = r->consumer_head - 1; + if (head < 0) + head = r->size - 1; + if (r->queue[head]) { + /* This batch entry will have to be destroyed. */ + goto done; + } + r->queue[head] = batch[--n]; + r->consumer_tail = r->consumer_head = head; + } + +done: + /* Destroy all entries left in the batch. */ + while (n) + destroy(batch[--n]); + spin_unlock(&r->producer_lock); + spin_unlock_irqrestore(&r->consumer_lock, flags); +} + +static inline void **__ptr_ring_swap_queue(struct ptr_ring *r, void **queue, + int size, gfp_t gfp, + void (*destroy)(void *)) +{ + int producer = 0; + void **old; + void *ptr; + + while ((ptr = __ptr_ring_consume(r))) + if (producer < size) + queue[producer++] = ptr; + else if (destroy) + destroy(ptr); + + __ptr_ring_set_size(r, size); + r->producer = producer; + r->consumer_head = 0; + r->consumer_tail = 0; + old = r->queue; + r->queue = queue; + + return old; +} + +/* + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline int ptr_ring_resize(struct ptr_ring *r, int size, gfp_t gfp, + void (*destroy)(void *)) +{ + unsigned long flags; + void **queue = __ptr_ring_init_queue_alloc(size, gfp); + void **old; + + if (!queue) + return -ENOMEM; + + spin_lock_irqsave(&(r)->consumer_lock, flags); + spin_lock(&(r)->producer_lock); + + old = __ptr_ring_swap_queue(r, queue, size, gfp, destroy); + + spin_unlock(&(r)->producer_lock); + spin_unlock_irqrestore(&(r)->consumer_lock, flags); + + kfree(old); + + return 0; +} + +/* + * Note: producer lock is nested within consumer lock, so if you + * resize you must make sure all uses nest correctly. + * In particular if you consume ring in interrupt or BH context, you must + * disable interrupts/BH when doing so. + */ +static inline int ptr_ring_resize_multiple(struct ptr_ring **rings, + unsigned int nrings, + int size, + gfp_t gfp, void (*destroy)(void *)) +{ + unsigned long flags; + void ***queues; + int i; + + queues = kmalloc_array(nrings, sizeof(*queues), gfp); + if (!queues) + goto noqueues; + + for (i = 0; i < nrings; ++i) { + queues[i] = __ptr_ring_init_queue_alloc(size, gfp); + if (!queues[i]) + goto nomem; + } + + for (i = 0; i < nrings; ++i) { + spin_lock_irqsave(&(rings[i])->consumer_lock, flags); + spin_lock(&(rings[i])->producer_lock); + queues[i] = __ptr_ring_swap_queue(rings[i], queues[i], + size, gfp, destroy); + spin_unlock(&(rings[i])->producer_lock); + spin_unlock_irqrestore(&(rings[i])->consumer_lock, flags); + } + + for (i = 0; i < nrings; ++i) + kfree(queues[i]); + + kfree(queues); + + return 0; + +nomem: + while (--i >= 0) + kfree(queues[i]); + + kfree(queues); + +noqueues: + return -ENOMEM; +} + +static inline void ptr_ring_cleanup(struct ptr_ring *r, void (*destroy)(void *)) +{ + void *ptr; + + if (destroy) + while ((ptr = ptr_ring_consume(r))) + destroy(ptr); + kfree(r->queue); +} + +#endif /* _LINUX_PTR_RING_H */ diff --git a/net/wireguard/compat/simd-asm/include/asm/simd.h b/net/wireguard/compat/simd-asm/include/asm/simd.h new file mode 100644 index 0000000000000..a975b38b55781 --- /dev/null +++ b/net/wireguard/compat/simd-asm/include/asm/simd.h @@ -0,0 +1,21 @@ +#ifndef _COMPAT_ASM_SIMD_H +#define _COMPAT_ASM_SIMD_H + +#if defined(CONFIG_X86_64) +#include +#endif + +static __must_check inline bool may_use_simd(void) +{ +#if defined(CONFIG_X86_64) + return irq_fpu_usable(); +#elif defined(CONFIG_ARM64) && defined(CONFIG_KERNEL_MODE_NEON) + return true; +#elif defined(CONFIG_ARM) && defined(CONFIG_KERNEL_MODE_NEON) + return !in_nmi() && !in_irq() && !in_serving_softirq(); +#else + return false; +#endif +} + +#endif diff --git a/net/wireguard/compat/simd/include/linux/simd.h b/net/wireguard/compat/simd/include/linux/simd.h new file mode 100644 index 0000000000000..c75c72471e9ec --- /dev/null +++ b/net/wireguard/compat/simd/include/linux/simd.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_SIMD_H +#define _WG_SIMD_H + +#include +#include +#if defined(CONFIG_X86_64) +#include +#include +#elif defined(CONFIG_KERNEL_MODE_NEON) +#include +#endif + +typedef enum { + HAVE_NO_SIMD = 1 << 0, + HAVE_FULL_SIMD = 1 << 1, + HAVE_SIMD_IN_USE = 1 << 31 +} simd_context_t; + +#define DONT_USE_SIMD ((simd_context_t []){ HAVE_NO_SIMD }) + +static inline void simd_get(simd_context_t *ctx) +{ + *ctx = !IS_ENABLED(CONFIG_PREEMPT_RT_BASE) && may_use_simd() ? HAVE_FULL_SIMD : HAVE_NO_SIMD; +} + +static inline void simd_put(simd_context_t *ctx) +{ +#if defined(CONFIG_X86_64) + if (*ctx & HAVE_SIMD_IN_USE) + kernel_fpu_end(); +#elif defined(CONFIG_KERNEL_MODE_NEON) + if (*ctx & HAVE_SIMD_IN_USE) + kernel_neon_end(); +#endif + *ctx = HAVE_NO_SIMD; +} + +static inline bool simd_relax(simd_context_t *ctx) +{ +#ifdef CONFIG_PREEMPT + if ((*ctx & HAVE_SIMD_IN_USE) && need_resched()) { + simd_put(ctx); + simd_get(ctx); + return true; + } +#endif + return false; +} + +static __must_check inline bool simd_use(simd_context_t *ctx) +{ + if (!(*ctx & HAVE_FULL_SIMD)) + return false; + if (*ctx & HAVE_SIMD_IN_USE) + return true; +#if defined(CONFIG_X86_64) + kernel_fpu_begin(); +#elif defined(CONFIG_KERNEL_MODE_NEON) + kernel_neon_begin(); +#endif + *ctx |= HAVE_SIMD_IN_USE; + return true; +} + +#endif /* _WG_SIMD_H */ diff --git a/net/wireguard/compat/siphash/include/linux/siphash.h b/net/wireguard/compat/siphash/include/linux/siphash.h new file mode 100644 index 0000000000000..1e5e337d15bf6 --- /dev/null +++ b/net/wireguard/compat/siphash/include/linux/siphash.h @@ -0,0 +1,140 @@ +/* Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This file is provided under a dual BSD/GPLv2 license. + * + * SipHash: a fast short-input PRF + * https://131002.net/siphash/ + * + * This implementation is specifically for SipHash2-4 for a secure PRF + * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for + * hashtables. + */ + +#ifndef _WG_LINUX_SIPHASH_H +#define _WG_LINUX_SIPHASH_H + +#include +#include + +#define SIPHASH_ALIGNMENT __alignof__(u64) +typedef struct { + u64 key[2]; +} siphash_key_t; + +u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key); +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key); +#endif + +u64 siphash_1u64(const u64 a, const siphash_key_t *key); +u64 siphash_2u64(const u64 a, const u64 b, const siphash_key_t *key); +u64 siphash_3u64(const u64 a, const u64 b, const u64 c, + const siphash_key_t *key); +u64 siphash_4u64(const u64 a, const u64 b, const u64 c, const u64 d, + const siphash_key_t *key); +u64 siphash_1u32(const u32 a, const siphash_key_t *key); +u64 siphash_3u32(const u32 a, const u32 b, const u32 c, + const siphash_key_t *key); + +static inline u64 siphash_2u32(const u32 a, const u32 b, + const siphash_key_t *key) +{ + return siphash_1u64((u64)b << 32 | a, key); +} +static inline u64 siphash_4u32(const u32 a, const u32 b, const u32 c, + const u32 d, const siphash_key_t *key) +{ + return siphash_2u64((u64)b << 32 | a, (u64)d << 32 | c, key); +} + + +static inline u64 ___siphash_aligned(const __le64 *data, size_t len, + const siphash_key_t *key) +{ + if (__builtin_constant_p(len) && len == 4) + return siphash_1u32(le32_to_cpup((const __le32 *)data), key); + if (__builtin_constant_p(len) && len == 8) + return siphash_1u64(le64_to_cpu(data[0]), key); + if (__builtin_constant_p(len) && len == 16) + return siphash_2u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + key); + if (__builtin_constant_p(len) && len == 24) + return siphash_3u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + le64_to_cpu(data[2]), key); + if (__builtin_constant_p(len) && len == 32) + return siphash_4u64(le64_to_cpu(data[0]), le64_to_cpu(data[1]), + le64_to_cpu(data[2]), le64_to_cpu(data[3]), + key); + return __siphash_aligned(data, len, key); +} + +/** + * siphash - compute 64-bit siphash PRF value + * @data: buffer to hash + * @size: size of @data + * @key: the siphash key + */ +static inline u64 siphash(const void *data, size_t len, + const siphash_key_t *key) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (!IS_ALIGNED((unsigned long)data, SIPHASH_ALIGNMENT)) + return __siphash_unaligned(data, len, key); +#endif + return ___siphash_aligned(data, len, key); +} + +#define HSIPHASH_ALIGNMENT __alignof__(unsigned long) +typedef struct { + unsigned long key[2]; +} hsiphash_key_t; + +u32 __hsiphash_aligned(const void *data, size_t len, + const hsiphash_key_t *key); +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key); +#endif + +u32 hsiphash_1u32(const u32 a, const hsiphash_key_t *key); +u32 hsiphash_2u32(const u32 a, const u32 b, const hsiphash_key_t *key); +u32 hsiphash_3u32(const u32 a, const u32 b, const u32 c, + const hsiphash_key_t *key); +u32 hsiphash_4u32(const u32 a, const u32 b, const u32 c, const u32 d, + const hsiphash_key_t *key); + +static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len, + const hsiphash_key_t *key) +{ + if (__builtin_constant_p(len) && len == 4) + return hsiphash_1u32(le32_to_cpu(data[0]), key); + if (__builtin_constant_p(len) && len == 8) + return hsiphash_2u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + key); + if (__builtin_constant_p(len) && len == 12) + return hsiphash_3u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + le32_to_cpu(data[2]), key); + if (__builtin_constant_p(len) && len == 16) + return hsiphash_4u32(le32_to_cpu(data[0]), le32_to_cpu(data[1]), + le32_to_cpu(data[2]), le32_to_cpu(data[3]), + key); + return __hsiphash_aligned(data, len, key); +} + +/** + * hsiphash - compute 32-bit hsiphash PRF value + * @data: buffer to hash + * @size: size of @data + * @key: the hsiphash key + */ +static inline u32 hsiphash(const void *data, size_t len, + const hsiphash_key_t *key) +{ +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (!IS_ALIGNED((unsigned long)data, HSIPHASH_ALIGNMENT)) + return __hsiphash_unaligned(data, len, key); +#endif + return ___hsiphash_aligned(data, len, key); +} + +#endif /* _WG_LINUX_SIPHASH_H */ diff --git a/net/wireguard/compat/siphash/siphash.c b/net/wireguard/compat/siphash/siphash.c new file mode 100644 index 0000000000000..58855328e6e0d --- /dev/null +++ b/net/wireguard/compat/siphash/siphash.c @@ -0,0 +1,539 @@ +/* Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This file is provided under a dual BSD/GPLv2 license. + * + * SipHash: a fast short-input PRF + * https://131002.net/siphash/ + * + * This implementation is specifically for SipHash2-4 for a secure PRF + * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for + * hashtables. + */ + +#include +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) +#ifdef __LITTLE_ENDIAN +#define bytemask_from_count(cnt) (~(~0ul << (cnt)*8)) +#else +#define bytemask_from_count(cnt) (~(~0ul >> (cnt)*8)) +#endif +#endif + +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 +#include +#include +#endif + +#define SIPROUND \ + do { \ + v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \ + v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \ + v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \ + v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \ + } while (0) + +#define PREAMBLE(len) \ + u64 v0 = 0x736f6d6570736575ULL; \ + u64 v1 = 0x646f72616e646f6dULL; \ + u64 v2 = 0x6c7967656e657261ULL; \ + u64 v3 = 0x7465646279746573ULL; \ + u64 b = ((u64)(len)) << 56; \ + v3 ^= key->key[1]; \ + v2 ^= key->key[0]; \ + v1 ^= key->key[1]; \ + v0 ^= key->key[0]; + +#define POSTAMBLE \ + v3 ^= b; \ + SIPROUND; \ + SIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + SIPROUND; \ + SIPROUND; \ + SIPROUND; \ + SIPROUND; \ + return (v0 ^ v1) ^ (v2 ^ v3); + +u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + PREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = le64_to_cpup(data); + v3 ^= m; + SIPROUND; + SIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= le32_to_cpup(data); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } +#endif + POSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + PREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = get_unaligned_le64(data); + v3 ^= m; + SIPROUND; + SIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= get_unaligned_le32(end); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } +#endif + POSTAMBLE +} +#endif + +/** + * siphash_1u64 - compute 64-bit siphash PRF value of a u64 + * @first: first u64 + * @key: the siphash key + */ +u64 siphash_1u64(const u64 first, const siphash_key_t *key) +{ + PREAMBLE(8) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + POSTAMBLE +} + +/** + * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64 + * @first: first u64 + * @second: second u64 + * @key: the siphash key + */ +u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key) +{ + PREAMBLE(16) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + POSTAMBLE +} + +/** + * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64 + * @first: first u64 + * @second: second u64 + * @third: third u64 + * @key: the siphash key + */ +u64 siphash_3u64(const u64 first, const u64 second, const u64 third, + const siphash_key_t *key) +{ + PREAMBLE(24) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + v3 ^= third; + SIPROUND; + SIPROUND; + v0 ^= third; + POSTAMBLE +} + +/** + * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64 + * @first: first u64 + * @second: second u64 + * @third: third u64 + * @forth: forth u64 + * @key: the siphash key + */ +u64 siphash_4u64(const u64 first, const u64 second, const u64 third, + const u64 forth, const siphash_key_t *key) +{ + PREAMBLE(32) + v3 ^= first; + SIPROUND; + SIPROUND; + v0 ^= first; + v3 ^= second; + SIPROUND; + SIPROUND; + v0 ^= second; + v3 ^= third; + SIPROUND; + SIPROUND; + v0 ^= third; + v3 ^= forth; + SIPROUND; + SIPROUND; + v0 ^= forth; + POSTAMBLE +} + +u64 siphash_1u32(const u32 first, const siphash_key_t *key) +{ + PREAMBLE(4) + b |= first; + POSTAMBLE +} + +u64 siphash_3u32(const u32 first, const u32 second, const u32 third, + const siphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + PREAMBLE(12) + v3 ^= combined; + SIPROUND; + SIPROUND; + v0 ^= combined; + b |= third; + POSTAMBLE +} + +#if BITS_PER_LONG == 64 +/* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for + * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3. + */ + +#define HSIPROUND SIPROUND +#define HPREAMBLE(len) PREAMBLE(len) +#define HPOSTAMBLE \ + v3 ^= b; \ + HSIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + HSIPROUND; \ + HSIPROUND; \ + HSIPROUND; \ + return (v0 ^ v1) ^ (v2 ^ v3); + +u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = le64_to_cpup(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= le32_to_cpup(data); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } +#endif + HPOSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u64)); + const u8 left = len & (sizeof(u64) - 1); + u64 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u64)) { + m = get_unaligned_le64(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } +#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64 + if (left) + b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) & + bytemask_from_count(left))); +#else + switch (left) { + case 7: b |= ((u64)end[6]) << 48; + case 6: b |= ((u64)end[5]) << 40; + case 5: b |= ((u64)end[4]) << 32; + case 4: b |= get_unaligned_le32(end); break; + case 3: b |= ((u64)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } +#endif + HPOSTAMBLE +} +#endif + +/** + * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32 + * @first: first u32 + * @key: the hsiphash key + */ +u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key) +{ + HPREAMBLE(4) + b |= first; + HPOSTAMBLE +} + +/** + * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32 + * @first: first u32 + * @second: second u32 + * @key: the hsiphash key + */ +u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(8) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + HPOSTAMBLE +} + +/** + * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @key: the hsiphash key + */ +u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third, + const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(12) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + b |= third; + HPOSTAMBLE +} + +/** + * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @forth: forth u32 + * @key: the hsiphash key + */ +u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, + const u32 forth, const hsiphash_key_t *key) +{ + u64 combined = (u64)second << 32 | first; + HPREAMBLE(16) + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + combined = (u64)forth << 32 | third; + v3 ^= combined; + HSIPROUND; + v0 ^= combined; + HPOSTAMBLE +} +#else +#define HSIPROUND \ + do { \ + v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \ + v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \ + v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \ + v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \ + } while (0) + +#define HPREAMBLE(len) \ + u32 v0 = 0; \ + u32 v1 = 0; \ + u32 v2 = 0x6c796765U; \ + u32 v3 = 0x74656462U; \ + u32 b = ((u32)(len)) << 24; \ + v3 ^= key->key[1]; \ + v2 ^= key->key[0]; \ + v1 ^= key->key[1]; \ + v0 ^= key->key[0]; + +#define HPOSTAMBLE \ + v3 ^= b; \ + HSIPROUND; \ + v0 ^= b; \ + v2 ^= 0xff; \ + HSIPROUND; \ + HSIPROUND; \ + HSIPROUND; \ + return v1 ^ v3; + +u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u32)); + const u8 left = len & (sizeof(u32) - 1); + u32 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u32)) { + m = le32_to_cpup(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } + switch (left) { + case 3: b |= ((u32)end[2]) << 16; + case 2: b |= le16_to_cpup(data); break; + case 1: b |= end[0]; + } + HPOSTAMBLE +} + +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +u32 __hsiphash_unaligned(const void *data, size_t len, + const hsiphash_key_t *key) +{ + const u8 *end = data + len - (len % sizeof(u32)); + const u8 left = len & (sizeof(u32) - 1); + u32 m; + HPREAMBLE(len) + for (; data != end; data += sizeof(u32)) { + m = get_unaligned_le32(data); + v3 ^= m; + HSIPROUND; + v0 ^= m; + } + switch (left) { + case 3: b |= ((u32)end[2]) << 16; + case 2: b |= get_unaligned_le16(end); break; + case 1: b |= end[0]; + } + HPOSTAMBLE +} +#endif + +/** + * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32 + * @first: first u32 + * @key: the hsiphash key + */ +u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key) +{ + HPREAMBLE(4) + v3 ^= first; + HSIPROUND; + v0 ^= first; + HPOSTAMBLE +} + +/** + * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32 + * @first: first u32 + * @second: second u32 + * @key: the hsiphash key + */ +u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key) +{ + HPREAMBLE(8) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + HPOSTAMBLE +} + +/** + * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @key: the hsiphash key + */ +u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third, + const hsiphash_key_t *key) +{ + HPREAMBLE(12) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + v3 ^= third; + HSIPROUND; + v0 ^= third; + HPOSTAMBLE +} + +/** + * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32 + * @first: first u32 + * @second: second u32 + * @third: third u32 + * @forth: forth u32 + * @key: the hsiphash key + */ +u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, + const u32 forth, const hsiphash_key_t *key) +{ + HPREAMBLE(16) + v3 ^= first; + HSIPROUND; + v0 ^= first; + v3 ^= second; + HSIPROUND; + v0 ^= second; + v3 ^= third; + HSIPROUND; + v0 ^= third; + v3 ^= forth; + HSIPROUND; + v0 ^= forth; + HPOSTAMBLE +} +#endif diff --git a/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h b/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h new file mode 100644 index 0000000000000..8999527d69521 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/include/net/udp_tunnel.h @@ -0,0 +1,94 @@ +#ifndef _WG_NET_UDP_TUNNEL_H +#define _WG_NET_UDP_TUNNEL_H + +#include +#include + +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#endif + +struct udp_port_cfg { + u8 family; + + /* Used only for kernel-created sockets */ + union { + struct in_addr local_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr local_ip6; +#endif + }; + + union { + struct in_addr peer_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr peer_ip6; +#endif + }; + + __be16 local_udp_port; + __be16 peer_udp_port; + unsigned int use_udp_checksums:1, + use_udp6_tx_checksums:1, + use_udp6_rx_checksums:1, + ipv6_v6only:1; +}; + +int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp); + +#if IS_ENABLED(CONFIG_IPV6) +int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp); +#else +static inline int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + return 0; +} +#endif + +static inline int udp_sock_create(struct net *net, + struct udp_port_cfg *cfg, + struct socket **sockp) +{ + if (cfg->family == AF_INET) + return udp_sock_create4(net, cfg, sockp); + + if (cfg->family == AF_INET6) + return udp_sock_create6(net, cfg, sockp); + + return -EPFNOSUPPORT; +} + +typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb); + +struct udp_tunnel_sock_cfg { + void *sk_user_data; + __u8 encap_type; + udp_tunnel_encap_rcv_t encap_rcv; +}; + +/* Setup the given (UDP) sock to receive UDP encapsulated packets */ +void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *sock_cfg); + +/* Transmit the skb using UDP encapsulation. */ +void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 tos, __u8 ttl, + __be16 df, __be16 src_port, __be16 dst_port, + bool xnet, bool nocheck); + +#if IS_ENABLED(CONFIG_IPV6) +int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk, + struct sk_buff *skb, + struct net_device *dev, struct in6_addr *saddr, + struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be32 label, + __be16 src_port, __be16 dst_port, bool nocheck); +#endif + +void udp_tunnel_sock_release(struct socket *sock); + +#endif /* _WG_NET_UDP_TUNNEL_H */ diff --git a/net/wireguard/compat/udp_tunnel/udp_tunnel.c b/net/wireguard/compat/udp_tunnel/udp_tunnel.c new file mode 100644 index 0000000000000..ae435660931c8 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/udp_tunnel.c @@ -0,0 +1,387 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data))) +#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk))) +#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr) +#endif + +/* This is global so, uh, only one real call site... This is the kind of horrific hack you'd expect to see in compat code. */ +static udp_tunnel_encap_rcv_t encap_rcv = NULL; +static void __compat_sk_data_ready(struct sock *sk +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 15, 0) + ,int unused_vulnerable_length_param +#endif + ) +{ + struct sk_buff *skb; + while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) { + skb_orphan(skb); + sk_mem_reclaim(sk); + encap_rcv(sk, skb); + } +} + +int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + int err; + struct socket *sock = NULL; + struct sockaddr_in udp_addr; + + err = __sock_create(net, AF_INET, SOCK_DGRAM, 0, &sock, 1); + if (err < 0) + goto error; + + udp_addr.sin_family = AF_INET; + udp_addr.sin_addr = cfg->local_ip; + udp_addr.sin_port = cfg->local_udp_port; + err = kernel_bind(sock, (struct sockaddr *)&udp_addr, + sizeof(udp_addr)); + if (err < 0) + goto error; + + if (cfg->peer_udp_port) { + udp_addr.sin_family = AF_INET; + udp_addr.sin_addr = cfg->peer_ip; + udp_addr.sin_port = cfg->peer_udp_port; + err = kernel_connect(sock, (struct sockaddr *)&udp_addr, + sizeof(udp_addr), 0); + if (err < 0) + goto error; + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) + sock->sk->sk_no_check = !cfg->use_udp_checksums; +#else + sock->sk->sk_no_check_tx = !cfg->use_udp_checksums; +#endif + + *sockp = sock; + return 0; + +error: + if (sock) { + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); + } + *sockp = NULL; + return err; +} + +void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *cfg) +{ + inet_sk(sock->sk)->mc_loop = 0; + encap_rcv = cfg->encap_rcv; + rcu_assign_sk_user_data(sock->sk, cfg->sk_user_data); + /* We force the cast in this awful way, due to various Android kernels + * backporting things stupidly. */ + *(void **)&sock->sk->sk_data_ready = (void *)__compat_sk_data_ready; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) +static inline __sum16 udp_v4_check(int len, __be32 saddr, + __be32 daddr, __wsum base) +{ + return csum_tcpudp_magic(saddr, daddr, len, IPPROTO_UDP, base); +} + +static void udp_set_csum(bool nocheck, struct sk_buff *skb, + __be32 saddr, __be32 daddr, int len) +{ + struct udphdr *uh = udp_hdr(skb); + + if (nocheck) + uh->check = 0; + else if (skb_is_gso(skb)) + uh->check = ~udp_v4_check(len, saddr, daddr, 0); + else if (skb_dst(skb) && skb_dst(skb)->dev && + (skb_dst(skb)->dev->features & NETIF_F_V4_CSUM)) { + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + skb->ip_summed = CHECKSUM_PARTIAL; + skb->csum_start = skb_transport_header(skb) - skb->head; + skb->csum_offset = offsetof(struct udphdr, check); + uh->check = ~udp_v4_check(len, saddr, daddr, 0); + } else { + __wsum csum; + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + uh->check = 0; + csum = skb_checksum(skb, 0, len, 0); + uh->check = udp_v4_check(len, saddr, daddr, csum); + if (uh->check == 0) + uh->check = CSUM_MANGLED_0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + } +} + +#endif + +static void __compat_fake_destructor(struct sk_buff *skb) +{ +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) +static void __compat_iptunnel_xmit(struct rtable *rt, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 proto, + __u8 tos, __u8 ttl, __be16 df, bool xnet) +{ + struct iphdr *iph; + struct pcpu_tstats *tstats = this_cpu_ptr(skb->dev->tstats); + + skb_scrub_packet(skb, xnet); + + skb->rxhash = 0; + skb_dst_set(skb, &rt->dst); + memset(IPCB(skb), 0, sizeof(*IPCB(skb))); + + /* Push down and install the IP header. */ + skb_push(skb, sizeof(struct iphdr)); + skb_reset_network_header(skb); + + iph = ip_hdr(skb); + + iph->version = 4; + iph->ihl = sizeof(struct iphdr) >> 2; + iph->frag_off = df; + iph->protocol = proto; + iph->tos = tos; + iph->daddr = dst; + iph->saddr = src; + iph->ttl = ttl; +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 10, 53) + __ip_select_ident(iph, &rt->dst, (skb_shinfo(skb)->gso_segs ?: 1) - 1); +#else + __ip_select_ident(iph, skb_shinfo(skb)->gso_segs ?: 1); +#endif + + iptunnel_xmit(skb, skb->dev); + u64_stats_update_begin(&tstats->syncp); + tstats->tx_bytes -= 8; + u64_stats_update_end(&tstats->syncp); +} +#define iptunnel_xmit __compat_iptunnel_xmit +#endif + +void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb, + __be32 src, __be32 dst, __u8 tos, __u8 ttl, + __be16 df, __be16 src_port, __be16 dst_port, + bool xnet, bool nocheck) +{ + struct udphdr *uh; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + struct net_device *dev = skb->dev; + int ret; +#endif + + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); + + udp_set_csum(nocheck, skb, src, dst, skb->len); + + if (!skb->sk) + skb->sk = sk; + if (!skb->destructor) + skb->destructor = __compat_fake_destructor; +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + ret = +#endif + iptunnel_xmit( +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0) + sk, +#endif + rt, skb, src, dst, IPPROTO_UDP, tos, ttl, df, xnet); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0) + if (ret) + iptunnel_xmit_stats(ret - 8, &dev->stats, dev->tstats); +#endif +} + +void udp_tunnel_sock_release(struct socket *sock) +{ + rcu_assign_sk_user_data(sock->sk, NULL); + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); +} + +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int udp_sock_create6(struct net *net, struct udp_port_cfg *cfg, + struct socket **sockp) +{ + struct sockaddr_in6 udp6_addr; + int err; + struct socket *sock = NULL; + + err = __sock_create(net, AF_INET6, SOCK_DGRAM, 0, &sock, 1); + if (err < 0) + goto error; + + if (cfg->ipv6_v6only) { + int val = 1; + + err = kernel_setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, + (char *) &val, sizeof(val)); + if (err < 0) + goto error; + } + + udp6_addr.sin6_family = AF_INET6; + memcpy(&udp6_addr.sin6_addr, &cfg->local_ip6, + sizeof(udp6_addr.sin6_addr)); + udp6_addr.sin6_port = cfg->local_udp_port; + err = kernel_bind(sock, (struct sockaddr *)&udp6_addr, + sizeof(udp6_addr)); + if (err < 0) + goto error; + + if (cfg->peer_udp_port) { + udp6_addr.sin6_family = AF_INET6; + memcpy(&udp6_addr.sin6_addr, &cfg->peer_ip6, + sizeof(udp6_addr.sin6_addr)); + udp6_addr.sin6_port = cfg->peer_udp_port; + err = kernel_connect(sock, + (struct sockaddr *)&udp6_addr, + sizeof(udp6_addr), 0); + } + if (err < 0) + goto error; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) + sock->sk->sk_no_check = !cfg->use_udp_checksums; +#else + udp_set_no_check6_tx(sock->sk, !cfg->use_udp6_tx_checksums); + udp_set_no_check6_rx(sock->sk, !cfg->use_udp6_rx_checksums); +#endif + + *sockp = sock; + return 0; + +error: + if (sock) { + kernel_sock_shutdown(sock, SHUT_RDWR); + sock_release(sock); + } + *sockp = NULL; + return err; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) +static inline __sum16 udp_v6_check(int len, + const struct in6_addr *saddr, + const struct in6_addr *daddr, + __wsum base) +{ + return csum_ipv6_magic(saddr, daddr, len, IPPROTO_UDP, base); +} +static void udp6_set_csum(bool nocheck, struct sk_buff *skb, + const struct in6_addr *saddr, + const struct in6_addr *daddr, int len) +{ + struct udphdr *uh = udp_hdr(skb); + + if (nocheck) + uh->check = 0; + else if (skb_is_gso(skb)) + uh->check = ~udp_v6_check(len, saddr, daddr, 0); + else if (skb_dst(skb) && skb_dst(skb)->dev && + (skb_dst(skb)->dev->features & NETIF_F_IPV6_CSUM)) { + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + skb->ip_summed = CHECKSUM_PARTIAL; + skb->csum_start = skb_transport_header(skb) - skb->head; + skb->csum_offset = offsetof(struct udphdr, check); + uh->check = ~udp_v6_check(len, saddr, daddr, 0); + } else { + __wsum csum; + + BUG_ON(skb->ip_summed == CHECKSUM_PARTIAL); + + uh->check = 0; + csum = skb_checksum(skb, 0, len, 0); + uh->check = udp_v6_check(len, saddr, daddr, csum); + if (uh->check == 0) + uh->check = CSUM_MANGLED_0; + + skb->ip_summed = CHECKSUM_UNNECESSARY; + } +} +#endif + +int udp_tunnel6_xmit_skb(struct dst_entry *dst, struct sock *sk, + struct sk_buff *skb, + struct net_device *dev, struct in6_addr *saddr, + struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be32 label, + __be16 src_port, __be16 dst_port, bool nocheck) +{ + struct udphdr *uh; + struct ipv6hdr *ip6h; + + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + + uh->dest = dst_port; + uh->source = src_port; + + uh->len = htons(skb->len); + + skb_dst_set(skb, dst); + + udp6_set_csum(nocheck, skb, saddr, daddr, skb->len); + + __skb_push(skb, sizeof(*ip6h)); + skb_reset_network_header(skb); + ip6h = ipv6_hdr(skb); + ip6_flow_hdr(ip6h, prio, label); + ip6h->payload_len = htons(skb->len); + ip6h->nexthdr = IPPROTO_UDP; + ip6h->hop_limit = ttl; + ip6h->daddr = *daddr; + ip6h->saddr = *saddr; + + if (!skb->sk) + skb->sk = sk; + if (!skb->destructor) + skb->destructor = __compat_fake_destructor; + + ip6tunnel_xmit(skb, dev); + return 0; +} +#endif diff --git a/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h b/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h new file mode 100644 index 0000000000000..0605896e902f7 --- /dev/null +++ b/net/wireguard/compat/udp_tunnel/udp_tunnel_partial_compat.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) +#define udp_sock_create4 udp_sock_create +#define udp_sock_create6 udp_sock_create +#include +#include +#include +#include +#include +#include +#if IS_ENABLED(CONFIG_IPV6) +#include +#include +#include +#include +#include +#endif +static inline void __compat_fake_destructor(struct sk_buff *skb) +{ +} +typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb); +struct udp_tunnel_sock_cfg { + void *sk_user_data; + __u8 encap_type; + udp_tunnel_encap_rcv_t encap_rcv; +}; +/* This is global so, uh, only one real call site... This is the kind of horrific hack you'd expect to see in compat code. */ +static udp_tunnel_encap_rcv_t encap_rcv = NULL; +static void __compat_sk_data_ready(struct sock *sk) +{ + struct sk_buff *skb; + while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) { + skb_orphan(skb); + sk_mem_reclaim(sk); + encap_rcv(sk, skb); + } +} +static inline void setup_udp_tunnel_sock(struct net *net, struct socket *sock, + struct udp_tunnel_sock_cfg *cfg) +{ + struct sock *sk = sock->sk; + inet_sk(sk)->mc_loop = 0; + encap_rcv = cfg->encap_rcv; + rcu_assign_sk_user_data(sk, cfg->sk_user_data); + sk->sk_data_ready = __compat_sk_data_ready; +} +static inline void udp_tunnel_sock_release(struct socket *sock) +{ + rcu_assign_sk_user_data(sock->sk, NULL); + kernel_sock_shutdown(sock, SHUT_RDWR); + sk_release_kernel(sock->sk); +} +static inline int udp_tunnel_xmit_skb(struct socket *sock, struct rtable *rt, + struct sk_buff *skb, __be32 src, __be32 dst, + __u8 tos, __u8 ttl, __be16 df, __be16 src_port, + __be16 dst_port, bool xnet) +{ + struct udphdr *uh; + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + udp_set_csum(sock->sk->sk_no_check_tx, skb, src, dst, skb->len); + return iptunnel_xmit(sock->sk, rt, skb, src, dst, IPPROTO_UDP, + tos, ttl, df, xnet); +} +#if IS_ENABLED(CONFIG_IPV6) +static inline int udp_tunnel6_xmit_skb(struct socket *sock, struct dst_entry *dst, + struct sk_buff *skb, struct net_device *dev, + struct in6_addr *saddr, struct in6_addr *daddr, + __u8 prio, __u8 ttl, __be16 src_port, + __be16 dst_port) +{ + struct udphdr *uh; + struct ipv6hdr *ip6h; + struct sock *sk = sock->sk; + __skb_push(skb, sizeof(*uh)); + skb_reset_transport_header(skb); + uh = udp_hdr(skb); + uh->dest = dst_port; + uh->source = src_port; + uh->len = htons(skb->len); + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); + IPCB(skb)->flags &= ~(IPSKB_XFRM_TUNNEL_SIZE | IPSKB_XFRM_TRANSFORMED + | IPSKB_REROUTED); + skb_dst_set(skb, dst); + udp6_set_csum(udp_get_no_check6_tx(sk), skb, &inet6_sk(sk)->saddr, + &sk->sk_v6_daddr, skb->len); + __skb_push(skb, sizeof(*ip6h)); + skb_reset_network_header(skb); + ip6h = ipv6_hdr(skb); + ip6_flow_hdr(ip6h, prio, htonl(0)); + ip6h->payload_len = htons(skb->len); + ip6h->nexthdr = IPPROTO_UDP; + ip6h->hop_limit = ttl; + ip6h->daddr = *daddr; + ip6h->saddr = *saddr; + ip6tunnel_xmit(skb, dev); + return 0; +} +#endif +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#include +#include +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__; ret__ = udp_tunnel_xmit_skb((b)->sk_socket, a, c, d, e, f, g, h, i, j, k); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#if IS_ENABLED(CONFIG_IPV6) +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) udp_tunnel6_xmit_skb((b)->sk_socket, a, c, d, e, f, g, h, j, k); +#endif +#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 18, 0) +static inline void __compat_fake_destructor(struct sk_buff *skb) +{ +} +#endif +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__; if (!(c)->destructor) (c)->destructor = __compat_fake_destructor; if (!(c)->sk) (c)->sk = (b); ret__ = udp_tunnel_xmit_skb(a, c, d, e, f, g, h, i, j, k, l); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#if IS_ENABLED(CONFIG_IPV6) +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { if (!(c)->destructor) (c)->destructor = __compat_fake_destructor; if (!(c)->sk) (c)->sk = (b); udp_tunnel6_xmit_skb(a, c, d, e, f, g, h, j, k, l); } while(0) +#endif +#else + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__ = udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l); if (ret__) iptunnel_xmit_stats(ret__ - 8, &dev__->stats, dev__->tstats); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0) +#include +#include +#define udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) do { struct net_device *dev__ = (c)->dev; int ret__ = udp_tunnel_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l); iptunnel_xmit_stats(ret__, &dev__->stats, dev__->tstats); } while (0) +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && IS_ENABLED(CONFIG_IPV6) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#define udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, i, j, k, l) udp_tunnel6_xmit_skb(a, b, c, d, e, f, g, h, j, k, l) +#endif + +#endif + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#include +#include +struct __compat_udp_port_cfg { + u8 family; + union { + struct in_addr local_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr local_ip6; +#endif + }; + union { + struct in_addr peer_ip; +#if IS_ENABLED(CONFIG_IPV6) + struct in6_addr peer_ip6; +#endif + }; + __be16 local_udp_port; + __be16 peer_udp_port; + unsigned int use_udp_checksums:1, use_udp6_tx_checksums:1, use_udp6_rx_checksums:1, ipv6_v6only:1; +}; +static inline int __maybe_unused __compat_udp_sock_create(struct net *net, struct __compat_udp_port_cfg *cfg, struct socket **sockp) +{ + struct udp_port_cfg old_cfg = { + .family = cfg->family, + .local_ip = cfg->local_ip, +#if IS_ENABLED(CONFIG_IPV6) + .local_ip6 = cfg->local_ip6, +#endif + .peer_ip = cfg->peer_ip, +#if IS_ENABLED(CONFIG_IPV6) + .peer_ip6 = cfg->peer_ip6, +#endif + .local_udp_port = cfg->local_udp_port, + .peer_udp_port = cfg->peer_udp_port, + .use_udp_checksums = cfg->use_udp_checksums, + .use_udp6_tx_checksums = cfg->use_udp6_tx_checksums, + .use_udp6_rx_checksums = cfg->use_udp6_rx_checksums + }; + if (cfg->family == AF_INET) + return udp_sock_create4(net, &old_cfg, sockp); + +#if IS_ENABLED(CONFIG_IPV6) + if (cfg->family == AF_INET6) { + int ret; + int old_bindv6only; + struct net *nobns; + + if (cfg->ipv6_v6only) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0) + nobns = &init_net; +#else + nobns = net; +#endif + /* Since udp_port_cfg only learned of ipv6_v6only in 4.3, we do this horrible + * hack here and set the sysctl variable temporarily to something that will + * set the right option for us in sock_create. It's super racey! */ + old_bindv6only = nobns->ipv6.sysctl.bindv6only; + nobns->ipv6.sysctl.bindv6only = 1; + } + ret = udp_sock_create6(net, &old_cfg, sockp); + if (cfg->ipv6_v6only) + nobns->ipv6.sysctl.bindv6only = old_bindv6only; + return ret; + } +#endif + return -EPFNOSUPPORT; +} +#define udp_port_cfg __compat_udp_port_cfg +#define udp_sock_create(a, b, c) __compat_udp_sock_create(a, b, c) +#endif diff --git a/net/wireguard/cookie.c b/net/wireguard/cookie.c new file mode 100644 index 0000000000000..8b7d1fe0cdf4b --- /dev/null +++ b/net/wireguard/cookie.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "cookie.h" +#include "peer.h" +#include "device.h" +#include "messages.h" +#include "ratelimiter.h" +#include "timers.h" + +#include +#include + +#include +#include + +void wg_cookie_checker_init(struct cookie_checker *checker, + struct wg_device *wg) +{ + init_rwsem(&checker->secret_lock); + checker->secret_birthdate = ktime_get_coarse_boottime_ns(); + get_random_bytes(checker->secret, NOISE_HASH_LEN); + checker->device = wg; +} + +enum { COOKIE_KEY_LABEL_LEN = 8 }; +static const u8 mac1_key_label[COOKIE_KEY_LABEL_LEN] = "mac1----"; +static const u8 cookie_key_label[COOKIE_KEY_LABEL_LEN] = "cookie--"; + +static void precompute_key(u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 pubkey[NOISE_PUBLIC_KEY_LEN], + const u8 label[COOKIE_KEY_LABEL_LEN]) +{ + struct blake2s_state blake; + + blake2s_init(&blake, NOISE_SYMMETRIC_KEY_LEN); + blake2s_update(&blake, label, COOKIE_KEY_LABEL_LEN); + blake2s_update(&blake, pubkey, NOISE_PUBLIC_KEY_LEN); + blake2s_final(&blake, key); +} + +/* Must hold peer->handshake.static_identity->lock */ +void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker) +{ + if (likely(checker->device->static_identity.has_identity)) { + precompute_key(checker->cookie_encryption_key, + checker->device->static_identity.static_public, + cookie_key_label); + precompute_key(checker->message_mac1_key, + checker->device->static_identity.static_public, + mac1_key_label); + } else { + memset(checker->cookie_encryption_key, 0, + NOISE_SYMMETRIC_KEY_LEN); + memset(checker->message_mac1_key, 0, NOISE_SYMMETRIC_KEY_LEN); + } +} + +void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer) +{ + precompute_key(peer->latest_cookie.cookie_decryption_key, + peer->handshake.remote_static, cookie_key_label); + precompute_key(peer->latest_cookie.message_mac1_key, + peer->handshake.remote_static, mac1_key_label); +} + +void wg_cookie_init(struct cookie *cookie) +{ + memset(cookie, 0, sizeof(*cookie)); + init_rwsem(&cookie->lock); +} + +static void compute_mac1(u8 mac1[COOKIE_LEN], const void *message, size_t len, + const u8 key[NOISE_SYMMETRIC_KEY_LEN]) +{ + len = len - sizeof(struct message_macs) + + offsetof(struct message_macs, mac1); + blake2s(mac1, message, key, COOKIE_LEN, len, NOISE_SYMMETRIC_KEY_LEN); +} + +static void compute_mac2(u8 mac2[COOKIE_LEN], const void *message, size_t len, + const u8 cookie[COOKIE_LEN]) +{ + len = len - sizeof(struct message_macs) + + offsetof(struct message_macs, mac2); + blake2s(mac2, message, cookie, COOKIE_LEN, len, COOKIE_LEN); +} + +static void make_cookie(u8 cookie[COOKIE_LEN], struct sk_buff *skb, + struct cookie_checker *checker) +{ + struct blake2s_state state; + + if (wg_birthdate_has_expired(checker->secret_birthdate, + COOKIE_SECRET_MAX_AGE)) { + down_write(&checker->secret_lock); + checker->secret_birthdate = ktime_get_coarse_boottime_ns(); + get_random_bytes(checker->secret, NOISE_HASH_LEN); + up_write(&checker->secret_lock); + } + + down_read(&checker->secret_lock); + + blake2s_init_key(&state, COOKIE_LEN, checker->secret, NOISE_HASH_LEN); + if (skb->protocol == htons(ETH_P_IP)) + blake2s_update(&state, (u8 *)&ip_hdr(skb)->saddr, + sizeof(struct in_addr)); + else if (skb->protocol == htons(ETH_P_IPV6)) + blake2s_update(&state, (u8 *)&ipv6_hdr(skb)->saddr, + sizeof(struct in6_addr)); + blake2s_update(&state, (u8 *)&udp_hdr(skb)->source, sizeof(__be16)); + blake2s_final(&state, cookie); + + up_read(&checker->secret_lock); +} + +enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, + struct sk_buff *skb, + bool check_cookie) +{ + struct message_macs *macs = (struct message_macs *) + (skb->data + skb->len - sizeof(*macs)); + enum cookie_mac_state ret; + u8 computed_mac[COOKIE_LEN]; + u8 cookie[COOKIE_LEN]; + + ret = INVALID_MAC; + compute_mac1(computed_mac, skb->data, skb->len, + checker->message_mac1_key); + if (crypto_memneq(computed_mac, macs->mac1, COOKIE_LEN)) + goto out; + + ret = VALID_MAC_BUT_NO_COOKIE; + + if (!check_cookie) + goto out; + + make_cookie(cookie, skb, checker); + + compute_mac2(computed_mac, skb->data, skb->len, cookie); + if (crypto_memneq(computed_mac, macs->mac2, COOKIE_LEN)) + goto out; + + ret = VALID_MAC_WITH_COOKIE_BUT_RATELIMITED; + if (!wg_ratelimiter_allow(skb, dev_net(checker->device->dev))) + goto out; + + ret = VALID_MAC_WITH_COOKIE; + +out: + return ret; +} + +void wg_cookie_add_mac_to_packet(void *message, size_t len, + struct wg_peer *peer) +{ + struct message_macs *macs = (struct message_macs *) + ((u8 *)message + len - sizeof(*macs)); + + down_write(&peer->latest_cookie.lock); + compute_mac1(macs->mac1, message, len, + peer->latest_cookie.message_mac1_key); + memcpy(peer->latest_cookie.last_mac1_sent, macs->mac1, COOKIE_LEN); + peer->latest_cookie.have_sent_mac1 = true; + up_write(&peer->latest_cookie.lock); + + down_read(&peer->latest_cookie.lock); + if (peer->latest_cookie.is_valid && + !wg_birthdate_has_expired(peer->latest_cookie.birthdate, + COOKIE_SECRET_MAX_AGE - COOKIE_SECRET_LATENCY)) + compute_mac2(macs->mac2, message, len, + peer->latest_cookie.cookie); + else + memset(macs->mac2, 0, COOKIE_LEN); + up_read(&peer->latest_cookie.lock); +} + +void wg_cookie_message_create(struct message_handshake_cookie *dst, + struct sk_buff *skb, __le32 index, + struct cookie_checker *checker) +{ + struct message_macs *macs = (struct message_macs *) + ((u8 *)skb->data + skb->len - sizeof(*macs)); + u8 cookie[COOKIE_LEN]; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE); + dst->receiver_index = index; + get_random_bytes_wait(dst->nonce, COOKIE_NONCE_LEN); + + make_cookie(cookie, skb, checker); + xchacha20poly1305_encrypt(dst->encrypted_cookie, cookie, COOKIE_LEN, + macs->mac1, COOKIE_LEN, dst->nonce, + checker->cookie_encryption_key); +} + +void wg_cookie_message_consume(struct message_handshake_cookie *src, + struct wg_device *wg) +{ + struct wg_peer *peer = NULL; + u8 cookie[COOKIE_LEN]; + bool ret; + + if (unlikely(!wg_index_hashtable_lookup(wg->index_hashtable, + INDEX_HASHTABLE_HANDSHAKE | + INDEX_HASHTABLE_KEYPAIR, + src->receiver_index, &peer))) + return; + + down_read(&peer->latest_cookie.lock); + if (unlikely(!peer->latest_cookie.have_sent_mac1)) { + up_read(&peer->latest_cookie.lock); + goto out; + } + ret = xchacha20poly1305_decrypt( + cookie, src->encrypted_cookie, sizeof(src->encrypted_cookie), + peer->latest_cookie.last_mac1_sent, COOKIE_LEN, src->nonce, + peer->latest_cookie.cookie_decryption_key); + up_read(&peer->latest_cookie.lock); + + if (ret) { + down_write(&peer->latest_cookie.lock); + memcpy(peer->latest_cookie.cookie, cookie, COOKIE_LEN); + peer->latest_cookie.birthdate = ktime_get_coarse_boottime_ns(); + peer->latest_cookie.is_valid = true; + peer->latest_cookie.have_sent_mac1 = false; + up_write(&peer->latest_cookie.lock); + } else { + net_dbg_ratelimited("%s: Could not decrypt invalid cookie response\n", + wg->dev->name); + } + +out: + wg_peer_put(peer); +} diff --git a/net/wireguard/cookie.h b/net/wireguard/cookie.h new file mode 100644 index 0000000000000..c4bd61ca03f24 --- /dev/null +++ b/net/wireguard/cookie.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COOKIE_H +#define _WG_COOKIE_H + +#include "messages.h" +#include + +struct wg_peer; + +struct cookie_checker { + u8 secret[NOISE_HASH_LEN]; + u8 cookie_encryption_key[NOISE_SYMMETRIC_KEY_LEN]; + u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; + u64 secret_birthdate; + struct rw_semaphore secret_lock; + struct wg_device *device; +}; + +struct cookie { + u64 birthdate; + bool is_valid; + u8 cookie[COOKIE_LEN]; + bool have_sent_mac1; + u8 last_mac1_sent[COOKIE_LEN]; + u8 cookie_decryption_key[NOISE_SYMMETRIC_KEY_LEN]; + u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; + struct rw_semaphore lock; +}; + +enum cookie_mac_state { + INVALID_MAC, + VALID_MAC_BUT_NO_COOKIE, + VALID_MAC_WITH_COOKIE_BUT_RATELIMITED, + VALID_MAC_WITH_COOKIE +}; + +void wg_cookie_checker_init(struct cookie_checker *checker, + struct wg_device *wg); +void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker); +void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer); +void wg_cookie_init(struct cookie *cookie); + +enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, + struct sk_buff *skb, + bool check_cookie); +void wg_cookie_add_mac_to_packet(void *message, size_t len, + struct wg_peer *peer); + +void wg_cookie_message_create(struct message_handshake_cookie *src, + struct sk_buff *skb, __le32 index, + struct cookie_checker *checker); +void wg_cookie_message_consume(struct message_handshake_cookie *src, + struct wg_device *wg); + +#endif /* _WG_COOKIE_H */ diff --git a/net/wireguard/crypto/Makefile.include b/net/wireguard/crypto/Makefile.include new file mode 100644 index 0000000000000..f2a312e96d889 --- /dev/null +++ b/net/wireguard/crypto/Makefile.include @@ -0,0 +1,57 @@ +ifeq ($(CONFIG_X86_64)$(if $(CONFIG_UML),y,n),yn) +CONFIG_ZINC_ARCH_X86_64 := y +endif +ifeq ($(CONFIG_ARM)$(if $(CONFIG_CPU_32v3),y,n),yn) +CONFIG_ZINC_ARCH_ARM := y +endif +ifeq ($(CONFIG_ARM64),y) +CONFIG_ZINC_ARCH_ARM64 := y +endif +ifeq ($(CONFIG_MIPS)$(CONFIG_CPU_MIPS32_R2),yy) +CONFIG_ZINC_ARCH_MIPS := y +endif +ifeq ($(CONFIG_MIPS)$(CONFIG_64BIT),yy) +CONFIG_ZINC_ARCH_MIPS64 := y +endif + +zinc-y += chacha20/chacha20.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += chacha20/chacha20-x86_64.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += chacha20/chacha20-arm.o chacha20/chacha20-unrolled-arm.o +zinc-$(CONFIG_ZINC_ARCH_ARM64) += chacha20/chacha20-arm64.o +zinc-$(CONFIG_ZINC_ARCH_MIPS) += chacha20/chacha20-mips.o +AFLAGS_chacha20-mips.o += -O2 # This is required to fill the branch delay slots + +zinc-y += poly1305/poly1305.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += poly1305/poly1305-x86_64.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += poly1305/poly1305-arm.o +zinc-$(CONFIG_ZINC_ARCH_ARM64) += poly1305/poly1305-arm64.o +zinc-$(CONFIG_ZINC_ARCH_MIPS) += poly1305/poly1305-mips.o +AFLAGS_poly1305-mips.o += -O2 # This is required to fill the branch delay slots +zinc-$(CONFIG_ZINC_ARCH_MIPS64) += poly1305/poly1305-mips64.o + +zinc-y += chacha20poly1305.o + +zinc-y += blake2s/blake2s.o +zinc-$(CONFIG_ZINC_ARCH_X86_64) += blake2s/blake2s-x86_64.o + +zinc-y += curve25519/curve25519.o +zinc-$(CONFIG_ZINC_ARCH_ARM) += curve25519/curve25519-arm.o + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $< > $@ +$(obj)/%.S: $(src)/%.pl FORCE + $(call if_changed,perlasm) +kbuild-dir := $(if $(filter /%,$(src)),$(src),$(srctree)/$(src)) +targets := $(patsubst $(kbuild-dir)/%.pl,%.S,$(wildcard $(patsubst %.o,$(kbuild-dir)/crypto/zinc/%.pl,$(zinc-y) $(zinc-m) $(zinc-)))) + +# Old kernels don't set this, which causes trouble. +.SECONDARY: + +wireguard-y += $(addprefix crypto/zinc/,$(zinc-y)) +ccflags-y += -I$(kbuild-dir)/crypto/include +ccflags-$(CONFIG_ZINC_ARCH_X86_64) += -DCONFIG_ZINC_ARCH_X86_64 +ccflags-$(CONFIG_ZINC_ARCH_ARM) += -DCONFIG_ZINC_ARCH_ARM +ccflags-$(CONFIG_ZINC_ARCH_ARM64) += -DCONFIG_ZINC_ARCH_ARM64 +ccflags-$(CONFIG_ZINC_ARCH_MIPS) += -DCONFIG_ZINC_ARCH_MIPS +ccflags-$(CONFIG_ZINC_ARCH_MIPS64) += -DCONFIG_ZINC_ARCH_MIPS64 +ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DCONFIG_ZINC_SELFTEST diff --git a/net/wireguard/crypto/include/zinc/blake2s.h b/net/wireguard/crypto/include/zinc/blake2s.h new file mode 100644 index 0000000000000..2ca0bc30750d9 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/blake2s.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_BLAKE2S_H +#define _ZINC_BLAKE2S_H + +#include +#include +#include + +enum blake2s_lengths { + BLAKE2S_BLOCK_SIZE = 64, + BLAKE2S_HASH_SIZE = 32, + BLAKE2S_KEY_SIZE = 32 +}; + +struct blake2s_state { + u32 h[8]; + u32 t[2]; + u32 f[2]; + u8 buf[BLAKE2S_BLOCK_SIZE]; + unsigned int buflen; + unsigned int outlen; +}; + +void blake2s_init(struct blake2s_state *state, const size_t outlen); +void blake2s_init_key(struct blake2s_state *state, const size_t outlen, + const void *key, const size_t keylen); +void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen); +void blake2s_final(struct blake2s_state *state, u8 *out); + +static inline void blake2s(u8 *out, const u8 *in, const u8 *key, + const size_t outlen, const size_t inlen, + const size_t keylen) +{ + struct blake2s_state state; + + WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen || + outlen > BLAKE2S_HASH_SIZE || keylen > BLAKE2S_KEY_SIZE || + (!key && keylen))); + + if (keylen) + blake2s_init_key(&state, outlen, key, keylen); + else + blake2s_init(&state, outlen); + + blake2s_update(&state, in, inlen); + blake2s_final(&state, out); +} + +void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen, + const size_t inlen, const size_t keylen); + +#endif /* _ZINC_BLAKE2S_H */ diff --git a/net/wireguard/crypto/include/zinc/chacha20.h b/net/wireguard/crypto/include/zinc/chacha20.h new file mode 100644 index 0000000000000..1b0083d871fb6 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/chacha20.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CHACHA20_H +#define _ZINC_CHACHA20_H + +#include +#include +#include +#include + +enum chacha20_lengths { + CHACHA20_NONCE_SIZE = 16, + CHACHA20_KEY_SIZE = 32, + CHACHA20_KEY_WORDS = CHACHA20_KEY_SIZE / sizeof(u32), + CHACHA20_BLOCK_SIZE = 64, + CHACHA20_BLOCK_WORDS = CHACHA20_BLOCK_SIZE / sizeof(u32), + HCHACHA20_NONCE_SIZE = CHACHA20_NONCE_SIZE, + HCHACHA20_KEY_SIZE = CHACHA20_KEY_SIZE +}; + +enum chacha20_constants { /* expand 32-byte k */ + CHACHA20_CONSTANT_EXPA = 0x61707865U, + CHACHA20_CONSTANT_ND_3 = 0x3320646eU, + CHACHA20_CONSTANT_2_BY = 0x79622d32U, + CHACHA20_CONSTANT_TE_K = 0x6b206574U +}; + +struct chacha20_ctx { + union { + u32 state[16]; + struct { + u32 constant[4]; + u32 key[8]; + u32 counter[4]; + }; + }; +}; + +static inline void chacha20_init(struct chacha20_ctx *ctx, + const u8 key[CHACHA20_KEY_SIZE], + const u64 nonce) +{ + ctx->constant[0] = CHACHA20_CONSTANT_EXPA; + ctx->constant[1] = CHACHA20_CONSTANT_ND_3; + ctx->constant[2] = CHACHA20_CONSTANT_2_BY; + ctx->constant[3] = CHACHA20_CONSTANT_TE_K; + ctx->key[0] = get_unaligned_le32(key + 0); + ctx->key[1] = get_unaligned_le32(key + 4); + ctx->key[2] = get_unaligned_le32(key + 8); + ctx->key[3] = get_unaligned_le32(key + 12); + ctx->key[4] = get_unaligned_le32(key + 16); + ctx->key[5] = get_unaligned_le32(key + 20); + ctx->key[6] = get_unaligned_le32(key + 24); + ctx->key[7] = get_unaligned_le32(key + 28); + ctx->counter[0] = 0; + ctx->counter[1] = 0; + ctx->counter[2] = nonce & U32_MAX; + ctx->counter[3] = nonce >> 32; +} +void chacha20(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, u32 len, + simd_context_t *simd_context); + +void hchacha20(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context); + +#endif /* _ZINC_CHACHA20_H */ diff --git a/net/wireguard/crypto/include/zinc/chacha20poly1305.h b/net/wireguard/crypto/include/zinc/chacha20poly1305.h new file mode 100644 index 0000000000000..e3339f02996e2 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/chacha20poly1305.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CHACHA20POLY1305_H +#define _ZINC_CHACHA20POLY1305_H + +#include +#include + +struct scatterlist; + +enum chacha20poly1305_lengths { + XCHACHA20POLY1305_NONCE_SIZE = 24, + CHACHA20POLY1305_KEY_SIZE = 32, + CHACHA20POLY1305_AUTHTAG_SIZE = 16 +}; + +void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check chacha20poly1305_encrypt_sg_inplace( + struct scatterlist *src, const size_t src_len, const u8 *ad, + const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context); + +bool __must_check +chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check chacha20poly1305_decrypt_sg_inplace( + struct scatterlist *src, size_t src_len, const u8 *ad, + const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], simd_context_t *simd_context); + +void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +bool __must_check xchacha20poly1305_decrypt( + u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, + const size_t ad_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]); + +#endif /* _ZINC_CHACHA20POLY1305_H */ diff --git a/net/wireguard/crypto/include/zinc/curve25519.h b/net/wireguard/crypto/include/zinc/curve25519.h new file mode 100644 index 0000000000000..127d8a3a1c828 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/curve25519.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_CURVE25519_H +#define _ZINC_CURVE25519_H + +#include + +enum curve25519_lengths { + CURVE25519_KEY_SIZE = 32 +}; + +bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]); +void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]); +bool __must_check curve25519_generate_public( + u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]); + +static inline void curve25519_clamp_secret(u8 secret[CURVE25519_KEY_SIZE]) +{ + secret[0] &= 248; + secret[31] = (secret[31] & 127) | 64; +} + +#endif /* _ZINC_CURVE25519_H */ diff --git a/net/wireguard/crypto/include/zinc/poly1305.h b/net/wireguard/crypto/include/zinc/poly1305.h new file mode 100644 index 0000000000000..8a16d19f81776 --- /dev/null +++ b/net/wireguard/crypto/include/zinc/poly1305.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_POLY1305_H +#define _ZINC_POLY1305_H + +#include +#include + +enum poly1305_lengths { + POLY1305_BLOCK_SIZE = 16, + POLY1305_KEY_SIZE = 32, + POLY1305_MAC_SIZE = 16 +}; + +struct poly1305_ctx { + u8 opaque[24 * sizeof(u64)]; + u32 nonce[4]; + u8 data[POLY1305_BLOCK_SIZE]; + size_t num; +} __aligned(8); + +void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]); +void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len, + simd_context_t *simd_context); +void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], + simd_context_t *simd_context); + +#endif /* _ZINC_POLY1305_H */ diff --git a/net/wireguard/crypto/zinc.h b/net/wireguard/crypto/zinc.h new file mode 100644 index 0000000000000..9aa1e8d59bf5e --- /dev/null +++ b/net/wireguard/crypto/zinc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_ZINC_H +#define _WG_ZINC_H + +int chacha20_mod_init(void); +int poly1305_mod_init(void); +int chacha20poly1305_mod_init(void); +int blake2s_mod_init(void); +int curve25519_mod_init(void); + +#endif diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c new file mode 100644 index 0000000000000..f8cda59bf297b --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64-glue.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include + +asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, + const u8 *block, const size_t nblocks, + const u32 inc); +asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, + const u8 *block, const size_t nblocks, + const u32 inc); + +static bool blake2s_use_ssse3 __ro_after_init; +static bool blake2s_use_avx512 __ro_after_init; +static bool *const blake2s_nobs[] __initconst = { &blake2s_use_ssse3, + &blake2s_use_avx512 }; + +static void __init blake2s_fpu_init(void) +{ + blake2s_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3); +#ifndef COMPAT_CANNOT_USE_AVX512 + blake2s_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL); +#endif +} + +static inline bool blake2s_compress_arch(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + simd_context_t simd_context; + bool used_arch = false; + + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE / BLAKE2S_BLOCK_SIZE < 8); + + simd_get(&simd_context); + + if (!IS_ENABLED(CONFIG_AS_SSSE3) || !blake2s_use_ssse3 || + !simd_use(&simd_context)) + goto out; + used_arch = true; + + for (;;) { + const size_t blocks = min_t(size_t, nblocks, + PAGE_SIZE / BLAKE2S_BLOCK_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && blake2s_use_avx512) + blake2s_compress_avx512(state, block, blocks, inc); + else + blake2s_compress_ssse3(state, block, blocks, inc); + + nblocks -= blocks; + if (!nblocks) + break; + block += blocks * BLAKE2S_BLOCK_SIZE; + simd_relax(&simd_context); + } +out: + simd_put(&simd_context); + return used_arch; +} diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S new file mode 100644 index 0000000000000..24910b766bdda --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s-x86_64.S @@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * Copyright (C) 2017-2019 Samuel Neves . All Rights Reserved. + */ + +#include + +.section .rodata.cst32.BLAKE2S_IV, "aM", @progbits, 32 +.align 32 +IV: .octa 0xA54FF53A3C6EF372BB67AE856A09E667 + .octa 0x5BE0CD191F83D9AB9B05688C510E527F +.section .rodata.cst16.ROT16, "aM", @progbits, 16 +.align 16 +ROT16: .octa 0x0D0C0F0E09080B0A0504070601000302 +.section .rodata.cst16.ROR328, "aM", @progbits, 16 +.align 16 +ROR328: .octa 0x0C0F0E0D080B0A090407060500030201 +.section .rodata.cst64.BLAKE2S_SIGMA, "aM", @progbits, 160 +.align 64 +SIGMA: +.byte 0, 2, 4, 6, 1, 3, 5, 7, 14, 8, 10, 12, 15, 9, 11, 13 +.byte 14, 4, 9, 13, 10, 8, 15, 6, 5, 1, 0, 11, 3, 12, 2, 7 +.byte 11, 12, 5, 15, 8, 0, 2, 13, 9, 10, 3, 7, 4, 14, 6, 1 +.byte 7, 3, 13, 11, 9, 1, 12, 14, 15, 2, 5, 4, 8, 6, 10, 0 +.byte 9, 5, 2, 10, 0, 7, 4, 15, 3, 14, 11, 6, 13, 1, 12, 8 +.byte 2, 6, 0, 8, 12, 10, 11, 3, 1, 4, 7, 15, 9, 13, 5, 14 +.byte 12, 1, 14, 4, 5, 15, 13, 10, 8, 0, 6, 9, 11, 7, 3, 2 +.byte 13, 7, 12, 3, 11, 14, 1, 9, 2, 5, 15, 8, 10, 0, 4, 6 +.byte 6, 14, 11, 0, 15, 9, 3, 8, 10, 12, 13, 1, 5, 2, 7, 4 +.byte 10, 8, 7, 1, 2, 4, 6, 5, 13, 15, 9, 3, 0, 11, 14, 12 +#ifdef CONFIG_AS_AVX512 +.section .rodata.cst64.BLAKE2S_SIGMA2, "aM", @progbits, 640 +.align 64 +SIGMA2: +.long 0, 2, 4, 6, 1, 3, 5, 7, 14, 8, 10, 12, 15, 9, 11, 13 +.long 8, 2, 13, 15, 10, 9, 12, 3, 6, 4, 0, 14, 5, 11, 1, 7 +.long 11, 13, 8, 6, 5, 10, 14, 3, 2, 4, 12, 15, 1, 0, 7, 9 +.long 11, 10, 7, 0, 8, 15, 1, 13, 3, 6, 2, 12, 4, 14, 9, 5 +.long 4, 10, 9, 14, 15, 0, 11, 8, 1, 7, 3, 13, 2, 5, 6, 12 +.long 2, 11, 4, 15, 14, 3, 10, 8, 13, 6, 5, 7, 0, 12, 1, 9 +.long 4, 8, 15, 9, 14, 11, 13, 5, 3, 2, 1, 12, 6, 10, 7, 0 +.long 6, 13, 0, 14, 12, 2, 1, 11, 15, 4, 5, 8, 7, 9, 3, 10 +.long 15, 5, 4, 13, 10, 7, 3, 11, 12, 2, 0, 6, 9, 8, 1, 14 +.long 8, 7, 14, 11, 13, 15, 0, 12, 10, 4, 5, 6, 3, 2, 1, 9 +#endif /* CONFIG_AS_AVX512 */ + +.text +#ifdef CONFIG_AS_SSSE3 +SYM_FUNC_START(blake2s_compress_ssse3) + testq %rdx,%rdx + je .Lendofloop + movdqu (%rdi),%xmm0 + movdqu 0x10(%rdi),%xmm1 + movdqa ROT16(%rip),%xmm12 + movdqa ROR328(%rip),%xmm13 + movdqu 0x20(%rdi),%xmm14 + movq %rcx,%xmm15 + leaq SIGMA+0xa0(%rip),%r8 + jmp .Lbeginofloop + .align 32 +.Lbeginofloop: + movdqa %xmm0,%xmm10 + movdqa %xmm1,%xmm11 + paddq %xmm15,%xmm14 + movdqa IV(%rip),%xmm2 + movdqa %xmm14,%xmm3 + pxor IV+0x10(%rip),%xmm3 + leaq SIGMA(%rip),%rcx +.Lroundloop: + movzbl (%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0x1(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0x2(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x3(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + punpckldq %xmm5,%xmm4 + punpckldq %xmm7,%xmm6 + punpcklqdq %xmm6,%xmm4 + paddd %xmm4,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm12,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0xc,%xmm1 + pslld $0x14,%xmm8 + por %xmm8,%xmm1 + movzbl 0x4(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0x5(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x6(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0x7(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + punpckldq %xmm6,%xmm5 + punpckldq %xmm4,%xmm7 + punpcklqdq %xmm7,%xmm5 + paddd %xmm5,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm13,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0x7,%xmm1 + pslld $0x19,%xmm8 + por %xmm8,%xmm1 + pshufd $0x93,%xmm0,%xmm0 + pshufd $0x4e,%xmm3,%xmm3 + pshufd $0x39,%xmm2,%xmm2 + movzbl 0x8(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + movzbl 0x9(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0xa(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0xb(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + punpckldq %xmm7,%xmm6 + punpckldq %xmm5,%xmm4 + punpcklqdq %xmm4,%xmm6 + paddd %xmm6,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm12,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0xc,%xmm1 + pslld $0x14,%xmm8 + por %xmm8,%xmm1 + movzbl 0xc(%rcx),%eax + movd (%rsi,%rax,4),%xmm7 + movzbl 0xd(%rcx),%eax + movd (%rsi,%rax,4),%xmm4 + movzbl 0xe(%rcx),%eax + movd (%rsi,%rax,4),%xmm5 + movzbl 0xf(%rcx),%eax + movd (%rsi,%rax,4),%xmm6 + punpckldq %xmm4,%xmm7 + punpckldq %xmm6,%xmm5 + punpcklqdq %xmm5,%xmm7 + paddd %xmm7,%xmm0 + paddd %xmm1,%xmm0 + pxor %xmm0,%xmm3 + pshufb %xmm13,%xmm3 + paddd %xmm3,%xmm2 + pxor %xmm2,%xmm1 + movdqa %xmm1,%xmm8 + psrld $0x7,%xmm1 + pslld $0x19,%xmm8 + por %xmm8,%xmm1 + pshufd $0x39,%xmm0,%xmm0 + pshufd $0x4e,%xmm3,%xmm3 + pshufd $0x93,%xmm2,%xmm2 + addq $0x10,%rcx + cmpq %r8,%rcx + jnz .Lroundloop + pxor %xmm2,%xmm0 + pxor %xmm3,%xmm1 + pxor %xmm10,%xmm0 + pxor %xmm11,%xmm1 + addq $0x40,%rsi + decq %rdx + jnz .Lbeginofloop + movdqu %xmm0,(%rdi) + movdqu %xmm1,0x10(%rdi) + movdqu %xmm14,0x20(%rdi) +.Lendofloop: + ret +SYM_FUNC_END(blake2s_compress_ssse3) +#endif /* CONFIG_AS_SSSE3 */ + +#ifdef CONFIG_AS_AVX512 +SYM_FUNC_START(blake2s_compress_avx512) + vmovdqu (%rdi),%xmm0 + vmovdqu 0x10(%rdi),%xmm1 + vmovdqu 0x20(%rdi),%xmm4 + vmovq %rcx,%xmm5 + vmovdqa IV(%rip),%xmm14 + vmovdqa IV+16(%rip),%xmm15 + jmp .Lblake2s_compress_avx512_mainloop +.align 32 +.Lblake2s_compress_avx512_mainloop: + vmovdqa %xmm0,%xmm10 + vmovdqa %xmm1,%xmm11 + vpaddq %xmm5,%xmm4,%xmm4 + vmovdqa %xmm14,%xmm2 + vpxor %xmm15,%xmm4,%xmm3 + vmovdqu (%rsi),%ymm6 + vmovdqu 0x20(%rsi),%ymm7 + addq $0x40,%rsi + leaq SIGMA2(%rip),%rax + movb $0xa,%cl +.Lblake2s_compress_avx512_roundloop: + addq $0x40,%rax + vmovdqa -0x40(%rax),%ymm8 + vmovdqa -0x20(%rax),%ymm9 + vpermi2d %ymm7,%ymm6,%ymm8 + vpermi2d %ymm7,%ymm6,%ymm9 + vmovdqa %ymm8,%ymm6 + vmovdqa %ymm9,%ymm7 + vpaddd %xmm8,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x10,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0xc,%xmm1,%xmm1 + vextracti128 $0x1,%ymm8,%xmm8 + vpaddd %xmm8,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x8,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0x7,%xmm1,%xmm1 + vpshufd $0x93,%xmm0,%xmm0 + vpshufd $0x4e,%xmm3,%xmm3 + vpshufd $0x39,%xmm2,%xmm2 + vpaddd %xmm9,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x10,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0xc,%xmm1,%xmm1 + vextracti128 $0x1,%ymm9,%xmm9 + vpaddd %xmm9,%xmm0,%xmm0 + vpaddd %xmm1,%xmm0,%xmm0 + vpxor %xmm0,%xmm3,%xmm3 + vprord $0x8,%xmm3,%xmm3 + vpaddd %xmm3,%xmm2,%xmm2 + vpxor %xmm2,%xmm1,%xmm1 + vprord $0x7,%xmm1,%xmm1 + vpshufd $0x39,%xmm0,%xmm0 + vpshufd $0x4e,%xmm3,%xmm3 + vpshufd $0x93,%xmm2,%xmm2 + decb %cl + jne .Lblake2s_compress_avx512_roundloop + vpxor %xmm10,%xmm0,%xmm0 + vpxor %xmm11,%xmm1,%xmm1 + vpxor %xmm2,%xmm0,%xmm0 + vpxor %xmm3,%xmm1,%xmm1 + decq %rdx + jne .Lblake2s_compress_avx512_mainloop + vmovdqu %xmm0,(%rdi) + vmovdqu %xmm1,0x10(%rdi) + vmovdqu %xmm4,0x20(%rdi) + vzeroupper + retq +SYM_FUNC_END(blake2s_compress_avx512) +#endif /* CONFIG_AS_AVX512 */ diff --git a/net/wireguard/crypto/zinc/blake2s/blake2s.c b/net/wireguard/crypto/zinc/blake2s/blake2s.c new file mode 100644 index 0000000000000..e9e0bdaffc82a --- /dev/null +++ b/net/wireguard/crypto/zinc/blake2s/blake2s.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the BLAKE2s hash and PRF functions. + * + * Information: https://blake2.net/ + * + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include +#include +#include + +static const u32 blake2s_iv[8] = { + 0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL, 0xA54FF53AUL, + 0x510E527FUL, 0x9B05688CUL, 0x1F83D9ABUL, 0x5BE0CD19UL +}; + +static const u8 blake2s_sigma[10][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, +}; + +static inline void blake2s_set_lastblock(struct blake2s_state *state) +{ + state->f[0] = -1; +} + +static inline void blake2s_increment_counter(struct blake2s_state *state, + const u32 inc) +{ + state->t[0] += inc; + state->t[1] += (state->t[0] < inc); +} + +static inline void blake2s_init_param(struct blake2s_state *state, + const u32 param) +{ + int i; + + memset(state, 0, sizeof(*state)); + for (i = 0; i < 8; ++i) + state->h[i] = blake2s_iv[i]; + state->h[0] ^= param; +} + +void blake2s_init(struct blake2s_state *state, const size_t outlen) +{ + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE)); + blake2s_init_param(state, 0x01010000 | outlen); + state->outlen = outlen; +} +EXPORT_SYMBOL(blake2s_init); + +void blake2s_init_key(struct blake2s_state *state, const size_t outlen, + const void *key, const size_t keylen) +{ + u8 block[BLAKE2S_BLOCK_SIZE] = { 0 }; + + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE || + !key || !keylen || keylen > BLAKE2S_KEY_SIZE)); + blake2s_init_param(state, 0x01010000 | keylen << 8 | outlen); + state->outlen = outlen; + memcpy(block, key, keylen); + blake2s_update(state, block, BLAKE2S_BLOCK_SIZE); + memzero_explicit(block, BLAKE2S_BLOCK_SIZE); +} +EXPORT_SYMBOL(blake2s_init_key); + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "blake2s-x86_64-glue.c" +#else +static bool *const blake2s_nobs[] __initconst = { }; +static void __init blake2s_fpu_init(void) +{ +} +static inline bool blake2s_compress_arch(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + return false; +} +#endif + +static inline void blake2s_compress(struct blake2s_state *state, + const u8 *block, size_t nblocks, + const u32 inc) +{ + u32 m[16]; + u32 v[16]; + int i; + + WARN_ON(IS_ENABLED(DEBUG) && + (nblocks > 1 && inc != BLAKE2S_BLOCK_SIZE)); + + if (blake2s_compress_arch(state, block, nblocks, inc)) + return; + + while (nblocks > 0) { + blake2s_increment_counter(state, inc); + memcpy(m, block, BLAKE2S_BLOCK_SIZE); + le32_to_cpu_array(m, ARRAY_SIZE(m)); + memcpy(v, state->h, 32); + v[ 8] = blake2s_iv[0]; + v[ 9] = blake2s_iv[1]; + v[10] = blake2s_iv[2]; + v[11] = blake2s_iv[3]; + v[12] = blake2s_iv[4] ^ state->t[0]; + v[13] = blake2s_iv[5] ^ state->t[1]; + v[14] = blake2s_iv[6] ^ state->f[0]; + v[15] = blake2s_iv[7] ^ state->f[1]; + +#define G(r, i, a, b, c, d) do { \ + a += b + m[blake2s_sigma[r][2 * i + 0]]; \ + d = ror32(d ^ a, 16); \ + c += d; \ + b = ror32(b ^ c, 12); \ + a += b + m[blake2s_sigma[r][2 * i + 1]]; \ + d = ror32(d ^ a, 8); \ + c += d; \ + b = ror32(b ^ c, 7); \ +} while (0) + +#define ROUND(r) do { \ + G(r, 0, v[0], v[ 4], v[ 8], v[12]); \ + G(r, 1, v[1], v[ 5], v[ 9], v[13]); \ + G(r, 2, v[2], v[ 6], v[10], v[14]); \ + G(r, 3, v[3], v[ 7], v[11], v[15]); \ + G(r, 4, v[0], v[ 5], v[10], v[15]); \ + G(r, 5, v[1], v[ 6], v[11], v[12]); \ + G(r, 6, v[2], v[ 7], v[ 8], v[13]); \ + G(r, 7, v[3], v[ 4], v[ 9], v[14]); \ +} while (0) + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + +#undef G +#undef ROUND + + for (i = 0; i < 8; ++i) + state->h[i] ^= v[i] ^ v[i + 8]; + + block += BLAKE2S_BLOCK_SIZE; + --nblocks; + } +} + +void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen) +{ + const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; + + if (unlikely(!inlen)) + return; + if (inlen > fill) { + memcpy(state->buf + state->buflen, in, fill); + blake2s_compress(state, state->buf, 1, BLAKE2S_BLOCK_SIZE); + state->buflen = 0; + in += fill; + inlen -= fill; + } + if (inlen > BLAKE2S_BLOCK_SIZE) { + const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); + /* Hash one less (full) block than strictly possible */ + blake2s_compress(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); + in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); + inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); + } + memcpy(state->buf + state->buflen, in, inlen); + state->buflen += inlen; +} +EXPORT_SYMBOL(blake2s_update); + +void blake2s_final(struct blake2s_state *state, u8 *out) +{ + WARN_ON(IS_ENABLED(DEBUG) && !out); + blake2s_set_lastblock(state); + memset(state->buf + state->buflen, 0, + BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ + blake2s_compress(state, state->buf, 1, state->buflen); + cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); + memcpy(out, state->h, state->outlen); + memzero_explicit(state, sizeof(*state)); +} +EXPORT_SYMBOL(blake2s_final); + +void blake2s_hmac(u8 *out, const u8 *in, const u8 *key, const size_t outlen, + const size_t inlen, const size_t keylen) +{ + struct blake2s_state state; + u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 }; + u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32)); + int i; + + if (keylen > BLAKE2S_BLOCK_SIZE) { + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, key, keylen); + blake2s_final(&state, x_key); + } else + memcpy(x_key, key, keylen); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + x_key[i] ^= 0x36; + + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&state, in, inlen); + blake2s_final(&state, i_hash); + + for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) + x_key[i] ^= 0x5c ^ 0x36; + + blake2s_init(&state, BLAKE2S_HASH_SIZE); + blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE); + blake2s_final(&state, i_hash); + + memcpy(out, i_hash, outlen); + memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE); + memzero_explicit(i_hash, BLAKE2S_HASH_SIZE); +} +EXPORT_SYMBOL(blake2s_hmac); + +#include "../selftest/blake2s.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init blake2s_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + blake2s_fpu_init(); + if (!selftest_run("blake2s", blake2s_selftest, blake2s_nobs, + ARRAY_SIZE(blake2s_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("BLAKE2s hash function"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c new file mode 100644 index 0000000000000..41e2e79abb2bc --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm-glue.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#if defined(CONFIG_ZINC_ARCH_ARM) +#include +#include +#endif + +asmlinkage void chacha20_arm(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void hchacha20_arm(const u32 state[16], u32 out[8]); +asmlinkage void chacha20_neon(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); + +static bool chacha20_use_neon __ro_after_init; +static bool *const chacha20_nobs[] __initconst = { &chacha20_use_neon }; +static void __init chacha20_fpu_init(void) +{ +#if defined(CONFIG_ZINC_ARCH_ARM64) + chacha20_use_neon = cpu_have_named_feature(ASIMD); +#elif defined(CONFIG_ZINC_ARCH_ARM) + switch (read_cpuid_part()) { + case ARM_CPU_PART_CORTEX_A7: + case ARM_CPU_PART_CORTEX_A5: + /* The Cortex-A7 and Cortex-A5 do not perform well with the NEON + * implementation but do incredibly with the scalar one and use + * less power. + */ + break; + default: + chacha20_use_neon = elf_hwcap & HWCAP_NEON; + } +#endif +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE || + PAGE_SIZE % CHACHA20_BLOCK_SIZE); + + for (;;) { + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && chacha20_use_neon && + len >= CHACHA20_BLOCK_SIZE * 3 && simd_use(simd_context)) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + chacha20_neon(dst, src, bytes, ctx->key, ctx->counter); + ctx->counter[0] += (bytes + 63) / 64; + len -= bytes; + if (!len) + break; + dst += bytes; + src += bytes; + simd_relax(simd_context); + } else { + chacha20_arm(dst, src, len, ctx->key, ctx->counter); + ctx->counter[0] += (len + 63) / 64; + break; + } + } + + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM)) { + u32 x[] = { CHACHA20_CONSTANT_EXPA, + CHACHA20_CONSTANT_ND_3, + CHACHA20_CONSTANT_2_BY, + CHACHA20_CONSTANT_TE_K, + get_unaligned_le32(key + 0), + get_unaligned_le32(key + 4), + get_unaligned_le32(key + 8), + get_unaligned_le32(key + 12), + get_unaligned_le32(key + 16), + get_unaligned_le32(key + 20), + get_unaligned_le32(key + 24), + get_unaligned_le32(key + 28), + get_unaligned_le32(nonce + 0), + get_unaligned_le32(nonce + 4), + get_unaligned_le32(nonce + 8), + get_unaligned_le32(nonce + 12) + }; + hchacha20_arm(x, derived_key); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl new file mode 100644 index 0000000000000..6785383ab7bba --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm.pl @@ -0,0 +1,1227 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# December 2014 +# +# ChaCha20 for ARMv4. +# +# September 2018 +# +# Improve scalar performance per Eric Biggers' suggestion to eliminate +# separate rotates. This requires b[0..3] and d[0..3] to be maintained +# pre-rotated, hence odd twists prior inner loop and when accumulating +# key material. Since amount of instructions is reduced as result, even +# NEON performance is improved somewhat, most notably by ~9% on low-end +# Cortex-A5/A7. Full unroll was shown to provide even better scalar +# performance on Cortex-A5/A7, naturally at the cost of manyfold size +# increase. We let it be. Oversized code works in benchmarks, but is not +# necessarily optimal in real life, when it's likely to be out-of-cache +# upon entry and evict significant part of cache upon completion. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc-4.4 1xNEON 3xNEON+1xIALU +# +# Cortex-A5 14.2(*)/+160% 21.8 12.9(**) +# Cortex-A8 10.2(*)/+190% 13.9 6.10 +# Cortex-A9 10.8(*)/+150% 14.3 6.50 +# Cortex-A15 11.0/+40% 16.0 4.90 +# Snapdragon S4 13.9(***)/+90% 13.6 4.90 +# +# (*) most "favourable" result for aligned data on little-endian +# processor, result for misaligned data is 10-15% lower; +# (**) pure 4xNEON [with "vertical" layout] was shown to provide ~8% +# better performance on Cortex-A5/A7, but not on others; +# (***) it's 17% slower than original, trade-off is considered +# acceptable, because of improvement on others, specifically +# +36% on Cortex-A5/A7 and +20% on Cortex-A9; + +$flavour = shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +my @x=map("r$_",(0..7,"x","x","x","x",12,"x",14,"x")); +my @t=map("r$_",(8..11)); + +sub ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my $odd = $d0&1; +my ($xc,$xc_) = (@t[0..1]); +my ($xd,$xd_) = $odd ? (@t[2],@x[$d1]) : (@x[$d0],@t[2]); +my @ret; + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' are permanently allocated in registers, @x[0..7], + # while 'c's and pair of 'd's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. If you observe 'd' column, you'll + # notice that 15 and 13 are reused in next pair of rounds. + # This is why these two are chosen for offloading to memory, + # to make loads count more. + push @ret,( + "&add (@x[$a0],@x[$a0],@x[$b0],'ror#13')", + "&add (@x[$a1],@x[$a1],@x[$b1],'ror#13')", + "&eor ($xd,@x[$a0],$xd,'ror#24')", + "&eor ($xd_,@x[$a1],$xd_,'ror#24')", + + "&add ($xc,$xc,$xd,'ror#16')", + "&add ($xc_,$xc_,$xd_,'ror#16')", + "&eor (@x[$b0],$xc, @x[$b0],'ror#13')", + "&eor (@x[$b1],$xc_,@x[$b1],'ror#13')", + + "&add (@x[$a0],@x[$a0],@x[$b0],'ror#20')", + "&add (@x[$a1],@x[$a1],@x[$b1],'ror#20')", + "&eor ($xd,@x[$a0],$xd,'ror#16')", + "&eor ($xd_,@x[$a1],$xd_,'ror#16')" ); + push @ret,( + "&str ($xd,'[sp,#4*(16+$d0)]')" ) if ($odd); + push @ret,( + "&add ($xc,$xc,$xd,'ror#24')" ); + push @ret,( + "&ldr ($xd,'[sp,#4*(16+$d2)]')" ) if ($odd); + push @ret,( + "&str ($xd_,'[sp,#4*(16+$d1)]')" ) if (!$odd); + push @ret,( + "&add ($xc_,$xc_,$xd_,'ror#24')" ); + push @ret,( + "&ldr ($xd_,'[sp,#4*(16+$d3)]')" ) if (!$odd); + push @ret,( + "&str ($xc,'[sp,#4*(16+$c0)]')", + "&eor (@x[$b0],@x[$b0],$xc,'ror#12')", + "&str ($xc_,'[sp,#4*(16+$c1)]')", + "&eor (@x[$b1],@x[$b1],$xc_,'ror#12')" ); + + $xd=@x[$d2] if (!$odd); + $xd_=@x[$d3] if ($odd); + push @ret,( + "&ldr ($xc,'[sp,#4*(16+$c2)]')", + "&add (@x[$a2],@x[$a2],@x[$b2],'ror#13')", + "&ldr ($xc_,'[sp,#4*(16+$c3)]')", + "&add (@x[$a3],@x[$a3],@x[$b3],'ror#13')", + "&eor ($xd,@x[$a2],$xd,'ror#24')", + "&eor ($xd_,@x[$a3],$xd_,'ror#24')", + + "&add ($xc,$xc,$xd,'ror#16')", + "&add ($xc_,$xc_,$xd_,'ror#16')", + "&eor (@x[$b2],$xc, @x[$b2],'ror#13')", + "&eor (@x[$b3],$xc_,@x[$b3],'ror#13')", + + "&add (@x[$a2],@x[$a2],@x[$b2],'ror#20')", + "&add (@x[$a3],@x[$a3],@x[$b3],'ror#20')", + "&eor ($xd,@x[$a2],$xd,'ror#16')", + "&eor ($xd_,@x[$a3],$xd_,'ror#16')", + + "&add ($xc,$xc,$xd,'ror#24')", + "&add ($xc_,$xc_,$xd_,'ror#24')", + "&eor (@x[$b2],@x[$b2],$xc,'ror#12')", + "&eor (@x[$b3],@x[$b3],$xc_,'ror#12')" ); + + @ret; +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ +# define ChaCha20_ctr32 chacha20_arm_cryptogams +# define ChaCha20_neon chacha20_neon +#endif + +.text +#if defined(__thumb2__) || defined(__clang__) +.syntax unified +# define ldrhsb ldrbhs +#endif +#if defined(__thumb2__) +.thumb +#else +.code 32 +#endif + +.align 5 +.Lsigma: +.long 0x61707865,0x3320646e,0x79622d32,0x6b206574 @ endian-neutral +.Lone: +.long 1,0,0,0 +.Lrot8: +.long 0x02010003,0x06050407 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-.LChaCha20_ctr32 +#else +.word -1 +#endif + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: +.LChaCha20_ctr32: + ldr r12,[sp,#0] @ pull pointer to counter and nonce + stmdb sp!,{r0-r2,r4-r11,lr} +#if __ARM_ARCH__<7 && !defined(__thumb2__) + sub r14,pc,#16 @ ChaCha20_ctr32 +#else + adr r14,.LChaCha20_ctr32 +#endif + cmp r2,#0 @ len==0? +#ifdef __thumb2__ + itt eq +#endif + addeq sp,sp,#4*3 + beq .Lno_data +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + cmp r2,#192 @ test len + bls .Lshort + ldr r4,[r14,#-24] + ldr r4,[r14,r4] +# ifdef __APPLE__ + ldr r4,[r4] +# endif + tst r4,#ARMV7_NEON + bne .LChaCha20_neon +.Lshort: +#endif + ldmia r12,{r4-r7} @ load counter and nonce + sub sp,sp,#4*(16) @ off-load area + sub r14,r14,#64 @ .Lsigma + stmdb sp!,{r4-r7} @ copy counter and nonce + ldmia r3,{r4-r11} @ load key + ldmia r14,{r0-r3} @ load sigma + stmdb sp!,{r4-r11} @ copy key + stmdb sp!,{r0-r3} @ copy sigma + str r10,[sp,#4*(16+10)] @ off-load "@x[10]" + str r11,[sp,#4*(16+11)] @ off-load "@x[11]" + b .Loop_outer_enter + +.align 4 +.Loop_outer: + ldmia sp,{r0-r9} @ load key material + str @t[3],[sp,#4*(32+2)] @ save len + str r12, [sp,#4*(32+1)] @ save inp + str r14, [sp,#4*(32+0)] @ save out +.Loop_outer_enter: + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(16+15)] + mov @t[3],#10 + b .Loop + +.align 4 +.Loop: + subs @t[3],@t[3],#1 +___ + foreach (&ROUND(0, 4, 8,12)) { eval; } + foreach (&ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + bne .Loop + + ldr @t[3],[sp,#4*(32+2)] @ load len + + str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store + str @t[1], [sp,#4*(16+9)] + str @x[12],[sp,#4*(16+12)] + str @t[2], [sp,#4*(16+13)] + str @x[14],[sp,#4*(16+14)] + + @ at this point we have first half of 512-bit result in + @ @x[0-7] and second half at sp+4*(16+8) + + cmp @t[3],#64 @ done yet? +#ifdef __thumb2__ + itete lo +#endif + addlo r12,sp,#4*(0) @ shortcut or ... + ldrhs r12,[sp,#4*(32+1)] @ ... load inp + addlo r14,sp,#4*(0) @ shortcut or ... + ldrhs r14,[sp,#4*(32+0)] @ ... load out + + ldr @t[0],[sp,#4*(0)] @ load key material + ldr @t[1],[sp,#4*(1)] + +#if __ARM_ARCH__>=6 || !defined(__ARMEB__) +# if __ARM_ARCH__<7 + orr @t[2],r12,r14 + tst @t[2],#3 @ are input and output aligned? + ldr @t[2],[sp,#4*(2)] + bne .Lunaligned + cmp @t[3],#64 @ restore flags +# else + ldr @t[2],[sp,#4*(2)] +# endif + ldr @t[3],[sp,#4*(3)] + + add @x[0],@x[0],@t[0] @ accumulate key material + add @x[1],@x[1],@t[1] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[0],@x[0],@t[0] @ xor with input + eorhs @x[1],@x[1],@t[1] + add @t[0],sp,#4*(4) + str @x[0],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[2],@x[2],@t[2] + eorhs @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[1],[r14,#-12] + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + add @x[5],@t[1],@x[5],ror#13 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#13 + add @x[7],@t[3],@x[7],ror#13 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[4],@x[4],@t[0] + eorhs @x[5],@x[5],@t[1] + add @t[0],sp,#4*(8) + str @x[4],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[6],@x[6],@t[2] + eorhs @x[7],@x[7],@t[3] + str @x[5],[r14,#-12] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[6],[r14,#-8] + add @x[0],sp,#4*(16+8) + str @x[7],[r14,#-4] + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + add @x[1],@x[1],@t[1] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] +# ifdef __thumb2__ + itt hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[0],@x[0],@t[0] + eorhs @x[1],@x[1],@t[1] + add @t[0],sp,#4*(12) + str @x[0],[r14],#16 @ store output +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[2],@x[2],@t[2] + eorhs @x[3],@x[3],@t[3] + str @x[1],[r14,#-12] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @x[5],@t[1],@x[5],ror#24 +# ifdef __thumb2__ + itt hi +# endif + addhi @t[0],@t[0],#1 @ next counter value + strhi @t[0],[sp,#4*(12)] @ save next counter value +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[0],[r12],#16 @ load input + ldrhs @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#24 + add @x[7],@t[3],@x[7],ror#24 +# ifdef __thumb2__ + itt hs +# endif + ldrhs @t[2],[r12,#-8] + ldrhs @t[3],[r12,#-4] +# if __ARM_ARCH__>=6 && defined(__ARMEB__) + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[4],@x[4],@t[0] + eorhs @x[5],@x[5],@t[1] +# ifdef __thumb2__ + it ne +# endif + ldrne @t[0],[sp,#4*(32+2)] @ re-load len +# ifdef __thumb2__ + itt hs +# endif + eorhs @x[6],@x[6],@t[2] + eorhs @x[7],@x[7],@t[3] + str @x[4],[r14],#16 @ store output + str @x[5],[r14,#-12] +# ifdef __thumb2__ + it hs +# endif + subhs @t[3],@t[0],#64 @ len-=64 + str @x[6],[r14,#-8] + str @x[7],[r14,#-4] + bhi .Loop_outer + + beq .Ldone +# if __ARM_ARCH__<7 + b .Ltail + +.align 4 +.Lunaligned: @ unaligned endian-neutral path + cmp @t[3],#64 @ restore flags +# endif +#endif +#if __ARM_ARCH__<7 + ldr @t[3],[sp,#4*(3)] +___ +for ($i=0;$i<16;$i+=4) { +my $j=$i&0x7; +my $twist=""; +if ($i==4) { $twist = ",ror#13"; } +elsif ($i==12) { $twist = ",ror#24"; } + +$code.=<<___ if ($i==4); + add @x[0],sp,#4*(16+8) +___ +$code.=<<___ if ($i==8); + ldmia @x[0],{@x[0]-@x[7]} @ load second half +# ifdef __thumb2__ + itt hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" +___ +$code.=<<___; + add @x[$j+0],@t[0],@x[$j+0]$twist @ accumulate key material +___ +$code.=<<___ if ($i==12); +# ifdef __thumb2__ + itt hi +# endif + addhi @t[0],@t[0],#1 @ next counter value + strhi @t[0],[sp,#4*(12)] @ save next counter value +___ +$code.=<<___; + add @x[$j+1],@t[1],@x[$j+1]$twist + add @x[$j+2],@t[2],@x[$j+2]$twist +# ifdef __thumb2__ + itete lo +# endif + eorlo @t[0],@t[0],@t[0] @ zero or ... + ldrhsb @t[0],[r12],#16 @ ... load input + eorlo @t[1],@t[1],@t[1] + ldrhsb @t[1],[r12,#-12] + + add @x[$j+3],@t[3],@x[$j+3]$twist +# ifdef __thumb2__ + itete lo +# endif + eorlo @t[2],@t[2],@t[2] + ldrhsb @t[2],[r12,#-8] + eorlo @t[3],@t[3],@t[3] + ldrhsb @t[3],[r12,#-4] + + eor @x[$j+0],@t[0],@x[$j+0] @ xor with input (or zero) + eor @x[$j+1],@t[1],@x[$j+1] +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-15] @ load more input + ldrhsb @t[1],[r12,#-11] + eor @x[$j+2],@t[2],@x[$j+2] + strb @x[$j+0],[r14],#16 @ store output + eor @x[$j+3],@t[3],@x[$j+3] +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-7] + ldrhsb @t[3],[r12,#-3] + strb @x[$j+1],[r14,#-12] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+2],[r14,#-8] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-14] @ load more input + ldrhsb @t[1],[r12,#-10] + strb @x[$j+3],[r14,#-4] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+0],[r14,#-15] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-6] + ldrhsb @t[3],[r12,#-2] + strb @x[$j+1],[r14,#-11] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+2],[r14,#-7] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[0],[r12,#-13] @ load more input + ldrhsb @t[1],[r12,#-9] + strb @x[$j+3],[r14,#-3] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+0],[r14,#-14] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 +# ifdef __thumb2__ + itt hs +# endif + ldrhsb @t[2],[r12,#-5] + ldrhsb @t[3],[r12,#-1] + strb @x[$j+1],[r14,#-10] + strb @x[$j+2],[r14,#-6] + eor @x[$j+0],@t[0],@x[$j+0],lsr#8 + strb @x[$j+3],[r14,#-2] + eor @x[$j+1],@t[1],@x[$j+1],lsr#8 + strb @x[$j+0],[r14,#-13] + eor @x[$j+2],@t[2],@x[$j+2],lsr#8 + strb @x[$j+1],[r14,#-9] + eor @x[$j+3],@t[3],@x[$j+3],lsr#8 + strb @x[$j+2],[r14,#-5] + strb @x[$j+3],[r14,#-1] +___ +$code.=<<___ if ($i<12); + add @t[0],sp,#4*(4+$i) + ldmia @t[0],{@t[0]-@t[3]} @ load key material +___ +} +$code.=<<___; +# ifdef __thumb2__ + it ne +# endif + ldrne @t[0],[sp,#4*(32+2)] @ re-load len +# ifdef __thumb2__ + it hs +# endif + subhs @t[3],@t[0],#64 @ len-=64 + bhi .Loop_outer + + beq .Ldone +#endif + +.Ltail: + ldr r12,[sp,#4*(32+1)] @ load inp + add @t[1],sp,#4*(0) + ldr r14,[sp,#4*(32+0)] @ load out + +.Loop_tail: + ldrb @t[2],[@t[1]],#1 @ read buffer on stack + ldrb @t[3],[r12],#1 @ read input + subs @t[0],@t[0],#1 + eor @t[3],@t[3],@t[2] + strb @t[3],[r14],#1 @ store output + bne .Loop_tail + +.Ldone: + add sp,sp,#4*(32+3) +.Lno_data: +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r11,pc} +#else + ldmia sp!,{r4-r12,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .long 0xe12fff1e @ interoperable with Thumb ISA:-) +#endif +.size ChaCha20_ctr32,.-ChaCha20_ctr32 +___ + +{{{ +my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$t0,$t1,$t2,$t3) = + map("q$_",(0..15)); + +# This can replace vshr-by-24+vsli-by-8. It gives ~3% improvement on +# Cortex-A5/A7, but hurts Cortex-A9 by 5% and Snapdragon S4 by 14%! +sub vperm() +{ my ($dst,$src,$tbl) = @_; + $code .= " vtbl.8 $dst#lo,{$src#lo},$tbl#lo\n"; + $code .= " vtbl.8 $dst#hi,{$src#hi},$tbl#lo\n"; +} + +sub NEONROUND { +my $odd = pop; +my ($a,$b,$c,$d,$t)=@_; + + ( + "&vadd_i32 ($a,$a,$b)", + "&veor ($d,$d,$a)", + "&vrev32_16 ($d,$d)", # vrot ($d,16) + + "&vadd_i32 ($c,$c,$d)", + "&veor ($t,$b,$c)", + "&vshr_u32 ($b,$t,20)", + "&vsli_32 ($b,$t,12)", + + "&vadd_i32 ($a,$a,$b)", + "&veor ($t,$d,$a)", + "&vshr_u32 ($d,$t,24)", + "&vsli_32 ($d,$t,8)", + #"&vperm ($d,$t,$t3)", + + "&vadd_i32 ($c,$c,$d)", + "&veor ($t,$b,$c)", + "&vshr_u32 ($b,$t,25)", + "&vsli_32 ($b,$t,7)", + + "&vext_8 ($a,$a,$a,$odd?4:12)", + "&vext_8 ($d,$d,$d,8)", + "&vext_8 ($c,$c,$c,$odd?12:4)" + ); +} + +$code.=<<___; +#if (defined(__KERNEL__) && defined(CONFIG_KERNEL_MODE_NEON)) || (!defined(__KERNEL__) && __ARM_MAX_ARCH__>=7) +.arch armv7-a +.fpu neon + +# ifdef __KERNEL__ +.globl ChaCha20_neon +@ For optimal performance it's appropriate for caller to enforce +@ minimum input length, 193 bytes is suggested. +# endif +.type ChaCha20_neon,%function +.align 5 +ChaCha20_neon: + ldr r12,[sp,#0] @ pull pointer to counter and nonce + stmdb sp!,{r0-r2,r4-r11,lr} +.LChaCha20_neon: + adr r14,.Lsigma + vstmdb sp!,{d8-d15} @ ABI spec says so + stmdb sp!,{r0-r3} + + vld1.32 {$b0-$c0},[r3] @ load key + ldmia r3,{r4-r11} @ load key + + sub sp,sp,#4*(16+16) + vld1.32 {$d0},[r12] @ load counter and nonce + add r12,sp,#4*8 + ldmia r14,{r0-r3} @ load sigma + vld1.32 {$a0},[r14]! @ load sigma + vld1.32 {$t0},[r14]! @ one + @ vld1.32 {$t3#lo},[r14] @ rot8 + vst1.32 {$c0-$d0},[r12] @ copy 1/2key|counter|nonce + vst1.32 {$a0-$b0},[sp] @ copy sigma|1/2key + + str r10,[sp,#4*(16+10)] @ off-load "@x[10]" + str r11,[sp,#4*(16+11)] @ off-load "@x[11]" + vshl.i32 $t1#lo,$t0#lo,#1 @ two + vstr $t0#lo,[sp,#4*(16+0)] + vshl.i32 $t2#lo,$t0#lo,#2 @ four + vstr $t1#lo,[sp,#4*(16+2)] + vmov $a1,$a0 + vstr $t2#lo,[sp,#4*(16+4)] + vmov $a2,$a0 + @ vstr $t3#lo,[sp,#4*(16+6)] + vmov $b1,$b0 + vmov $b2,$b0 + b .Loop_neon_enter + +.align 4 +.Loop_neon_outer: + ldmia sp,{r0-r9} @ load key material + cmp @t[3],#64*2 @ if len<=64*2 + bls .Lbreak_neon @ switch to integer-only + @ vldr $t3#lo,[sp,#4*(16+6)] @ rot8 + vmov $a1,$a0 + str @t[3],[sp,#4*(32+2)] @ save len + vmov $a2,$a0 + str r12, [sp,#4*(32+1)] @ save inp + vmov $b1,$b0 + str r14, [sp,#4*(32+0)] @ save out + vmov $b2,$b0 +.Loop_neon_enter: + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + vadd.i32 $d1,$d0,$t0 @ counter+1 + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + vmov $c1,$c0 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + vmov $c2,$c0 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + vadd.i32 $d2,$d1,$t0 @ counter+2 + add @x[12],@x[12],#3 @ counter+3 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(16+15)] + mov @t[3],#10 + b .Loop_neon + +.align 4 +.Loop_neon: + subs @t[3],@t[3],#1 +___ + my @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,0); + my @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,0); + my @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,0); + my @thread3=&ROUND(0,4,8,12); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } + + @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,1); + @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,1); + @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,1); + @thread3=&ROUND(0,5,10,15); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } +$code.=<<___; + bne .Loop_neon + + add @t[3],sp,#32 + vld1.32 {$t0-$t1},[sp] @ load key material + vld1.32 {$t2-$t3},[@t[3]] + + ldr @t[3],[sp,#4*(32+2)] @ load len + + str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store + str @t[1], [sp,#4*(16+9)] + str @x[12],[sp,#4*(16+12)] + str @t[2], [sp,#4*(16+13)] + str @x[14],[sp,#4*(16+14)] + + @ at this point we have first half of 512-bit result in + @ @x[0-7] and second half at sp+4*(16+8) + + ldr r12,[sp,#4*(32+1)] @ load inp + ldr r14,[sp,#4*(32+0)] @ load out + + vadd.i32 $a0,$a0,$t0 @ accumulate key material + vadd.i32 $a1,$a1,$t0 + vadd.i32 $a2,$a2,$t0 + vldr $t0#lo,[sp,#4*(16+0)] @ one + + vadd.i32 $b0,$b0,$t1 + vadd.i32 $b1,$b1,$t1 + vadd.i32 $b2,$b2,$t1 + vldr $t1#lo,[sp,#4*(16+2)] @ two + + vadd.i32 $c0,$c0,$t2 + vadd.i32 $c1,$c1,$t2 + vadd.i32 $c2,$c2,$t2 + vadd.i32 $d1#lo,$d1#lo,$t0#lo @ counter+1 + vadd.i32 $d2#lo,$d2#lo,$t1#lo @ counter+2 + + vadd.i32 $d0,$d0,$t3 + vadd.i32 $d1,$d1,$t3 + vadd.i32 $d2,$d2,$t3 + + cmp @t[3],#64*4 + blo .Ltail_neon + + vld1.8 {$t0-$t1},[r12]! @ load input + mov @t[3],sp + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 @ xor with input + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + vst1.8 {$a0-$b0},[r14]! @ store output + veor $b1,$b1,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c1,$c1,$t2 + vst1.8 {$c0-$d0},[r14]! + veor $d1,$d1,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a2,$a2,$t0 + vld1.32 {$a0-$b0},[@t[3]]! @ load for next iteration + veor $t0#hi,$t0#hi,$t0#hi + vldr $t0#lo,[sp,#4*(16+4)] @ four + veor $b2,$b2,$t1 + vld1.32 {$c0-$d0},[@t[3]] + veor $c2,$c2,$t2 + vst1.8 {$a1-$b1},[r14]! + veor $d2,$d2,$t3 + vst1.8 {$c1-$d1},[r14]! + + vadd.i32 $d0#lo,$d0#lo,$t0#lo @ next counter value + vldr $t0#lo,[sp,#4*(16+0)] @ one + + ldmia sp,{@t[0]-@t[3]} @ load key material + add @x[0],@x[0],@t[0] @ accumulate key material + ldr @t[0],[r12],#16 @ load input + vst1.8 {$a2-$b2},[r14]! + add @x[1],@x[1],@t[1] + ldr @t[1],[r12,#-12] + vst1.8 {$c2-$d2},[r14]! + add @x[2],@x[2],@t[2] + ldr @t[2],[r12,#-8] + add @x[3],@x[3],@t[3] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif + eor @x[0],@x[0],@t[0] @ xor with input + add @t[0],sp,#4*(4) + eor @x[1],@x[1],@t[1] + str @x[0],[r14],#16 @ store output + eor @x[2],@x[2],@t[2] + str @x[1],[r14,#-12] + eor @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + ldr @t[0],[r12],#16 @ load input + add @x[5],@t[1],@x[5],ror#13 + ldr @t[1],[r12,#-12] + add @x[6],@t[2],@x[6],ror#13 + ldr @t[2],[r12,#-8] + add @x[7],@t[3],@x[7],ror#13 + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + eor @x[4],@x[4],@t[0] + add @t[0],sp,#4*(8) + eor @x[5],@x[5],@t[1] + str @x[4],[r14],#16 @ store output + eor @x[6],@x[6],@t[2] + str @x[5],[r14,#-12] + eor @x[7],@x[7],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[6],[r14,#-8] + add @x[0],sp,#4*(16+8) + str @x[7],[r14,#-4] + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + ldr @t[0],[r12],#16 @ load input + add @x[1],@x[1],@t[1] + ldr @t[1],[r12,#-12] +# ifdef __thumb2__ + it hi +# endif + strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it + add @x[2],@x[2],@t[2] + ldr @t[2],[r12,#-8] +# ifdef __thumb2__ + it hi +# endif + strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it + add @x[3],@x[3],@t[3] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] +# endif + eor @x[0],@x[0],@t[0] + add @t[0],sp,#4*(12) + eor @x[1],@x[1],@t[1] + str @x[0],[r14],#16 @ store output + eor @x[2],@x[2],@t[2] + str @x[1],[r14,#-12] + eor @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + str @x[2],[r14,#-8] + str @x[3],[r14,#-4] + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @t[0],@t[0],#4 @ next counter value + add @x[5],@t[1],@x[5],ror#24 + str @t[0],[sp,#4*(12)] @ save next counter value + ldr @t[0],[r12],#16 @ load input + add @x[6],@t[2],@x[6],ror#24 + add @x[4],@x[4],#3 @ counter+3 + ldr @t[1],[r12,#-12] + add @x[7],@t[3],@x[7],ror#24 + ldr @t[2],[r12,#-8] + ldr @t[3],[r12,#-4] +# ifdef __ARMEB__ + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + eor @x[4],@x[4],@t[0] +# ifdef __thumb2__ + it hi +# endif + ldrhi @t[0],[sp,#4*(32+2)] @ re-load len + eor @x[5],@x[5],@t[1] + eor @x[6],@x[6],@t[2] + str @x[4],[r14],#16 @ store output + eor @x[7],@x[7],@t[3] + str @x[5],[r14,#-12] + sub @t[3],@t[0],#64*4 @ len-=64*4 + str @x[6],[r14,#-8] + str @x[7],[r14,#-4] + bhi .Loop_neon_outer + + b .Ldone_neon + +.align 4 +.Lbreak_neon: + @ harmonize NEON and integer-only stack frames: load data + @ from NEON frame, but save to integer-only one; distance + @ between the two is 4*(32+4+16-32)=4*(20). + + str @t[3], [sp,#4*(20+32+2)] @ save len + add @t[3],sp,#4*(32+4) + str r12, [sp,#4*(20+32+1)] @ save inp + str r14, [sp,#4*(20+32+0)] @ save out + + ldr @x[12],[sp,#4*(16+10)] + ldr @x[14],[sp,#4*(16+11)] + vldmia @t[3],{d8-d15} @ fulfill ABI requirement + str @x[12],[sp,#4*(20+16+10)] @ copy "@x[10]" + str @x[14],[sp,#4*(20+16+11)] @ copy "@x[11]" + + ldr @t[3], [sp,#4*(15)] + mov @x[4],@x[4],ror#19 @ twist b[0..3] + ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load + mov @x[5],@x[5],ror#19 + ldr @t[2], [sp,#4*(13)] + mov @x[6],@x[6],ror#19 + ldr @x[14],[sp,#4*(14)] + mov @x[7],@x[7],ror#19 + mov @t[3],@t[3],ror#8 @ twist d[0..3] + mov @x[12],@x[12],ror#8 + mov @t[2],@t[2],ror#8 + mov @x[14],@x[14],ror#8 + str @t[3], [sp,#4*(20+16+15)] + add @t[3],sp,#4*(20) + vst1.32 {$a0-$b0},[@t[3]]! @ copy key + add sp,sp,#4*(20) @ switch frame + vst1.32 {$c0-$d0},[@t[3]] + mov @t[3],#10 + b .Loop @ go integer-only + +.align 4 +.Ltail_neon: + cmp @t[3],#64*3 + bhs .L192_or_more_neon + cmp @t[3],#64*2 + bhs .L128_or_more_neon + cmp @t[3],#64*1 + bhs .L64_or_more_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a0-$b0},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c0-$d0},[@t[0]] + b .Loop_tail_neon + +.align 4 +.L64_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vst1.8 {$a0-$b0},[r14]! + vst1.8 {$c0-$d0},[r14]! + + beq .Ldone_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a1-$b1},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c1-$d1},[@t[0]] + sub @t[3],@t[3],#64*1 @ len-=64*1 + b .Loop_tail_neon + +.align 4 +.L128_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + veor $b1,$b1,$t1 + vst1.8 {$a0-$b0},[r14]! + veor $c1,$c1,$t2 + vst1.8 {$c0-$d0},[r14]! + veor $d1,$d1,$t3 + vst1.8 {$a1-$b1},[r14]! + vst1.8 {$c1-$d1},[r14]! + + beq .Ldone_neon + + add @t[0],sp,#4*(8) + vst1.8 {$a2-$b2},[sp] + add @t[2],sp,#4*(0) + vst1.8 {$c2-$d2},[@t[0]] + sub @t[3],@t[3],#64*2 @ len-=64*2 + b .Loop_tail_neon + +.align 4 +.L192_or_more_neon: + vld1.8 {$t0-$t1},[r12]! + vld1.8 {$t2-$t3},[r12]! + veor $a0,$a0,$t0 + veor $b0,$b0,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c0,$c0,$t2 + veor $d0,$d0,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a1,$a1,$t0 + veor $b1,$b1,$t1 + vld1.8 {$t0-$t1},[r12]! + veor $c1,$c1,$t2 + vst1.8 {$a0-$b0},[r14]! + veor $d1,$d1,$t3 + vld1.8 {$t2-$t3},[r12]! + + veor $a2,$a2,$t0 + vst1.8 {$c0-$d0},[r14]! + veor $b2,$b2,$t1 + vst1.8 {$a1-$b1},[r14]! + veor $c2,$c2,$t2 + vst1.8 {$c1-$d1},[r14]! + veor $d2,$d2,$t3 + vst1.8 {$a2-$b2},[r14]! + vst1.8 {$c2-$d2},[r14]! + + beq .Ldone_neon + + ldmia sp,{@t[0]-@t[3]} @ load key material + add @x[0],@x[0],@t[0] @ accumulate key material + add @t[0],sp,#4*(4) + add @x[1],@x[1],@t[1] + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + + add @x[4],@t[0],@x[4],ror#13 @ accumulate key material + add @t[0],sp,#4*(8) + add @x[5],@t[1],@x[5],ror#13 + add @x[6],@t[2],@x[6],ror#13 + add @x[7],@t[3],@x[7],ror#13 + ldmia @t[0],{@t[0]-@t[3]} @ load key material +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + stmia sp,{@x[0]-@x[7]} + add @x[0],sp,#4*(16+8) + + ldmia @x[0],{@x[0]-@x[7]} @ load second half + + add @x[0],@x[0],@t[0] @ accumulate key material + add @t[0],sp,#4*(12) + add @x[1],@x[1],@t[1] + add @x[2],@x[2],@t[2] + add @x[3],@x[3],@t[3] + ldmia @t[0],{@t[0]-@t[3]} @ load key material + + add @x[4],@t[0],@x[4],ror#24 @ accumulate key material + add @t[0],sp,#4*(8) + add @x[5],@t[1],@x[5],ror#24 + add @x[4],@x[4],#3 @ counter+3 + add @x[6],@t[2],@x[6],ror#24 + add @x[7],@t[3],@x[7],ror#24 + ldr @t[3],[sp,#4*(32+2)] @ re-load len +# ifdef __ARMEB__ + rev @x[0],@x[0] + rev @x[1],@x[1] + rev @x[2],@x[2] + rev @x[3],@x[3] + rev @x[4],@x[4] + rev @x[5],@x[5] + rev @x[6],@x[6] + rev @x[7],@x[7] +# endif + stmia @t[0],{@x[0]-@x[7]} + add @t[2],sp,#4*(0) + sub @t[3],@t[3],#64*3 @ len-=64*3 + +.Loop_tail_neon: + ldrb @t[0],[@t[2]],#1 @ read buffer on stack + ldrb @t[1],[r12],#1 @ read input + subs @t[3],@t[3],#1 + eor @t[0],@t[0],@t[1] + strb @t[0],[r14],#1 @ store output + bne .Loop_tail_neon + +.Ldone_neon: + add sp,sp,#4*(32+4) + vldmia sp,{d8-d15} + add sp,sp,#4*(16+3) + ldmia sp!,{r4-r11,pc} +.size ChaCha20_neon,.-ChaCha20_neon +# ifndef __KERNEL__ +.comm OPENSSL_armcap_P,4,4 +# endif +#endif +___ +}}} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo; + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl new file mode 100644 index 0000000000000..ac14a99241652 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-arm64.pl @@ -0,0 +1,1163 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# June 2015 +# +# ChaCha20 for ARMv8. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc-4.9 3xNEON+1xIALU 6xNEON+2xIALU(*) +# +# Apple A7 5.50/+49% 3.33 1.70 +# Cortex-A53 8.40/+80% 4.72 4.72(**) +# Cortex-A57 8.06/+43% 4.90 4.43(***) +# Denver 4.50/+82% 2.63 2.67(**) +# X-Gene 9.50/+46% 8.82 8.89(**) +# Mongoose 8.00/+44% 3.64 3.25(***) +# Kryo 8.17/+50% 4.83 4.65(***) +# +# (*) since no non-Apple processor exhibits significantly better +# performance, the code path is #ifdef __APPLE__-ed; +# (**) it's expected that doubling interleave factor doesn't help +# all processors, only those with higher NEON latency and +# higher instruction issue rate; +# (***) expected improvement was actually higher; + +$flavour=shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +my ($out,$inp,$len,$key,$ctr) = map("x$_",(0..4)); + +my @x=map("x$_",(5..17,19..21)); +my @d=map("x$_",(22..28,30)); + +sub ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); + + ( + "&add_32 (@x[$a0],@x[$a0],@x[$b0])", + "&add_32 (@x[$a1],@x[$a1],@x[$b1])", + "&add_32 (@x[$a2],@x[$a2],@x[$b2])", + "&add_32 (@x[$a3],@x[$a3],@x[$b3])", + "&eor_32 (@x[$d0],@x[$d0],@x[$a0])", + "&eor_32 (@x[$d1],@x[$d1],@x[$a1])", + "&eor_32 (@x[$d2],@x[$d2],@x[$a2])", + "&eor_32 (@x[$d3],@x[$d3],@x[$a3])", + "&ror_32 (@x[$d0],@x[$d0],16)", + "&ror_32 (@x[$d1],@x[$d1],16)", + "&ror_32 (@x[$d2],@x[$d2],16)", + "&ror_32 (@x[$d3],@x[$d3],16)", + + "&add_32 (@x[$c0],@x[$c0],@x[$d0])", + "&add_32 (@x[$c1],@x[$c1],@x[$d1])", + "&add_32 (@x[$c2],@x[$c2],@x[$d2])", + "&add_32 (@x[$c3],@x[$c3],@x[$d3])", + "&eor_32 (@x[$b0],@x[$b0],@x[$c0])", + "&eor_32 (@x[$b1],@x[$b1],@x[$c1])", + "&eor_32 (@x[$b2],@x[$b2],@x[$c2])", + "&eor_32 (@x[$b3],@x[$b3],@x[$c3])", + "&ror_32 (@x[$b0],@x[$b0],20)", + "&ror_32 (@x[$b1],@x[$b1],20)", + "&ror_32 (@x[$b2],@x[$b2],20)", + "&ror_32 (@x[$b3],@x[$b3],20)", + + "&add_32 (@x[$a0],@x[$a0],@x[$b0])", + "&add_32 (@x[$a1],@x[$a1],@x[$b1])", + "&add_32 (@x[$a2],@x[$a2],@x[$b2])", + "&add_32 (@x[$a3],@x[$a3],@x[$b3])", + "&eor_32 (@x[$d0],@x[$d0],@x[$a0])", + "&eor_32 (@x[$d1],@x[$d1],@x[$a1])", + "&eor_32 (@x[$d2],@x[$d2],@x[$a2])", + "&eor_32 (@x[$d3],@x[$d3],@x[$a3])", + "&ror_32 (@x[$d0],@x[$d0],24)", + "&ror_32 (@x[$d1],@x[$d1],24)", + "&ror_32 (@x[$d2],@x[$d2],24)", + "&ror_32 (@x[$d3],@x[$d3],24)", + + "&add_32 (@x[$c0],@x[$c0],@x[$d0])", + "&add_32 (@x[$c1],@x[$c1],@x[$d1])", + "&add_32 (@x[$c2],@x[$c2],@x[$d2])", + "&add_32 (@x[$c3],@x[$c3],@x[$d3])", + "&eor_32 (@x[$b0],@x[$b0],@x[$c0])", + "&eor_32 (@x[$b1],@x[$b1],@x[$c1])", + "&eor_32 (@x[$b2],@x[$b2],@x[$c2])", + "&eor_32 (@x[$b3],@x[$b3],@x[$c3])", + "&ror_32 (@x[$b0],@x[$b0],25)", + "&ror_32 (@x[$b1],@x[$b1],25)", + "&ror_32 (@x[$b2],@x[$b2],25)", + "&ror_32 (@x[$b3],@x[$b3],25)" + ); +} + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +.extern OPENSSL_armcap_P +#else +# define ChaCha20_ctr32 chacha20_arm +# define ChaCha20_neon chacha20_neon +#endif + +.text + +.align 5 +.Lsigma: +.quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral +.Lone: +.long 1,0,0,0 +#ifndef __KERNEL__ +.LOPENSSL_armcap_P: +# ifdef __ILP32__ +.long OPENSSL_armcap_P-. +# else +.quad OPENSSL_armcap_P-. +# endif +#endif + +.globl ChaCha20_ctr32 +.type ChaCha20_ctr32,%function +.align 5 +ChaCha20_ctr32: + cbz $len,.Labort +#ifndef __KERNEL__ + adr @x[0],.LOPENSSL_armcap_P + cmp $len,#192 + b.lo .Lshort +# ifdef __ILP32__ + ldrsw @x[1],[@x[0]] +# else + ldr @x[1],[@x[0]] +# endif + ldr w17,[@x[1],@x[0]] + tst w17,#ARMV7_NEON + b.ne ChaCha20_neon + +.Lshort: +#endif + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ldp @d[6],@d[7],[$ctr] // load counter +#ifdef __AARCH64EB__ + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +#endif + +.Loop_outer: + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + mov.32 @x[6],@d[3] + lsr @x[7],@d[3],#32 + mov.32 @x[8],@d[4] + lsr @x[9],@d[4],#32 + mov.32 @x[10],@d[5] + lsr @x[11],@d[5],#32 + mov.32 @x[12],@d[6] + lsr @x[13],@d[6],#32 + mov.32 @x[14],@d[7] + lsr @x[15],@d[7],#32 + + mov $ctr,#10 + subs $len,$len,#64 +.Loop: + sub $ctr,$ctr,#1 +___ + foreach (&ROUND(0, 4, 8,12)) { eval; } + foreach (&ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + cbnz $ctr,.Loop + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add @x[1],@x[1],@d[0],lsr#32 + add.32 @x[2],@x[2],@d[1] + add @x[3],@x[3],@d[1],lsr#32 + add.32 @x[4],@x[4],@d[2] + add @x[5],@x[5],@d[2],lsr#32 + add.32 @x[6],@x[6],@d[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add @x[15],@x[15],@d[7],lsr#32 + + b.lo .Ltail + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#1 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + + b.hi .Loop_outer + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 +.Labort: + ret + +.align 4 +.Ltail: + add $len,$len,#64 +.Less_than_64: + sub $out,$out,#1 + add $inp,$inp,$len + add $out,$out,$len + add $ctr,sp,$len + neg $len,$len + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + stp @x[0],@x[2],[sp,#0] + stp @x[4],@x[6],[sp,#16] + stp @x[8],@x[10],[sp,#32] + stp @x[12],@x[14],[sp,#48] + +.Loop_tail: + ldrb w10,[$inp,$len] + ldrb w11,[$ctr,$len] + add $len,$len,#1 + eor w10,w10,w11 + strb w10,[$out,$len] + cbnz $len,.Loop_tail + + stp xzr,xzr,[sp,#0] + stp xzr,xzr,[sp,#16] + stp xzr,xzr,[sp,#32] + stp xzr,xzr,[sp,#48] + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_ctr32,.-ChaCha20_ctr32 +___ + +{{{ +my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2,$T3) = + map("v$_.4s",(0..7,16..23)); +my (@K)=map("v$_.4s",(24..30)); +my $ONE="v31.4s"; + +sub NEONROUND { +my $odd = pop; +my ($a,$b,$c,$d,$t)=@_; + + ( + "&add ('$a','$a','$b')", + "&eor ('$d','$d','$a')", + "&rev32_16 ('$d','$d')", # vrot ($d,16) + + "&add ('$c','$c','$d')", + "&eor ('$t','$b','$c')", + "&ushr ('$b','$t',20)", + "&sli ('$b','$t',12)", + + "&add ('$a','$a','$b')", + "&eor ('$t','$d','$a')", + "&ushr ('$d','$t',24)", + "&sli ('$d','$t',8)", + + "&add ('$c','$c','$d')", + "&eor ('$t','$b','$c')", + "&ushr ('$b','$t',25)", + "&sli ('$b','$t',7)", + + "&ext ('$a','$a','$a',$odd?4:12)", + "&ext ('$d','$d','$d',8)", + "&ext ('$c','$c','$c',$odd?12:4)" + ); +} + +$code.=<<___; +#if !defined(__KERNEL__) || defined(CONFIG_KERNEL_MODE_NEON) +#ifdef __KERNEL__ +.globl ChaCha20_neon +.type ChaCha20_neon,%function +#endif +.type ChaCha20_neon,%function +.align 5 +ChaCha20_neon: + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] +#ifdef __APPLE__ + cmp $len,#512 + b.hs .L512_or_more_neon +#endif + + sub sp,sp,#64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ld1 {@K[0]},[@x[0]],#16 + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ld1 {@K[1],@K[2]},[$key] + ldp @d[6],@d[7],[$ctr] // load counter + ld1 {@K[3]},[$ctr] + ld1 {$ONE},[@x[0]] +#ifdef __AARCH64EB__ + rev64 @K[0],@K[0] + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +#endif + add @K[3],@K[3],$ONE // += 1 + add @K[4],@K[3],$ONE + add @K[5],@K[4],$ONE + shl $ONE,$ONE,#2 // 1 -> 4 + +.Loop_outer_neon: + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + mov $A0,@K[0] + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + mov $A1,@K[0] + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + mov $A2,@K[0] + mov.32 @x[6],@d[3] + mov $B0,@K[1] + lsr @x[7],@d[3],#32 + mov $B1,@K[1] + mov.32 @x[8],@d[4] + mov $B2,@K[1] + lsr @x[9],@d[4],#32 + mov $D0,@K[3] + mov.32 @x[10],@d[5] + mov $D1,@K[4] + lsr @x[11],@d[5],#32 + mov $D2,@K[5] + mov.32 @x[12],@d[6] + mov $C0,@K[2] + lsr @x[13],@d[6],#32 + mov $C1,@K[2] + mov.32 @x[14],@d[7] + mov $C2,@K[2] + lsr @x[15],@d[7],#32 + + mov $ctr,#10 + subs $len,$len,#256 +.Loop_neon: + sub $ctr,$ctr,#1 +___ + my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + my @thread3=&ROUND(0,4,8,12); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&ROUND(0,5,10,15); + + foreach (@thread0) { + eval; eval(shift(@thread3)); + eval(shift(@thread1)); eval(shift(@thread3)); + eval(shift(@thread2)); eval(shift(@thread3)); + } +$code.=<<___; + cbnz $ctr,.Loop_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add $A0,$A0,@K[0] + add @x[1],@x[1],@d[0],lsr#32 + add $A1,$A1,@K[0] + add.32 @x[2],@x[2],@d[1] + add $A2,$A2,@K[0] + add @x[3],@x[3],@d[1],lsr#32 + add $C0,$C0,@K[2] + add.32 @x[4],@x[4],@d[2] + add $C1,$C1,@K[2] + add @x[5],@x[5],@d[2],lsr#32 + add $C2,$C2,@K[2] + add.32 @x[6],@x[6],@d[3] + add $D0,$D0,@K[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add $D1,$D1,@K[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add $D2,$D2,@K[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add $B0,$B0,@K[1] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add $B1,$B1,@K[1] + add @x[15],@x[15],@d[7],lsr#32 + add $B2,$B2,@K[1] + + b.lo .Ltail_neon + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + ld1.8 {$T0-$T3},[$inp],#64 + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor $A0,$A0,$T0 + eor @x[10],@x[10],@x[11] + eor $B0,$B0,$T1 + eor @x[12],@x[12],@x[13] + eor $C0,$C0,$T2 + eor @x[14],@x[14],@x[15] + eor $D0,$D0,$T3 + ld1.8 {$T0-$T3},[$inp],#64 + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#4 // increment counter + stp @x[4],@x[6],[$out,#16] + add @K[3],@K[3],$ONE // += 4 + stp @x[8],@x[10],[$out,#32] + add @K[4],@K[4],$ONE + stp @x[12],@x[14],[$out,#48] + add @K[5],@K[5],$ONE + add $out,$out,#64 + + st1.8 {$A0-$D0},[$out],#64 + ld1.8 {$A0-$D0},[$inp],#64 + + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + + eor $A2,$A2,$A0 + eor $B2,$B2,$B0 + eor $C2,$C2,$C0 + eor $D2,$D2,$D0 + st1.8 {$A2-$D2},[$out],#64 + + b.hi .Loop_outer_neon + + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret + +.Ltail_neon: + add $len,$len,#256 + cmp $len,#64 + b.lo .Less_than_64 + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +#ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +#endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#4 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + b.eq .Ldone_neon + sub $len,$len,#64 + cmp $len,#64 + b.lo .Less_than_128 + + ld1.8 {$T0-$T3},[$inp],#64 + eor $A0,$A0,$T0 + eor $B0,$B0,$T1 + eor $C0,$C0,$T2 + eor $D0,$D0,$T3 + st1.8 {$A0-$D0},[$out],#64 + b.eq .Ldone_neon + sub $len,$len,#64 + cmp $len,#64 + b.lo .Less_than_192 + + ld1.8 {$T0-$T3},[$inp],#64 + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + b.eq .Ldone_neon + sub $len,$len,#64 + + st1.8 {$A2-$D2},[sp] + b .Last_neon + +.Less_than_128: + st1.8 {$A0-$D0},[sp] + b .Last_neon +.Less_than_192: + st1.8 {$A1-$D1},[sp] + b .Last_neon + +.align 4 +.Last_neon: + sub $out,$out,#1 + add $inp,$inp,$len + add $out,$out,$len + add $ctr,sp,$len + neg $len,$len + +.Loop_tail_neon: + ldrb w10,[$inp,$len] + ldrb w11,[$ctr,$len] + add $len,$len,#1 + eor w10,w10,w11 + strb w10,[$out,$len] + cbnz $len,.Loop_tail_neon + + stp xzr,xzr,[sp,#0] + stp xzr,xzr,[sp,#16] + stp xzr,xzr,[sp,#32] + stp xzr,xzr,[sp,#48] + +.Ldone_neon: + ldp x19,x20,[x29,#16] + add sp,sp,#64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_neon,.-ChaCha20_neon +___ +{ +my ($T0,$T1,$T2,$T3,$T4,$T5)=@K; +my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2, + $A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(0..23)); + +$code.=<<___; +#ifdef __APPLE__ +.type ChaCha20_512_neon,%function +.align 5 +ChaCha20_512_neon: + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + + adr @x[0],.Lsigma + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + +.L512_or_more_neon: + sub sp,sp,#128+64 + + ldp @d[0],@d[1],[@x[0]] // load sigma + ld1 {@K[0]},[@x[0]],#16 + ldp @d[2],@d[3],[$key] // load key + ldp @d[4],@d[5],[$key,#16] + ld1 {@K[1],@K[2]},[$key] + ldp @d[6],@d[7],[$ctr] // load counter + ld1 {@K[3]},[$ctr] + ld1 {$ONE},[@x[0]] +# ifdef __AARCH64EB__ + rev64 @K[0],@K[0] + ror @d[2],@d[2],#32 + ror @d[3],@d[3],#32 + ror @d[4],@d[4],#32 + ror @d[5],@d[5],#32 + ror @d[6],@d[6],#32 + ror @d[7],@d[7],#32 +# endif + add @K[3],@K[3],$ONE // += 1 + stp @K[0],@K[1],[sp,#0] // off-load key block, invariant part + add @K[3],@K[3],$ONE // not typo + str @K[2],[sp,#32] + add @K[4],@K[3],$ONE + add @K[5],@K[4],$ONE + add @K[6],@K[5],$ONE + shl $ONE,$ONE,#2 // 1 -> 4 + + stp d8,d9,[sp,#128+0] // meet ABI requirements + stp d10,d11,[sp,#128+16] + stp d12,d13,[sp,#128+32] + stp d14,d15,[sp,#128+48] + + sub $len,$len,#512 // not typo + +.Loop_outer_512_neon: + mov $A0,@K[0] + mov $A1,@K[0] + mov $A2,@K[0] + mov $A3,@K[0] + mov $A4,@K[0] + mov $A5,@K[0] + mov $B0,@K[1] + mov.32 @x[0],@d[0] // unpack key block + mov $B1,@K[1] + lsr @x[1],@d[0],#32 + mov $B2,@K[1] + mov.32 @x[2],@d[1] + mov $B3,@K[1] + lsr @x[3],@d[1],#32 + mov $B4,@K[1] + mov.32 @x[4],@d[2] + mov $B5,@K[1] + lsr @x[5],@d[2],#32 + mov $D0,@K[3] + mov.32 @x[6],@d[3] + mov $D1,@K[4] + lsr @x[7],@d[3],#32 + mov $D2,@K[5] + mov.32 @x[8],@d[4] + mov $D3,@K[6] + lsr @x[9],@d[4],#32 + mov $C0,@K[2] + mov.32 @x[10],@d[5] + mov $C1,@K[2] + lsr @x[11],@d[5],#32 + add $D4,$D0,$ONE // +4 + mov.32 @x[12],@d[6] + add $D5,$D1,$ONE // +4 + lsr @x[13],@d[6],#32 + mov $C2,@K[2] + mov.32 @x[14],@d[7] + mov $C3,@K[2] + lsr @x[15],@d[7],#32 + mov $C4,@K[2] + stp @K[3],@K[4],[sp,#48] // off-load key block, variable part + mov $C5,@K[2] + str @K[5],[sp,#80] + + mov $ctr,#5 + subs $len,$len,#512 +.Loop_upper_neon: + sub $ctr,$ctr,#1 +___ + my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + my @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0); + my @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0); + my @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0); + my @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + my $diff = ($#thread0+1)*6 - $#thread67 - 1; + my $i = 0; + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } +$code.=<<___; + cbnz $ctr,.Loop_upper_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + add @x[1],@x[1],@d[0],lsr#32 + add.32 @x[2],@x[2],@d[1] + add @x[3],@x[3],@d[1],lsr#32 + add.32 @x[4],@x[4],@d[2] + add @x[5],@x[5],@d[2],lsr#32 + add.32 @x[6],@x[6],@d[3] + add @x[7],@x[7],@d[3],lsr#32 + add.32 @x[8],@x[8],@d[4] + add @x[9],@x[9],@d[4],lsr#32 + add.32 @x[10],@x[10],@d[5] + add @x[11],@x[11],@d[5],lsr#32 + add.32 @x[12],@x[12],@d[6] + add @x[13],@x[13],@d[6],lsr#32 + add.32 @x[14],@x[14],@d[7] + add @x[15],@x[15],@d[7],lsr#32 + + add @x[0],@x[0],@x[1],lsl#32 // pack + add @x[2],@x[2],@x[3],lsl#32 + ldp @x[1],@x[3],[$inp,#0] // load input + add @x[4],@x[4],@x[5],lsl#32 + add @x[6],@x[6],@x[7],lsl#32 + ldp @x[5],@x[7],[$inp,#16] + add @x[8],@x[8],@x[9],lsl#32 + add @x[10],@x[10],@x[11],lsl#32 + ldp @x[9],@x[11],[$inp,#32] + add @x[12],@x[12],@x[13],lsl#32 + add @x[14],@x[14],@x[15],lsl#32 + ldp @x[13],@x[15],[$inp,#48] + add $inp,$inp,#64 +# ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +# endif + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor @x[10],@x[10],@x[11] + eor @x[12],@x[12],@x[13] + eor @x[14],@x[14],@x[15] + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#1 // increment counter + mov.32 @x[0],@d[0] // unpack key block + lsr @x[1],@d[0],#32 + stp @x[4],@x[6],[$out,#16] + mov.32 @x[2],@d[1] + lsr @x[3],@d[1],#32 + stp @x[8],@x[10],[$out,#32] + mov.32 @x[4],@d[2] + lsr @x[5],@d[2],#32 + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + mov.32 @x[6],@d[3] + lsr @x[7],@d[3],#32 + mov.32 @x[8],@d[4] + lsr @x[9],@d[4],#32 + mov.32 @x[10],@d[5] + lsr @x[11],@d[5],#32 + mov.32 @x[12],@d[6] + lsr @x[13],@d[6],#32 + mov.32 @x[14],@d[7] + lsr @x[15],@d[7],#32 + + mov $ctr,#5 +.Loop_lower_neon: + sub $ctr,$ctr,#1 +___ + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } + + @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1); + @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1); + @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1); + @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1); + @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1); + @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1); + @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15)); + + foreach (@thread0) { + eval; eval(shift(@thread67)); + eval(shift(@thread1)); eval(shift(@thread67)); + eval(shift(@thread2)); eval(shift(@thread67)); + eval(shift(@thread3)); eval(shift(@thread67)); + eval(shift(@thread4)); eval(shift(@thread67)); + eval(shift(@thread5)); eval(shift(@thread67)); + } +$code.=<<___; + cbnz $ctr,.Loop_lower_neon + + add.32 @x[0],@x[0],@d[0] // accumulate key block + ldp @K[0],@K[1],[sp,#0] + add @x[1],@x[1],@d[0],lsr#32 + ldp @K[2],@K[3],[sp,#32] + add.32 @x[2],@x[2],@d[1] + ldp @K[4],@K[5],[sp,#64] + add @x[3],@x[3],@d[1],lsr#32 + add $A0,$A0,@K[0] + add.32 @x[4],@x[4],@d[2] + add $A1,$A1,@K[0] + add @x[5],@x[5],@d[2],lsr#32 + add $A2,$A2,@K[0] + add.32 @x[6],@x[6],@d[3] + add $A3,$A3,@K[0] + add @x[7],@x[7],@d[3],lsr#32 + add $A4,$A4,@K[0] + add.32 @x[8],@x[8],@d[4] + add $A5,$A5,@K[0] + add @x[9],@x[9],@d[4],lsr#32 + add $C0,$C0,@K[2] + add.32 @x[10],@x[10],@d[5] + add $C1,$C1,@K[2] + add @x[11],@x[11],@d[5],lsr#32 + add $C2,$C2,@K[2] + add.32 @x[12],@x[12],@d[6] + add $C3,$C3,@K[2] + add @x[13],@x[13],@d[6],lsr#32 + add $C4,$C4,@K[2] + add.32 @x[14],@x[14],@d[7] + add $C5,$C5,@K[2] + add @x[15],@x[15],@d[7],lsr#32 + add $D4,$D4,$ONE // +4 + add @x[0],@x[0],@x[1],lsl#32 // pack + add $D5,$D5,$ONE // +4 + add @x[2],@x[2],@x[3],lsl#32 + add $D0,$D0,@K[3] + ldp @x[1],@x[3],[$inp,#0] // load input + add $D1,$D1,@K[4] + add @x[4],@x[4],@x[5],lsl#32 + add $D2,$D2,@K[5] + add @x[6],@x[6],@x[7],lsl#32 + add $D3,$D3,@K[6] + ldp @x[5],@x[7],[$inp,#16] + add $D4,$D4,@K[3] + add @x[8],@x[8],@x[9],lsl#32 + add $D5,$D5,@K[4] + add @x[10],@x[10],@x[11],lsl#32 + add $B0,$B0,@K[1] + ldp @x[9],@x[11],[$inp,#32] + add $B1,$B1,@K[1] + add @x[12],@x[12],@x[13],lsl#32 + add $B2,$B2,@K[1] + add @x[14],@x[14],@x[15],lsl#32 + add $B3,$B3,@K[1] + ldp @x[13],@x[15],[$inp,#48] + add $B4,$B4,@K[1] + add $inp,$inp,#64 + add $B5,$B5,@K[1] + +# ifdef __AARCH64EB__ + rev @x[0],@x[0] + rev @x[2],@x[2] + rev @x[4],@x[4] + rev @x[6],@x[6] + rev @x[8],@x[8] + rev @x[10],@x[10] + rev @x[12],@x[12] + rev @x[14],@x[14] +# endif + ld1.8 {$T0-$T3},[$inp],#64 + eor @x[0],@x[0],@x[1] + eor @x[2],@x[2],@x[3] + eor @x[4],@x[4],@x[5] + eor @x[6],@x[6],@x[7] + eor @x[8],@x[8],@x[9] + eor $A0,$A0,$T0 + eor @x[10],@x[10],@x[11] + eor $B0,$B0,$T1 + eor @x[12],@x[12],@x[13] + eor $C0,$C0,$T2 + eor @x[14],@x[14],@x[15] + eor $D0,$D0,$T3 + ld1.8 {$T0-$T3},[$inp],#64 + + stp @x[0],@x[2],[$out,#0] // store output + add @d[6],@d[6],#7 // increment counter + stp @x[4],@x[6],[$out,#16] + stp @x[8],@x[10],[$out,#32] + stp @x[12],@x[14],[$out,#48] + add $out,$out,#64 + st1.8 {$A0-$D0},[$out],#64 + + ld1.8 {$A0-$D0},[$inp],#64 + eor $A1,$A1,$T0 + eor $B1,$B1,$T1 + eor $C1,$C1,$T2 + eor $D1,$D1,$T3 + st1.8 {$A1-$D1},[$out],#64 + + ld1.8 {$A1-$D1},[$inp],#64 + eor $A2,$A2,$A0 + ldp @K[0],@K[1],[sp,#0] + eor $B2,$B2,$B0 + ldp @K[2],@K[3],[sp,#32] + eor $C2,$C2,$C0 + eor $D2,$D2,$D0 + st1.8 {$A2-$D2},[$out],#64 + + ld1.8 {$A2-$D2},[$inp],#64 + eor $A3,$A3,$A1 + eor $B3,$B3,$B1 + eor $C3,$C3,$C1 + eor $D3,$D3,$D1 + st1.8 {$A3-$D3},[$out],#64 + + ld1.8 {$A3-$D3},[$inp],#64 + eor $A4,$A4,$A2 + eor $B4,$B4,$B2 + eor $C4,$C4,$C2 + eor $D4,$D4,$D2 + st1.8 {$A4-$D4},[$out],#64 + + shl $A0,$ONE,#1 // 4 -> 8 + eor $A5,$A5,$A3 + eor $B5,$B5,$B3 + eor $C5,$C5,$C3 + eor $D5,$D5,$D3 + st1.8 {$A5-$D5},[$out],#64 + + add @K[3],@K[3],$A0 // += 8 + add @K[4],@K[4],$A0 + add @K[5],@K[5],$A0 + add @K[6],@K[6],$A0 + + b.hs .Loop_outer_512_neon + + adds $len,$len,#512 + ushr $A0,$ONE,#2 // 4 -> 1 + + ldp d8,d9,[sp,#128+0] // meet ABI requirements + ldp d10,d11,[sp,#128+16] + ldp d12,d13,[sp,#128+32] + ldp d14,d15,[sp,#128+48] + + stp @K[0],$ONE,[sp,#0] // wipe off-load area + stp @K[0],$ONE,[sp,#32] + stp @K[0],$ONE,[sp,#64] + + b.eq .Ldone_512_neon + + cmp $len,#192 + sub @K[3],@K[3],$A0 // -= 1 + sub @K[4],@K[4],$A0 + sub @K[5],@K[5],$A0 + add sp,sp,#128 + b.hs .Loop_outer_neon + + eor @K[1],@K[1],@K[1] + eor @K[2],@K[2],@K[2] + eor @K[3],@K[3],@K[3] + eor @K[4],@K[4],@K[4] + eor @K[5],@K[5],@K[5] + eor @K[6],@K[6],@K[6] + b .Loop_outer + +.Ldone_512_neon: + ldp x19,x20,[x29,#16] + add sp,sp,#128+64 + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + ret +.size ChaCha20_512_neon,.-ChaCha20_512_neon +#endif +#endif +___ +} +}}} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + (s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1)) or + (m/\b(eor|ext|mov)\b/ and (s/\.4s/\.16b/g or 1)) or + (s/\b((?:ld|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1)) or + (m/\b(ld|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1)) or + (s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1)); + + print $_,"\n"; +} +close STDOUT; # flush diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c new file mode 100644 index 0000000000000..96ce01e2c1333 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-mips-glue.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +asmlinkage void chacha20_mips(u32 state[16], u8 *out, const u8 *in, + const size_t len); +static bool *const chacha20_nobs[] __initconst = { }; +static void __init chacha20_fpu_init(void) +{ +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + chacha20_mips(ctx->state, dst, src, len); + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S b/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S new file mode 100644 index 0000000000000..a81e02db95e73 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-mips.S @@ -0,0 +1,424 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2016-2018 René van Dorst . All Rights Reserved. + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#define MASK_U32 0x3c +#define CHACHA20_BLOCK_SIZE 64 +#define STACK_SIZE 32 + +#define X0 $t0 +#define X1 $t1 +#define X2 $t2 +#define X3 $t3 +#define X4 $t4 +#define X5 $t5 +#define X6 $t6 +#define X7 $t7 +#define X8 $t8 +#define X9 $t9 +#define X10 $v1 +#define X11 $s6 +#define X12 $s5 +#define X13 $s4 +#define X14 $s3 +#define X15 $s2 +/* Use regs which are overwritten on exit for Tx so we don't leak clear data. */ +#define T0 $s1 +#define T1 $s0 +#define T(n) T ## n +#define X(n) X ## n + +/* Input arguments */ +#define STATE $a0 +#define OUT $a1 +#define IN $a2 +#define BYTES $a3 + +/* Output argument */ +/* NONCE[0] is kept in a register and not in memory. + * We don't want to touch original value in memory. + * Must be incremented every loop iteration. + */ +#define NONCE_0 $v0 + +/* SAVED_X and SAVED_CA are set in the jump table. + * Use regs which are overwritten on exit else we don't leak clear data. + * They are used to handling the last bytes which are not multiple of 4. + */ +#define SAVED_X X15 +#define SAVED_CA $s7 + +#define IS_UNALIGNED $s7 + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MSB 0 +#define LSB 3 +#define ROTx rotl +#define ROTR(n) rotr n, 24 +#define CPU_TO_LE32(n) \ + wsbh n; \ + rotr n, 16; +#else +#define MSB 3 +#define LSB 0 +#define ROTx rotr +#define CPU_TO_LE32(n) +#define ROTR(n) +#endif + +#define FOR_EACH_WORD(x) \ + x( 0); \ + x( 1); \ + x( 2); \ + x( 3); \ + x( 4); \ + x( 5); \ + x( 6); \ + x( 7); \ + x( 8); \ + x( 9); \ + x(10); \ + x(11); \ + x(12); \ + x(13); \ + x(14); \ + x(15); + +#define FOR_EACH_WORD_REV(x) \ + x(15); \ + x(14); \ + x(13); \ + x(12); \ + x(11); \ + x(10); \ + x( 9); \ + x( 8); \ + x( 7); \ + x( 6); \ + x( 5); \ + x( 4); \ + x( 3); \ + x( 2); \ + x( 1); \ + x( 0); + +#define PLUS_ONE_0 1 +#define PLUS_ONE_1 2 +#define PLUS_ONE_2 3 +#define PLUS_ONE_3 4 +#define PLUS_ONE_4 5 +#define PLUS_ONE_5 6 +#define PLUS_ONE_6 7 +#define PLUS_ONE_7 8 +#define PLUS_ONE_8 9 +#define PLUS_ONE_9 10 +#define PLUS_ONE_10 11 +#define PLUS_ONE_11 12 +#define PLUS_ONE_12 13 +#define PLUS_ONE_13 14 +#define PLUS_ONE_14 15 +#define PLUS_ONE_15 16 +#define PLUS_ONE(x) PLUS_ONE_ ## x +#define _CONCAT3(a,b,c) a ## b ## c +#define CONCAT3(a,b,c) _CONCAT3(a,b,c) + +#define STORE_UNALIGNED(x) \ +CONCAT3(.Lchacha20_mips_xor_unaligned_, PLUS_ONE(x), _b: ;) \ + .if (x != 12); \ + lw T0, (x*4)(STATE); \ + .endif; \ + lwl T1, (x*4)+MSB ## (IN); \ + lwr T1, (x*4)+LSB ## (IN); \ + .if (x == 12); \ + addu X ## x, NONCE_0; \ + .else; \ + addu X ## x, T0; \ + .endif; \ + CPU_TO_LE32(X ## x); \ + xor X ## x, T1; \ + swl X ## x, (x*4)+MSB ## (OUT); \ + swr X ## x, (x*4)+LSB ## (OUT); + +#define STORE_ALIGNED(x) \ +CONCAT3(.Lchacha20_mips_xor_aligned_, PLUS_ONE(x), _b: ;) \ + .if (x != 12); \ + lw T0, (x*4)(STATE); \ + .endif; \ + lw T1, (x*4) ## (IN); \ + .if (x == 12); \ + addu X ## x, NONCE_0; \ + .else; \ + addu X ## x, T0; \ + .endif; \ + CPU_TO_LE32(X ## x); \ + xor X ## x, T1; \ + sw X ## x, (x*4) ## (OUT); + +/* Jump table macro. + * Used for setup and handling the last bytes, which are not multiple of 4. + * X15 is free to store Xn + * Every jumptable entry must be equal in size. + */ +#define JMPTBL_ALIGNED(x) \ +.Lchacha20_mips_jmptbl_aligned_ ## x: ; \ + .set noreorder; \ + b .Lchacha20_mips_xor_aligned_ ## x ## _b; \ + .if (x == 12); \ + addu SAVED_X, X ## x, NONCE_0; \ + .else; \ + addu SAVED_X, X ## x, SAVED_CA; \ + .endif; \ + .set reorder + +#define JMPTBL_UNALIGNED(x) \ +.Lchacha20_mips_jmptbl_unaligned_ ## x: ; \ + .set noreorder; \ + b .Lchacha20_mips_xor_unaligned_ ## x ## _b; \ + .if (x == 12); \ + addu SAVED_X, X ## x, NONCE_0; \ + .else; \ + addu SAVED_X, X ## x, SAVED_CA; \ + .endif; \ + .set reorder + +#define AXR(A, B, C, D, K, L, M, N, V, W, Y, Z, S) \ + addu X(A), X(K); \ + addu X(B), X(L); \ + addu X(C), X(M); \ + addu X(D), X(N); \ + xor X(V), X(A); \ + xor X(W), X(B); \ + xor X(Y), X(C); \ + xor X(Z), X(D); \ + rotl X(V), S; \ + rotl X(W), S; \ + rotl X(Y), S; \ + rotl X(Z), S; + +.text +.set reorder +.set noat +.globl chacha20_mips +.ent chacha20_mips +chacha20_mips: + .frame $sp, STACK_SIZE, $ra + + addiu $sp, -STACK_SIZE + + /* Return bytes = 0. */ + beqz BYTES, .Lchacha20_mips_end + + lw NONCE_0, 48(STATE) + + /* Save s0-s7 */ + sw $s0, 0($sp) + sw $s1, 4($sp) + sw $s2, 8($sp) + sw $s3, 12($sp) + sw $s4, 16($sp) + sw $s5, 20($sp) + sw $s6, 24($sp) + sw $s7, 28($sp) + + /* Test IN or OUT is unaligned. + * IS_UNALIGNED = ( IN | OUT ) & 0x00000003 + */ + or IS_UNALIGNED, IN, OUT + andi IS_UNALIGNED, 0x3 + + /* Set number of rounds */ + li $at, 20 + + b .Lchacha20_rounds_start + +.align 4 +.Loop_chacha20_rounds: + addiu IN, CHACHA20_BLOCK_SIZE + addiu OUT, CHACHA20_BLOCK_SIZE + addiu NONCE_0, 1 + +.Lchacha20_rounds_start: + lw X0, 0(STATE) + lw X1, 4(STATE) + lw X2, 8(STATE) + lw X3, 12(STATE) + + lw X4, 16(STATE) + lw X5, 20(STATE) + lw X6, 24(STATE) + lw X7, 28(STATE) + lw X8, 32(STATE) + lw X9, 36(STATE) + lw X10, 40(STATE) + lw X11, 44(STATE) + + move X12, NONCE_0 + lw X13, 52(STATE) + lw X14, 56(STATE) + lw X15, 60(STATE) + +.Loop_chacha20_xor_rounds: + addiu $at, -2 + AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 16); + AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 12); + AXR( 0, 1, 2, 3, 4, 5, 6, 7, 12,13,14,15, 8); + AXR( 8, 9,10,11, 12,13,14,15, 4, 5, 6, 7, 7); + AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 16); + AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 12); + AXR( 0, 1, 2, 3, 5, 6, 7, 4, 15,12,13,14, 8); + AXR(10,11, 8, 9, 15,12,13,14, 5, 6, 7, 4, 7); + bnez $at, .Loop_chacha20_xor_rounds + + addiu BYTES, -(CHACHA20_BLOCK_SIZE) + + /* Is data src/dst unaligned? Jump */ + bnez IS_UNALIGNED, .Loop_chacha20_unaligned + + /* Set number rounds here to fill delayslot. */ + li $at, 20 + + /* BYTES < 0, it has no full block. */ + bltz BYTES, .Lchacha20_mips_no_full_block_aligned + + FOR_EACH_WORD_REV(STORE_ALIGNED) + + /* BYTES > 0? Loop again. */ + bgtz BYTES, .Loop_chacha20_rounds + + /* Place this here to fill delay slot */ + addiu NONCE_0, 1 + + /* BYTES < 0? Handle last bytes */ + bltz BYTES, .Lchacha20_mips_xor_bytes + +.Lchacha20_mips_xor_done: + /* Restore used registers */ + lw $s0, 0($sp) + lw $s1, 4($sp) + lw $s2, 8($sp) + lw $s3, 12($sp) + lw $s4, 16($sp) + lw $s5, 20($sp) + lw $s6, 24($sp) + lw $s7, 28($sp) + + /* Write NONCE_0 back to right location in state */ + sw NONCE_0, 48(STATE) + +.Lchacha20_mips_end: + addiu $sp, STACK_SIZE + jr $ra + +.Lchacha20_mips_no_full_block_aligned: + /* Restore the offset on BYTES */ + addiu BYTES, CHACHA20_BLOCK_SIZE + + /* Get number of full WORDS */ + andi $at, BYTES, MASK_U32 + + /* Load upper half of jump table addr */ + lui T0, %hi(.Lchacha20_mips_jmptbl_aligned_0) + + /* Calculate lower half jump table offset */ + ins T0, $at, 1, 6 + + /* Add offset to STATE */ + addu T1, STATE, $at + + /* Add lower half jump table addr */ + addiu T0, %lo(.Lchacha20_mips_jmptbl_aligned_0) + + /* Read value from STATE */ + lw SAVED_CA, 0(T1) + + /* Store remaining bytecounter as negative value */ + subu BYTES, $at, BYTES + + jr T0 + + /* Jump table */ + FOR_EACH_WORD(JMPTBL_ALIGNED) + + +.Loop_chacha20_unaligned: + /* Set number rounds here to fill delayslot. */ + li $at, 20 + + /* BYTES > 0, it has no full block. */ + bltz BYTES, .Lchacha20_mips_no_full_block_unaligned + + FOR_EACH_WORD_REV(STORE_UNALIGNED) + + /* BYTES > 0? Loop again. */ + bgtz BYTES, .Loop_chacha20_rounds + + /* Write NONCE_0 back to right location in state */ + sw NONCE_0, 48(STATE) + + .set noreorder + /* Fall through to byte handling */ + bgez BYTES, .Lchacha20_mips_xor_done +.Lchacha20_mips_xor_unaligned_0_b: +.Lchacha20_mips_xor_aligned_0_b: + /* Place this here to fill delay slot */ + addiu NONCE_0, 1 + .set reorder + +.Lchacha20_mips_xor_bytes: + addu IN, $at + addu OUT, $at + /* First byte */ + lbu T1, 0(IN) + addiu $at, BYTES, 1 + CPU_TO_LE32(SAVED_X) + ROTR(SAVED_X) + xor T1, SAVED_X + sb T1, 0(OUT) + beqz $at, .Lchacha20_mips_xor_done + /* Second byte */ + lbu T1, 1(IN) + addiu $at, BYTES, 2 + ROTx SAVED_X, 8 + xor T1, SAVED_X + sb T1, 1(OUT) + beqz $at, .Lchacha20_mips_xor_done + /* Third byte */ + lbu T1, 2(IN) + ROTx SAVED_X, 8 + xor T1, SAVED_X + sb T1, 2(OUT) + b .Lchacha20_mips_xor_done + +.Lchacha20_mips_no_full_block_unaligned: + /* Restore the offset on BYTES */ + addiu BYTES, CHACHA20_BLOCK_SIZE + + /* Get number of full WORDS */ + andi $at, BYTES, MASK_U32 + + /* Load upper half of jump table addr */ + lui T0, %hi(.Lchacha20_mips_jmptbl_unaligned_0) + + /* Calculate lower half jump table offset */ + ins T0, $at, 1, 6 + + /* Add offset to STATE */ + addu T1, STATE, $at + + /* Add lower half jump table addr */ + addiu T0, %lo(.Lchacha20_mips_jmptbl_unaligned_0) + + /* Read value from STATE */ + lw SAVED_CA, 0(T1) + + /* Store remaining bytecounter as negative value */ + subu BYTES, $at, BYTES + + jr T0 + + /* Jump table */ + FOR_EACH_WORD(JMPTBL_UNALIGNED) +.end chacha20_mips +.set at diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S b/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S new file mode 100644 index 0000000000000..8fb4bc2e7b5b2 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S @@ -0,0 +1,461 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018 Google, Inc. + */ + +#include +#include + +/* + * Design notes: + * + * 16 registers would be needed to hold the state matrix, but only 14 are + * available because 'sp' and 'pc' cannot be used. So we spill the elements + * (x8, x9) to the stack and swap them out with (x10, x11). This adds one + * 'ldrd' and one 'strd' instruction per round. + * + * All rotates are performed using the implicit rotate operand accepted by the + * 'add' and 'eor' instructions. This is faster than using explicit rotate + * instructions. To make this work, we allow the values in the second and last + * rows of the ChaCha state matrix (rows 'b' and 'd') to temporarily have the + * wrong rotation amount. The rotation amount is then fixed up just in time + * when the values are used. 'brot' is the number of bits the values in row 'b' + * need to be rotated right to arrive at the correct values, and 'drot' + * similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such + * that they end up as (25, 24) after every round. + */ + + // ChaCha state registers + X0 .req r0 + X1 .req r1 + X2 .req r2 + X3 .req r3 + X4 .req r4 + X5 .req r5 + X6 .req r6 + X7 .req r7 + X8_X10 .req r8 // shared by x8 and x10 + X9_X11 .req r9 // shared by x9 and x11 + X12 .req r10 + X13 .req r11 + X14 .req r12 + X15 .req r14 + +.Lexpand_32byte_k: + // "expand 32-byte k" + .word 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + +#ifdef __thumb2__ +# define adrl adr +#endif + +.macro __rev out, in, t0, t1, t2 +.if __LINUX_ARM_ARCH__ >= 6 + rev \out, \in +.else + lsl \t0, \in, #24 + and \t1, \in, #0xff00 + and \t2, \in, #0xff0000 + orr \out, \t0, \in, lsr #24 + orr \out, \out, \t1, lsl #8 + orr \out, \out, \t2, lsr #8 +.endif +.endm + +.macro _le32_bswap x, t0, t1, t2 +#ifdef __ARMEB__ + __rev \x, \x, \t0, \t1, \t2 +#endif +.endm + +.macro _le32_bswap_4x a, b, c, d, t0, t1, t2 + _le32_bswap \a, \t0, \t1, \t2 + _le32_bswap \b, \t0, \t1, \t2 + _le32_bswap \c, \t0, \t1, \t2 + _le32_bswap \d, \t0, \t1, \t2 +.endm + +.macro __ldrd a, b, src, offset +#if __LINUX_ARM_ARCH__ >= 6 + ldrd \a, \b, [\src, #\offset] +#else + ldr \a, [\src, #\offset] + ldr \b, [\src, #\offset + 4] +#endif +.endm + +.macro __strd a, b, dst, offset +#if __LINUX_ARM_ARCH__ >= 6 + strd \a, \b, [\dst, #\offset] +#else + str \a, [\dst, #\offset] + str \b, [\dst, #\offset + 4] +#endif +.endm + +.macro _halfround a1, b1, c1, d1, a2, b2, c2, d2 + + // a += b; d ^= a; d = rol(d, 16); + add \a1, \a1, \b1, ror #brot + add \a2, \a2, \b2, ror #brot + eor \d1, \a1, \d1, ror #drot + eor \d2, \a2, \d2, ror #drot + // drot == 32 - 16 == 16 + + // c += d; b ^= c; b = rol(b, 12); + add \c1, \c1, \d1, ror #16 + add \c2, \c2, \d2, ror #16 + eor \b1, \c1, \b1, ror #brot + eor \b2, \c2, \b2, ror #brot + // brot == 32 - 12 == 20 + + // a += b; d ^= a; d = rol(d, 8); + add \a1, \a1, \b1, ror #20 + add \a2, \a2, \b2, ror #20 + eor \d1, \a1, \d1, ror #16 + eor \d2, \a2, \d2, ror #16 + // drot == 32 - 8 == 24 + + // c += d; b ^= c; b = rol(b, 7); + add \c1, \c1, \d1, ror #24 + add \c2, \c2, \d2, ror #24 + eor \b1, \c1, \b1, ror #20 + eor \b2, \c2, \b2, ror #20 + // brot == 32 - 7 == 25 +.endm + +.macro _doubleround + + // column round + + // quarterrounds: (x0, x4, x8, x12) and (x1, x5, x9, x13) + _halfround X0, X4, X8_X10, X12, X1, X5, X9_X11, X13 + + // save (x8, x9); restore (x10, x11) + __strd X8_X10, X9_X11, sp, 0 + __ldrd X8_X10, X9_X11, sp, 8 + + // quarterrounds: (x2, x6, x10, x14) and (x3, x7, x11, x15) + _halfround X2, X6, X8_X10, X14, X3, X7, X9_X11, X15 + + .set brot, 25 + .set drot, 24 + + // diagonal round + + // quarterrounds: (x0, x5, x10, x15) and (x1, x6, x11, x12) + _halfround X0, X5, X8_X10, X15, X1, X6, X9_X11, X12 + + // save (x10, x11); restore (x8, x9) + __strd X8_X10, X9_X11, sp, 8 + __ldrd X8_X10, X9_X11, sp, 0 + + // quarterrounds: (x2, x7, x8, x13) and (x3, x4, x9, x14) + _halfround X2, X7, X8_X10, X13, X3, X4, X9_X11, X14 +.endm + +.macro _chacha_permute nrounds + .set brot, 0 + .set drot, 0 + .rept \nrounds / 2 + _doubleround + .endr +.endm + +.macro _chacha nrounds + +.Lnext_block\@: + // Stack: unused0-unused1 x10-x11 x0-x15 OUT IN LEN + // Registers contain x0-x9,x12-x15. + + // Do the core ChaCha permutation to update x0-x15. + _chacha_permute \nrounds + + add sp, #8 + // Stack: x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers contain x0-x9,x12-x15. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // Free up some registers (r8-r12,r14) by pushing (x8-x9,x12-x15). + push {X8_X10, X9_X11, X12, X13, X14, X15} + + // Load (OUT, IN, LEN). + ldr r14, [sp, #96] + ldr r12, [sp, #100] + ldr r11, [sp, #104] + + orr r10, r14, r12 + + // Use slow path if fewer than 64 bytes remain. + cmp r11, #64 + blt .Lxor_slowpath\@ + + // Use slow path if IN and/or OUT isn't 4-byte aligned. Needed even on + // ARMv6+, since ldmia and stmia (used below) still require alignment. + tst r10, #3 + bne .Lxor_slowpath\@ + + // Fast path: XOR 64 bytes of aligned data. + + // Stack: x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is OUT. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // x0-x3 + __ldrd r8, r9, sp, 32 + __ldrd r10, r11, sp, 40 + add X0, X0, r8 + add X1, X1, r9 + add X2, X2, r10 + add X3, X3, r11 + _le32_bswap_4x X0, X1, X2, X3, r8, r9, r10 + ldmia r12!, {r8-r11} + eor X0, X0, r8 + eor X1, X1, r9 + eor X2, X2, r10 + eor X3, X3, r11 + stmia r14!, {X0-X3} + + // x4-x7 + __ldrd r8, r9, sp, 48 + __ldrd r10, r11, sp, 56 + add X4, r8, X4, ror #brot + add X5, r9, X5, ror #brot + ldmia r12!, {X0-X3} + add X6, r10, X6, ror #brot + add X7, r11, X7, ror #brot + _le32_bswap_4x X4, X5, X6, X7, r8, r9, r10 + eor X4, X4, X0 + eor X5, X5, X1 + eor X6, X6, X2 + eor X7, X7, X3 + stmia r14!, {X4-X7} + + // x8-x15 + pop {r0-r7} // (x8-x9,x12-x15,x10-x11) + __ldrd r8, r9, sp, 32 + __ldrd r10, r11, sp, 40 + add r0, r0, r8 // x8 + add r1, r1, r9 // x9 + add r6, r6, r10 // x10 + add r7, r7, r11 // x11 + _le32_bswap_4x r0, r1, r6, r7, r8, r9, r10 + ldmia r12!, {r8-r11} + eor r0, r0, r8 // x8 + eor r1, r1, r9 // x9 + eor r6, r6, r10 // x10 + eor r7, r7, r11 // x11 + stmia r14!, {r0,r1,r6,r7} + ldmia r12!, {r0,r1,r6,r7} + __ldrd r8, r9, sp, 48 + __ldrd r10, r11, sp, 56 + add r2, r8, r2, ror #drot // x12 + add r3, r9, r3, ror #drot // x13 + add r4, r10, r4, ror #drot // x14 + add r5, r11, r5, ror #drot // x15 + _le32_bswap_4x r2, r3, r4, r5, r9, r10, r11 + ldr r9, [sp, #72] // load LEN + eor r2, r2, r0 // x12 + eor r3, r3, r1 // x13 + eor r4, r4, r6 // x14 + eor r5, r5, r7 // x15 + subs r9, #64 // decrement and check LEN + stmia r14!, {r2-r5} + + beq .Ldone\@ + +.Lprepare_for_next_block\@: + + // Stack: x0-x15 OUT IN LEN + + // Increment block counter (x12) + add r8, #1 + + // Store updated (OUT, IN, LEN) + str r14, [sp, #64] + str r12, [sp, #68] + str r9, [sp, #72] + + mov r14, sp + + // Store updated block counter (x12) + str r8, [sp, #48] + + sub sp, #16 + + // Reload state and do next block + ldmia r14!, {r0-r11} // load x0-x11 + __strd r10, r11, sp, 8 // store x10-x11 before state + ldmia r14, {r10-r12,r14} // load x12-x15 + b .Lnext_block\@ + +.Lxor_slowpath\@: + // Slow path: < 64 bytes remaining, or unaligned input or output buffer. + // We handle it by storing the 64 bytes of keystream to the stack, then + // XOR-ing the needed portion with the data. + + // Allocate keystream buffer + sub sp, #64 + mov r14, sp + + // Stack: ks0-ks15 x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN + // Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is &ks0. + // x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'. + + // Save keystream for x0-x3 + __ldrd r8, r9, sp, 96 + __ldrd r10, r11, sp, 104 + add X0, X0, r8 + add X1, X1, r9 + add X2, X2, r10 + add X3, X3, r11 + _le32_bswap_4x X0, X1, X2, X3, r8, r9, r10 + stmia r14!, {X0-X3} + + // Save keystream for x4-x7 + __ldrd r8, r9, sp, 112 + __ldrd r10, r11, sp, 120 + add X4, r8, X4, ror #brot + add X5, r9, X5, ror #brot + add X6, r10, X6, ror #brot + add X7, r11, X7, ror #brot + _le32_bswap_4x X4, X5, X6, X7, r8, r9, r10 + add r8, sp, #64 + stmia r14!, {X4-X7} + + // Save keystream for x8-x15 + ldm r8, {r0-r7} // (x8-x9,x12-x15,x10-x11) + __ldrd r8, r9, sp, 128 + __ldrd r10, r11, sp, 136 + add r0, r0, r8 // x8 + add r1, r1, r9 // x9 + add r6, r6, r10 // x10 + add r7, r7, r11 // x11 + _le32_bswap_4x r0, r1, r6, r7, r8, r9, r10 + stmia r14!, {r0,r1,r6,r7} + __ldrd r8, r9, sp, 144 + __ldrd r10, r11, sp, 152 + add r2, r8, r2, ror #drot // x12 + add r3, r9, r3, ror #drot // x13 + add r4, r10, r4, ror #drot // x14 + add r5, r11, r5, ror #drot // x15 + _le32_bswap_4x r2, r3, r4, r5, r9, r10, r11 + stmia r14, {r2-r5} + + // Stack: ks0-ks15 unused0-unused7 x0-x15 OUT IN LEN + // Registers: r8 is block counter, r12 is IN. + + ldr r9, [sp, #168] // LEN + ldr r14, [sp, #160] // OUT + cmp r9, #64 + mov r0, sp + movle r1, r9 + movgt r1, #64 + // r1 is number of bytes to XOR, in range [1, 64] + +.if __LINUX_ARM_ARCH__ < 6 + orr r2, r12, r14 + tst r2, #3 // IN or OUT misaligned? + bne .Lxor_next_byte\@ +.endif + + // XOR a word at a time +.rept 16 + subs r1, #4 + blt .Lxor_words_done\@ + ldr r2, [r12], #4 + ldr r3, [r0], #4 + eor r2, r2, r3 + str r2, [r14], #4 +.endr + b .Lxor_slowpath_done\@ +.Lxor_words_done\@: + ands r1, r1, #3 + beq .Lxor_slowpath_done\@ + + // XOR a byte at a time +.Lxor_next_byte\@: + ldrb r2, [r12], #1 + ldrb r3, [r0], #1 + eor r2, r2, r3 + strb r2, [r14], #1 + subs r1, #1 + bne .Lxor_next_byte\@ + +.Lxor_slowpath_done\@: + subs r9, #64 + add sp, #96 + bgt .Lprepare_for_next_block\@ + +.Ldone\@: +.endm // _chacha + +/* + * void chacha20_arm(u8 *out, const u8 *in, size_t len, const u32 key[8], + * const u32 iv[4]); + */ +SYM_FUNC_START(chacha20_arm) + cmp r2, #0 // len == 0? + reteq lr + + push {r0-r2,r4-r11,lr} + + // Push state x0-x15 onto stack. + // Also store an extra copy of x10-x11 just before the state. + + ldr r4, [sp, #48] // iv + mov r0, sp + sub sp, #80 + + // iv: x12-x15 + ldm r4, {X12,X13,X14,X15} + stmdb r0!, {X12,X13,X14,X15} + + // key: x4-x11 + __ldrd X8_X10, X9_X11, r3, 24 + __strd X8_X10, X9_X11, sp, 8 + stmdb r0!, {X8_X10, X9_X11} + ldm r3, {X4-X9_X11} + stmdb r0!, {X4-X9_X11} + + // constants: x0-x3 + adrl X3, .Lexpand_32byte_k + ldm X3, {X0-X3} + __strd X0, X1, sp, 16 + __strd X2, X3, sp, 24 + + _chacha 20 + + add sp, #76 + pop {r4-r11, pc} +SYM_FUNC_END(chacha20_arm) + +/* + * void hchacha20_arm(const u32 state[16], u32 out[8]); + */ +SYM_FUNC_START(hchacha20_arm) + push {r1,r4-r11,lr} + + mov r14, r0 + ldmia r14!, {r0-r11} // load x0-x11 + push {r10-r11} // store x10-x11 to stack + ldm r14, {r10-r12,r14} // load x12-x15 + sub sp, #8 + + _chacha_permute 20 + + // Skip over (unused0-unused1, x10-x11) + add sp, #16 + + // Fix up rotations of x12-x15 + ror X12, X12, #drot + ror X13, X13, #drot + pop {r4} // load 'out' + ror X14, X14, #drot + ror X15, X15, #drot + + // Store (x0-x3,x12-x15) to 'out' + stm r4, {X0,X1,X2,X3,X12,X13,X14,X15} + + pop {r4-r11,pc} +SYM_FUNC_END(hchacha20_arm) diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c new file mode 100644 index 0000000000000..5ac5f686a641a --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64-glue.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include +#include + +asmlinkage void hchacha20_ssse3(u32 *derived_key, const u8 *nonce, + const u8 *key); +asmlinkage void chacha20_ssse3(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx2(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx512(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); +asmlinkage void chacha20_avx512vl(u8 *out, const u8 *in, const size_t len, + const u32 key[8], const u32 counter[4]); + +static bool chacha20_use_ssse3 __ro_after_init; +static bool chacha20_use_avx2 __ro_after_init; +static bool chacha20_use_avx512 __ro_after_init; +static bool chacha20_use_avx512vl __ro_after_init; +static bool *const chacha20_nobs[] __initconst = { + &chacha20_use_ssse3, &chacha20_use_avx2, &chacha20_use_avx512, + &chacha20_use_avx512vl }; + +static void __init chacha20_fpu_init(void) +{ + chacha20_use_ssse3 = boot_cpu_has(X86_FEATURE_SSSE3); + chacha20_use_avx2 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + chacha20_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; + chacha20_use_avx512vl = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL); +#endif +} + +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < CHACHA20_BLOCK_SIZE || + PAGE_SIZE % CHACHA20_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_AS_SSSE3) || !chacha20_use_ssse3 || + len <= CHACHA20_BLOCK_SIZE || !simd_use(simd_context)) + return false; + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512 && + len >= CHACHA20_BLOCK_SIZE * 8) + chacha20_avx512(dst, src, bytes, ctx->key, ctx->counter); + else if (IS_ENABLED(CONFIG_AS_AVX512) && chacha20_use_avx512vl && + len >= CHACHA20_BLOCK_SIZE * 4) + chacha20_avx512vl(dst, src, bytes, ctx->key, ctx->counter); + else if (IS_ENABLED(CONFIG_AS_AVX2) && chacha20_use_avx2 && + len >= CHACHA20_BLOCK_SIZE * 4) + chacha20_avx2(dst, src, bytes, ctx->key, ctx->counter); + else + chacha20_ssse3(dst, src, bytes, ctx->key, ctx->counter); + ctx->counter[0] += (bytes + 63) / 64; + len -= bytes; + if (!len) + break; + dst += bytes; + src += bytes; + simd_relax(simd_context); + } + + return true; +} + +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + if (IS_ENABLED(CONFIG_AS_SSSE3) && chacha20_use_ssse3 && + simd_use(simd_context)) { + hchacha20_ssse3(derived_key, nonce, key); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl new file mode 100644 index 0000000000000..29906a66b8b73 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20-x86_64.pl @@ -0,0 +1,4106 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# Copyright (C) 2017-2019 Samuel Neves . All Rights Reserved. +# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. +# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# November 2014 +# +# ChaCha20 for x86_64. +# +# December 2016 +# +# Add AVX512F code path. +# +# December 2017 +# +# Add AVX512VL code path. +# +# Performance in cycles per byte out of large buffer. +# +# IALU/gcc 4.8(i) 1x/2xSSSE3(ii) 4xSSSE3 NxAVX(v) +# +# P4 9.48/+99% - - +# Core2 7.83/+55% 7.90/5.76 4.35 +# Westmere 7.19/+50% 5.60/4.50 3.00 +# Sandy Bridge 8.31/+42% 5.45/4.00 2.72 +# Ivy Bridge 6.71/+46% 5.40/? 2.41 +# Haswell 5.92/+43% 5.20/3.45 2.42 1.23 +# Skylake[-X] 5.87/+39% 4.70/3.22 2.31 1.19[0.80(vi)] +# Silvermont 12.0/+33% 7.75/6.90 7.03(iii) +# Knights L 11.7/- ? 9.60(iii) 0.80 +# Goldmont 10.6/+17% 5.10/3.52 3.28 +# Sledgehammer 7.28/+52% - - +# Bulldozer 9.66/+28% 9.85/5.35(iv) 3.06(iv) +# Ryzen 5.96/+50% 5.19/3.00 2.40 2.09 +# VIA Nano 10.5/+46% 6.72/6.88 6.05 +# +# (i) compared to older gcc 3.x one can observe >2x improvement on +# most platforms; +# (ii) 2xSSSE3 is code path optimized specifically for 128 bytes used +# by chacha20_poly1305_tls_cipher, results are EVP-free; +# (iii) this is not optimal result for Atom because of MSROM +# limitations, SSE2 can do better, but gain is considered too +# low to justify the [maintenance] effort; +# (iv) Bulldozer actually executes 4xXOP code path that delivers 2.20 +# and 4.85 for 128-byte inputs; +# (v) 8xAVX2, 8xAVX512VL or 16xAVX512F, whichever best applicable; +# (vi) even though Skylake-X can execute AVX512F code and deliver 0.57 +# cpb in single thread, the corresponding capability is suppressed; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$kernel=0; $kernel=1 if (!$flavour && !$output); + +if (!$kernel) { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or + die "can't locate x86_64-xlate.pl"; + + open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; + *STDOUT=*OUT; + + if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); + } + + if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); + $avx += 1 if ($1==2.11 && $2>=8); + } + + if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + } + + if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); + } +} else { + $avx = 4; # The kernel uses ifdefs for this. +} + +# input parameter block +($out,$inp,$len,$key,$counter)=("%rdi","%rsi","%rdx","%rcx","%r8"); + +$code.=<<___ if $kernel; +#include +___ + +sub declare_variable() { + my ($name, $size, $type, $payload) = @_; + if($kernel) { + $code.=".section .rodata.cst$size.L$name, \"aM\", \@progbits, $size\n"; + $code.=".align $size\n"; + $code.=".L$name:\n"; + $code.=".$type $payload\n"; + } else { + $code.=".L$name:\n"; + $code.=".$type $payload\n"; + } +} + +sub declare_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= ".align $align\n"; + $code .= "SYM_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } +} + +sub end_function() { + my ($name) = @_; + if($kernel) { + $code .= "SYM_FUNC_END($name)\n"; + } else { + $code .= ".size $name,.-$name\n"; + } +} + +if(!$kernel) { + $code .= ".text\n"; +} +&declare_variable('zero', 16, 'long', '0,0,0,0'); +&declare_variable('one', 16, 'long', '1,0,0,0'); +&declare_variable('inc', 16, 'long', '0,1,2,3'); +&declare_variable('four', 16, 'long', '4,4,4,4'); +&declare_variable('incy', 32, 'long', '0,2,4,6,1,3,5,7'); +&declare_variable('eight', 32, 'long', '8,8,8,8,8,8,8,8'); +&declare_variable('rot16', 16, 'byte', '0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd'); +&declare_variable('rot24', 16, 'byte', '0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe'); +&declare_variable('twoy', 32, 'long', '2,0,0,0, 2,0,0,0'); +&declare_variable('zeroz', 64, 'long', '0,0,0,0, 1,0,0,0, 2,0,0,0, 3,0,0,0'); +&declare_variable('fourz', 64, 'long', '4,0,0,0, 4,0,0,0, 4,0,0,0, 4,0,0,0'); +&declare_variable('incz', 64, 'long', '0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15'); +&declare_variable('sixteen', 64, 'long', '16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16'); +&declare_variable('sigma', 16, 'ascii', '"expand 32-byte k"'); + +$code.=<<___ if !$kernel; +.asciz "ChaCha20 for x86_64, CRYPTOGAMS by " +___ +$code.=".text\n"; + +sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; + my $arg = pop; + $arg = "\$$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n"; +} + +@x=("%eax","%ebx","%ecx","%edx",map("%r${_}d",(8..11)), + "%nox","%nox","%nox","%nox",map("%r${_}d",(12..15))); +@t=("%esi","%edi"); + +sub ROUND { # critical path is 24 cycles per round +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_)=map("\"$_\"",@t); +my @x=map("\"$_\"",@x); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + # Normally instructions would be interleaved to favour in-order + # execution. Generally out-of-order cores manage it gracefully, + # but not this time for some reason. As in-order execution + # cores are dying breed, old Atom is the only one around, + # instructions are left uninterleaved. Besides, Atom is better + # off executing 1xSSSE3 code anyway... + + ( + "&add (@x[$a0],@x[$b0])", # Q1 + "&xor (@x[$d0],@x[$a0])", + "&rol (@x[$d0],16)", + "&add (@x[$a1],@x[$b1])", # Q2 + "&xor (@x[$d1],@x[$a1])", + "&rol (@x[$d1],16)", + + "&add ($xc,@x[$d0])", + "&xor (@x[$b0],$xc)", + "&rol (@x[$b0],12)", + "&add ($xc_,@x[$d1])", + "&xor (@x[$b1],$xc_)", + "&rol (@x[$b1],12)", + + "&add (@x[$a0],@x[$b0])", + "&xor (@x[$d0],@x[$a0])", + "&rol (@x[$d0],8)", + "&add (@x[$a1],@x[$b1])", + "&xor (@x[$d1],@x[$a1])", + "&rol (@x[$d1],8)", + + "&add ($xc,@x[$d0])", + "&xor (@x[$b0],$xc)", + "&rol (@x[$b0],7)", + "&add ($xc_,@x[$d1])", + "&xor (@x[$b1],$xc_)", + "&rol (@x[$b1],7)", + + "&mov (\"4*$c0(%rsp)\",$xc)", # reload pair of 'c's + "&mov (\"4*$c1(%rsp)\",$xc_)", + "&mov ($xc,\"4*$c2(%rsp)\")", + "&mov ($xc_,\"4*$c3(%rsp)\")", + + "&add (@x[$a2],@x[$b2])", # Q3 + "&xor (@x[$d2],@x[$a2])", + "&rol (@x[$d2],16)", + "&add (@x[$a3],@x[$b3])", # Q4 + "&xor (@x[$d3],@x[$a3])", + "&rol (@x[$d3],16)", + + "&add ($xc,@x[$d2])", + "&xor (@x[$b2],$xc)", + "&rol (@x[$b2],12)", + "&add ($xc_,@x[$d3])", + "&xor (@x[$b3],$xc_)", + "&rol (@x[$b3],12)", + + "&add (@x[$a2],@x[$b2])", + "&xor (@x[$d2],@x[$a2])", + "&rol (@x[$d2],8)", + "&add (@x[$a3],@x[$b3])", + "&xor (@x[$d3],@x[$a3])", + "&rol (@x[$d3],8)", + + "&add ($xc,@x[$d2])", + "&xor (@x[$b2],$xc)", + "&rol (@x[$b2],7)", + "&add ($xc_,@x[$d3])", + "&xor (@x[$b3],$xc_)", + "&rol (@x[$b3],7)" + ); +} + +######################################################################## +# Generic code path that handles all lengths on pre-SSSE3 processors. +if(!$kernel) { +&declare_function("chacha20_ctr32", 64, 5); +$code.=<<___; +.cfi_startproc + cmp \$0,$len + je .Lno_data + mov OPENSSL_ia32cap_P+4(%rip),%r9 +___ +$code.=<<___ if ($avx>2); + bt \$48,%r9 # check for AVX512F + jc .Lchacha20_avx512 + test %r9,%r9 # check for AVX512VL + js .Lchacha20_avx512vl +___ +$code.=<<___; + test \$`1<<(41-32)`,%r9d + jnz .Lchacha20_ssse3 +___ +$code.=<<___; + push %rbx +.cfi_push %rbx + push %rbp +.cfi_push %rbp + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + sub \$64+24,%rsp +.cfi_adjust_cfa_offset 64+24 +.Lctr32_body: + + #movdqa .Lsigma(%rip),%xmm0 + movdqu ($key),%xmm1 + movdqu 16($key),%xmm2 + movdqu ($counter),%xmm3 + movdqa .Lone(%rip),%xmm4 + + #movdqa %xmm0,4*0(%rsp) # key[0] + movdqa %xmm1,4*4(%rsp) # key[1] + movdqa %xmm2,4*8(%rsp) # key[2] + movdqa %xmm3,4*12(%rsp) # key[3] + mov $len,%rbp # reassign $len + jmp .Loop_outer + +.align 32 +.Loop_outer: + mov \$0x61707865,@x[0] # 'expa' + mov \$0x3320646e,@x[1] # 'nd 3' + mov \$0x79622d32,@x[2] # '2-by' + mov \$0x6b206574,@x[3] # 'te k' + mov 4*4(%rsp),@x[4] + mov 4*5(%rsp),@x[5] + mov 4*6(%rsp),@x[6] + mov 4*7(%rsp),@x[7] + movd %xmm3,@x[12] + mov 4*13(%rsp),@x[13] + mov 4*14(%rsp),@x[14] + mov 4*15(%rsp),@x[15] + + mov %rbp,64+0(%rsp) # save len + mov \$10,%ebp + mov $inp,64+8(%rsp) # save inp + movq %xmm2,%rsi # "@x[8]" + mov $out,64+16(%rsp) # save out + mov %rsi,%rdi + shr \$32,%rdi # "@x[9]" + jmp .Loop + +.align 32 +.Loop: +___ + foreach (&ROUND (0, 4, 8,12)) { eval; } + foreach (&ROUND (0, 5,10,15)) { eval; } + &dec ("%ebp"); + &jnz (".Loop"); + +$code.=<<___; + mov @t[1],4*9(%rsp) # modulo-scheduled + mov @t[0],4*8(%rsp) + mov 64(%rsp),%rbp # load len + movdqa %xmm2,%xmm1 + mov 64+8(%rsp),$inp # load inp + paddd %xmm4,%xmm3 # increment counter + mov 64+16(%rsp),$out # load out + + add \$0x61707865,@x[0] # 'expa' + add \$0x3320646e,@x[1] # 'nd 3' + add \$0x79622d32,@x[2] # '2-by' + add \$0x6b206574,@x[3] # 'te k' + add 4*4(%rsp),@x[4] + add 4*5(%rsp),@x[5] + add 4*6(%rsp),@x[6] + add 4*7(%rsp),@x[7] + add 4*12(%rsp),@x[12] + add 4*13(%rsp),@x[13] + add 4*14(%rsp),@x[14] + add 4*15(%rsp),@x[15] + paddd 4*8(%rsp),%xmm1 + + cmp \$64,%rbp + jb .Ltail + + xor 4*0($inp),@x[0] # xor with input + xor 4*1($inp),@x[1] + xor 4*2($inp),@x[2] + xor 4*3($inp),@x[3] + xor 4*4($inp),@x[4] + xor 4*5($inp),@x[5] + xor 4*6($inp),@x[6] + xor 4*7($inp),@x[7] + movdqu 4*8($inp),%xmm0 + xor 4*12($inp),@x[12] + xor 4*13($inp),@x[13] + xor 4*14($inp),@x[14] + xor 4*15($inp),@x[15] + lea 4*16($inp),$inp # inp+=64 + pxor %xmm1,%xmm0 + + movdqa %xmm2,4*8(%rsp) + movd %xmm3,4*12(%rsp) + + mov @x[0],4*0($out) # write output + mov @x[1],4*1($out) + mov @x[2],4*2($out) + mov @x[3],4*3($out) + mov @x[4],4*4($out) + mov @x[5],4*5($out) + mov @x[6],4*6($out) + mov @x[7],4*7($out) + movdqu %xmm0,4*8($out) + mov @x[12],4*12($out) + mov @x[13],4*13($out) + mov @x[14],4*14($out) + mov @x[15],4*15($out) + lea 4*16($out),$out # out+=64 + + sub \$64,%rbp + jnz .Loop_outer + + jmp .Ldone + +.align 16 +.Ltail: + mov @x[0],4*0(%rsp) + mov @x[1],4*1(%rsp) + xor %rbx,%rbx + mov @x[2],4*2(%rsp) + mov @x[3],4*3(%rsp) + mov @x[4],4*4(%rsp) + mov @x[5],4*5(%rsp) + mov @x[6],4*6(%rsp) + mov @x[7],4*7(%rsp) + movdqa %xmm1,4*8(%rsp) + mov @x[12],4*12(%rsp) + mov @x[13],4*13(%rsp) + mov @x[14],4*14(%rsp) + mov @x[15],4*15(%rsp) + +.Loop_tail: + movzb ($inp,%rbx),%eax + movzb (%rsp,%rbx),%edx + lea 1(%rbx),%rbx + xor %edx,%eax + mov %al,-1($out,%rbx) + dec %rbp + jnz .Loop_tail + +.Ldone: + add \$64+24,%rsp +.cfi_adjust_cfa_offset -64-24 + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbp +.cfi_restore %rbp + pop %rbx +.cfi_restore %rbx +.Lno_data: + ret +.cfi_endproc +___ +&end_function("chacha20_ctr32"); +} + +######################################################################## +# SSSE3 code path that handles shorter lengths +{ +my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(0..7)); + +sub SSSE3ROUND { # critical path is 20 "SIMD ticks" per round + &paddd ($a,$b); + &pxor ($d,$a); + &pshufb ($d,$rot16); + + &paddd ($c,$d); + &pxor ($b,$c); + &movdqa ($t,$b); + &psrld ($b,20); + &pslld ($t,12); + &por ($b,$t); + + &paddd ($a,$b); + &pxor ($d,$a); + &pshufb ($d,$rot24); + + &paddd ($c,$d); + &pxor ($b,$c); + &movdqa ($t,$b); + &psrld ($b,25); + &pslld ($t,7); + &por ($b,$t); +} + +my $xframe = $win64 ? 32+8 : 8; + +if($kernel) { + $code .= "#ifdef CONFIG_AS_SSSE3\n"; +} + +if($kernel) { +&declare_function("hchacha20_ssse3", 32, 5); +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($len),$b + movdqu 16($len),$c + movdqu ($inp),$d + # This code is only used when targeting kernel. + # If targeting win64, xmm{6,7} preserving needs to be added. + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + mov \$10,$counter # reuse $counter + jmp 1f +.align 32 +1: +___ + &SSSE3ROUND(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &nop (); + + &SSSE3ROUND(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + + &dec ($counter); + &jnz ("1b"); + +$code.=<<___; + movdqu $a, ($out) + movdqu $d, 16($out) + ret +___ +&end_function("hchacha20_ssse3"); +} + +&declare_function("chacha20_ssse3", 32, 5); +$code.=<<___; +.cfi_startproc + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 +___ +$code.=<<___ if ($avx && !$kernel); + test \$`1<<(43-32)`,%r10d + jnz .Lchacha20_4xop # XOP is fastest even if we use 1/4 +___ +$code.=<<___; + cmp \$128,$len # we might throw away some data, + je .Lchacha20_128 + ja .Lchacha20_4x # but overall it won't be slower + +.Ldo_ssse3_after_all: + sub \$64+$xframe,%rsp + and \$-16,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lssse3_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($key),$b + movdqu 16($key),$c + movdqu ($counter),$d + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + + movdqa $a,0x00(%rsp) + movdqa $b,0x10(%rsp) + movdqa $c,0x20(%rsp) + movdqa $d,0x30(%rsp) + mov \$10,$counter # reuse $counter + jmp .Loop_ssse3 + +.align 32 +.Loop_outer_ssse3: + movdqa .Lone(%rip),$d + movdqa 0x00(%rsp),$a + movdqa 0x10(%rsp),$b + movdqa 0x20(%rsp),$c + paddd 0x30(%rsp),$d + mov \$10,$counter + movdqa $d,0x30(%rsp) + jmp .Loop_ssse3 + +.align 32 +.Loop_ssse3: +___ + &SSSE3ROUND(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &nop (); + + &SSSE3ROUND(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + + &dec ($counter); + &jnz (".Loop_ssse3"); + +$code.=<<___; + paddd 0x00(%rsp),$a + paddd 0x10(%rsp),$b + paddd 0x20(%rsp),$c + paddd 0x30(%rsp),$d + + cmp \$64,$len + jb .Ltail_ssse3 + + movdqu 0x00($inp),$t + movdqu 0x10($inp),$t1 + pxor $t,$a # xor with input + movdqu 0x20($inp),$t + pxor $t1,$b + movdqu 0x30($inp),$t1 + lea 0x40($inp),$inp # inp+=64 + pxor $t,$c + pxor $t1,$d + + movdqu $a,0x00($out) # write output + movdqu $b,0x10($out) + movdqu $c,0x20($out) + movdqu $d,0x30($out) + lea 0x40($out),$out # out+=64 + + sub \$64,$len + jnz .Loop_outer_ssse3 + + jmp .Ldone_ssse3 + +.align 16 +.Ltail_ssse3: + movdqa $a,0x00(%rsp) + movdqa $b,0x10(%rsp) + movdqa $c,0x20(%rsp) + movdqa $d,0x30(%rsp) + xor $counter,$counter + +.Loop_tail_ssse3: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_ssse3 + +.Ldone_ssse3: +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lssse3_epilogue: + ret +.cfi_endproc +___ +} +&end_function("chacha20_ssse3"); + +######################################################################## +# SSSE3 code path that handles 128-byte inputs +{ +my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(8,9,2..7)); +my ($a1,$b1,$c1,$d1)=map("%xmm$_",(10,11,0,1)); + +sub SSSE3ROUND_2x { + &paddd ($a,$b); + &pxor ($d,$a); + &paddd ($a1,$b1); + &pxor ($d1,$a1); + &pshufb ($d,$rot16); + &pshufb($d1,$rot16); + + &paddd ($c,$d); + &paddd ($c1,$d1); + &pxor ($b,$c); + &pxor ($b1,$c1); + &movdqa ($t,$b); + &psrld ($b,20); + &movdqa($t1,$b1); + &pslld ($t,12); + &psrld ($b1,20); + &por ($b,$t); + &pslld ($t1,12); + &por ($b1,$t1); + + &paddd ($a,$b); + &pxor ($d,$a); + &paddd ($a1,$b1); + &pxor ($d1,$a1); + &pshufb ($d,$rot24); + &pshufb($d1,$rot24); + + &paddd ($c,$d); + &paddd ($c1,$d1); + &pxor ($b,$c); + &pxor ($b1,$c1); + &movdqa ($t,$b); + &psrld ($b,25); + &movdqa($t1,$b1); + &pslld ($t,7); + &psrld ($b1,25); + &por ($b,$t); + &pslld ($t1,7); + &por ($b1,$t1); +} + +my $xframe = $win64 ? 0x68 : 8; + +$code.=<<___; +.type chacha20_128,\@function,5 +.align 32 +chacha20_128: +.cfi_startproc +.Lchacha20_128: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-16,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x70(%r10) + movaps %xmm7,-0x60(%r10) + movaps %xmm8,-0x50(%r10) + movaps %xmm9,-0x40(%r10) + movaps %xmm10,-0x30(%r10) + movaps %xmm11,-0x20(%r10) +.L128_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$a + movdqu ($key),$b + movdqu 16($key),$c + movdqu ($counter),$d + movdqa .Lone(%rip),$d1 + movdqa .Lrot16(%rip),$rot16 + movdqa .Lrot24(%rip),$rot24 + + movdqa $a,$a1 + movdqa $a,0x00(%rsp) + movdqa $b,$b1 + movdqa $b,0x10(%rsp) + movdqa $c,$c1 + movdqa $c,0x20(%rsp) + paddd $d,$d1 + movdqa $d,0x30(%rsp) + mov \$10,$counter # reuse $counter + jmp .Loop_128 + +.align 32 +.Loop_128: +___ + &SSSE3ROUND_2x(); + &pshufd ($a,$a,0b10010011); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b00111001); + &pshufd ($a1,$a1,0b10010011); + &pshufd ($d1,$d1,0b01001110); + &pshufd ($c1,$c1,0b00111001); + + &SSSE3ROUND_2x(); + &pshufd ($a,$a,0b00111001); + &pshufd ($d,$d,0b01001110); + &pshufd ($c,$c,0b10010011); + &pshufd ($a1,$a1,0b00111001); + &pshufd ($d1,$d1,0b01001110); + &pshufd ($c1,$c1,0b10010011); + + &dec ($counter); + &jnz (".Loop_128"); + +$code.=<<___; + paddd 0x00(%rsp),$a + paddd 0x10(%rsp),$b + paddd 0x20(%rsp),$c + paddd 0x30(%rsp),$d + paddd .Lone(%rip),$d1 + paddd 0x00(%rsp),$a1 + paddd 0x10(%rsp),$b1 + paddd 0x20(%rsp),$c1 + paddd 0x30(%rsp),$d1 + + movdqu 0x00($inp),$t + movdqu 0x10($inp),$t1 + pxor $t,$a # xor with input + movdqu 0x20($inp),$t + pxor $t1,$b + movdqu 0x30($inp),$t1 + pxor $t,$c + movdqu 0x40($inp),$t + pxor $t1,$d + movdqu 0x50($inp),$t1 + pxor $t,$a1 + movdqu 0x60($inp),$t + pxor $t1,$b1 + movdqu 0x70($inp),$t1 + pxor $t,$c1 + pxor $t1,$d1 + + movdqu $a,0x00($out) # write output + movdqu $b,0x10($out) + movdqu $c,0x20($out) + movdqu $d,0x30($out) + movdqu $a1,0x40($out) + movdqu $b1,0x50($out) + movdqu $c1,0x60($out) + movdqu $d1,0x70($out) +___ +$code.=<<___ if ($win64); + movaps -0x70(%r10),%xmm6 + movaps -0x60(%r10),%xmm7 + movaps -0x50(%r10),%xmm8 + movaps -0x40(%r10),%xmm9 + movaps -0x30(%r10),%xmm10 + movaps -0x20(%r10),%xmm11 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L128_epilogue: + ret +.cfi_endproc +.size chacha20_128,.-chacha20_128 +___ +} + +######################################################################## +# SSSE3 code path that handles longer messages. +{ +# assign variables to favor Atom front-end +my ($xd0,$xd1,$xd2,$xd3, $xt0,$xt1,$xt2,$xt3, + $xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3)=map("%xmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3); + +sub SSSE3_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3); +my @x=map("\"$_\"",@xx); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + ( + "&paddd (@x[$a0],@x[$b0])", # Q1 + "&paddd (@x[$a1],@x[$b1])", # Q2 + "&pxor (@x[$d0],@x[$a0])", + "&pxor (@x[$d1],@x[$a1])", + "&pshufb (@x[$d0],$t1)", + "&pshufb (@x[$d1],$t1)", + + "&paddd ($xc,@x[$d0])", + "&paddd ($xc_,@x[$d1])", + "&pxor (@x[$b0],$xc)", + "&pxor (@x[$b1],$xc_)", + "&movdqa ($t0,@x[$b0])", + "&pslld (@x[$b0],12)", + "&psrld ($t0,20)", + "&movdqa ($t1,@x[$b1])", + "&pslld (@x[$b1],12)", + "&por (@x[$b0],$t0)", + "&psrld ($t1,20)", + "&movdqa ($t0,'(%r11)')", # .Lrot24(%rip) + "&por (@x[$b1],$t1)", + + "&paddd (@x[$a0],@x[$b0])", + "&paddd (@x[$a1],@x[$b1])", + "&pxor (@x[$d0],@x[$a0])", + "&pxor (@x[$d1],@x[$a1])", + "&pshufb (@x[$d0],$t0)", + "&pshufb (@x[$d1],$t0)", + + "&paddd ($xc,@x[$d0])", + "&paddd ($xc_,@x[$d1])", + "&pxor (@x[$b0],$xc)", + "&pxor (@x[$b1],$xc_)", + "&movdqa ($t1,@x[$b0])", + "&pslld (@x[$b0],7)", + "&psrld ($t1,25)", + "&movdqa ($t0,@x[$b1])", + "&pslld (@x[$b1],7)", + "&por (@x[$b0],$t1)", + "&psrld ($t0,25)", + "&movdqa ($t1,'(%r9)')", # .Lrot16(%rip) + "&por (@x[$b1],$t0)", + + "&movdqa (\"`16*($c0-8)`(%rsp)\",$xc)", # reload pair of 'c's + "&movdqa (\"`16*($c1-8)`(%rsp)\",$xc_)", + "&movdqa ($xc,\"`16*($c2-8)`(%rsp)\")", + "&movdqa ($xc_,\"`16*($c3-8)`(%rsp)\")", + + "&paddd (@x[$a2],@x[$b2])", # Q3 + "&paddd (@x[$a3],@x[$b3])", # Q4 + "&pxor (@x[$d2],@x[$a2])", + "&pxor (@x[$d3],@x[$a3])", + "&pshufb (@x[$d2],$t1)", + "&pshufb (@x[$d3],$t1)", + + "&paddd ($xc,@x[$d2])", + "&paddd ($xc_,@x[$d3])", + "&pxor (@x[$b2],$xc)", + "&pxor (@x[$b3],$xc_)", + "&movdqa ($t0,@x[$b2])", + "&pslld (@x[$b2],12)", + "&psrld ($t0,20)", + "&movdqa ($t1,@x[$b3])", + "&pslld (@x[$b3],12)", + "&por (@x[$b2],$t0)", + "&psrld ($t1,20)", + "&movdqa ($t0,'(%r11)')", # .Lrot24(%rip) + "&por (@x[$b3],$t1)", + + "&paddd (@x[$a2],@x[$b2])", + "&paddd (@x[$a3],@x[$b3])", + "&pxor (@x[$d2],@x[$a2])", + "&pxor (@x[$d3],@x[$a3])", + "&pshufb (@x[$d2],$t0)", + "&pshufb (@x[$d3],$t0)", + + "&paddd ($xc,@x[$d2])", + "&paddd ($xc_,@x[$d3])", + "&pxor (@x[$b2],$xc)", + "&pxor (@x[$b3],$xc_)", + "&movdqa ($t1,@x[$b2])", + "&pslld (@x[$b2],7)", + "&psrld ($t1,25)", + "&movdqa ($t0,@x[$b3])", + "&pslld (@x[$b3],7)", + "&por (@x[$b2],$t1)", + "&psrld ($t0,25)", + "&movdqa ($t1,'(%r9)')", # .Lrot16(%rip) + "&por (@x[$b3],$t0)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +$code.=<<___; +.type chacha20_4x,\@function,5 +.align 32 +chacha20_4x: +.cfi_startproc +.Lchacha20_4x: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 +___ +$code.=<<___ if (!$kernel); + mov %r9,%r11 +___ +$code.=<<___ if ($avx>1 && !$kernel); + shr \$32,%r9 # OPENSSL_ia32cap_P+8 + test \$`1<<5`,%r9 # test AVX2 + jnz .Lchacha20_8x +___ +$code.=<<___; + cmp \$192,$len + ja .Lproceed4x +___ +$code.=<<___ if (!$kernel); + and \$`1<<26|1<<22`,%r11 # isolate XSAVE+MOVBE + cmp \$`1<<22`,%r11 # check for MOVBE without XSAVE + je .Ldo_ssse3_after_all # to detect Atom +___ +$code.=<<___; +.Lproceed4x: + sub \$0x140+$xframe,%rsp + and \$-16,%rsp +___ + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x40 constant copy of key[0-2] smashed by lanes + # ... + # +0x100 SIMD counters (with nonce smashed by lanes) + # ... + # +0x140 +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L4x_body: +___ +$code.=<<___; + movdqa .Lsigma(%rip),$xa3 # key[0] + movdqu ($key),$xb3 # key[1] + movdqu 16($key),$xt3 # key[2] + movdqu ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + lea .Lrot16(%rip),%r9 + lea .Lrot24(%rip),%r11 + + pshufd \$0x00,$xa3,$xa0 # smash key by lanes... + pshufd \$0x55,$xa3,$xa1 + movdqa $xa0,0x40(%rsp) # ... and offload + pshufd \$0xaa,$xa3,$xa2 + movdqa $xa1,0x50(%rsp) + pshufd \$0xff,$xa3,$xa3 + movdqa $xa2,0x60(%rsp) + movdqa $xa3,0x70(%rsp) + + pshufd \$0x00,$xb3,$xb0 + pshufd \$0x55,$xb3,$xb1 + movdqa $xb0,0x80-0x100(%rcx) + pshufd \$0xaa,$xb3,$xb2 + movdqa $xb1,0x90-0x100(%rcx) + pshufd \$0xff,$xb3,$xb3 + movdqa $xb2,0xa0-0x100(%rcx) + movdqa $xb3,0xb0-0x100(%rcx) + + pshufd \$0x00,$xt3,$xt0 # "$xc0" + pshufd \$0x55,$xt3,$xt1 # "$xc1" + movdqa $xt0,0xc0-0x100(%rcx) + pshufd \$0xaa,$xt3,$xt2 # "$xc2" + movdqa $xt1,0xd0-0x100(%rcx) + pshufd \$0xff,$xt3,$xt3 # "$xc3" + movdqa $xt2,0xe0-0x100(%rcx) + movdqa $xt3,0xf0-0x100(%rcx) + + pshufd \$0x00,$xd3,$xd0 + pshufd \$0x55,$xd3,$xd1 + paddd .Linc(%rip),$xd0 # don't save counters yet + pshufd \$0xaa,$xd3,$xd2 + movdqa $xd1,0x110-0x100(%rcx) + pshufd \$0xff,$xd3,$xd3 + movdqa $xd2,0x120-0x100(%rcx) + movdqa $xd3,0x130-0x100(%rcx) + + jmp .Loop_enter4x + +.align 32 +.Loop_outer4x: + movdqa 0x40(%rsp),$xa0 # re-load smashed key + movdqa 0x50(%rsp),$xa1 + movdqa 0x60(%rsp),$xa2 + movdqa 0x70(%rsp),$xa3 + movdqa 0x80-0x100(%rcx),$xb0 + movdqa 0x90-0x100(%rcx),$xb1 + movdqa 0xa0-0x100(%rcx),$xb2 + movdqa 0xb0-0x100(%rcx),$xb3 + movdqa 0xc0-0x100(%rcx),$xt0 # "$xc0" + movdqa 0xd0-0x100(%rcx),$xt1 # "$xc1" + movdqa 0xe0-0x100(%rcx),$xt2 # "$xc2" + movdqa 0xf0-0x100(%rcx),$xt3 # "$xc3" + movdqa 0x100-0x100(%rcx),$xd0 + movdqa 0x110-0x100(%rcx),$xd1 + movdqa 0x120-0x100(%rcx),$xd2 + movdqa 0x130-0x100(%rcx),$xd3 + paddd .Lfour(%rip),$xd0 # next SIMD counters + +.Loop_enter4x: + movdqa $xt2,0x20(%rsp) # SIMD equivalent of "@x[10]" + movdqa $xt3,0x30(%rsp) # SIMD equivalent of "@x[11]" + movdqa (%r9),$xt3 # .Lrot16(%rip) + mov \$10,%eax + movdqa $xd0,0x100-0x100(%rcx) # save SIMD counters + jmp .Loop4x + +.align 32 +.Loop4x: +___ + foreach (&SSSE3_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&SSSE3_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop4x + + paddd 0x40(%rsp),$xa0 # accumulate key material + paddd 0x50(%rsp),$xa1 + paddd 0x60(%rsp),$xa2 + paddd 0x70(%rsp),$xa3 + + movdqa $xa0,$xt2 # "de-interlace" data + punpckldq $xa1,$xa0 + movdqa $xa2,$xt3 + punpckldq $xa3,$xa2 + punpckhdq $xa1,$xt2 + punpckhdq $xa3,$xt3 + movdqa $xa0,$xa1 + punpcklqdq $xa2,$xa0 # "a0" + movdqa $xt2,$xa3 + punpcklqdq $xt3,$xt2 # "a2" + punpckhqdq $xa2,$xa1 # "a1" + punpckhqdq $xt3,$xa3 # "a3" +___ + ($xa2,$xt2)=($xt2,$xa2); +$code.=<<___; + paddd 0x80-0x100(%rcx),$xb0 + paddd 0x90-0x100(%rcx),$xb1 + paddd 0xa0-0x100(%rcx),$xb2 + paddd 0xb0-0x100(%rcx),$xb3 + + movdqa $xa0,0x00(%rsp) # offload $xaN + movdqa $xa1,0x10(%rsp) + movdqa 0x20(%rsp),$xa0 # "xc2" + movdqa 0x30(%rsp),$xa1 # "xc3" + + movdqa $xb0,$xt2 + punpckldq $xb1,$xb0 + movdqa $xb2,$xt3 + punpckldq $xb3,$xb2 + punpckhdq $xb1,$xt2 + punpckhdq $xb3,$xt3 + movdqa $xb0,$xb1 + punpcklqdq $xb2,$xb0 # "b0" + movdqa $xt2,$xb3 + punpcklqdq $xt3,$xt2 # "b2" + punpckhqdq $xb2,$xb1 # "b1" + punpckhqdq $xt3,$xb3 # "b3" +___ + ($xb2,$xt2)=($xt2,$xb2); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + paddd 0xc0-0x100(%rcx),$xc0 + paddd 0xd0-0x100(%rcx),$xc1 + paddd 0xe0-0x100(%rcx),$xc2 + paddd 0xf0-0x100(%rcx),$xc3 + + movdqa $xa2,0x20(%rsp) # keep offloading $xaN + movdqa $xa3,0x30(%rsp) + + movdqa $xc0,$xt2 + punpckldq $xc1,$xc0 + movdqa $xc2,$xt3 + punpckldq $xc3,$xc2 + punpckhdq $xc1,$xt2 + punpckhdq $xc3,$xt3 + movdqa $xc0,$xc1 + punpcklqdq $xc2,$xc0 # "c0" + movdqa $xt2,$xc3 + punpcklqdq $xt3,$xt2 # "c2" + punpckhqdq $xc2,$xc1 # "c1" + punpckhqdq $xt3,$xc3 # "c3" +___ + ($xc2,$xt2)=($xt2,$xc2); + ($xt0,$xt1)=($xa2,$xa3); # use $xaN as temporary +$code.=<<___; + paddd 0x100-0x100(%rcx),$xd0 + paddd 0x110-0x100(%rcx),$xd1 + paddd 0x120-0x100(%rcx),$xd2 + paddd 0x130-0x100(%rcx),$xd3 + + movdqa $xd0,$xt2 + punpckldq $xd1,$xd0 + movdqa $xd2,$xt3 + punpckldq $xd3,$xd2 + punpckhdq $xd1,$xt2 + punpckhdq $xd3,$xt3 + movdqa $xd0,$xd1 + punpcklqdq $xd2,$xd0 # "d0" + movdqa $xt2,$xd3 + punpcklqdq $xt3,$xt2 # "d2" + punpckhqdq $xd2,$xd1 # "d1" + punpckhqdq $xt3,$xd3 # "d3" +___ + ($xd2,$xt2)=($xt2,$xd2); +$code.=<<___; + cmp \$64*4,$len + jb .Ltail4x + + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # size optimization + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + + movdqu $xt0,0x40($out) + movdqu 0x00($inp),$xt0 + movdqu $xt1,0x50($out) + movdqu 0x10($inp),$xt1 + movdqu $xt2,0x60($out) + movdqu 0x20($inp),$xt2 + movdqu $xt3,0x70($out) + lea 0x80($out),$out # size optimization + movdqu 0x30($inp),$xt3 + pxor 0x20(%rsp),$xt0 + pxor $xb2,$xt1 + pxor $xc2,$xt2 + pxor $xd2,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # inp+=64*4 + pxor 0x30(%rsp),$xt0 + pxor $xb3,$xt1 + pxor $xc3,$xt2 + pxor $xd3,$xt3 + movdqu $xt0,0x40($out) + movdqu $xt1,0x50($out) + movdqu $xt2,0x60($out) + movdqu $xt3,0x70($out) + lea 0x80($out),$out # out+=64*4 + + sub \$64*4,$len + jnz .Loop_outer4x + + jmp .Ldone4x + +.Ltail4x: + cmp \$192,$len + jae .L192_or_more4x + cmp \$128,$len + jae .L128_or_more4x + cmp \$64,$len + jae .L64_or_more4x + + #movdqa 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + xor %r9,%r9 + #movdqa $xt0,0x00(%rsp) + movdqa $xb0,0x10(%rsp) + movdqa $xc0,0x20(%rsp) + movdqa $xd0,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L64_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaxN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + movdqu $xt0,0x00($out) + movdqu $xt1,0x10($out) + movdqu $xt2,0x20($out) + movdqu $xt3,0x30($out) + je .Ldone4x + + movdqa 0x10(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x40($inp),$inp # inp+=64*1 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb1,0x10(%rsp) + lea 0x40($out),$out # out+=64*1 + movdqa $xc1,0x20(%rsp) + sub \$64,$len # len-=64*1 + movdqa $xd1,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L128_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + movdqu $xt0,0x40($out) + movdqu $xt1,0x50($out) + movdqu $xt2,0x60($out) + movdqu $xt3,0x70($out) + je .Ldone4x + + movdqa 0x20(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x80($inp),$inp # inp+=64*2 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb2,0x10(%rsp) + lea 0x80($out),$out # out+=64*2 + movdqa $xc2,0x20(%rsp) + sub \$128,$len # len-=64*2 + movdqa $xd2,0x30(%rsp) + jmp .Loop_tail4x + +.align 32 +.L192_or_more4x: + movdqu 0x00($inp),$xt0 # xor with input + movdqu 0x10($inp),$xt1 + movdqu 0x20($inp),$xt2 + movdqu 0x30($inp),$xt3 + pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember? + pxor $xb0,$xt1 + pxor $xc0,$xt2 + pxor $xd0,$xt3 + + movdqu $xt0,0x00($out) + movdqu 0x40($inp),$xt0 + movdqu $xt1,0x10($out) + movdqu 0x50($inp),$xt1 + movdqu $xt2,0x20($out) + movdqu 0x60($inp),$xt2 + movdqu $xt3,0x30($out) + movdqu 0x70($inp),$xt3 + lea 0x80($inp),$inp # size optimization + pxor 0x10(%rsp),$xt0 + pxor $xb1,$xt1 + pxor $xc1,$xt2 + pxor $xd1,$xt3 + + movdqu $xt0,0x40($out) + movdqu 0x00($inp),$xt0 + movdqu $xt1,0x50($out) + movdqu 0x10($inp),$xt1 + movdqu $xt2,0x60($out) + movdqu 0x20($inp),$xt2 + movdqu $xt3,0x70($out) + lea 0x80($out),$out # size optimization + movdqu 0x30($inp),$xt3 + pxor 0x20(%rsp),$xt0 + pxor $xb2,$xt1 + pxor $xc2,$xt2 + pxor $xd2,$xt3 + movdqu $xt0,0x00($out) + movdqu $xt1,0x10($out) + movdqu $xt2,0x20($out) + movdqu $xt3,0x30($out) + je .Ldone4x + + movdqa 0x30(%rsp),$xt0 # $xaN is offloaded, remember? + lea 0x40($inp),$inp # inp+=64*3 + xor %r9,%r9 + movdqa $xt0,0x00(%rsp) + movdqa $xb3,0x10(%rsp) + lea 0x40($out),$out # out+=64*3 + movdqa $xc3,0x20(%rsp) + sub \$192,$len # len-=64*3 + movdqa $xd3,0x30(%rsp) + +.Loop_tail4x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail4x + +.Ldone4x: +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L4x_epilogue: + ret +.cfi_endproc +.size chacha20_4x,.-chacha20_4x +___ +} +if($kernel) { + $code .= "#endif\n"; +} + +######################################################################## +# XOP code path that handles all lengths. +if ($avx && !$kernel) { +# There is some "anomaly" observed depending on instructions' size or +# alignment. If you look closely at below code you'll notice that +# sometimes argument order varies. The order affects instruction +# encoding by making it larger, and such fiddling gives 5% performance +# improvement. This is on FX-4100... + +my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3, + $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%xmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xt0,$xt1,$xt2,$xt3, $xd0,$xd1,$xd2,$xd3); + +sub XOP_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my @x=map("\"$_\"",@xx); + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vprotd (@x[$d0],@x[$d0],16)", + "&vprotd (@x[$d1],@x[$d1],16)", + "&vprotd (@x[$d2],@x[$d2],16)", + "&vprotd (@x[$d3],@x[$d3],16)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxor (@x[$b0],@x[$c0],@x[$b0])", + "&vpxor (@x[$b1],@x[$c1],@x[$b1])", + "&vpxor (@x[$b2],@x[$b2],@x[$c2])", # flip + "&vpxor (@x[$b3],@x[$b3],@x[$c3])", # flip + "&vprotd (@x[$b0],@x[$b0],12)", + "&vprotd (@x[$b1],@x[$b1],12)", + "&vprotd (@x[$b2],@x[$b2],12)", + "&vprotd (@x[$b3],@x[$b3],12)", + + "&vpaddd (@x[$a0],@x[$b0],@x[$a0])", # flip + "&vpaddd (@x[$a1],@x[$b1],@x[$a1])", # flip + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vprotd (@x[$d0],@x[$d0],8)", + "&vprotd (@x[$d1],@x[$d1],8)", + "&vprotd (@x[$d2],@x[$d2],8)", + "&vprotd (@x[$d3],@x[$d3],8)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxor (@x[$b0],@x[$c0],@x[$b0])", + "&vpxor (@x[$b1],@x[$c1],@x[$b1])", + "&vpxor (@x[$b2],@x[$b2],@x[$c2])", # flip + "&vpxor (@x[$b3],@x[$b3],@x[$c3])", # flip + "&vprotd (@x[$b0],@x[$b0],7)", + "&vprotd (@x[$b1],@x[$b1],7)", + "&vprotd (@x[$b2],@x[$b2],7)", + "&vprotd (@x[$b3],@x[$b3],7)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +&declare_function("chacha20_xop", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_4xop: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + sub \$0x140+$xframe,%rsp + and \$-16,%rsp +___ + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x40 constant copy of key[0-2] smashed by lanes + # ... + # +0x100 SIMD counters (with nonce smashed by lanes) + # ... + # +0x140 +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L4xop_body: +___ +$code.=<<___; + vzeroupper + + vmovdqa .Lsigma(%rip),$xa3 # key[0] + vmovdqu ($key),$xb3 # key[1] + vmovdqu 16($key),$xt3 # key[2] + vmovdqu ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vmovdqa $xa0,0x40(%rsp) # ... and offload + vpshufd \$0xaa,$xa3,$xa2 + vmovdqa $xa1,0x50(%rsp) + vpshufd \$0xff,$xa3,$xa3 + vmovdqa $xa2,0x60(%rsp) + vmovdqa $xa3,0x70(%rsp) + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vmovdqa $xb0,0x80-0x100(%rcx) + vpshufd \$0xaa,$xb3,$xb2 + vmovdqa $xb1,0x90-0x100(%rcx) + vpshufd \$0xff,$xb3,$xb3 + vmovdqa $xb2,0xa0-0x100(%rcx) + vmovdqa $xb3,0xb0-0x100(%rcx) + + vpshufd \$0x00,$xt3,$xt0 # "$xc0" + vpshufd \$0x55,$xt3,$xt1 # "$xc1" + vmovdqa $xt0,0xc0-0x100(%rcx) + vpshufd \$0xaa,$xt3,$xt2 # "$xc2" + vmovdqa $xt1,0xd0-0x100(%rcx) + vpshufd \$0xff,$xt3,$xt3 # "$xc3" + vmovdqa $xt2,0xe0-0x100(%rcx) + vmovdqa $xt3,0xf0-0x100(%rcx) + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpaddd .Linc(%rip),$xd0,$xd0 # don't save counters yet + vpshufd \$0xaa,$xd3,$xd2 + vmovdqa $xd1,0x110-0x100(%rcx) + vpshufd \$0xff,$xd3,$xd3 + vmovdqa $xd2,0x120-0x100(%rcx) + vmovdqa $xd3,0x130-0x100(%rcx) + + jmp .Loop_enter4xop + +.align 32 +.Loop_outer4xop: + vmovdqa 0x40(%rsp),$xa0 # re-load smashed key + vmovdqa 0x50(%rsp),$xa1 + vmovdqa 0x60(%rsp),$xa2 + vmovdqa 0x70(%rsp),$xa3 + vmovdqa 0x80-0x100(%rcx),$xb0 + vmovdqa 0x90-0x100(%rcx),$xb1 + vmovdqa 0xa0-0x100(%rcx),$xb2 + vmovdqa 0xb0-0x100(%rcx),$xb3 + vmovdqa 0xc0-0x100(%rcx),$xt0 # "$xc0" + vmovdqa 0xd0-0x100(%rcx),$xt1 # "$xc1" + vmovdqa 0xe0-0x100(%rcx),$xt2 # "$xc2" + vmovdqa 0xf0-0x100(%rcx),$xt3 # "$xc3" + vmovdqa 0x100-0x100(%rcx),$xd0 + vmovdqa 0x110-0x100(%rcx),$xd1 + vmovdqa 0x120-0x100(%rcx),$xd2 + vmovdqa 0x130-0x100(%rcx),$xd3 + vpaddd .Lfour(%rip),$xd0,$xd0 # next SIMD counters + +.Loop_enter4xop: + mov \$10,%eax + vmovdqa $xd0,0x100-0x100(%rcx) # save SIMD counters + jmp .Loop4xop + +.align 32 +.Loop4xop: +___ + foreach (&XOP_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&XOP_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop4xop + + vpaddd 0x40(%rsp),$xa0,$xa0 # accumulate key material + vpaddd 0x50(%rsp),$xa1,$xa1 + vpaddd 0x60(%rsp),$xa2,$xa2 + vpaddd 0x70(%rsp),$xa3,$xa3 + + vmovdqa $xt2,0x20(%rsp) # offload $xc2,3 + vmovdqa $xt3,0x30(%rsp) + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd 0x80-0x100(%rcx),$xb0,$xb0 + vpaddd 0x90-0x100(%rcx),$xb1,$xb1 + vpaddd 0xa0-0x100(%rcx),$xb2,$xb2 + vpaddd 0xb0-0x100(%rcx),$xb3,$xb3 + + vmovdqa $xa0,0x00(%rsp) # offload $xa0,1 + vmovdqa $xa1,0x10(%rsp) + vmovdqa 0x20(%rsp),$xa0 # "xc2" + vmovdqa 0x30(%rsp),$xa1 # "xc3" + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + vpaddd 0xc0-0x100(%rcx),$xc0,$xc0 + vpaddd 0xd0-0x100(%rcx),$xc1,$xc1 + vpaddd 0xe0-0x100(%rcx),$xc2,$xc2 + vpaddd 0xf0-0x100(%rcx),$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd 0x100-0x100(%rcx),$xd0,$xd0 + vpaddd 0x110-0x100(%rcx),$xd1,$xd1 + vpaddd 0x120-0x100(%rcx),$xd2,$xd2 + vpaddd 0x130-0x100(%rcx),$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); + ($xa0,$xa1)=($xt2,$xt3); +$code.=<<___; + vmovdqa 0x00(%rsp),$xa0 # restore $xa0,1 + vmovdqa 0x10(%rsp),$xa1 + + cmp \$64*4,$len + jb .Ltail4xop + + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x10($inp),$xb2,$xb2 + vpxor 0x20($inp),$xc2,$xc2 + vpxor 0x30($inp),$xd2,$xd2 + vpxor 0x40($inp),$xa3,$xa3 + vpxor 0x50($inp),$xb3,$xb3 + vpxor 0x60($inp),$xc3,$xc3 + vpxor 0x70($inp),$xd3,$xd3 + lea 0x80($inp),$inp # inp+=64*4 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + lea 0x80($out),$out # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x10($out) + vmovdqu $xc2,0x20($out) + vmovdqu $xd2,0x30($out) + vmovdqu $xa3,0x40($out) + vmovdqu $xb3,0x50($out) + vmovdqu $xc3,0x60($out) + vmovdqu $xd3,0x70($out) + lea 0x80($out),$out # out+=64*4 + + sub \$64*4,$len + jnz .Loop_outer4xop + + jmp .Ldone4xop + +.align 32 +.Ltail4xop: + cmp \$192,$len + jae .L192_or_more4xop + cmp \$128,$len + jae .L128_or_more4xop + cmp \$64,$len + jae .L64_or_more4xop + + xor %r9,%r9 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x10(%rsp) + vmovdqa $xc0,0x20(%rsp) + vmovdqa $xd0,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L64_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + je .Ldone4xop + + lea 0x40($inp),$inp # inp+=64*1 + vmovdqa $xa1,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb1,0x10(%rsp) + lea 0x40($out),$out # out+=64*1 + vmovdqa $xc1,0x20(%rsp) + sub \$64,$len # len-=64*1 + vmovdqa $xd1,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L128_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + je .Ldone4xop + + lea 0x80($inp),$inp # inp+=64*2 + vmovdqa $xa2,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb2,0x10(%rsp) + lea 0x80($out),$out # out+=64*2 + vmovdqa $xc2,0x20(%rsp) + sub \$128,$len # len-=64*2 + vmovdqa $xd2,0x30(%rsp) + jmp .Loop_tail4xop + +.align 32 +.L192_or_more4xop: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x10($inp),$xb0,$xb0 + vpxor 0x20($inp),$xc0,$xc0 + vpxor 0x30($inp),$xd0,$xd0 + vpxor 0x40($inp),$xa1,$xa1 + vpxor 0x50($inp),$xb1,$xb1 + vpxor 0x60($inp),$xc1,$xc1 + vpxor 0x70($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x10($inp),$xb2,$xb2 + vpxor 0x20($inp),$xc2,$xc2 + vpxor 0x30($inp),$xd2,$xd2 + + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x10($out) + vmovdqu $xc0,0x20($out) + vmovdqu $xd0,0x30($out) + vmovdqu $xa1,0x40($out) + vmovdqu $xb1,0x50($out) + vmovdqu $xc1,0x60($out) + vmovdqu $xd1,0x70($out) + lea 0x80($out),$out # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x10($out) + vmovdqu $xc2,0x20($out) + vmovdqu $xd2,0x30($out) + je .Ldone4xop + + lea 0x40($inp),$inp # inp+=64*3 + vmovdqa $xa3,0x00(%rsp) + xor %r9,%r9 + vmovdqa $xb3,0x10(%rsp) + lea 0x40($out),$out # out+=64*3 + vmovdqa $xc3,0x20(%rsp) + sub \$192,$len # len-=64*3 + vmovdqa $xd3,0x30(%rsp) + +.Loop_tail4xop: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail4xop + +.Ldone4xop: + vzeroupper +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L4xop_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_xop"); +} + +######################################################################## +# AVX2 code path +if ($avx>1) { + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX2\n"; +} + +my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3, + $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%ymm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3); + +sub AVX2_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3); +my @x=map("\"$_\"",@xx); + + # Consider order in which variables are addressed by their + # index: + # + # a b c d + # + # 0 4 8 12 < even round + # 1 5 9 13 + # 2 6 10 14 + # 3 7 11 15 + # 0 5 10 15 < odd round + # 1 6 11 12 + # 2 7 8 13 + # 3 4 9 14 + # + # 'a', 'b' and 'd's are permanently allocated in registers, + # @x[0..7,12..15], while 'c's are maintained in memory. If + # you observe 'c' column, you'll notice that pair of 'c's is + # invariant between rounds. This means that we have to reload + # them once per round, in the middle. This is why you'll see + # bunch of 'c' stores and loads in the middle, but none in + # the beginning or end. + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpshufb (@x[$d0],@x[$d0],$t1)", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpshufb (@x[$d1],@x[$d1],$t1)", + + "&vpaddd ($xc,$xc,@x[$d0])", + "&vpxor (@x[$b0],$xc,@x[$b0])", + "&vpslld ($t0,@x[$b0],12)", + "&vpsrld (@x[$b0],@x[$b0],20)", + "&vpor (@x[$b0],$t0,@x[$b0])", + "&vbroadcasti128($t0,'(%r11)')", # .Lrot24(%rip) + "&vpaddd ($xc_,$xc_,@x[$d1])", + "&vpxor (@x[$b1],$xc_,@x[$b1])", + "&vpslld ($t1,@x[$b1],12)", + "&vpsrld (@x[$b1],@x[$b1],20)", + "&vpor (@x[$b1],$t1,@x[$b1])", + + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", + "&vpxor (@x[$d0],@x[$a0],@x[$d0])", + "&vpshufb (@x[$d0],@x[$d0],$t0)", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", + "&vpxor (@x[$d1],@x[$a1],@x[$d1])", + "&vpshufb (@x[$d1],@x[$d1],$t0)", + + "&vpaddd ($xc,$xc,@x[$d0])", + "&vpxor (@x[$b0],$xc,@x[$b0])", + "&vpslld ($t1,@x[$b0],7)", + "&vpsrld (@x[$b0],@x[$b0],25)", + "&vpor (@x[$b0],$t1,@x[$b0])", + "&vbroadcasti128($t1,'(%r9)')", # .Lrot16(%rip) + "&vpaddd ($xc_,$xc_,@x[$d1])", + "&vpxor (@x[$b1],$xc_,@x[$b1])", + "&vpslld ($t0,@x[$b1],7)", + "&vpsrld (@x[$b1],@x[$b1],25)", + "&vpor (@x[$b1],$t0,@x[$b1])", + + "&vmovdqa (\"`32*($c0-8)`(%rsp)\",$xc)", # reload pair of 'c's + "&vmovdqa (\"`32*($c1-8)`(%rsp)\",$xc_)", + "&vmovdqa ($xc,\"`32*($c2-8)`(%rsp)\")", + "&vmovdqa ($xc_,\"`32*($c3-8)`(%rsp)\")", + + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpshufb (@x[$d2],@x[$d2],$t1)", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vpshufb (@x[$d3],@x[$d3],$t1)", + + "&vpaddd ($xc,$xc,@x[$d2])", + "&vpxor (@x[$b2],$xc,@x[$b2])", + "&vpslld ($t0,@x[$b2],12)", + "&vpsrld (@x[$b2],@x[$b2],20)", + "&vpor (@x[$b2],$t0,@x[$b2])", + "&vbroadcasti128($t0,'(%r11)')", # .Lrot24(%rip) + "&vpaddd ($xc_,$xc_,@x[$d3])", + "&vpxor (@x[$b3],$xc_,@x[$b3])", + "&vpslld ($t1,@x[$b3],12)", + "&vpsrld (@x[$b3],@x[$b3],20)", + "&vpor (@x[$b3],$t1,@x[$b3])", + + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpxor (@x[$d2],@x[$a2],@x[$d2])", + "&vpshufb (@x[$d2],@x[$d2],$t0)", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxor (@x[$d3],@x[$a3],@x[$d3])", + "&vpshufb (@x[$d3],@x[$d3],$t0)", + + "&vpaddd ($xc,$xc,@x[$d2])", + "&vpxor (@x[$b2],$xc,@x[$b2])", + "&vpslld ($t1,@x[$b2],7)", + "&vpsrld (@x[$b2],@x[$b2],25)", + "&vpor (@x[$b2],$t1,@x[$b2])", + "&vbroadcasti128($t1,'(%r9)')", # .Lrot16(%rip) + "&vpaddd ($xc_,$xc_,@x[$d3])", + "&vpxor (@x[$b3],$xc_,@x[$b3])", + "&vpslld ($t0,@x[$b3],7)", + "&vpsrld (@x[$b3],@x[$b3],25)", + "&vpor (@x[$b3],$t0,@x[$b3])" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +&declare_function("chacha20_avx2", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_8x: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$0x280+$xframe,%rsp + and \$-32,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L8x_body: +___ +$code.=<<___; + vzeroupper + + ################ stack layout + # +0x00 SIMD equivalent of @x[8-12] + # ... + # +0x80 constant copy of key[0-2] smashed by lanes + # ... + # +0x200 SIMD counters (with nonce smashed by lanes) + # ... + # +0x280 + + vbroadcasti128 .Lsigma(%rip),$xa3 # key[0] + vbroadcasti128 ($key),$xb3 # key[1] + vbroadcasti128 16($key),$xt3 # key[2] + vbroadcasti128 ($counter),$xd3 # key[3] + lea 0x100(%rsp),%rcx # size optimization + lea 0x200(%rsp),%rax # size optimization + lea .Lrot16(%rip),%r9 + lea .Lrot24(%rip),%r11 + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vmovdqa $xa0,0x80-0x100(%rcx) # ... and offload + vpshufd \$0xaa,$xa3,$xa2 + vmovdqa $xa1,0xa0-0x100(%rcx) + vpshufd \$0xff,$xa3,$xa3 + vmovdqa $xa2,0xc0-0x100(%rcx) + vmovdqa $xa3,0xe0-0x100(%rcx) + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vmovdqa $xb0,0x100-0x100(%rcx) + vpshufd \$0xaa,$xb3,$xb2 + vmovdqa $xb1,0x120-0x100(%rcx) + vpshufd \$0xff,$xb3,$xb3 + vmovdqa $xb2,0x140-0x100(%rcx) + vmovdqa $xb3,0x160-0x100(%rcx) + + vpshufd \$0x00,$xt3,$xt0 # "xc0" + vpshufd \$0x55,$xt3,$xt1 # "xc1" + vmovdqa $xt0,0x180-0x200(%rax) + vpshufd \$0xaa,$xt3,$xt2 # "xc2" + vmovdqa $xt1,0x1a0-0x200(%rax) + vpshufd \$0xff,$xt3,$xt3 # "xc3" + vmovdqa $xt2,0x1c0-0x200(%rax) + vmovdqa $xt3,0x1e0-0x200(%rax) + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpaddd .Lincy(%rip),$xd0,$xd0 # don't save counters yet + vpshufd \$0xaa,$xd3,$xd2 + vmovdqa $xd1,0x220-0x200(%rax) + vpshufd \$0xff,$xd3,$xd3 + vmovdqa $xd2,0x240-0x200(%rax) + vmovdqa $xd3,0x260-0x200(%rax) + + jmp .Loop_enter8x + +.align 32 +.Loop_outer8x: + vmovdqa 0x80-0x100(%rcx),$xa0 # re-load smashed key + vmovdqa 0xa0-0x100(%rcx),$xa1 + vmovdqa 0xc0-0x100(%rcx),$xa2 + vmovdqa 0xe0-0x100(%rcx),$xa3 + vmovdqa 0x100-0x100(%rcx),$xb0 + vmovdqa 0x120-0x100(%rcx),$xb1 + vmovdqa 0x140-0x100(%rcx),$xb2 + vmovdqa 0x160-0x100(%rcx),$xb3 + vmovdqa 0x180-0x200(%rax),$xt0 # "xc0" + vmovdqa 0x1a0-0x200(%rax),$xt1 # "xc1" + vmovdqa 0x1c0-0x200(%rax),$xt2 # "xc2" + vmovdqa 0x1e0-0x200(%rax),$xt3 # "xc3" + vmovdqa 0x200-0x200(%rax),$xd0 + vmovdqa 0x220-0x200(%rax),$xd1 + vmovdqa 0x240-0x200(%rax),$xd2 + vmovdqa 0x260-0x200(%rax),$xd3 + vpaddd .Leight(%rip),$xd0,$xd0 # next SIMD counters + +.Loop_enter8x: + vmovdqa $xt2,0x40(%rsp) # SIMD equivalent of "@x[10]" + vmovdqa $xt3,0x60(%rsp) # SIMD equivalent of "@x[11]" + vbroadcasti128 (%r9),$xt3 + vmovdqa $xd0,0x200-0x200(%rax) # save SIMD counters + mov \$10,%eax + jmp .Loop8x + +.align 32 +.Loop8x: +___ + foreach (&AVX2_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX2_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop8x + + lea 0x200(%rsp),%rax # size optimization + vpaddd 0x80-0x100(%rcx),$xa0,$xa0 # accumulate key + vpaddd 0xa0-0x100(%rcx),$xa1,$xa1 + vpaddd 0xc0-0x100(%rcx),$xa2,$xa2 + vpaddd 0xe0-0x100(%rcx),$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd 0x100-0x100(%rcx),$xb0,$xb0 + vpaddd 0x120-0x100(%rcx),$xb1,$xb1 + vpaddd 0x140-0x100(%rcx),$xb2,$xb2 + vpaddd 0x160-0x100(%rcx),$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vperm2i128 \$0x20,$xb0,$xa0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xb0,$xa0,$xb0 + vperm2i128 \$0x20,$xb1,$xa1,$xa0 + vperm2i128 \$0x31,$xb1,$xa1,$xb1 + vperm2i128 \$0x20,$xb2,$xa2,$xa1 + vperm2i128 \$0x31,$xb2,$xa2,$xb2 + vperm2i128 \$0x20,$xb3,$xa3,$xa2 + vperm2i128 \$0x31,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); + my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1); +$code.=<<___; + vmovdqa $xa0,0x00(%rsp) # offload $xaN + vmovdqa $xa1,0x20(%rsp) + vmovdqa 0x40(%rsp),$xc2 # $xa0 + vmovdqa 0x60(%rsp),$xc3 # $xa1 + + vpaddd 0x180-0x200(%rax),$xc0,$xc0 + vpaddd 0x1a0-0x200(%rax),$xc1,$xc1 + vpaddd 0x1c0-0x200(%rax),$xc2,$xc2 + vpaddd 0x1e0-0x200(%rax),$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd 0x200-0x200(%rax),$xd0,$xd0 + vpaddd 0x220-0x200(%rax),$xd1,$xd1 + vpaddd 0x240-0x200(%rax),$xd2,$xd2 + vpaddd 0x260-0x200(%rax),$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vperm2i128 \$0x20,$xd0,$xc0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xd0,$xc0,$xd0 + vperm2i128 \$0x20,$xd1,$xc1,$xc0 + vperm2i128 \$0x31,$xd1,$xc1,$xd1 + vperm2i128 \$0x20,$xd2,$xc2,$xc1 + vperm2i128 \$0x31,$xd2,$xc2,$xd2 + vperm2i128 \$0x20,$xd3,$xc3,$xc2 + vperm2i128 \$0x31,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); + ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)= + ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3); + ($xa0,$xa1)=($xt2,$xt3); +$code.=<<___; + vmovdqa 0x00(%rsp),$xa0 # $xaN was offloaded, remember? + vmovdqa 0x20(%rsp),$xa1 + + cmp \$64*8,$len + jb .Ltail8x + + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vpxor 0x40($inp),$xc1,$xc1 + vpxor 0x60($inp),$xd1,$xd1 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa1,0x00($out) + vmovdqu $xb1,0x20($out) + vmovdqu $xc1,0x40($out) + vmovdqu $xd1,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vpxor 0x40($inp),$xc2,$xc2 + vpxor 0x60($inp),$xd2,$xd2 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa2,0x00($out) + vmovdqu $xb2,0x20($out) + vmovdqu $xc2,0x40($out) + vmovdqu $xd2,0x60($out) + lea 0x80($out),$out # size optimization + + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vpxor 0x40($inp),$xc3,$xc3 + vpxor 0x60($inp),$xd3,$xd3 + lea 0x80($inp),$inp # size optimization + vmovdqu $xa3,0x00($out) + vmovdqu $xb3,0x20($out) + vmovdqu $xc3,0x40($out) + vmovdqu $xd3,0x60($out) + lea 0x80($out),$out # size optimization + + sub \$64*8,$len + jnz .Loop_outer8x + + jmp .Ldone8x + +.Ltail8x: + cmp \$448,$len + jae .L448_or_more8x + cmp \$384,$len + jae .L384_or_more8x + cmp \$320,$len + jae .L320_or_more8x + cmp \$256,$len + jae .L256_or_more8x + cmp \$192,$len + jae .L192_or_more8x + cmp \$128,$len + jae .L128_or_more8x + cmp \$64,$len + jae .L64_or_more8x + + xor %r9,%r9 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L64_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + je .Ldone8x + + lea 0x40($inp),$inp # inp+=64*1 + xor %r9,%r9 + vmovdqa $xc0,0x00(%rsp) + lea 0x40($out),$out # out+=64*1 + sub \$64,$len # len-=64*1 + vmovdqa $xd0,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L128_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + je .Ldone8x + + lea 0x80($inp),$inp # inp+=64*2 + xor %r9,%r9 + vmovdqa $xa1,0x00(%rsp) + lea 0x80($out),$out # out+=64*2 + sub \$128,$len # len-=64*2 + vmovdqa $xb1,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L192_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + je .Ldone8x + + lea 0xc0($inp),$inp # inp+=64*3 + xor %r9,%r9 + vmovdqa $xc1,0x00(%rsp) + lea 0xc0($out),$out # out+=64*3 + sub \$192,$len # len-=64*3 + vmovdqa $xd1,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L256_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + je .Ldone8x + + lea 0x100($inp),$inp # inp+=64*4 + xor %r9,%r9 + vmovdqa $xa2,0x00(%rsp) + lea 0x100($out),$out # out+=64*4 + sub \$256,$len # len-=64*4 + vmovdqa $xb2,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L320_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + je .Ldone8x + + lea 0x140($inp),$inp # inp+=64*5 + xor %r9,%r9 + vmovdqa $xc2,0x00(%rsp) + lea 0x140($out),$out # out+=64*5 + sub \$320,$len # len-=64*5 + vmovdqa $xd2,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L384_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vpxor 0x140($inp),$xc2,$xc2 + vpxor 0x160($inp),$xd2,$xd2 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + vmovdqu $xc2,0x140($out) + vmovdqu $xd2,0x160($out) + je .Ldone8x + + lea 0x180($inp),$inp # inp+=64*6 + xor %r9,%r9 + vmovdqa $xa3,0x00(%rsp) + lea 0x180($out),$out # out+=64*6 + sub \$384,$len # len-=64*6 + vmovdqa $xb3,0x20(%rsp) + jmp .Loop_tail8x + +.align 32 +.L448_or_more8x: + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + vpxor 0x80($inp),$xa1,$xa1 + vpxor 0xa0($inp),$xb1,$xb1 + vpxor 0xc0($inp),$xc1,$xc1 + vpxor 0xe0($inp),$xd1,$xd1 + vpxor 0x100($inp),$xa2,$xa2 + vpxor 0x120($inp),$xb2,$xb2 + vpxor 0x140($inp),$xc2,$xc2 + vpxor 0x160($inp),$xd2,$xd2 + vpxor 0x180($inp),$xa3,$xa3 + vpxor 0x1a0($inp),$xb3,$xb3 + vmovdqu $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + vmovdqu $xa1,0x80($out) + vmovdqu $xb1,0xa0($out) + vmovdqu $xc1,0xc0($out) + vmovdqu $xd1,0xe0($out) + vmovdqu $xa2,0x100($out) + vmovdqu $xb2,0x120($out) + vmovdqu $xc2,0x140($out) + vmovdqu $xd2,0x160($out) + vmovdqu $xa3,0x180($out) + vmovdqu $xb3,0x1a0($out) + je .Ldone8x + + lea 0x1c0($inp),$inp # inp+=64*7 + xor %r9,%r9 + vmovdqa $xc3,0x00(%rsp) + lea 0x1c0($out),$out # out+=64*7 + sub \$448,$len # len-=64*7 + vmovdqa $xd3,0x20(%rsp) + +.Loop_tail8x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail8x + +.Ldone8x: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L8x_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx2"); +if($kernel) { + $code .= "#endif\n"; +} +} + +######################################################################## +# AVX512 code paths +if ($avx>2) { +# This one handles shorter inputs... +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX512\n"; +} + +my ($a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz) = map("%zmm$_",(0..3,16..20)); +my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7)); + +sub vpxord() # size optimization +{ my $opcode = "vpxor"; # adhere to vpxor when possible + + foreach (@_) { + if (/%([zy])mm([0-9]+)/ && ($1 eq "z" || $2>=16)) { + $opcode = "vpxord"; + last; + } + } + + $code .= "\t$opcode\t".join(',',reverse @_)."\n"; +} + +sub AVX512ROUND { # critical path is 14 "SIMD ticks" per round + &vpaddd ($a,$a,$b); + &vpxord ($d,$d,$a); + &vprold ($d,$d,16); + + &vpaddd ($c,$c,$d); + &vpxord ($b,$b,$c); + &vprold ($b,$b,12); + + &vpaddd ($a,$a,$b); + &vpxord ($d,$d,$a); + &vprold ($d,$d,8); + + &vpaddd ($c,$c,$d); + &vpxord ($b,$b,$c); + &vprold ($b,$b,7); +} + +my $xframe = $win64 ? 32+8 : 8; + +&declare_function("chacha20_avx512", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_avx512: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + cmp \$512,$len + ja .Lchacha20_16x + + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lavx512_body: +___ +$code.=<<___; + vbroadcasti32x4 .Lsigma(%rip),$a + vbroadcasti32x4 ($key),$b + vbroadcasti32x4 16($key),$c + vbroadcasti32x4 ($counter),$d + + vmovdqa32 $a,$a_ + vmovdqa32 $b,$b_ + vmovdqa32 $c,$c_ + vpaddd .Lzeroz(%rip),$d,$d + vmovdqa32 .Lfourz(%rip),$fourz + mov \$10,$counter # reuse $counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512 + +.align 16 +.Loop_outer_avx512: + vmovdqa32 $a_,$a + vmovdqa32 $b_,$b + vmovdqa32 $c_,$c + vpaddd $fourz,$d_,$d + mov \$10,$counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512 + +.align 32 +.Loop_avx512: +___ + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b00111001); + &vpshufd ($d,$d,0b10010011); + + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b10010011); + &vpshufd ($d,$d,0b00111001); + + &dec ($counter); + &jnz (".Loop_avx512"); + +$code.=<<___; + vpaddd $a_,$a,$a + vpaddd $b_,$b,$b + vpaddd $c_,$c,$c + vpaddd $d_,$d,$d + + sub \$64,$len + jb .Ltail64_avx512 + + vpxor 0x00($inp),%x#$a,$t0 # xor with input + vpxor 0x10($inp),%x#$b,$t1 + vpxor 0x20($inp),%x#$c,$t2 + vpxor 0x30($inp),%x#$d,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$1,$a,$t0 + vextracti32x4 \$1,$b,$t1 + vextracti32x4 \$1,$c,$t2 + vextracti32x4 \$1,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$2,$a,$t0 + vextracti32x4 \$2,$b,$t1 + vextracti32x4 \$2,$c,$t2 + vextracti32x4 \$2,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512 + + vextracti32x4 \$3,$a,$t0 + vextracti32x4 \$3,$b,$t1 + vextracti32x4 \$3,$c,$t2 + vextracti32x4 \$3,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512 + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jnz .Loop_outer_avx512 + + jmp .Ldone_avx512 + +.align 16 +.Ltail64_avx512: + vmovdqa %x#$a,0x00(%rsp) + vmovdqa %x#$b,0x10(%rsp) + vmovdqa %x#$c,0x20(%rsp) + vmovdqa %x#$d,0x30(%rsp) + add \$64,$len + jmp .Loop_tail_avx512 + +.align 16 +.Ltail_avx512: + vmovdqa $t0,0x00(%rsp) + vmovdqa $t1,0x10(%rsp) + vmovdqa $t2,0x20(%rsp) + vmovdqa $t3,0x30(%rsp) + add \$64,$len + +.Loop_tail_avx512: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_avx512 + + vmovdqu32 $a_,0x00(%rsp) + +.Ldone_avx512: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lavx512_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx512"); + +map(s/%z/%y/, $a,$b,$c,$d, $a_,$b_,$c_,$d_,$fourz); + +&declare_function("chacha20_avx512vl", 32, 5); +$code.=<<___; +.cfi_startproc +.Lchacha20_avx512vl: + lea 8(%rsp),%r10 # frame pointer +.cfi_def_cfa_register %r10 + cmp \$128,$len + ja .Lchacha20_8xvl + + sub \$64+$xframe,%rsp + and \$-32,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0x30(%r10) + movaps %xmm7,-0x20(%r10) +.Lavx512vl_body: +___ +$code.=<<___; + vbroadcasti128 .Lsigma(%rip),$a + vbroadcasti128 ($key),$b + vbroadcasti128 16($key),$c + vbroadcasti128 ($counter),$d + + vmovdqa32 $a,$a_ + vmovdqa32 $b,$b_ + vmovdqa32 $c,$c_ + vpaddd .Lzeroz(%rip),$d,$d + vmovdqa32 .Ltwoy(%rip),$fourz + mov \$10,$counter # reuse $counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512vl + +.align 16 +.Loop_outer_avx512vl: + vmovdqa32 $c_,$c + vpaddd $fourz,$d_,$d + mov \$10,$counter + vmovdqa32 $d,$d_ + jmp .Loop_avx512vl + +.align 32 +.Loop_avx512vl: +___ + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b00111001); + &vpshufd ($d,$d,0b10010011); + + &AVX512ROUND(); + &vpshufd ($c,$c,0b01001110); + &vpshufd ($b,$b,0b10010011); + &vpshufd ($d,$d,0b00111001); + + &dec ($counter); + &jnz (".Loop_avx512vl"); + +$code.=<<___; + vpaddd $a_,$a,$a + vpaddd $b_,$b,$b + vpaddd $c_,$c,$c + vpaddd $d_,$d,$d + + sub \$64,$len + jb .Ltail64_avx512vl + + vpxor 0x00($inp),%x#$a,$t0 # xor with input + vpxor 0x10($inp),%x#$b,$t1 + vpxor 0x20($inp),%x#$c,$t2 + vpxor 0x30($inp),%x#$d,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + jz .Ldone_avx512vl + + vextracti128 \$1,$a,$t0 + vextracti128 \$1,$b,$t1 + vextracti128 \$1,$c,$t2 + vextracti128 \$1,$d,$t3 + + sub \$64,$len + jb .Ltail_avx512vl + + vpxor 0x00($inp),$t0,$t0 # xor with input + vpxor 0x10($inp),$t1,$t1 + vpxor 0x20($inp),$t2,$t2 + vpxor 0x30($inp),$t3,$t3 + lea 0x40($inp),$inp # inp+=64 + + vmovdqu $t0,0x00($out) # write output + vmovdqu $t1,0x10($out) + vmovdqu $t2,0x20($out) + vmovdqu $t3,0x30($out) + lea 0x40($out),$out # out+=64 + + vmovdqa32 $a_,$a + vmovdqa32 $b_,$b + jnz .Loop_outer_avx512vl + + jmp .Ldone_avx512vl + +.align 16 +.Ltail64_avx512vl: + vmovdqa %x#$a,0x00(%rsp) + vmovdqa %x#$b,0x10(%rsp) + vmovdqa %x#$c,0x20(%rsp) + vmovdqa %x#$d,0x30(%rsp) + add \$64,$len + jmp .Loop_tail_avx512vl + +.align 16 +.Ltail_avx512vl: + vmovdqa $t0,0x00(%rsp) + vmovdqa $t1,0x10(%rsp) + vmovdqa $t2,0x20(%rsp) + vmovdqa $t3,0x30(%rsp) + add \$64,$len + +.Loop_tail_avx512vl: + movzb ($inp,$counter),%eax + movzb (%rsp,$counter),%ecx + lea 1($counter),$counter + xor %ecx,%eax + mov %al,-1($out,$counter) + dec $len + jnz .Loop_tail_avx512vl + + vmovdqu32 $a_,0x00(%rsp) + vmovdqu32 $a_,0x20(%rsp) + +.Ldone_avx512vl: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0x30(%r10),%xmm6 + movaps -0x20(%r10),%xmm7 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.Lavx512vl_epilogue: + ret +.cfi_endproc +___ +&end_function("chacha20_avx512vl"); + +# This one handles longer inputs... + +my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%zmm$_",(0..15)); +my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +my @key=map("%zmm$_",(16..31)); +my ($xt0,$xt1,$xt2,$xt3)=@key[0..3]; + +sub AVX512_lane_ROUND { +my ($a0,$b0,$c0,$d0)=@_; +my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0)); +my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1)); +my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2)); +my @x=map("\"$_\"",@xx); + + ( + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1 + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2 + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3 + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4 + "&vpxord (@x[$d0],@x[$d0],@x[$a0])", + "&vpxord (@x[$d1],@x[$d1],@x[$a1])", + "&vpxord (@x[$d2],@x[$d2],@x[$a2])", + "&vpxord (@x[$d3],@x[$d3],@x[$a3])", + "&vprold (@x[$d0],@x[$d0],16)", + "&vprold (@x[$d1],@x[$d1],16)", + "&vprold (@x[$d2],@x[$d2],16)", + "&vprold (@x[$d3],@x[$d3],16)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxord (@x[$b0],@x[$b0],@x[$c0])", + "&vpxord (@x[$b1],@x[$b1],@x[$c1])", + "&vpxord (@x[$b2],@x[$b2],@x[$c2])", + "&vpxord (@x[$b3],@x[$b3],@x[$c3])", + "&vprold (@x[$b0],@x[$b0],12)", + "&vprold (@x[$b1],@x[$b1],12)", + "&vprold (@x[$b2],@x[$b2],12)", + "&vprold (@x[$b3],@x[$b3],12)", + + "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", + "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", + "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", + "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", + "&vpxord (@x[$d0],@x[$d0],@x[$a0])", + "&vpxord (@x[$d1],@x[$d1],@x[$a1])", + "&vpxord (@x[$d2],@x[$d2],@x[$a2])", + "&vpxord (@x[$d3],@x[$d3],@x[$a3])", + "&vprold (@x[$d0],@x[$d0],8)", + "&vprold (@x[$d1],@x[$d1],8)", + "&vprold (@x[$d2],@x[$d2],8)", + "&vprold (@x[$d3],@x[$d3],8)", + + "&vpaddd (@x[$c0],@x[$c0],@x[$d0])", + "&vpaddd (@x[$c1],@x[$c1],@x[$d1])", + "&vpaddd (@x[$c2],@x[$c2],@x[$d2])", + "&vpaddd (@x[$c3],@x[$c3],@x[$d3])", + "&vpxord (@x[$b0],@x[$b0],@x[$c0])", + "&vpxord (@x[$b1],@x[$b1],@x[$c1])", + "&vpxord (@x[$b2],@x[$b2],@x[$c2])", + "&vpxord (@x[$b3],@x[$b3],@x[$c3])", + "&vprold (@x[$b0],@x[$b0],7)", + "&vprold (@x[$b1],@x[$b1],7)", + "&vprold (@x[$b2],@x[$b2],7)", + "&vprold (@x[$b3],@x[$b3],7)" + ); +} + +my $xframe = $win64 ? 0xa8 : 8; + +$code.=<<___; +.type chacha20_16x,\@function,5 +.align 32 +chacha20_16x: +.cfi_startproc +.Lchacha20_16x: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L16x_body: +___ +$code.=<<___; + vzeroupper + + lea .Lsigma(%rip),%r9 + vbroadcasti32x4 (%r9),$xa3 # key[0] + vbroadcasti32x4 ($key),$xb3 # key[1] + vbroadcasti32x4 16($key),$xc3 # key[2] + vbroadcasti32x4 ($counter),$xd3 # key[3] + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vpshufd \$0xaa,$xa3,$xa2 + vpshufd \$0xff,$xa3,$xa3 + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vpshufd \$0xaa,$xb3,$xb2 + vpshufd \$0xff,$xb3,$xb3 + vmovdqa64 $xb0,@key[4] + vmovdqa64 $xb1,@key[5] + vmovdqa64 $xb2,@key[6] + vmovdqa64 $xb3,@key[7] + + vpshufd \$0x00,$xc3,$xc0 + vpshufd \$0x55,$xc3,$xc1 + vpshufd \$0xaa,$xc3,$xc2 + vpshufd \$0xff,$xc3,$xc3 + vmovdqa64 $xc0,@key[8] + vmovdqa64 $xc1,@key[9] + vmovdqa64 $xc2,@key[10] + vmovdqa64 $xc3,@key[11] + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpshufd \$0xaa,$xd3,$xd2 + vpshufd \$0xff,$xd3,$xd3 + vpaddd .Lincz(%rip),$xd0,$xd0 # don't save counters yet + vmovdqa64 $xd0,@key[12] + vmovdqa64 $xd1,@key[13] + vmovdqa64 $xd2,@key[14] + vmovdqa64 $xd3,@key[15] + + mov \$10,%eax + jmp .Loop16x + +.align 32 +.Loop_outer16x: + vpbroadcastd 0(%r9),$xa0 # reload key + vpbroadcastd 4(%r9),$xa1 + vpbroadcastd 8(%r9),$xa2 + vpbroadcastd 12(%r9),$xa3 + vpaddd .Lsixteen(%rip),@key[12],@key[12] # next SIMD counters + vmovdqa64 @key[4],$xb0 + vmovdqa64 @key[5],$xb1 + vmovdqa64 @key[6],$xb2 + vmovdqa64 @key[7],$xb3 + vmovdqa64 @key[8],$xc0 + vmovdqa64 @key[9],$xc1 + vmovdqa64 @key[10],$xc2 + vmovdqa64 @key[11],$xc3 + vmovdqa64 @key[12],$xd0 + vmovdqa64 @key[13],$xd1 + vmovdqa64 @key[14],$xd2 + vmovdqa64 @key[15],$xd3 + + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + mov \$10,%eax + jmp .Loop16x + +.align 32 +.Loop16x: +___ + foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop16x + + vpaddd @key[0],$xa0,$xa0 # accumulate key + vpaddd @key[1],$xa1,$xa1 + vpaddd @key[2],$xa2,$xa2 + vpaddd @key[3],$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd @key[4],$xb0,$xb0 + vpaddd @key[5],$xb1,$xb1 + vpaddd @key[6],$xb2,$xb2 + vpaddd @key[7],$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vshufi32x4 \$0x44,$xb0,$xa0,$xt3 # "de-interlace" further + vshufi32x4 \$0xee,$xb0,$xa0,$xb0 + vshufi32x4 \$0x44,$xb1,$xa1,$xa0 + vshufi32x4 \$0xee,$xb1,$xa1,$xb1 + vshufi32x4 \$0x44,$xb2,$xa2,$xa1 + vshufi32x4 \$0xee,$xb2,$xa2,$xb2 + vshufi32x4 \$0x44,$xb3,$xa3,$xa2 + vshufi32x4 \$0xee,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); +$code.=<<___; + vpaddd @key[8],$xc0,$xc0 + vpaddd @key[9],$xc1,$xc1 + vpaddd @key[10],$xc2,$xc2 + vpaddd @key[11],$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd @key[12],$xd0,$xd0 + vpaddd @key[13],$xd1,$xd1 + vpaddd @key[14],$xd2,$xd2 + vpaddd @key[15],$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vshufi32x4 \$0x44,$xd0,$xc0,$xt3 # "de-interlace" further + vshufi32x4 \$0xee,$xd0,$xc0,$xd0 + vshufi32x4 \$0x44,$xd1,$xc1,$xc0 + vshufi32x4 \$0xee,$xd1,$xc1,$xd1 + vshufi32x4 \$0x44,$xd2,$xc2,$xc1 + vshufi32x4 \$0xee,$xd2,$xc2,$xd2 + vshufi32x4 \$0x44,$xd3,$xc3,$xc2 + vshufi32x4 \$0xee,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); +$code.=<<___; + vshufi32x4 \$0x88,$xc0,$xa0,$xt0 # "de-interlace" further + vshufi32x4 \$0xdd,$xc0,$xa0,$xa0 + vshufi32x4 \$0x88,$xd0,$xb0,$xc0 + vshufi32x4 \$0xdd,$xd0,$xb0,$xd0 + vshufi32x4 \$0x88,$xc1,$xa1,$xt1 + vshufi32x4 \$0xdd,$xc1,$xa1,$xa1 + vshufi32x4 \$0x88,$xd1,$xb1,$xc1 + vshufi32x4 \$0xdd,$xd1,$xb1,$xd1 + vshufi32x4 \$0x88,$xc2,$xa2,$xt2 + vshufi32x4 \$0xdd,$xc2,$xa2,$xa2 + vshufi32x4 \$0x88,$xd2,$xb2,$xc2 + vshufi32x4 \$0xdd,$xd2,$xb2,$xd2 + vshufi32x4 \$0x88,$xc3,$xa3,$xt3 + vshufi32x4 \$0xdd,$xc3,$xa3,$xa3 + vshufi32x4 \$0x88,$xd3,$xb3,$xc3 + vshufi32x4 \$0xdd,$xd3,$xb3,$xd3 +___ + ($xa0,$xa1,$xa2,$xa3,$xb0,$xb1,$xb2,$xb3)= + ($xt0,$xt1,$xt2,$xt3,$xa0,$xa1,$xa2,$xa3); + + ($xa0,$xb0,$xc0,$xd0, $xa1,$xb1,$xc1,$xd1, + $xa2,$xb2,$xc2,$xd2, $xa3,$xb3,$xc3,$xd3) = + ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +$code.=<<___; + cmp \$64*16,$len + jb .Ltail16x + + vpxord 0x00($inp),$xa0,$xa0 # xor with input + vpxord 0x40($inp),$xb0,$xb0 + vpxord 0x80($inp),$xc0,$xc0 + vpxord 0xc0($inp),$xd0,$xd0 + vmovdqu32 $xa0,0x00($out) + vmovdqu32 $xb0,0x40($out) + vmovdqu32 $xc0,0x80($out) + vmovdqu32 $xd0,0xc0($out) + + vpxord 0x100($inp),$xa1,$xa1 + vpxord 0x140($inp),$xb1,$xb1 + vpxord 0x180($inp),$xc1,$xc1 + vpxord 0x1c0($inp),$xd1,$xd1 + vmovdqu32 $xa1,0x100($out) + vmovdqu32 $xb1,0x140($out) + vmovdqu32 $xc1,0x180($out) + vmovdqu32 $xd1,0x1c0($out) + + vpxord 0x200($inp),$xa2,$xa2 + vpxord 0x240($inp),$xb2,$xb2 + vpxord 0x280($inp),$xc2,$xc2 + vpxord 0x2c0($inp),$xd2,$xd2 + vmovdqu32 $xa2,0x200($out) + vmovdqu32 $xb2,0x240($out) + vmovdqu32 $xc2,0x280($out) + vmovdqu32 $xd2,0x2c0($out) + + vpxord 0x300($inp),$xa3,$xa3 + vpxord 0x340($inp),$xb3,$xb3 + vpxord 0x380($inp),$xc3,$xc3 + vpxord 0x3c0($inp),$xd3,$xd3 + lea 0x400($inp),$inp + vmovdqu32 $xa3,0x300($out) + vmovdqu32 $xb3,0x340($out) + vmovdqu32 $xc3,0x380($out) + vmovdqu32 $xd3,0x3c0($out) + lea 0x400($out),$out + + sub \$64*16,$len + jnz .Loop_outer16x + + jmp .Ldone16x + +.align 32 +.Ltail16x: + xor %r9,%r9 + sub $inp,$out + cmp \$64*1,$len + jb .Less_than_64_16x + vpxord ($inp),$xa0,$xa0 # xor with input + vmovdqu32 $xa0,($out,$inp) + je .Ldone16x + vmovdqa32 $xb0,$xa0 + lea 64($inp),$inp + + cmp \$64*2,$len + jb .Less_than_64_16x + vpxord ($inp),$xb0,$xb0 + vmovdqu32 $xb0,($out,$inp) + je .Ldone16x + vmovdqa32 $xc0,$xa0 + lea 64($inp),$inp + + cmp \$64*3,$len + jb .Less_than_64_16x + vpxord ($inp),$xc0,$xc0 + vmovdqu32 $xc0,($out,$inp) + je .Ldone16x + vmovdqa32 $xd0,$xa0 + lea 64($inp),$inp + + cmp \$64*4,$len + jb .Less_than_64_16x + vpxord ($inp),$xd0,$xd0 + vmovdqu32 $xd0,($out,$inp) + je .Ldone16x + vmovdqa32 $xa1,$xa0 + lea 64($inp),$inp + + cmp \$64*5,$len + jb .Less_than_64_16x + vpxord ($inp),$xa1,$xa1 + vmovdqu32 $xa1,($out,$inp) + je .Ldone16x + vmovdqa32 $xb1,$xa0 + lea 64($inp),$inp + + cmp \$64*6,$len + jb .Less_than_64_16x + vpxord ($inp),$xb1,$xb1 + vmovdqu32 $xb1,($out,$inp) + je .Ldone16x + vmovdqa32 $xc1,$xa0 + lea 64($inp),$inp + + cmp \$64*7,$len + jb .Less_than_64_16x + vpxord ($inp),$xc1,$xc1 + vmovdqu32 $xc1,($out,$inp) + je .Ldone16x + vmovdqa32 $xd1,$xa0 + lea 64($inp),$inp + + cmp \$64*8,$len + jb .Less_than_64_16x + vpxord ($inp),$xd1,$xd1 + vmovdqu32 $xd1,($out,$inp) + je .Ldone16x + vmovdqa32 $xa2,$xa0 + lea 64($inp),$inp + + cmp \$64*9,$len + jb .Less_than_64_16x + vpxord ($inp),$xa2,$xa2 + vmovdqu32 $xa2,($out,$inp) + je .Ldone16x + vmovdqa32 $xb2,$xa0 + lea 64($inp),$inp + + cmp \$64*10,$len + jb .Less_than_64_16x + vpxord ($inp),$xb2,$xb2 + vmovdqu32 $xb2,($out,$inp) + je .Ldone16x + vmovdqa32 $xc2,$xa0 + lea 64($inp),$inp + + cmp \$64*11,$len + jb .Less_than_64_16x + vpxord ($inp),$xc2,$xc2 + vmovdqu32 $xc2,($out,$inp) + je .Ldone16x + vmovdqa32 $xd2,$xa0 + lea 64($inp),$inp + + cmp \$64*12,$len + jb .Less_than_64_16x + vpxord ($inp),$xd2,$xd2 + vmovdqu32 $xd2,($out,$inp) + je .Ldone16x + vmovdqa32 $xa3,$xa0 + lea 64($inp),$inp + + cmp \$64*13,$len + jb .Less_than_64_16x + vpxord ($inp),$xa3,$xa3 + vmovdqu32 $xa3,($out,$inp) + je .Ldone16x + vmovdqa32 $xb3,$xa0 + lea 64($inp),$inp + + cmp \$64*14,$len + jb .Less_than_64_16x + vpxord ($inp),$xb3,$xb3 + vmovdqu32 $xb3,($out,$inp) + je .Ldone16x + vmovdqa32 $xc3,$xa0 + lea 64($inp),$inp + + cmp \$64*15,$len + jb .Less_than_64_16x + vpxord ($inp),$xc3,$xc3 + vmovdqu32 $xc3,($out,$inp) + je .Ldone16x + vmovdqa32 $xd3,$xa0 + lea 64($inp),$inp + +.Less_than_64_16x: + vmovdqa32 $xa0,0x00(%rsp) + lea ($out,$inp),$out + and \$63,$len + +.Loop_tail16x: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail16x + + vpxord $xa0,$xa0,$xa0 + vmovdqa32 $xa0,0(%rsp) + +.Ldone16x: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L16x_epilogue: + ret +.cfi_endproc +.size chacha20_16x,.-chacha20_16x +___ + +# switch to %ymm domain +($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%ymm$_",(0..15)); +@xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3, + $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3); +@key=map("%ymm$_",(16..31)); +($xt0,$xt1,$xt2,$xt3)=@key[0..3]; + +$code.=<<___; +.type chacha20_8xvl,\@function,5 +.align 32 +chacha20_8xvl: +.cfi_startproc +.Lchacha20_8xvl: + lea 8(%rsp),%r10 # frame register +.cfi_def_cfa_register %r10 + sub \$64+$xframe,%rsp + and \$-64,%rsp +___ +$code.=<<___ if ($win64); + movaps %xmm6,-0xb0(%r10) + movaps %xmm7,-0xa0(%r10) + movaps %xmm8,-0x90(%r10) + movaps %xmm9,-0x80(%r10) + movaps %xmm10,-0x70(%r10) + movaps %xmm11,-0x60(%r10) + movaps %xmm12,-0x50(%r10) + movaps %xmm13,-0x40(%r10) + movaps %xmm14,-0x30(%r10) + movaps %xmm15,-0x20(%r10) +.L8xvl_body: +___ +$code.=<<___; + vzeroupper + + lea .Lsigma(%rip),%r9 + vbroadcasti128 (%r9),$xa3 # key[0] + vbroadcasti128 ($key),$xb3 # key[1] + vbroadcasti128 16($key),$xc3 # key[2] + vbroadcasti128 ($counter),$xd3 # key[3] + + vpshufd \$0x00,$xa3,$xa0 # smash key by lanes... + vpshufd \$0x55,$xa3,$xa1 + vpshufd \$0xaa,$xa3,$xa2 + vpshufd \$0xff,$xa3,$xa3 + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + vpshufd \$0x00,$xb3,$xb0 + vpshufd \$0x55,$xb3,$xb1 + vpshufd \$0xaa,$xb3,$xb2 + vpshufd \$0xff,$xb3,$xb3 + vmovdqa64 $xb0,@key[4] + vmovdqa64 $xb1,@key[5] + vmovdqa64 $xb2,@key[6] + vmovdqa64 $xb3,@key[7] + + vpshufd \$0x00,$xc3,$xc0 + vpshufd \$0x55,$xc3,$xc1 + vpshufd \$0xaa,$xc3,$xc2 + vpshufd \$0xff,$xc3,$xc3 + vmovdqa64 $xc0,@key[8] + vmovdqa64 $xc1,@key[9] + vmovdqa64 $xc2,@key[10] + vmovdqa64 $xc3,@key[11] + + vpshufd \$0x00,$xd3,$xd0 + vpshufd \$0x55,$xd3,$xd1 + vpshufd \$0xaa,$xd3,$xd2 + vpshufd \$0xff,$xd3,$xd3 + vpaddd .Lincy(%rip),$xd0,$xd0 # don't save counters yet + vmovdqa64 $xd0,@key[12] + vmovdqa64 $xd1,@key[13] + vmovdqa64 $xd2,@key[14] + vmovdqa64 $xd3,@key[15] + + mov \$10,%eax + jmp .Loop8xvl + +.align 32 +.Loop_outer8xvl: + #vpbroadcastd 0(%r9),$xa0 # reload key + #vpbroadcastd 4(%r9),$xa1 + vpbroadcastd 8(%r9),$xa2 + vpbroadcastd 12(%r9),$xa3 + vpaddd .Leight(%rip),@key[12],@key[12] # next SIMD counters + vmovdqa64 @key[4],$xb0 + vmovdqa64 @key[5],$xb1 + vmovdqa64 @key[6],$xb2 + vmovdqa64 @key[7],$xb3 + vmovdqa64 @key[8],$xc0 + vmovdqa64 @key[9],$xc1 + vmovdqa64 @key[10],$xc2 + vmovdqa64 @key[11],$xc3 + vmovdqa64 @key[12],$xd0 + vmovdqa64 @key[13],$xd1 + vmovdqa64 @key[14],$xd2 + vmovdqa64 @key[15],$xd3 + + vmovdqa64 $xa0,@key[0] + vmovdqa64 $xa1,@key[1] + vmovdqa64 $xa2,@key[2] + vmovdqa64 $xa3,@key[3] + + mov \$10,%eax + jmp .Loop8xvl + +.align 32 +.Loop8xvl: +___ + foreach (&AVX512_lane_ROUND(0, 4, 8,12)) { eval; } + foreach (&AVX512_lane_ROUND(0, 5,10,15)) { eval; } +$code.=<<___; + dec %eax + jnz .Loop8xvl + + vpaddd @key[0],$xa0,$xa0 # accumulate key + vpaddd @key[1],$xa1,$xa1 + vpaddd @key[2],$xa2,$xa2 + vpaddd @key[3],$xa3,$xa3 + + vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data + vpunpckldq $xa3,$xa2,$xt3 + vpunpckhdq $xa1,$xa0,$xa0 + vpunpckhdq $xa3,$xa2,$xa2 + vpunpcklqdq $xt3,$xt2,$xa1 # "a0" + vpunpckhqdq $xt3,$xt2,$xt2 # "a1" + vpunpcklqdq $xa2,$xa0,$xa3 # "a2" + vpunpckhqdq $xa2,$xa0,$xa0 # "a3" +___ + ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2); +$code.=<<___; + vpaddd @key[4],$xb0,$xb0 + vpaddd @key[5],$xb1,$xb1 + vpaddd @key[6],$xb2,$xb2 + vpaddd @key[7],$xb3,$xb3 + + vpunpckldq $xb1,$xb0,$xt2 + vpunpckldq $xb3,$xb2,$xt3 + vpunpckhdq $xb1,$xb0,$xb0 + vpunpckhdq $xb3,$xb2,$xb2 + vpunpcklqdq $xt3,$xt2,$xb1 # "b0" + vpunpckhqdq $xt3,$xt2,$xt2 # "b1" + vpunpcklqdq $xb2,$xb0,$xb3 # "b2" + vpunpckhqdq $xb2,$xb0,$xb0 # "b3" +___ + ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2); +$code.=<<___; + vshufi32x4 \$0,$xb0,$xa0,$xt3 # "de-interlace" further + vshufi32x4 \$3,$xb0,$xa0,$xb0 + vshufi32x4 \$0,$xb1,$xa1,$xa0 + vshufi32x4 \$3,$xb1,$xa1,$xb1 + vshufi32x4 \$0,$xb2,$xa2,$xa1 + vshufi32x4 \$3,$xb2,$xa2,$xb2 + vshufi32x4 \$0,$xb3,$xa3,$xa2 + vshufi32x4 \$3,$xb3,$xa3,$xb3 +___ + ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3); +$code.=<<___; + vpaddd @key[8],$xc0,$xc0 + vpaddd @key[9],$xc1,$xc1 + vpaddd @key[10],$xc2,$xc2 + vpaddd @key[11],$xc3,$xc3 + + vpunpckldq $xc1,$xc0,$xt2 + vpunpckldq $xc3,$xc2,$xt3 + vpunpckhdq $xc1,$xc0,$xc0 + vpunpckhdq $xc3,$xc2,$xc2 + vpunpcklqdq $xt3,$xt2,$xc1 # "c0" + vpunpckhqdq $xt3,$xt2,$xt2 # "c1" + vpunpcklqdq $xc2,$xc0,$xc3 # "c2" + vpunpckhqdq $xc2,$xc0,$xc0 # "c3" +___ + ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2); +$code.=<<___; + vpaddd @key[12],$xd0,$xd0 + vpaddd @key[13],$xd1,$xd1 + vpaddd @key[14],$xd2,$xd2 + vpaddd @key[15],$xd3,$xd3 + + vpunpckldq $xd1,$xd0,$xt2 + vpunpckldq $xd3,$xd2,$xt3 + vpunpckhdq $xd1,$xd0,$xd0 + vpunpckhdq $xd3,$xd2,$xd2 + vpunpcklqdq $xt3,$xt2,$xd1 # "d0" + vpunpckhqdq $xt3,$xt2,$xt2 # "d1" + vpunpcklqdq $xd2,$xd0,$xd3 # "d2" + vpunpckhqdq $xd2,$xd0,$xd0 # "d3" +___ + ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2); +$code.=<<___; + vperm2i128 \$0x20,$xd0,$xc0,$xt3 # "de-interlace" further + vperm2i128 \$0x31,$xd0,$xc0,$xd0 + vperm2i128 \$0x20,$xd1,$xc1,$xc0 + vperm2i128 \$0x31,$xd1,$xc1,$xd1 + vperm2i128 \$0x20,$xd2,$xc2,$xc1 + vperm2i128 \$0x31,$xd2,$xc2,$xd2 + vperm2i128 \$0x20,$xd3,$xc3,$xc2 + vperm2i128 \$0x31,$xd3,$xc3,$xd3 +___ + ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3); + ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)= + ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3); +$code.=<<___; + cmp \$64*8,$len + jb .Ltail8xvl + + mov \$0x80,%eax # size optimization + vpxord 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vpxor 0x40($inp),$xc0,$xc0 + vpxor 0x60($inp),$xd0,$xd0 + lea ($inp,%rax),$inp # size optimization + vmovdqu32 $xa0,0x00($out) + vmovdqu $xb0,0x20($out) + vmovdqu $xc0,0x40($out) + vmovdqu $xd0,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vpxor 0x40($inp),$xc1,$xc1 + vpxor 0x60($inp),$xd1,$xd1 + lea ($inp,%rax),$inp # size optimization + vmovdqu $xa1,0x00($out) + vmovdqu $xb1,0x20($out) + vmovdqu $xc1,0x40($out) + vmovdqu $xd1,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxord 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vpxor 0x40($inp),$xc2,$xc2 + vpxor 0x60($inp),$xd2,$xd2 + lea ($inp,%rax),$inp # size optimization + vmovdqu32 $xa2,0x00($out) + vmovdqu $xb2,0x20($out) + vmovdqu $xc2,0x40($out) + vmovdqu $xd2,0x60($out) + lea ($out,%rax),$out # size optimization + + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vpxor 0x40($inp),$xc3,$xc3 + vpxor 0x60($inp),$xd3,$xd3 + lea ($inp,%rax),$inp # size optimization + vmovdqu $xa3,0x00($out) + vmovdqu $xb3,0x20($out) + vmovdqu $xc3,0x40($out) + vmovdqu $xd3,0x60($out) + lea ($out,%rax),$out # size optimization + + vpbroadcastd 0(%r9),%ymm0 # reload key + vpbroadcastd 4(%r9),%ymm1 + + sub \$64*8,$len + jnz .Loop_outer8xvl + + jmp .Ldone8xvl + +.align 32 +.Ltail8xvl: + vmovdqa64 $xa0,%ymm8 # size optimization +___ +$xa0 = "%ymm8"; +$code.=<<___; + xor %r9,%r9 + sub $inp,$out + cmp \$64*1,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa0,$xa0 # xor with input + vpxor 0x20($inp),$xb0,$xb0 + vmovdqu $xa0,0x00($out,$inp) + vmovdqu $xb0,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc0,$xa0 + vmovdqa $xd0,$xb0 + lea 64($inp),$inp + + cmp \$64*2,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc0,$xc0 + vpxor 0x20($inp),$xd0,$xd0 + vmovdqu $xc0,0x00($out,$inp) + vmovdqu $xd0,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xa1,$xa0 + vmovdqa $xb1,$xb0 + lea 64($inp),$inp + + cmp \$64*3,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa1,$xa1 + vpxor 0x20($inp),$xb1,$xb1 + vmovdqu $xa1,0x00($out,$inp) + vmovdqu $xb1,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc1,$xa0 + vmovdqa $xd1,$xb0 + lea 64($inp),$inp + + cmp \$64*4,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc1,$xc1 + vpxor 0x20($inp),$xd1,$xd1 + vmovdqu $xc1,0x00($out,$inp) + vmovdqu $xd1,0x20($out,$inp) + je .Ldone8xvl + vmovdqa32 $xa2,$xa0 + vmovdqa $xb2,$xb0 + lea 64($inp),$inp + + cmp \$64*5,$len + jb .Less_than_64_8xvl + vpxord 0x00($inp),$xa2,$xa2 + vpxor 0x20($inp),$xb2,$xb2 + vmovdqu32 $xa2,0x00($out,$inp) + vmovdqu $xb2,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc2,$xa0 + vmovdqa $xd2,$xb0 + lea 64($inp),$inp + + cmp \$64*6,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xc2,$xc2 + vpxor 0x20($inp),$xd2,$xd2 + vmovdqu $xc2,0x00($out,$inp) + vmovdqu $xd2,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xa3,$xa0 + vmovdqa $xb3,$xb0 + lea 64($inp),$inp + + cmp \$64*7,$len + jb .Less_than_64_8xvl + vpxor 0x00($inp),$xa3,$xa3 + vpxor 0x20($inp),$xb3,$xb3 + vmovdqu $xa3,0x00($out,$inp) + vmovdqu $xb3,0x20($out,$inp) + je .Ldone8xvl + vmovdqa $xc3,$xa0 + vmovdqa $xd3,$xb0 + lea 64($inp),$inp + +.Less_than_64_8xvl: + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xb0,0x20(%rsp) + lea ($out,$inp),$out + and \$63,$len + +.Loop_tail8xvl: + movzb ($inp,%r9),%eax + movzb (%rsp,%r9),%ecx + lea 1(%r9),%r9 + xor %ecx,%eax + mov %al,-1($out,%r9) + dec $len + jnz .Loop_tail8xvl + + vpxor $xa0,$xa0,$xa0 + vmovdqa $xa0,0x00(%rsp) + vmovdqa $xa0,0x20(%rsp) + +.Ldone8xvl: + vzeroall +___ +$code.=<<___ if ($win64); + movaps -0xb0(%r10),%xmm6 + movaps -0xa0(%r10),%xmm7 + movaps -0x90(%r10),%xmm8 + movaps -0x80(%r10),%xmm9 + movaps -0x70(%r10),%xmm10 + movaps -0x60(%r10),%xmm11 + movaps -0x50(%r10),%xmm12 + movaps -0x40(%r10),%xmm13 + movaps -0x30(%r10),%xmm14 + movaps -0x20(%r10),%xmm15 +___ +$code.=<<___; + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +.L8xvl_epilogue: + ret +.cfi_endproc +.size chacha20_8xvl,.-chacha20_8xvl +___ +if($kernel) { + $code .= "#endif\n"; +} +} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + lea .Lctr32_body(%rip),%r10 + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + lea .Lno_data(%rip),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea 64+24+48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size se_handler,.-se_handler + +.type simd_handler,\@abi-omnipotent +.align 16 +simd_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipR9 + + mov 4(%r11),%r10d # HandlerData[1] + mov 8(%r11),%ecx # HandlerData[2] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + neg %rcx + lea -8(%rax,%rcx),%rsi + lea 512($context),%rdi # &context.Xmm6 + neg %ecx + shr \$3,%ecx + .long 0xa548f3fc # cld; rep movsq + + jmp .Lcommon_seh_tail +.size simd_handler,.-simd_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_chacha20_ctr32 + .rva .LSEH_end_chacha20_ctr32 + .rva .LSEH_info_chacha20_ctr32 + + .rva .LSEH_begin_chacha20_ssse3 + .rva .LSEH_end_chacha20_ssse3 + .rva .LSEH_info_chacha20_ssse3 + + .rva .LSEH_begin_chacha20_128 + .rva .LSEH_end_chacha20_128 + .rva .LSEH_info_chacha20_128 + + .rva .LSEH_begin_chacha20_4x + .rva .LSEH_end_chacha20_4x + .rva .LSEH_info_chacha20_4x +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_chacha20_xop + .rva .LSEH_end_chacha20_xop + .rva .LSEH_info_chacha20_xop +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_chacha20_avx2 + .rva .LSEH_end_chacha20_avx2 + .rva .LSEH_info_chacha20_avx2 +___ +$code.=<<___ if ($avx>2); + .rva .LSEH_begin_chacha20_avx512 + .rva .LSEH_end_chacha20_avx512 + .rva .LSEH_info_chacha20_avx512 + + .rva .LSEH_begin_chacha20_avx512vl + .rva .LSEH_end_chacha20_avx512vl + .rva .LSEH_info_chacha20_avx512vl + + .rva .LSEH_begin_chacha20_16x + .rva .LSEH_end_chacha20_16x + .rva .LSEH_info_chacha20_16x + + .rva .LSEH_begin_chacha20_8xvl + .rva .LSEH_end_chacha20_8xvl + .rva .LSEH_info_chacha20_8xvl +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_chacha20_ctr32: + .byte 9,0,0,0 + .rva se_handler + +.LSEH_info_chacha20_ssse3: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lssse3_body,.Lssse3_epilogue + .long 0x20,0 + +.LSEH_info_chacha20_128: + .byte 9,0,0,0 + .rva simd_handler + .rva .L128_body,.L128_epilogue + .long 0x60,0 + +.LSEH_info_chacha20_4x: + .byte 9,0,0,0 + .rva simd_handler + .rva .L4x_body,.L4x_epilogue + .long 0xa0,0 +___ +$code.=<<___ if ($avx); +.LSEH_info_chacha20_xop: + .byte 9,0,0,0 + .rva simd_handler + .rva .L4xop_body,.L4xop_epilogue # HandlerData[] + .long 0xa0,0 +___ +$code.=<<___ if ($avx>1); +.LSEH_info_chacha20_avx2: + .byte 9,0,0,0 + .rva simd_handler + .rva .L8x_body,.L8x_epilogue # HandlerData[] + .long 0xa0,0 +___ +$code.=<<___ if ($avx>2); +.LSEH_info_chacha20_avx512: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lavx512_body,.Lavx512_epilogue # HandlerData[] + .long 0x20,0 + +.LSEH_info_chacha20_avx512vl: + .byte 9,0,0,0 + .rva simd_handler + .rva .Lavx512vl_body,.Lavx512vl_epilogue # HandlerData[] + .long 0x20,0 + +.LSEH_info_chacha20_16x: + .byte 9,0,0,0 + .rva simd_handler + .rva .L16x_body,.L16x_epilogue # HandlerData[] + .long 0xa0,0 + +.LSEH_info_chacha20_8xvl: + .byte 9,0,0,0 + .rva simd_handler + .rva .L8xvl_body,.L8xvl_epilogue # HandlerData[] + .long 0xa0,0 +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/ge; + + s/%x#%[yz]/%x/g; # "down-shift" + + if ($kernel) { + s/(^\.type.*),[0-9]+$/\1/; + next if /^\.cfi.*/; + } + + print $_,"\n"; +} + +close STDOUT; diff --git a/net/wireguard/crypto/zinc/chacha20/chacha20.c b/net/wireguard/crypto/zinc/chacha20/chacha20.c new file mode 100644 index 0000000000000..b4763c81e0826 --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20/chacha20.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Implementation of the ChaCha20 stream cipher. + * + * Information: https://cr.yp.to/chacha.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include // For crypto_xor_cpy. + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "chacha20-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64) +#include "chacha20-arm-glue.c" +#elif defined(CONFIG_ZINC_ARCH_MIPS) +#include "chacha20-mips-glue.c" +#else +static bool *const chacha20_nobs[] __initconst = { }; +static void __init chacha20_fpu_init(void) +{ +} +static inline bool chacha20_arch(struct chacha20_ctx *ctx, u8 *dst, + const u8 *src, size_t len, + simd_context_t *simd_context) +{ + return false; +} +static inline bool hchacha20_arch(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], + simd_context_t *simd_context) +{ + return false; +} +#endif + +#define QUARTER_ROUND(x, a, b, c, d) ( \ + x[a] += x[b], \ + x[d] = rol32((x[d] ^ x[a]), 16), \ + x[c] += x[d], \ + x[b] = rol32((x[b] ^ x[c]), 12), \ + x[a] += x[b], \ + x[d] = rol32((x[d] ^ x[a]), 8), \ + x[c] += x[d], \ + x[b] = rol32((x[b] ^ x[c]), 7) \ +) + +#define C(i, j) (i * 4 + j) + +#define DOUBLE_ROUND(x) ( \ + /* Column Round */ \ + QUARTER_ROUND(x, C(0, 0), C(1, 0), C(2, 0), C(3, 0)), \ + QUARTER_ROUND(x, C(0, 1), C(1, 1), C(2, 1), C(3, 1)), \ + QUARTER_ROUND(x, C(0, 2), C(1, 2), C(2, 2), C(3, 2)), \ + QUARTER_ROUND(x, C(0, 3), C(1, 3), C(2, 3), C(3, 3)), \ + /* Diagonal Round */ \ + QUARTER_ROUND(x, C(0, 0), C(1, 1), C(2, 2), C(3, 3)), \ + QUARTER_ROUND(x, C(0, 1), C(1, 2), C(2, 3), C(3, 0)), \ + QUARTER_ROUND(x, C(0, 2), C(1, 3), C(2, 0), C(3, 1)), \ + QUARTER_ROUND(x, C(0, 3), C(1, 0), C(2, 1), C(3, 2)) \ +) + +#define TWENTY_ROUNDS(x) ( \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x), \ + DOUBLE_ROUND(x) \ +) + +static void chacha20_block_generic(struct chacha20_ctx *ctx, __le32 *stream) +{ + u32 x[CHACHA20_BLOCK_WORDS]; + int i; + + for (i = 0; i < ARRAY_SIZE(x); ++i) + x[i] = ctx->state[i]; + + TWENTY_ROUNDS(x); + + for (i = 0; i < ARRAY_SIZE(x); ++i) + stream[i] = cpu_to_le32(x[i] + ctx->state[i]); + + ctx->counter[0] += 1; +} + +static void chacha20_generic(struct chacha20_ctx *ctx, u8 *out, const u8 *in, + u32 len) +{ + __le32 buf[CHACHA20_BLOCK_WORDS]; + + while (len >= CHACHA20_BLOCK_SIZE) { + chacha20_block_generic(ctx, buf); + crypto_xor_cpy(out, in, (u8 *)buf, CHACHA20_BLOCK_SIZE); + len -= CHACHA20_BLOCK_SIZE; + out += CHACHA20_BLOCK_SIZE; + in += CHACHA20_BLOCK_SIZE; + } + if (len) { + chacha20_block_generic(ctx, buf); + crypto_xor_cpy(out, in, (u8 *)buf, len); + } +} + +void chacha20(struct chacha20_ctx *ctx, u8 *dst, const u8 *src, u32 len, + simd_context_t *simd_context) +{ + if (!chacha20_arch(ctx, dst, src, len, simd_context)) + chacha20_generic(ctx, dst, src, len); +} +EXPORT_SYMBOL(chacha20); + +static void hchacha20_generic(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE]) +{ + u32 x[] = { CHACHA20_CONSTANT_EXPA, + CHACHA20_CONSTANT_ND_3, + CHACHA20_CONSTANT_2_BY, + CHACHA20_CONSTANT_TE_K, + get_unaligned_le32(key + 0), + get_unaligned_le32(key + 4), + get_unaligned_le32(key + 8), + get_unaligned_le32(key + 12), + get_unaligned_le32(key + 16), + get_unaligned_le32(key + 20), + get_unaligned_le32(key + 24), + get_unaligned_le32(key + 28), + get_unaligned_le32(nonce + 0), + get_unaligned_le32(nonce + 4), + get_unaligned_le32(nonce + 8), + get_unaligned_le32(nonce + 12) + }; + + TWENTY_ROUNDS(x); + + memcpy(derived_key + 0, x + 0, sizeof(u32) * 4); + memcpy(derived_key + 4, x + 12, sizeof(u32) * 4); +} + +/* Derived key should be 32-bit aligned */ +void hchacha20(u32 derived_key[CHACHA20_KEY_WORDS], + const u8 nonce[HCHACHA20_NONCE_SIZE], + const u8 key[HCHACHA20_KEY_SIZE], simd_context_t *simd_context) +{ + if (!hchacha20_arch(derived_key, nonce, key, simd_context)) + hchacha20_generic(derived_key, nonce, key); +} +EXPORT_SYMBOL(hchacha20); + +#include "../selftest/chacha20.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init chacha20_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + chacha20_fpu_init(); + if (!selftest_run("chacha20", chacha20_selftest, chacha20_nobs, + ARRAY_SIZE(chacha20_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ChaCha20 stream cipher"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/chacha20poly1305.c b/net/wireguard/crypto/zinc/chacha20poly1305.c new file mode 100644 index 0000000000000..571a64e95a83a --- /dev/null +++ b/net/wireguard/crypto/zinc/chacha20poly1305.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the ChaCha20Poly1305 AEAD construction. + * + * Information: https://tools.ietf.org/html/rfc8439 + */ + +#include +#include +#include +#include "selftest/run.h" + +#include +#include +#include +#include +#include // For blkcipher_walk. + +static const u8 pad0[CHACHA20_BLOCK_SIZE] = { 0 }; + +static inline void +__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = { { 0 } }; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + chacha20(&chacha20_state, dst, src, src_len, simd_context); + + poly1305_update(&poly1305_state, dst, src_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + poly1305_final(&poly1305_state, dst + src_len, simd_context); + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); +} + +void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + + simd_get(&simd_context); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, nonce, key, + &simd_context); + simd_put(&simd_context); +} +EXPORT_SYMBOL(chacha20poly1305_encrypt); + +bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, + const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + struct sg_mapping_iter miter; + size_t partial = 0; + ssize_t sl; + union { + u8 chacha20_stream[CHACHA20_BLOCK_SIZE]; + u8 block0[POLY1305_KEY_SIZE]; + u8 mac[POLY1305_MAC_SIZE]; + __le64 lens[2]; + } b __aligned(16) = { { 0 } }; + + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC); + for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { + u8 *addr = miter.addr; + size_t length = min_t(size_t, sl, miter.length); + + if (unlikely(partial)) { + size_t l = min(length, CHACHA20_BLOCK_SIZE - partial); + + crypto_xor(addr, b.chacha20_stream + partial, l); + partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1); + + addr += l; + length -= l; + } + + if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) { + size_t l = length; + + if (unlikely(length < sl)) + l &= ~(CHACHA20_BLOCK_SIZE - 1); + chacha20(&chacha20_state, addr, addr, l, simd_context); + addr += l; + length -= l; + } + + if (unlikely(length > 0)) { + chacha20(&chacha20_state, b.chacha20_stream, pad0, + CHACHA20_BLOCK_SIZE, simd_context); + crypto_xor(addr, b.chacha20_stream, length); + partial = length; + } + + poly1305_update(&poly1305_state, miter.addr, + min_t(size_t, sl, miter.length), simd_context); + + simd_relax(simd_context); + } + + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + if (likely(sl <= -POLY1305_MAC_SIZE)) + poly1305_final(&poly1305_state, miter.addr + miter.length + sl, + simd_context); + + sg_miter_stop(&miter); + + if (unlikely(sl > -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.mac, simd_context); + scatterwalk_map_and_copy(b.mac, src, src_len, sizeof(b.mac), 1); + } + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + return true; +} +EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); + +static inline bool +__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + int ret; + size_t dst_len; + union { + u8 block0[POLY1305_KEY_SIZE]; + u8 mac[POLY1305_MAC_SIZE]; + __le64 lens[2]; + } b = { { 0 } }; + + if (unlikely(src_len < POLY1305_MAC_SIZE)) + return false; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + dst_len = src_len - POLY1305_MAC_SIZE; + poly1305_update(&poly1305_state, src, dst_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - dst_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(dst_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + poly1305_final(&poly1305_state, b.mac, simd_context); + + ret = crypto_memneq(b.mac, src + dst_len, POLY1305_MAC_SIZE); + if (likely(!ret)) + chacha20(&chacha20_state, dst, src, dst_len, simd_context); + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + + return !ret; +} + +bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context, ret; + + simd_get(&simd_context); + ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, nonce, + key, &simd_context); + simd_put(&simd_context); + return ret; +} +EXPORT_SYMBOL(chacha20poly1305_decrypt); + +bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, + size_t src_len, + const u8 *ad, const size_t ad_len, + const u64 nonce, + const u8 key[CHACHA20POLY1305_KEY_SIZE], + simd_context_t *simd_context) +{ + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + struct sg_mapping_iter miter; + size_t partial = 0; + ssize_t sl; + union { + u8 chacha20_stream[CHACHA20_BLOCK_SIZE]; + u8 block0[POLY1305_KEY_SIZE]; + struct { + u8 read_mac[POLY1305_MAC_SIZE]; + u8 computed_mac[POLY1305_MAC_SIZE]; + }; + __le64 lens[2]; + } b __aligned(16) = { { 0 } }; + bool ret = false; + + if (unlikely(src_len < POLY1305_MAC_SIZE)) + return ret; + src_len -= POLY1305_MAC_SIZE; + + chacha20_init(&chacha20_state, key, nonce); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + simd_context); + poly1305_init(&poly1305_state, b.block0); + + poly1305_update(&poly1305_state, ad, ad_len, simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + simd_context); + + sg_miter_start(&miter, src, sg_nents(src), SG_MITER_TO_SG | SG_MITER_ATOMIC); + for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) { + u8 *addr = miter.addr; + size_t length = min_t(size_t, sl, miter.length); + + poly1305_update(&poly1305_state, addr, length, simd_context); + + if (unlikely(partial)) { + size_t l = min(length, CHACHA20_BLOCK_SIZE - partial); + + crypto_xor(addr, b.chacha20_stream + partial, l); + partial = (partial + l) & (CHACHA20_BLOCK_SIZE - 1); + + addr += l; + length -= l; + } + + if (likely(length >= CHACHA20_BLOCK_SIZE || length == sl)) { + size_t l = length; + + if (unlikely(length < sl)) + l &= ~(CHACHA20_BLOCK_SIZE - 1); + chacha20(&chacha20_state, addr, addr, l, simd_context); + addr += l; + length -= l; + } + + if (unlikely(length > 0)) { + chacha20(&chacha20_state, b.chacha20_stream, pad0, + CHACHA20_BLOCK_SIZE, simd_context); + crypto_xor(addr, b.chacha20_stream, length); + partial = length; + } + + simd_relax(simd_context); + } + + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + simd_context); + + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + simd_context); + + if (likely(sl <= -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.computed_mac, simd_context); + ret = !crypto_memneq(b.computed_mac, + miter.addr + miter.length + sl, + POLY1305_MAC_SIZE); + } + + sg_miter_stop(&miter); + + if (unlikely(sl > -POLY1305_MAC_SIZE)) { + poly1305_final(&poly1305_state, b.computed_mac, simd_context); + scatterwalk_map_and_copy(b.read_mac, src, src_len, + sizeof(b.read_mac), 0); + ret = !crypto_memneq(b.read_mac, b.computed_mac, + POLY1305_MAC_SIZE); + + } + + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); + return ret; +} +EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); + +void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + u32 derived_key[CHACHA20_KEY_WORDS] __aligned(16); + + simd_get(&simd_context); + hchacha20(derived_key, nonce, key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce + 16), + (u8 *)derived_key, &simd_context); + memzero_explicit(derived_key, CHACHA20POLY1305_KEY_SIZE); + simd_put(&simd_context); +} +EXPORT_SYMBOL(xchacha20poly1305_encrypt); + +bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + bool ret; + simd_context_t simd_context; + u32 derived_key[CHACHA20_KEY_WORDS] __aligned(16); + + simd_get(&simd_context); + hchacha20(derived_key, nonce, key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce + 16), + (u8 *)derived_key, &simd_context); + memzero_explicit(derived_key, CHACHA20POLY1305_KEY_SIZE); + simd_put(&simd_context); + return ret; +} +EXPORT_SYMBOL(xchacha20poly1305_decrypt); + +#include "selftest/chacha20poly1305.c" + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init chacha20poly1305_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!selftest_run("chacha20poly1305", chacha20poly1305_selftest, + NULL, 0)) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c b/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c new file mode 100644 index 0000000000000..e0c5a5d297c01 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-arm-glue.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include + +asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]); + +static bool curve25519_use_neon __ro_after_init; +static bool *const curve25519_nobs[] __initconst = { &curve25519_use_neon }; +static void __init curve25519_fpu_init(void) +{ + curve25519_use_neon = elf_hwcap & HWCAP_NEON; +} + +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + simd_context_t simd_context; + bool used_arch = false; + + simd_get(&simd_context); + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && + !IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) && curve25519_use_neon && + simd_use(&simd_context)) { + curve25519_neon(mypublic, secret, basepoint); + used_arch = true; + } + simd_put(&simd_context); + return used_arch; +} + +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + return false; +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S b/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S new file mode 100644 index 0000000000000..8eca8a11ef28f --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-arm.S @@ -0,0 +1,2064 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Based on public domain code from Daniel J. Bernstein and Peter Schwabe. This + * began from SUPERCOP's curve25519/neon2/scalarmult.s, but has subsequently been + * manually reworked for use in kernel space. + */ + +#if defined(CONFIG_KERNEL_MODE_NEON) && !defined(__ARMEB__) +#include + +.text +.fpu neon +.arch armv7-a +.align 4 + +SYM_FUNC_START(curve25519_neon) + push {r4-r11, lr} + mov ip, sp + sub r3, sp, #704 + and r3, r3, #0xfffffff0 + mov sp, r3 + movw r4, #0 + movw r5, #254 + vmov.i32 q0, #1 + vshr.u64 q1, q0, #7 + vshr.u64 q0, q0, #8 + vmov.i32 d4, #19 + vmov.i32 d5, #38 + add r6, sp, #480 + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d4-d5}, [r6, : 128] + add r6, r3, #0 + vmov.i32 q2, #0 + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 d4, [r6, : 64] + add r6, r3, #0 + movw r7, #960 + sub r7, r7, #2 + neg r7, r7 + sub r7, r7, r7, LSL #7 + str r7, [r6] + add r6, sp, #672 + vld1.8 {d4-d5}, [r1]! + vld1.8 {d6-d7}, [r1] + vst1.8 {d4-d5}, [r6, : 128]! + vst1.8 {d6-d7}, [r6, : 128] + sub r1, r6, #16 + ldrb r6, [r1] + and r6, r6, #248 + strb r6, [r1] + ldrb r6, [r1, #31] + and r6, r6, #127 + orr r6, r6, #64 + strb r6, [r1, #31] + vmov.i64 q2, #0xffffffff + vshr.u64 q3, q2, #7 + vshr.u64 q2, q2, #6 + vld1.8 {d8}, [r2] + vld1.8 {d10}, [r2] + add r2, r2, #6 + vld1.8 {d12}, [r2] + vld1.8 {d14}, [r2] + add r2, r2, #6 + vld1.8 {d16}, [r2] + add r2, r2, #4 + vld1.8 {d18}, [r2] + vld1.8 {d20}, [r2] + add r2, r2, #6 + vld1.8 {d22}, [r2] + add r2, r2, #2 + vld1.8 {d24}, [r2] + vld1.8 {d26}, [r2] + vshr.u64 q5, q5, #26 + vshr.u64 q6, q6, #3 + vshr.u64 q7, q7, #29 + vshr.u64 q8, q8, #6 + vshr.u64 q10, q10, #25 + vshr.u64 q11, q11, #3 + vshr.u64 q12, q12, #12 + vshr.u64 q13, q13, #38 + vand q4, q4, q2 + vand q6, q6, q2 + vand q8, q8, q2 + vand q10, q10, q2 + vand q2, q12, q2 + vand q5, q5, q3 + vand q7, q7, q3 + vand q9, q9, q3 + vand q11, q11, q3 + vand q3, q13, q3 + add r2, r3, #48 + vadd.i64 q12, q4, q1 + vadd.i64 q13, q10, q1 + vshr.s64 q12, q12, #26 + vshr.s64 q13, q13, #26 + vadd.i64 q5, q5, q12 + vshl.i64 q12, q12, #26 + vadd.i64 q14, q5, q0 + vadd.i64 q11, q11, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q11, q0 + vsub.i64 q4, q4, q12 + vshr.s64 q12, q14, #25 + vsub.i64 q10, q10, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q12 + vshl.i64 q12, q12, #25 + vadd.i64 q14, q6, q1 + vadd.i64 q2, q2, q13 + vsub.i64 q5, q5, q12 + vshr.s64 q12, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q1 + vadd.i64 q7, q7, q12 + vshl.i64 q12, q12, #26 + vadd.i64 q15, q7, q0 + vsub.i64 q11, q11, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q12 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q3, q0 + vadd.i64 q8, q8, q12 + vshl.i64 q12, q12, #25 + vadd.i64 q15, q8, q1 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q7, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q9, q9, q12 + vtrn.32 d12, d14 + vshl.i64 q12, q12, #26 + vtrn.32 d13, d15 + vadd.i64 q0, q9, q0 + vadd.i64 q4, q4, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q6, q13, #4 + vsub.i64 q7, q8, q12 + vshr.s64 q0, q0, #25 + vadd.i64 q4, q4, q6 + vadd.i64 q6, q10, q0 + vshl.i64 q0, q0, #25 + vadd.i64 q8, q6, q1 + vadd.i64 q4, q4, q13 + vshl.i64 q10, q13, #25 + vadd.i64 q1, q4, q1 + vsub.i64 q0, q9, q0 + vshr.s64 q8, q8, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d14, d0 + vshr.s64 q1, q1, #26 + vtrn.32 d15, d1 + vadd.i64 q0, q11, q8 + vst1.8 d14, [r2, : 64] + vshl.i64 q7, q8, #26 + vadd.i64 q5, q5, q1 + vtrn.32 d4, d6 + vshl.i64 q1, q1, #26 + vtrn.32 d5, d7 + vsub.i64 q3, q6, q7 + add r2, r2, #16 + vsub.i64 q1, q4, q1 + vst1.8 d4, [r2, : 64] + vtrn.32 d6, d0 + vtrn.32 d7, d1 + sub r2, r2, #8 + vtrn.32 d2, d10 + vtrn.32 d3, d11 + vst1.8 d6, [r2, : 64] + sub r2, r2, #24 + vst1.8 d2, [r2, : 64] + add r2, r3, #96 + vmov.i32 q0, #0 + vmov.i64 d2, #0xff + vmov.i64 d3, #0 + vshr.u32 q1, q1, #7 + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #144 + vmov.i32 q0, #0 + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #240 + vmov.i32 q0, #0 + vmov.i64 d2, #0xff + vmov.i64 d3, #0 + vshr.u32 q1, q1, #7 + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 d0, [r2, : 64] + add r2, r3, #48 + add r6, r3, #192 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 d4, [r6, : 64] +.Lmainloop: + mov r2, r5, LSR #3 + and r6, r5, #7 + ldrb r2, [r1, r2] + mov r2, r2, LSR r6 + and r2, r2, #1 + str r5, [sp, #456] + eor r4, r4, r2 + str r2, [sp, #460] + neg r2, r4 + add r4, r3, #96 + add r5, r3, #192 + add r6, r3, #144 + vld1.8 {d8-d9}, [r4, : 128]! + add r7, r3, #240 + vld1.8 {d10-d11}, [r5, : 128]! + veor q6, q4, q5 + vld1.8 {d14-d15}, [r6, : 128]! + vdup.i32 q8, r2 + vld1.8 {d18-d19}, [r7, : 128]! + veor q10, q7, q9 + vld1.8 {d22-d23}, [r4, : 128]! + vand q6, q6, q8 + vld1.8 {d24-d25}, [r5, : 128]! + vand q10, q10, q8 + vld1.8 {d26-d27}, [r6, : 128]! + veor q4, q4, q6 + vld1.8 {d28-d29}, [r7, : 128]! + veor q5, q5, q6 + vld1.8 {d0}, [r4, : 64] + veor q6, q7, q10 + vld1.8 {d2}, [r5, : 64] + veor q7, q9, q10 + vld1.8 {d4}, [r6, : 64] + veor q9, q11, q12 + vld1.8 {d6}, [r7, : 64] + veor q10, q0, q1 + sub r2, r4, #32 + vand q9, q9, q8 + sub r4, r5, #32 + vand q10, q10, q8 + sub r5, r6, #32 + veor q11, q11, q9 + sub r6, r7, #32 + veor q0, q0, q10 + veor q9, q12, q9 + veor q1, q1, q10 + veor q10, q13, q14 + veor q12, q2, q3 + vand q10, q10, q8 + vand q8, q12, q8 + veor q12, q13, q10 + veor q2, q2, q8 + veor q10, q14, q10 + veor q3, q3, q8 + vadd.i32 q8, q4, q6 + vsub.i32 q4, q4, q6 + vst1.8 {d16-d17}, [r2, : 128]! + vadd.i32 q6, q11, q12 + vst1.8 {d8-d9}, [r5, : 128]! + vsub.i32 q4, q11, q12 + vst1.8 {d12-d13}, [r2, : 128]! + vadd.i32 q6, q0, q2 + vst1.8 {d8-d9}, [r5, : 128]! + vsub.i32 q0, q0, q2 + vst1.8 d12, [r2, : 64] + vadd.i32 q2, q5, q7 + vst1.8 d0, [r5, : 64] + vsub.i32 q0, q5, q7 + vst1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q2, q9, q10 + vst1.8 {d0-d1}, [r6, : 128]! + vsub.i32 q0, q9, q10 + vst1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q2, q1, q3 + vst1.8 {d0-d1}, [r6, : 128]! + vsub.i32 q0, q1, q3 + vst1.8 d4, [r4, : 64] + vst1.8 d0, [r6, : 64] + add r2, sp, #512 + add r4, r3, #96 + add r5, r3, #144 + vld1.8 {d0-d1}, [r2, : 128] + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4-d5}, [r5, : 128]! + vzip.i32 q1, q2 + vld1.8 {d6-d7}, [r4, : 128]! + vld1.8 {d8-d9}, [r5, : 128]! + vshl.i32 q5, q1, #1 + vzip.i32 q3, q4 + vshl.i32 q6, q2, #1 + vld1.8 {d14}, [r4, : 64] + vshl.i32 q8, q3, #1 + vld1.8 {d15}, [r5, : 64] + vshl.i32 q9, q4, #1 + vmul.i32 d21, d7, d1 + vtrn.32 d14, d15 + vmul.i32 q11, q4, q0 + vmul.i32 q0, q7, q0 + vmull.s32 q12, d2, d2 + vmlal.s32 q12, d11, d1 + vmlal.s32 q12, d12, d0 + vmlal.s32 q12, d13, d23 + vmlal.s32 q12, d16, d22 + vmlal.s32 q12, d7, d21 + vmull.s32 q10, d2, d11 + vmlal.s32 q10, d4, d1 + vmlal.s32 q10, d13, d0 + vmlal.s32 q10, d6, d23 + vmlal.s32 q10, d17, d22 + vmull.s32 q13, d10, d4 + vmlal.s32 q13, d11, d3 + vmlal.s32 q13, d13, d1 + vmlal.s32 q13, d16, d0 + vmlal.s32 q13, d17, d23 + vmlal.s32 q13, d8, d22 + vmull.s32 q1, d10, d5 + vmlal.s32 q1, d11, d4 + vmlal.s32 q1, d6, d1 + vmlal.s32 q1, d17, d0 + vmlal.s32 q1, d8, d23 + vmull.s32 q14, d10, d6 + vmlal.s32 q14, d11, d13 + vmlal.s32 q14, d4, d4 + vmlal.s32 q14, d17, d1 + vmlal.s32 q14, d18, d0 + vmlal.s32 q14, d9, d23 + vmull.s32 q11, d10, d7 + vmlal.s32 q11, d11, d6 + vmlal.s32 q11, d12, d5 + vmlal.s32 q11, d8, d1 + vmlal.s32 q11, d19, d0 + vmull.s32 q15, d10, d8 + vmlal.s32 q15, d11, d17 + vmlal.s32 q15, d12, d6 + vmlal.s32 q15, d13, d5 + vmlal.s32 q15, d19, d1 + vmlal.s32 q15, d14, d0 + vmull.s32 q2, d10, d9 + vmlal.s32 q2, d11, d8 + vmlal.s32 q2, d12, d7 + vmlal.s32 q2, d13, d6 + vmlal.s32 q2, d14, d1 + vmull.s32 q0, d15, d1 + vmlal.s32 q0, d10, d14 + vmlal.s32 q0, d11, d19 + vmlal.s32 q0, d12, d8 + vmlal.s32 q0, d13, d17 + vmlal.s32 q0, d6, d6 + add r2, sp, #480 + vld1.8 {d18-d19}, [r2, : 128]! + vmull.s32 q3, d16, d7 + vmlal.s32 q3, d10, d15 + vmlal.s32 q3, d11, d14 + vmlal.s32 q3, d12, d9 + vmlal.s32 q3, d13, d8 + vld1.8 {d8-d9}, [r2, : 128] + vadd.i64 q5, q12, q9 + vadd.i64 q6, q15, q9 + vshr.s64 q5, q5, #26 + vshr.s64 q6, q6, #26 + vadd.i64 q7, q10, q5 + vshl.i64 q5, q5, #26 + vadd.i64 q8, q7, q4 + vadd.i64 q2, q2, q6 + vshl.i64 q6, q6, #26 + vadd.i64 q10, q2, q4 + vsub.i64 q5, q12, q5 + vshr.s64 q8, q8, #25 + vsub.i64 q6, q15, q6 + vshr.s64 q10, q10, #25 + vadd.i64 q12, q13, q8 + vshl.i64 q8, q8, #25 + vadd.i64 q13, q12, q9 + vadd.i64 q0, q0, q10 + vsub.i64 q7, q7, q8 + vshr.s64 q8, q13, #26 + vshl.i64 q10, q10, #25 + vadd.i64 q13, q0, q9 + vadd.i64 q1, q1, q8 + vshl.i64 q8, q8, #26 + vadd.i64 q15, q1, q4 + vsub.i64 q2, q2, q10 + vshr.s64 q10, q13, #26 + vsub.i64 q8, q12, q8 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q10 + vshl.i64 q10, q10, #26 + vadd.i64 q13, q3, q4 + vadd.i64 q14, q14, q12 + add r2, r3, #288 + vshl.i64 q12, q12, #25 + add r4, r3, #336 + vadd.i64 q15, q14, q9 + add r2, r2, #8 + vsub.i64 q0, q0, q10 + add r4, r4, #8 + vshr.s64 q10, q13, #25 + vsub.i64 q1, q1, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q13, q10, q10 + vadd.i64 q11, q11, q12 + vtrn.32 d16, d2 + vshl.i64 q12, q12, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q11, q4 + vadd.i64 q4, q5, q13 + vst1.8 d16, [r2, : 64]! + vshl.i64 q5, q10, #4 + vst1.8 d17, [r4, : 64]! + vsub.i64 q8, q14, q12 + vshr.s64 q1, q1, #25 + vadd.i64 q4, q4, q5 + vadd.i64 q5, q6, q1 + vshl.i64 q1, q1, #25 + vadd.i64 q6, q5, q9 + vadd.i64 q4, q4, q10 + vshl.i64 q10, q10, #25 + vadd.i64 q9, q4, q9 + vsub.i64 q1, q11, q1 + vshr.s64 q6, q6, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d16, d2 + vshr.s64 q9, q9, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q2, q6 + vst1.8 d16, [r2, : 64] + vshl.i64 q2, q6, #26 + vst1.8 d17, [r4, : 64] + vadd.i64 q6, q7, q9 + vtrn.32 d0, d6 + vshl.i64 q7, q9, #26 + vtrn.32 d1, d7 + vsub.i64 q2, q5, q2 + add r2, r2, #16 + vsub.i64 q3, q4, q7 + vst1.8 d0, [r2, : 64] + add r4, r4, #16 + vst1.8 d1, [r4, : 64] + vtrn.32 d4, d2 + vtrn.32 d5, d3 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d6, d12 + vtrn.32 d7, d13 + vst1.8 d4, [r2, : 64] + vst1.8 d5, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d6, [r2, : 64] + vst1.8 d7, [r4, : 64] + add r2, r3, #240 + add r4, r3, #96 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #144 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #192 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128] + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #144 + vshl.i64 q7, q7, #25 + add r4, r3, #96 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + add r2, r3, #288 + add r4, r3, #336 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vsub.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4-d5}, [r4, : 128]! + vsub.i32 q1, q1, q2 + add r5, r3, #240 + vld1.8 {d4}, [r2, : 64] + vld1.8 {d6}, [r4, : 64] + vsub.i32 q2, q2, q3 + vst1.8 {d0-d1}, [r5, : 128]! + vst1.8 {d2-d3}, [r5, : 128]! + vst1.8 d4, [r5, : 64] + add r2, r3, #144 + add r4, r3, #96 + add r5, r3, #144 + add r6, r3, #192 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vsub.i32 q2, q0, q1 + vadd.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d6-d7}, [r4, : 128]! + vsub.i32 q4, q1, q3 + vadd.i32 q1, q1, q3 + vld1.8 {d6}, [r2, : 64] + vld1.8 {d10}, [r4, : 64] + vsub.i32 q6, q3, q5 + vadd.i32 q3, q3, q5 + vst1.8 {d4-d5}, [r5, : 128]! + vst1.8 {d0-d1}, [r6, : 128]! + vst1.8 {d8-d9}, [r5, : 128]! + vst1.8 {d2-d3}, [r6, : 128]! + vst1.8 d12, [r5, : 64] + vst1.8 d6, [r6, : 64] + add r2, r3, #0 + add r4, r3, #240 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #336 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #288 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #288 + vshl.i64 q7, q7, #25 + add r4, r3, #96 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + add r2, sp, #512 + add r4, r3, #144 + add r5, r3, #192 + vld1.8 {d0-d1}, [r2, : 128] + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4-d5}, [r5, : 128]! + vzip.i32 q1, q2 + vld1.8 {d6-d7}, [r4, : 128]! + vld1.8 {d8-d9}, [r5, : 128]! + vshl.i32 q5, q1, #1 + vzip.i32 q3, q4 + vshl.i32 q6, q2, #1 + vld1.8 {d14}, [r4, : 64] + vshl.i32 q8, q3, #1 + vld1.8 {d15}, [r5, : 64] + vshl.i32 q9, q4, #1 + vmul.i32 d21, d7, d1 + vtrn.32 d14, d15 + vmul.i32 q11, q4, q0 + vmul.i32 q0, q7, q0 + vmull.s32 q12, d2, d2 + vmlal.s32 q12, d11, d1 + vmlal.s32 q12, d12, d0 + vmlal.s32 q12, d13, d23 + vmlal.s32 q12, d16, d22 + vmlal.s32 q12, d7, d21 + vmull.s32 q10, d2, d11 + vmlal.s32 q10, d4, d1 + vmlal.s32 q10, d13, d0 + vmlal.s32 q10, d6, d23 + vmlal.s32 q10, d17, d22 + vmull.s32 q13, d10, d4 + vmlal.s32 q13, d11, d3 + vmlal.s32 q13, d13, d1 + vmlal.s32 q13, d16, d0 + vmlal.s32 q13, d17, d23 + vmlal.s32 q13, d8, d22 + vmull.s32 q1, d10, d5 + vmlal.s32 q1, d11, d4 + vmlal.s32 q1, d6, d1 + vmlal.s32 q1, d17, d0 + vmlal.s32 q1, d8, d23 + vmull.s32 q14, d10, d6 + vmlal.s32 q14, d11, d13 + vmlal.s32 q14, d4, d4 + vmlal.s32 q14, d17, d1 + vmlal.s32 q14, d18, d0 + vmlal.s32 q14, d9, d23 + vmull.s32 q11, d10, d7 + vmlal.s32 q11, d11, d6 + vmlal.s32 q11, d12, d5 + vmlal.s32 q11, d8, d1 + vmlal.s32 q11, d19, d0 + vmull.s32 q15, d10, d8 + vmlal.s32 q15, d11, d17 + vmlal.s32 q15, d12, d6 + vmlal.s32 q15, d13, d5 + vmlal.s32 q15, d19, d1 + vmlal.s32 q15, d14, d0 + vmull.s32 q2, d10, d9 + vmlal.s32 q2, d11, d8 + vmlal.s32 q2, d12, d7 + vmlal.s32 q2, d13, d6 + vmlal.s32 q2, d14, d1 + vmull.s32 q0, d15, d1 + vmlal.s32 q0, d10, d14 + vmlal.s32 q0, d11, d19 + vmlal.s32 q0, d12, d8 + vmlal.s32 q0, d13, d17 + vmlal.s32 q0, d6, d6 + add r2, sp, #480 + vld1.8 {d18-d19}, [r2, : 128]! + vmull.s32 q3, d16, d7 + vmlal.s32 q3, d10, d15 + vmlal.s32 q3, d11, d14 + vmlal.s32 q3, d12, d9 + vmlal.s32 q3, d13, d8 + vld1.8 {d8-d9}, [r2, : 128] + vadd.i64 q5, q12, q9 + vadd.i64 q6, q15, q9 + vshr.s64 q5, q5, #26 + vshr.s64 q6, q6, #26 + vadd.i64 q7, q10, q5 + vshl.i64 q5, q5, #26 + vadd.i64 q8, q7, q4 + vadd.i64 q2, q2, q6 + vshl.i64 q6, q6, #26 + vadd.i64 q10, q2, q4 + vsub.i64 q5, q12, q5 + vshr.s64 q8, q8, #25 + vsub.i64 q6, q15, q6 + vshr.s64 q10, q10, #25 + vadd.i64 q12, q13, q8 + vshl.i64 q8, q8, #25 + vadd.i64 q13, q12, q9 + vadd.i64 q0, q0, q10 + vsub.i64 q7, q7, q8 + vshr.s64 q8, q13, #26 + vshl.i64 q10, q10, #25 + vadd.i64 q13, q0, q9 + vadd.i64 q1, q1, q8 + vshl.i64 q8, q8, #26 + vadd.i64 q15, q1, q4 + vsub.i64 q2, q2, q10 + vshr.s64 q10, q13, #26 + vsub.i64 q8, q12, q8 + vshr.s64 q12, q15, #25 + vadd.i64 q3, q3, q10 + vshl.i64 q10, q10, #26 + vadd.i64 q13, q3, q4 + vadd.i64 q14, q14, q12 + add r2, r3, #144 + vshl.i64 q12, q12, #25 + add r4, r3, #192 + vadd.i64 q15, q14, q9 + add r2, r2, #8 + vsub.i64 q0, q0, q10 + add r4, r4, #8 + vshr.s64 q10, q13, #25 + vsub.i64 q1, q1, q12 + vshr.s64 q12, q15, #26 + vadd.i64 q13, q10, q10 + vadd.i64 q11, q11, q12 + vtrn.32 d16, d2 + vshl.i64 q12, q12, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q11, q4 + vadd.i64 q4, q5, q13 + vst1.8 d16, [r2, : 64]! + vshl.i64 q5, q10, #4 + vst1.8 d17, [r4, : 64]! + vsub.i64 q8, q14, q12 + vshr.s64 q1, q1, #25 + vadd.i64 q4, q4, q5 + vadd.i64 q5, q6, q1 + vshl.i64 q1, q1, #25 + vadd.i64 q6, q5, q9 + vadd.i64 q4, q4, q10 + vshl.i64 q10, q10, #25 + vadd.i64 q9, q4, q9 + vsub.i64 q1, q11, q1 + vshr.s64 q6, q6, #26 + vsub.i64 q3, q3, q10 + vtrn.32 d16, d2 + vshr.s64 q9, q9, #26 + vtrn.32 d17, d3 + vadd.i64 q1, q2, q6 + vst1.8 d16, [r2, : 64] + vshl.i64 q2, q6, #26 + vst1.8 d17, [r4, : 64] + vadd.i64 q6, q7, q9 + vtrn.32 d0, d6 + vshl.i64 q7, q9, #26 + vtrn.32 d1, d7 + vsub.i64 q2, q5, q2 + add r2, r2, #16 + vsub.i64 q3, q4, q7 + vst1.8 d0, [r2, : 64] + add r4, r4, #16 + vst1.8 d1, [r4, : 64] + vtrn.32 d4, d2 + vtrn.32 d5, d3 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d6, d12 + vtrn.32 d7, d13 + vst1.8 d4, [r2, : 64] + vst1.8 d5, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d6, [r2, : 64] + vst1.8 d7, [r4, : 64] + add r2, r3, #336 + add r4, r3, #288 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vadd.i32 q0, q0, q1 + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4-d5}, [r4, : 128]! + vadd.i32 q1, q1, q2 + add r5, r3, #288 + vld1.8 {d4}, [r2, : 64] + vld1.8 {d6}, [r4, : 64] + vadd.i32 q2, q2, q3 + vst1.8 {d0-d1}, [r5, : 128]! + vst1.8 {d2-d3}, [r5, : 128]! + vst1.8 d4, [r5, : 64] + add r2, r3, #48 + add r4, r3, #144 + vld1.8 {d0-d1}, [r4, : 128]! + vld1.8 {d2-d3}, [r4, : 128]! + vld1.8 {d4}, [r4, : 64] + add r4, r3, #288 + vld1.8 {d6-d7}, [r4, : 128]! + vtrn.32 q0, q3 + vld1.8 {d8-d9}, [r4, : 128]! + vshl.i32 q5, q0, #4 + vtrn.32 q1, q4 + vshl.i32 q6, q3, #4 + vadd.i32 q5, q5, q0 + vadd.i32 q6, q6, q3 + vshl.i32 q7, q1, #4 + vld1.8 {d5}, [r4, : 64] + vshl.i32 q8, q4, #4 + vtrn.32 d4, d5 + vadd.i32 q7, q7, q1 + vadd.i32 q8, q8, q4 + vld1.8 {d18-d19}, [r2, : 128]! + vshl.i32 q10, q2, #4 + vld1.8 {d22-d23}, [r2, : 128]! + vadd.i32 q10, q10, q2 + vld1.8 {d24}, [r2, : 64] + vadd.i32 q5, q5, q0 + add r2, r3, #240 + vld1.8 {d26-d27}, [r2, : 128]! + vadd.i32 q6, q6, q3 + vld1.8 {d28-d29}, [r2, : 128]! + vadd.i32 q8, q8, q4 + vld1.8 {d25}, [r2, : 64] + vadd.i32 q10, q10, q2 + vtrn.32 q9, q13 + vadd.i32 q7, q7, q1 + vadd.i32 q5, q5, q0 + vtrn.32 q11, q14 + vadd.i32 q6, q6, q3 + add r2, sp, #528 + vadd.i32 q10, q10, q2 + vtrn.32 d24, d25 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q6, q13, #1 + vst1.8 {d20-d21}, [r2, : 128]! + vshl.i32 q10, q14, #1 + vst1.8 {d12-d13}, [r2, : 128]! + vshl.i32 q15, q12, #1 + vadd.i32 q8, q8, q4 + vext.32 d10, d31, d30, #0 + vadd.i32 q7, q7, q1 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q8, d18, d5 + vmlal.s32 q8, d26, d4 + vmlal.s32 q8, d19, d9 + vmlal.s32 q8, d27, d3 + vmlal.s32 q8, d22, d8 + vmlal.s32 q8, d28, d2 + vmlal.s32 q8, d23, d7 + vmlal.s32 q8, d29, d1 + vmlal.s32 q8, d24, d6 + vmlal.s32 q8, d25, d0 + vst1.8 {d14-d15}, [r2, : 128]! + vmull.s32 q2, d18, d4 + vmlal.s32 q2, d12, d9 + vmlal.s32 q2, d13, d8 + vmlal.s32 q2, d19, d3 + vmlal.s32 q2, d22, d2 + vmlal.s32 q2, d23, d1 + vmlal.s32 q2, d24, d0 + vst1.8 {d20-d21}, [r2, : 128]! + vmull.s32 q7, d18, d9 + vmlal.s32 q7, d26, d3 + vmlal.s32 q7, d19, d8 + vmlal.s32 q7, d27, d2 + vmlal.s32 q7, d22, d7 + vmlal.s32 q7, d28, d1 + vmlal.s32 q7, d23, d6 + vmlal.s32 q7, d29, d0 + vst1.8 {d10-d11}, [r2, : 128]! + vmull.s32 q5, d18, d3 + vmlal.s32 q5, d19, d2 + vmlal.s32 q5, d22, d1 + vmlal.s32 q5, d23, d0 + vmlal.s32 q5, d12, d8 + vst1.8 {d16-d17}, [r2, : 128]! + vmull.s32 q4, d18, d8 + vmlal.s32 q4, d26, d2 + vmlal.s32 q4, d19, d7 + vmlal.s32 q4, d27, d1 + vmlal.s32 q4, d22, d6 + vmlal.s32 q4, d28, d0 + vmull.s32 q8, d18, d7 + vmlal.s32 q8, d26, d1 + vmlal.s32 q8, d19, d6 + vmlal.s32 q8, d27, d0 + add r2, sp, #544 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q7, d24, d21 + vmlal.s32 q7, d25, d20 + vmlal.s32 q4, d23, d21 + vmlal.s32 q4, d29, d20 + vmlal.s32 q8, d22, d21 + vmlal.s32 q8, d28, d20 + vmlal.s32 q5, d24, d20 + vst1.8 {d14-d15}, [r2, : 128] + vmull.s32 q7, d18, d6 + vmlal.s32 q7, d26, d0 + add r2, sp, #624 + vld1.8 {d30-d31}, [r2, : 128] + vmlal.s32 q2, d30, d21 + vmlal.s32 q7, d19, d21 + vmlal.s32 q7, d27, d20 + add r2, sp, #592 + vld1.8 {d26-d27}, [r2, : 128] + vmlal.s32 q4, d25, d27 + vmlal.s32 q8, d29, d27 + vmlal.s32 q8, d25, d26 + vmlal.s32 q7, d28, d27 + vmlal.s32 q7, d29, d26 + add r2, sp, #576 + vld1.8 {d28-d29}, [r2, : 128] + vmlal.s32 q4, d24, d29 + vmlal.s32 q8, d23, d29 + vmlal.s32 q8, d24, d28 + vmlal.s32 q7, d22, d29 + vmlal.s32 q7, d23, d28 + vst1.8 {d8-d9}, [r2, : 128] + add r2, sp, #528 + vld1.8 {d8-d9}, [r2, : 128] + vmlal.s32 q7, d24, d9 + vmlal.s32 q7, d25, d31 + vmull.s32 q1, d18, d2 + vmlal.s32 q1, d19, d1 + vmlal.s32 q1, d22, d0 + vmlal.s32 q1, d24, d27 + vmlal.s32 q1, d23, d20 + vmlal.s32 q1, d12, d7 + vmlal.s32 q1, d13, d6 + vmull.s32 q6, d18, d1 + vmlal.s32 q6, d19, d0 + vmlal.s32 q6, d23, d27 + vmlal.s32 q6, d22, d20 + vmlal.s32 q6, d24, d26 + vmull.s32 q0, d18, d0 + vmlal.s32 q0, d22, d27 + vmlal.s32 q0, d23, d26 + vmlal.s32 q0, d24, d31 + vmlal.s32 q0, d19, d20 + add r2, sp, #608 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q2, d18, d7 + vmlal.s32 q5, d18, d6 + vmlal.s32 q1, d18, d21 + vmlal.s32 q0, d18, d28 + vmlal.s32 q6, d18, d29 + vmlal.s32 q2, d19, d6 + vmlal.s32 q5, d19, d21 + vmlal.s32 q1, d19, d29 + vmlal.s32 q0, d19, d9 + vmlal.s32 q6, d19, d28 + add r2, sp, #560 + vld1.8 {d18-d19}, [r2, : 128] + add r2, sp, #480 + vld1.8 {d22-d23}, [r2, : 128] + vmlal.s32 q5, d19, d7 + vmlal.s32 q0, d18, d21 + vmlal.s32 q0, d19, d29 + vmlal.s32 q6, d18, d6 + add r2, sp, #496 + vld1.8 {d6-d7}, [r2, : 128] + vmlal.s32 q6, d19, d21 + add r2, sp, #544 + vld1.8 {d18-d19}, [r2, : 128] + vmlal.s32 q0, d30, d8 + add r2, sp, #640 + vld1.8 {d20-d21}, [r2, : 128] + vmlal.s32 q5, d30, d29 + add r2, sp, #576 + vld1.8 {d24-d25}, [r2, : 128] + vmlal.s32 q1, d30, d28 + vadd.i64 q13, q0, q11 + vadd.i64 q14, q5, q11 + vmlal.s32 q6, d30, d9 + vshr.s64 q4, q13, #26 + vshr.s64 q13, q14, #26 + vadd.i64 q7, q7, q4 + vshl.i64 q4, q4, #26 + vadd.i64 q14, q7, q3 + vadd.i64 q9, q9, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q15, q9, q3 + vsub.i64 q0, q0, q4 + vshr.s64 q4, q14, #25 + vsub.i64 q5, q5, q13 + vshr.s64 q13, q15, #25 + vadd.i64 q6, q6, q4 + vshl.i64 q4, q4, #25 + vadd.i64 q14, q6, q11 + vadd.i64 q2, q2, q13 + vsub.i64 q4, q7, q4 + vshr.s64 q7, q14, #26 + vshl.i64 q13, q13, #25 + vadd.i64 q14, q2, q11 + vadd.i64 q8, q8, q7 + vshl.i64 q7, q7, #26 + vadd.i64 q15, q8, q3 + vsub.i64 q9, q9, q13 + vshr.s64 q13, q14, #26 + vsub.i64 q6, q6, q7 + vshr.s64 q7, q15, #25 + vadd.i64 q10, q10, q13 + vshl.i64 q13, q13, #26 + vadd.i64 q14, q10, q3 + vadd.i64 q1, q1, q7 + add r2, r3, #240 + vshl.i64 q7, q7, #25 + add r4, r3, #144 + vadd.i64 q15, q1, q11 + add r2, r2, #8 + vsub.i64 q2, q2, q13 + add r4, r4, #8 + vshr.s64 q13, q14, #25 + vsub.i64 q7, q8, q7 + vshr.s64 q8, q15, #26 + vadd.i64 q14, q13, q13 + vadd.i64 q12, q12, q8 + vtrn.32 d12, d14 + vshl.i64 q8, q8, #26 + vtrn.32 d13, d15 + vadd.i64 q3, q12, q3 + vadd.i64 q0, q0, q14 + vst1.8 d12, [r2, : 64]! + vshl.i64 q7, q13, #4 + vst1.8 d13, [r4, : 64]! + vsub.i64 q1, q1, q8 + vshr.s64 q3, q3, #25 + vadd.i64 q0, q0, q7 + vadd.i64 q5, q5, q3 + vshl.i64 q3, q3, #25 + vadd.i64 q6, q5, q11 + vadd.i64 q0, q0, q13 + vshl.i64 q7, q13, #25 + vadd.i64 q8, q0, q11 + vsub.i64 q3, q12, q3 + vshr.s64 q6, q6, #26 + vsub.i64 q7, q10, q7 + vtrn.32 d2, d6 + vshr.s64 q8, q8, #26 + vtrn.32 d3, d7 + vadd.i64 q3, q9, q6 + vst1.8 d2, [r2, : 64] + vshl.i64 q6, q6, #26 + vst1.8 d3, [r4, : 64] + vadd.i64 q1, q4, q8 + vtrn.32 d4, d14 + vshl.i64 q4, q8, #26 + vtrn.32 d5, d15 + vsub.i64 q5, q5, q6 + add r2, r2, #16 + vsub.i64 q0, q0, q4 + vst1.8 d4, [r2, : 64] + add r4, r4, #16 + vst1.8 d5, [r4, : 64] + vtrn.32 d10, d6 + vtrn.32 d11, d7 + sub r2, r2, #8 + sub r4, r4, #8 + vtrn.32 d0, d2 + vtrn.32 d1, d3 + vst1.8 d10, [r2, : 64] + vst1.8 d11, [r4, : 64] + sub r2, r2, #24 + sub r4, r4, #24 + vst1.8 d0, [r2, : 64] + vst1.8 d1, [r4, : 64] + ldr r2, [sp, #456] + ldr r4, [sp, #460] + subs r5, r2, #1 + bge .Lmainloop + add r1, r3, #144 + add r2, r3, #336 + vld1.8 {d0-d1}, [r1, : 128]! + vld1.8 {d2-d3}, [r1, : 128]! + vld1.8 {d4}, [r1, : 64] + vst1.8 {d0-d1}, [r2, : 128]! + vst1.8 {d2-d3}, [r2, : 128]! + vst1.8 d4, [r2, : 64] + movw r1, #0 +.Linvertloop: + add r2, r3, #144 + movw r4, #0 + movw r5, #2 + cmp r1, #1 + moveq r5, #1 + addeq r2, r3, #336 + addeq r4, r3, #48 + cmp r1, #2 + moveq r5, #1 + addeq r2, r3, #48 + cmp r1, #3 + moveq r5, #5 + addeq r4, r3, #336 + cmp r1, #4 + moveq r5, #10 + cmp r1, #5 + moveq r5, #20 + cmp r1, #6 + moveq r5, #10 + addeq r2, r3, #336 + addeq r4, r3, #336 + cmp r1, #7 + moveq r5, #50 + cmp r1, #8 + moveq r5, #100 + cmp r1, #9 + moveq r5, #50 + addeq r2, r3, #336 + cmp r1, #10 + moveq r5, #5 + addeq r2, r3, #48 + cmp r1, #11 + moveq r5, #0 + addeq r2, r3, #96 + add r6, r3, #144 + add r7, r3, #288 + vld1.8 {d0-d1}, [r6, : 128]! + vld1.8 {d2-d3}, [r6, : 128]! + vld1.8 {d4}, [r6, : 64] + vst1.8 {d0-d1}, [r7, : 128]! + vst1.8 {d2-d3}, [r7, : 128]! + vst1.8 d4, [r7, : 64] + cmp r5, #0 + beq .Lskipsquaringloop +.Lsquaringloop: + add r6, r3, #288 + add r7, r3, #288 + add r8, r3, #288 + vmov.i32 q0, #19 + vmov.i32 q1, #0 + vmov.i32 q2, #1 + vzip.i32 q1, q2 + vld1.8 {d4-d5}, [r7, : 128]! + vld1.8 {d6-d7}, [r7, : 128]! + vld1.8 {d9}, [r7, : 64] + vld1.8 {d10-d11}, [r6, : 128]! + add r7, sp, #384 + vld1.8 {d12-d13}, [r6, : 128]! + vmul.i32 q7, q2, q0 + vld1.8 {d8}, [r6, : 64] + vext.32 d17, d11, d10, #1 + vmul.i32 q9, q3, q0 + vext.32 d16, d10, d8, #1 + vshl.u32 q10, q5, q1 + vext.32 d22, d14, d4, #1 + vext.32 d24, d18, d6, #1 + vshl.u32 q13, q6, q1 + vshl.u32 d28, d8, d2 + vrev64.i32 d22, d22 + vmul.i32 d1, d9, d1 + vrev64.i32 d24, d24 + vext.32 d29, d8, d13, #1 + vext.32 d0, d1, d9, #1 + vrev64.i32 d0, d0 + vext.32 d2, d9, d1, #1 + vext.32 d23, d15, d5, #1 + vmull.s32 q4, d20, d4 + vrev64.i32 d23, d23 + vmlal.s32 q4, d21, d1 + vrev64.i32 d2, d2 + vmlal.s32 q4, d26, d19 + vext.32 d3, d5, d15, #1 + vmlal.s32 q4, d27, d18 + vrev64.i32 d3, d3 + vmlal.s32 q4, d28, d15 + vext.32 d14, d12, d11, #1 + vmull.s32 q5, d16, d23 + vext.32 d15, d13, d12, #1 + vmlal.s32 q5, d17, d4 + vst1.8 d8, [r7, : 64]! + vmlal.s32 q5, d14, d1 + vext.32 d12, d9, d8, #0 + vmlal.s32 q5, d15, d19 + vmov.i64 d13, #0 + vmlal.s32 q5, d29, d18 + vext.32 d25, d19, d7, #1 + vmlal.s32 q6, d20, d5 + vrev64.i32 d25, d25 + vmlal.s32 q6, d21, d4 + vst1.8 d11, [r7, : 64]! + vmlal.s32 q6, d26, d1 + vext.32 d9, d10, d10, #0 + vmlal.s32 q6, d27, d19 + vmov.i64 d8, #0 + vmlal.s32 q6, d28, d18 + vmlal.s32 q4, d16, d24 + vmlal.s32 q4, d17, d5 + vmlal.s32 q4, d14, d4 + vst1.8 d12, [r7, : 64]! + vmlal.s32 q4, d15, d1 + vext.32 d10, d13, d12, #0 + vmlal.s32 q4, d29, d19 + vmov.i64 d11, #0 + vmlal.s32 q5, d20, d6 + vmlal.s32 q5, d21, d5 + vmlal.s32 q5, d26, d4 + vext.32 d13, d8, d8, #0 + vmlal.s32 q5, d27, d1 + vmov.i64 d12, #0 + vmlal.s32 q5, d28, d19 + vst1.8 d9, [r7, : 64]! + vmlal.s32 q6, d16, d25 + vmlal.s32 q6, d17, d6 + vst1.8 d10, [r7, : 64] + vmlal.s32 q6, d14, d5 + vext.32 d8, d11, d10, #0 + vmlal.s32 q6, d15, d4 + vmov.i64 d9, #0 + vmlal.s32 q6, d29, d1 + vmlal.s32 q4, d20, d7 + vmlal.s32 q4, d21, d6 + vmlal.s32 q4, d26, d5 + vext.32 d11, d12, d12, #0 + vmlal.s32 q4, d27, d4 + vmov.i64 d10, #0 + vmlal.s32 q4, d28, d1 + vmlal.s32 q5, d16, d0 + sub r6, r7, #32 + vmlal.s32 q5, d17, d7 + vmlal.s32 q5, d14, d6 + vext.32 d30, d9, d8, #0 + vmlal.s32 q5, d15, d5 + vld1.8 {d31}, [r6, : 64]! + vmlal.s32 q5, d29, d4 + vmlal.s32 q15, d20, d0 + vext.32 d0, d6, d18, #1 + vmlal.s32 q15, d21, d25 + vrev64.i32 d0, d0 + vmlal.s32 q15, d26, d24 + vext.32 d1, d7, d19, #1 + vext.32 d7, d10, d10, #0 + vmlal.s32 q15, d27, d23 + vrev64.i32 d1, d1 + vld1.8 {d6}, [r6, : 64] + vmlal.s32 q15, d28, d22 + vmlal.s32 q3, d16, d4 + add r6, r6, #24 + vmlal.s32 q3, d17, d2 + vext.32 d4, d31, d30, #0 + vmov d17, d11 + vmlal.s32 q3, d14, d1 + vext.32 d11, d13, d13, #0 + vext.32 d13, d30, d30, #0 + vmlal.s32 q3, d15, d0 + vext.32 d1, d8, d8, #0 + vmlal.s32 q3, d29, d3 + vld1.8 {d5}, [r6, : 64] + sub r6, r6, #16 + vext.32 d10, d6, d6, #0 + vmov.i32 q1, #0xffffffff + vshl.i64 q4, q1, #25 + add r7, sp, #480 + vld1.8 {d14-d15}, [r7, : 128] + vadd.i64 q9, q2, q7 + vshl.i64 q1, q1, #26 + vshr.s64 q10, q9, #26 + vld1.8 {d0}, [r6, : 64]! + vadd.i64 q5, q5, q10 + vand q9, q9, q1 + vld1.8 {d16}, [r6, : 64]! + add r6, sp, #496 + vld1.8 {d20-d21}, [r6, : 128] + vadd.i64 q11, q5, q10 + vsub.i64 q2, q2, q9 + vshr.s64 q9, q11, #25 + vext.32 d12, d5, d4, #0 + vand q11, q11, q4 + vadd.i64 q0, q0, q9 + vmov d19, d7 + vadd.i64 q3, q0, q7 + vsub.i64 q5, q5, q11 + vshr.s64 q11, q3, #26 + vext.32 d18, d11, d10, #0 + vand q3, q3, q1 + vadd.i64 q8, q8, q11 + vadd.i64 q11, q8, q10 + vsub.i64 q0, q0, q3 + vshr.s64 q3, q11, #25 + vand q11, q11, q4 + vadd.i64 q3, q6, q3 + vadd.i64 q6, q3, q7 + vsub.i64 q8, q8, q11 + vshr.s64 q11, q6, #26 + vand q6, q6, q1 + vadd.i64 q9, q9, q11 + vadd.i64 d25, d19, d21 + vsub.i64 q3, q3, q6 + vshr.s64 d23, d25, #25 + vand q4, q12, q4 + vadd.i64 d21, d23, d23 + vshl.i64 d25, d23, #4 + vadd.i64 d21, d21, d23 + vadd.i64 d25, d25, d21 + vadd.i64 d4, d4, d25 + vzip.i32 q0, q8 + vadd.i64 d12, d4, d14 + add r6, r8, #8 + vst1.8 d0, [r6, : 64] + vsub.i64 d19, d19, d9 + add r6, r6, #16 + vst1.8 d16, [r6, : 64] + vshr.s64 d22, d12, #26 + vand q0, q6, q1 + vadd.i64 d10, d10, d22 + vzip.i32 q3, q9 + vsub.i64 d4, d4, d0 + sub r6, r6, #8 + vst1.8 d6, [r6, : 64] + add r6, r6, #16 + vst1.8 d18, [r6, : 64] + vzip.i32 q2, q5 + sub r6, r6, #32 + vst1.8 d4, [r6, : 64] + subs r5, r5, #1 + bhi .Lsquaringloop +.Lskipsquaringloop: + mov r2, r2 + add r5, r3, #288 + add r6, r3, #144 + vmov.i32 q0, #19 + vmov.i32 q1, #0 + vmov.i32 q2, #1 + vzip.i32 q1, q2 + vld1.8 {d4-d5}, [r5, : 128]! + vld1.8 {d6-d7}, [r5, : 128]! + vld1.8 {d9}, [r5, : 64] + vld1.8 {d10-d11}, [r2, : 128]! + add r5, sp, #384 + vld1.8 {d12-d13}, [r2, : 128]! + vmul.i32 q7, q2, q0 + vld1.8 {d8}, [r2, : 64] + vext.32 d17, d11, d10, #1 + vmul.i32 q9, q3, q0 + vext.32 d16, d10, d8, #1 + vshl.u32 q10, q5, q1 + vext.32 d22, d14, d4, #1 + vext.32 d24, d18, d6, #1 + vshl.u32 q13, q6, q1 + vshl.u32 d28, d8, d2 + vrev64.i32 d22, d22 + vmul.i32 d1, d9, d1 + vrev64.i32 d24, d24 + vext.32 d29, d8, d13, #1 + vext.32 d0, d1, d9, #1 + vrev64.i32 d0, d0 + vext.32 d2, d9, d1, #1 + vext.32 d23, d15, d5, #1 + vmull.s32 q4, d20, d4 + vrev64.i32 d23, d23 + vmlal.s32 q4, d21, d1 + vrev64.i32 d2, d2 + vmlal.s32 q4, d26, d19 + vext.32 d3, d5, d15, #1 + vmlal.s32 q4, d27, d18 + vrev64.i32 d3, d3 + vmlal.s32 q4, d28, d15 + vext.32 d14, d12, d11, #1 + vmull.s32 q5, d16, d23 + vext.32 d15, d13, d12, #1 + vmlal.s32 q5, d17, d4 + vst1.8 d8, [r5, : 64]! + vmlal.s32 q5, d14, d1 + vext.32 d12, d9, d8, #0 + vmlal.s32 q5, d15, d19 + vmov.i64 d13, #0 + vmlal.s32 q5, d29, d18 + vext.32 d25, d19, d7, #1 + vmlal.s32 q6, d20, d5 + vrev64.i32 d25, d25 + vmlal.s32 q6, d21, d4 + vst1.8 d11, [r5, : 64]! + vmlal.s32 q6, d26, d1 + vext.32 d9, d10, d10, #0 + vmlal.s32 q6, d27, d19 + vmov.i64 d8, #0 + vmlal.s32 q6, d28, d18 + vmlal.s32 q4, d16, d24 + vmlal.s32 q4, d17, d5 + vmlal.s32 q4, d14, d4 + vst1.8 d12, [r5, : 64]! + vmlal.s32 q4, d15, d1 + vext.32 d10, d13, d12, #0 + vmlal.s32 q4, d29, d19 + vmov.i64 d11, #0 + vmlal.s32 q5, d20, d6 + vmlal.s32 q5, d21, d5 + vmlal.s32 q5, d26, d4 + vext.32 d13, d8, d8, #0 + vmlal.s32 q5, d27, d1 + vmov.i64 d12, #0 + vmlal.s32 q5, d28, d19 + vst1.8 d9, [r5, : 64]! + vmlal.s32 q6, d16, d25 + vmlal.s32 q6, d17, d6 + vst1.8 d10, [r5, : 64] + vmlal.s32 q6, d14, d5 + vext.32 d8, d11, d10, #0 + vmlal.s32 q6, d15, d4 + vmov.i64 d9, #0 + vmlal.s32 q6, d29, d1 + vmlal.s32 q4, d20, d7 + vmlal.s32 q4, d21, d6 + vmlal.s32 q4, d26, d5 + vext.32 d11, d12, d12, #0 + vmlal.s32 q4, d27, d4 + vmov.i64 d10, #0 + vmlal.s32 q4, d28, d1 + vmlal.s32 q5, d16, d0 + sub r2, r5, #32 + vmlal.s32 q5, d17, d7 + vmlal.s32 q5, d14, d6 + vext.32 d30, d9, d8, #0 + vmlal.s32 q5, d15, d5 + vld1.8 {d31}, [r2, : 64]! + vmlal.s32 q5, d29, d4 + vmlal.s32 q15, d20, d0 + vext.32 d0, d6, d18, #1 + vmlal.s32 q15, d21, d25 + vrev64.i32 d0, d0 + vmlal.s32 q15, d26, d24 + vext.32 d1, d7, d19, #1 + vext.32 d7, d10, d10, #0 + vmlal.s32 q15, d27, d23 + vrev64.i32 d1, d1 + vld1.8 {d6}, [r2, : 64] + vmlal.s32 q15, d28, d22 + vmlal.s32 q3, d16, d4 + add r2, r2, #24 + vmlal.s32 q3, d17, d2 + vext.32 d4, d31, d30, #0 + vmov d17, d11 + vmlal.s32 q3, d14, d1 + vext.32 d11, d13, d13, #0 + vext.32 d13, d30, d30, #0 + vmlal.s32 q3, d15, d0 + vext.32 d1, d8, d8, #0 + vmlal.s32 q3, d29, d3 + vld1.8 {d5}, [r2, : 64] + sub r2, r2, #16 + vext.32 d10, d6, d6, #0 + vmov.i32 q1, #0xffffffff + vshl.i64 q4, q1, #25 + add r5, sp, #480 + vld1.8 {d14-d15}, [r5, : 128] + vadd.i64 q9, q2, q7 + vshl.i64 q1, q1, #26 + vshr.s64 q10, q9, #26 + vld1.8 {d0}, [r2, : 64]! + vadd.i64 q5, q5, q10 + vand q9, q9, q1 + vld1.8 {d16}, [r2, : 64]! + add r2, sp, #496 + vld1.8 {d20-d21}, [r2, : 128] + vadd.i64 q11, q5, q10 + vsub.i64 q2, q2, q9 + vshr.s64 q9, q11, #25 + vext.32 d12, d5, d4, #0 + vand q11, q11, q4 + vadd.i64 q0, q0, q9 + vmov d19, d7 + vadd.i64 q3, q0, q7 + vsub.i64 q5, q5, q11 + vshr.s64 q11, q3, #26 + vext.32 d18, d11, d10, #0 + vand q3, q3, q1 + vadd.i64 q8, q8, q11 + vadd.i64 q11, q8, q10 + vsub.i64 q0, q0, q3 + vshr.s64 q3, q11, #25 + vand q11, q11, q4 + vadd.i64 q3, q6, q3 + vadd.i64 q6, q3, q7 + vsub.i64 q8, q8, q11 + vshr.s64 q11, q6, #26 + vand q6, q6, q1 + vadd.i64 q9, q9, q11 + vadd.i64 d25, d19, d21 + vsub.i64 q3, q3, q6 + vshr.s64 d23, d25, #25 + vand q4, q12, q4 + vadd.i64 d21, d23, d23 + vshl.i64 d25, d23, #4 + vadd.i64 d21, d21, d23 + vadd.i64 d25, d25, d21 + vadd.i64 d4, d4, d25 + vzip.i32 q0, q8 + vadd.i64 d12, d4, d14 + add r2, r6, #8 + vst1.8 d0, [r2, : 64] + vsub.i64 d19, d19, d9 + add r2, r2, #16 + vst1.8 d16, [r2, : 64] + vshr.s64 d22, d12, #26 + vand q0, q6, q1 + vadd.i64 d10, d10, d22 + vzip.i32 q3, q9 + vsub.i64 d4, d4, d0 + sub r2, r2, #8 + vst1.8 d6, [r2, : 64] + add r2, r2, #16 + vst1.8 d18, [r2, : 64] + vzip.i32 q2, q5 + sub r2, r2, #32 + vst1.8 d4, [r2, : 64] + cmp r4, #0 + beq .Lskippostcopy + add r2, r3, #144 + mov r4, r4 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r4, : 128]! + vst1.8 {d2-d3}, [r4, : 128]! + vst1.8 d4, [r4, : 64] +.Lskippostcopy: + cmp r1, #1 + bne .Lskipfinalcopy + add r2, r3, #288 + add r4, r3, #144 + vld1.8 {d0-d1}, [r2, : 128]! + vld1.8 {d2-d3}, [r2, : 128]! + vld1.8 {d4}, [r2, : 64] + vst1.8 {d0-d1}, [r4, : 128]! + vst1.8 {d2-d3}, [r4, : 128]! + vst1.8 d4, [r4, : 64] +.Lskipfinalcopy: + add r1, r1, #1 + cmp r1, #12 + blo .Linvertloop + add r1, r3, #144 + ldr r2, [r1], #4 + ldr r3, [r1], #4 + ldr r4, [r1], #4 + ldr r5, [r1], #4 + ldr r6, [r1], #4 + ldr r7, [r1], #4 + ldr r8, [r1], #4 + ldr r9, [r1], #4 + ldr r10, [r1], #4 + ldr r1, [r1] + add r11, r1, r1, LSL #4 + add r11, r11, r1, LSL #1 + add r11, r11, #16777216 + mov r11, r11, ASR #25 + add r11, r11, r2 + mov r11, r11, ASR #26 + add r11, r11, r3 + mov r11, r11, ASR #25 + add r11, r11, r4 + mov r11, r11, ASR #26 + add r11, r11, r5 + mov r11, r11, ASR #25 + add r11, r11, r6 + mov r11, r11, ASR #26 + add r11, r11, r7 + mov r11, r11, ASR #25 + add r11, r11, r8 + mov r11, r11, ASR #26 + add r11, r11, r9 + mov r11, r11, ASR #25 + add r11, r11, r10 + mov r11, r11, ASR #26 + add r11, r11, r1 + mov r11, r11, ASR #25 + add r2, r2, r11 + add r2, r2, r11, LSL #1 + add r2, r2, r11, LSL #4 + mov r11, r2, ASR #26 + add r3, r3, r11 + sub r2, r2, r11, LSL #26 + mov r11, r3, ASR #25 + add r4, r4, r11 + sub r3, r3, r11, LSL #25 + mov r11, r4, ASR #26 + add r5, r5, r11 + sub r4, r4, r11, LSL #26 + mov r11, r5, ASR #25 + add r6, r6, r11 + sub r5, r5, r11, LSL #25 + mov r11, r6, ASR #26 + add r7, r7, r11 + sub r6, r6, r11, LSL #26 + mov r11, r7, ASR #25 + add r8, r8, r11 + sub r7, r7, r11, LSL #25 + mov r11, r8, ASR #26 + add r9, r9, r11 + sub r8, r8, r11, LSL #26 + mov r11, r9, ASR #25 + add r10, r10, r11 + sub r9, r9, r11, LSL #25 + mov r11, r10, ASR #26 + add r1, r1, r11 + sub r10, r10, r11, LSL #26 + mov r11, r1, ASR #25 + sub r1, r1, r11, LSL #25 + add r2, r2, r3, LSL #26 + mov r3, r3, LSR #6 + add r3, r3, r4, LSL #19 + mov r4, r4, LSR #13 + add r4, r4, r5, LSL #13 + mov r5, r5, LSR #19 + add r5, r5, r6, LSL #6 + add r6, r7, r8, LSL #25 + mov r7, r8, LSR #7 + add r7, r7, r9, LSL #19 + mov r8, r9, LSR #13 + add r8, r8, r10, LSL #12 + mov r9, r10, LSR #20 + add r1, r9, r1, LSL #6 + str r2, [r0] + str r3, [r0, #4] + str r4, [r0, #8] + str r5, [r0, #12] + str r6, [r0, #16] + str r7, [r0, #20] + str r8, [r0, #24] + str r1, [r0, #28] + movw r0, #0 + mov sp, ip + pop {r4-r11, pc} +SYM_FUNC_END(curve25519_neon) +#endif diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c b/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c new file mode 100644 index 0000000000000..42cfb6c00f984 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-fiat32.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2016 The fiat-crypto Authors. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is a machine-generated formally verified implementation of Curve25519 + * ECDH from: . Though originally + * machine generated, it has been tweaked to be suitable for use in the kernel. + * It is optimized for 32-bit machines and machines that cannot work efficiently + * with 128-bit integer types. + */ + +/* fe means field element. Here the field is \Z/(2^255-19). An element t, + * entries t[0]...t[9], represents the integer t[0]+2^26 t[1]+2^51 t[2]+2^77 + * t[3]+2^102 t[4]+...+2^230 t[9]. + * fe limbs are bounded by 1.125*2^26,1.125*2^25,1.125*2^26,1.125*2^25,etc. + * Multiplication and carrying produce fe from fe_loose. + */ +typedef struct fe { u32 v[10]; } fe; + +/* fe_loose limbs are bounded by 3.375*2^26,3.375*2^25,3.375*2^26,3.375*2^25,etc + * Addition and subtraction produce fe_loose from (fe, fe). + */ +typedef struct fe_loose { u32 v[10]; } fe_loose; + +static __always_inline void fe_frombytes_impl(u32 h[10], const u8 *s) +{ + /* Ignores top bit of s. */ + u32 a0 = get_unaligned_le32(s); + u32 a1 = get_unaligned_le32(s+4); + u32 a2 = get_unaligned_le32(s+8); + u32 a3 = get_unaligned_le32(s+12); + u32 a4 = get_unaligned_le32(s+16); + u32 a5 = get_unaligned_le32(s+20); + u32 a6 = get_unaligned_le32(s+24); + u32 a7 = get_unaligned_le32(s+28); + h[0] = a0&((1<<26)-1); /* 26 used, 32-26 left. 26 */ + h[1] = (a0>>26) | ((a1&((1<<19)-1))<< 6); /* (32-26) + 19 = 6+19 = 25 */ + h[2] = (a1>>19) | ((a2&((1<<13)-1))<<13); /* (32-19) + 13 = 13+13 = 26 */ + h[3] = (a2>>13) | ((a3&((1<< 6)-1))<<19); /* (32-13) + 6 = 19+ 6 = 25 */ + h[4] = (a3>> 6); /* (32- 6) = 26 */ + h[5] = a4&((1<<25)-1); /* 25 */ + h[6] = (a4>>25) | ((a5&((1<<19)-1))<< 7); /* (32-25) + 19 = 7+19 = 26 */ + h[7] = (a5>>19) | ((a6&((1<<12)-1))<<13); /* (32-19) + 12 = 13+12 = 25 */ + h[8] = (a6>>12) | ((a7&((1<< 6)-1))<<20); /* (32-12) + 6 = 20+ 6 = 26 */ + h[9] = (a7>> 6)&((1<<25)-1); /* 25 */ +} + +static __always_inline void fe_frombytes(fe *h, const u8 *s) +{ + fe_frombytes_impl(h->v, s); +} + +static __always_inline u8 /*bool*/ +addcarryx_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 25 bits of result and 1 bit of carry + * (26 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a + b + c; + *low = x & ((1 << 25) - 1); + return (x >> 25) & 1; +} + +static __always_inline u8 /*bool*/ +addcarryx_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 26 bits of result and 1 bit of carry + * (27 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a + b + c; + *low = x & ((1 << 26) - 1); + return (x >> 26) & 1; +} + +static __always_inline u8 /*bool*/ +subborrow_u25(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 25 bits of result and 1 bit of borrow + * (26 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a - b - c; + *low = x & ((1 << 25) - 1); + return x >> 31; +} + +static __always_inline u8 /*bool*/ +subborrow_u26(u8 /*bool*/ c, u32 a, u32 b, u32 *low) +{ + /* This function extracts 26 bits of result and 1 bit of borrow + *(27 total), so a 32-bit intermediate is sufficient. + */ + u32 x = a - b - c; + *low = x & ((1 << 26) - 1); + return x >> 31; +} + +static __always_inline u32 cmovznz32(u32 t, u32 z, u32 nz) +{ + t = -!!t; /* all set if nonzero, 0 if 0 */ + return (t&nz) | ((~t)&z); +} + +static __always_inline void fe_freeze(u32 out[10], const u32 in1[10]) +{ + { const u32 x17 = in1[9]; + { const u32 x18 = in1[8]; + { const u32 x16 = in1[7]; + { const u32 x14 = in1[6]; + { const u32 x12 = in1[5]; + { const u32 x10 = in1[4]; + { const u32 x8 = in1[3]; + { const u32 x6 = in1[2]; + { const u32 x4 = in1[1]; + { const u32 x2 = in1[0]; + { u32 x20; u8/*bool*/ x21 = subborrow_u26(0x0, x2, 0x3ffffed, &x20); + { u32 x23; u8/*bool*/ x24 = subborrow_u25(x21, x4, 0x1ffffff, &x23); + { u32 x26; u8/*bool*/ x27 = subborrow_u26(x24, x6, 0x3ffffff, &x26); + { u32 x29; u8/*bool*/ x30 = subborrow_u25(x27, x8, 0x1ffffff, &x29); + { u32 x32; u8/*bool*/ x33 = subborrow_u26(x30, x10, 0x3ffffff, &x32); + { u32 x35; u8/*bool*/ x36 = subborrow_u25(x33, x12, 0x1ffffff, &x35); + { u32 x38; u8/*bool*/ x39 = subborrow_u26(x36, x14, 0x3ffffff, &x38); + { u32 x41; u8/*bool*/ x42 = subborrow_u25(x39, x16, 0x1ffffff, &x41); + { u32 x44; u8/*bool*/ x45 = subborrow_u26(x42, x18, 0x3ffffff, &x44); + { u32 x47; u8/*bool*/ x48 = subborrow_u25(x45, x17, 0x1ffffff, &x47); + { u32 x49 = cmovznz32(x48, 0x0, 0xffffffff); + { u32 x50 = (x49 & 0x3ffffed); + { u32 x52; u8/*bool*/ x53 = addcarryx_u26(0x0, x20, x50, &x52); + { u32 x54 = (x49 & 0x1ffffff); + { u32 x56; u8/*bool*/ x57 = addcarryx_u25(x53, x23, x54, &x56); + { u32 x58 = (x49 & 0x3ffffff); + { u32 x60; u8/*bool*/ x61 = addcarryx_u26(x57, x26, x58, &x60); + { u32 x62 = (x49 & 0x1ffffff); + { u32 x64; u8/*bool*/ x65 = addcarryx_u25(x61, x29, x62, &x64); + { u32 x66 = (x49 & 0x3ffffff); + { u32 x68; u8/*bool*/ x69 = addcarryx_u26(x65, x32, x66, &x68); + { u32 x70 = (x49 & 0x1ffffff); + { u32 x72; u8/*bool*/ x73 = addcarryx_u25(x69, x35, x70, &x72); + { u32 x74 = (x49 & 0x3ffffff); + { u32 x76; u8/*bool*/ x77 = addcarryx_u26(x73, x38, x74, &x76); + { u32 x78 = (x49 & 0x1ffffff); + { u32 x80; u8/*bool*/ x81 = addcarryx_u25(x77, x41, x78, &x80); + { u32 x82 = (x49 & 0x3ffffff); + { u32 x84; u8/*bool*/ x85 = addcarryx_u26(x81, x44, x82, &x84); + { u32 x86 = (x49 & 0x1ffffff); + { u32 x88; addcarryx_u25(x85, x47, x86, &x88); + out[0] = x52; + out[1] = x56; + out[2] = x60; + out[3] = x64; + out[4] = x68; + out[5] = x72; + out[6] = x76; + out[7] = x80; + out[8] = x84; + out[9] = x88; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_tobytes(u8 s[32], const fe *f) +{ + u32 h[10]; + fe_freeze(h, f->v); + s[0] = h[0] >> 0; + s[1] = h[0] >> 8; + s[2] = h[0] >> 16; + s[3] = (h[0] >> 24) | (h[1] << 2); + s[4] = h[1] >> 6; + s[5] = h[1] >> 14; + s[6] = (h[1] >> 22) | (h[2] << 3); + s[7] = h[2] >> 5; + s[8] = h[2] >> 13; + s[9] = (h[2] >> 21) | (h[3] << 5); + s[10] = h[3] >> 3; + s[11] = h[3] >> 11; + s[12] = (h[3] >> 19) | (h[4] << 6); + s[13] = h[4] >> 2; + s[14] = h[4] >> 10; + s[15] = h[4] >> 18; + s[16] = h[5] >> 0; + s[17] = h[5] >> 8; + s[18] = h[5] >> 16; + s[19] = (h[5] >> 24) | (h[6] << 1); + s[20] = h[6] >> 7; + s[21] = h[6] >> 15; + s[22] = (h[6] >> 23) | (h[7] << 3); + s[23] = h[7] >> 5; + s[24] = h[7] >> 13; + s[25] = (h[7] >> 21) | (h[8] << 4); + s[26] = h[8] >> 4; + s[27] = h[8] >> 12; + s[28] = (h[8] >> 20) | (h[9] << 6); + s[29] = h[9] >> 2; + s[30] = h[9] >> 10; + s[31] = h[9] >> 18; +} + +/* h = f */ +static __always_inline void fe_copy(fe *h, const fe *f) +{ + memmove(h, f, sizeof(u32) * 10); +} + +static __always_inline void fe_copy_lt(fe_loose *h, const fe *f) +{ + memmove(h, f, sizeof(u32) * 10); +} + +/* h = 0 */ +static __always_inline void fe_0(fe *h) +{ + memset(h, 0, sizeof(u32) * 10); +} + +/* h = 1 */ +static __always_inline void fe_1(fe *h) +{ + memset(h, 0, sizeof(u32) * 10); + h->v[0] = 1; +} + +static void fe_add_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + out[0] = (x5 + x23); + out[1] = (x7 + x25); + out[2] = (x9 + x27); + out[3] = (x11 + x29); + out[4] = (x13 + x31); + out[5] = (x15 + x33); + out[6] = (x17 + x35); + out[7] = (x19 + x37); + out[8] = (x21 + x39); + out[9] = (x20 + x38); + }}}}}}}}}}}}}}}}}}}} +} + +/* h = f + g + * Can overlap h with f or g. + */ +static __always_inline void fe_add(fe_loose *h, const fe *f, const fe *g) +{ + fe_add_impl(h->v, f->v, g->v); +} + +static void fe_sub_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + out[0] = ((0x7ffffda + x5) - x23); + out[1] = ((0x3fffffe + x7) - x25); + out[2] = ((0x7fffffe + x9) - x27); + out[3] = ((0x3fffffe + x11) - x29); + out[4] = ((0x7fffffe + x13) - x31); + out[5] = ((0x3fffffe + x15) - x33); + out[6] = ((0x7fffffe + x17) - x35); + out[7] = ((0x3fffffe + x19) - x37); + out[8] = ((0x7fffffe + x21) - x39); + out[9] = ((0x3fffffe + x20) - x38); + }}}}}}}}}}}}}}}}}}}} +} + +/* h = f - g + * Can overlap h with f or g. + */ +static __always_inline void fe_sub(fe_loose *h, const fe *f, const fe *g) +{ + fe_sub_impl(h->v, f->v, g->v); +} + +static void fe_mul_impl(u32 out[10], const u32 in1[10], const u32 in2[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = in2[9]; + { const u32 x39 = in2[8]; + { const u32 x37 = in2[7]; + { const u32 x35 = in2[6]; + { const u32 x33 = in2[5]; + { const u32 x31 = in2[4]; + { const u32 x29 = in2[3]; + { const u32 x27 = in2[2]; + { const u32 x25 = in2[1]; + { const u32 x23 = in2[0]; + { u64 x40 = ((u64)x23 * x5); + { u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5)); + { u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5)); + { u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5)); + { u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5)); + { u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5)); + { u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5)); + { u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5)); + { u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5)); + { u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5)); + { u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9)); + { u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9)); + { u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13)); + { u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13)); + { u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17)); + { u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17)); + { u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19)))); + { u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21)); + { u64 x58 = ((u64)(0x2 * x38) * x20); + { u64 x59 = (x48 + (x58 << 0x4)); + { u64 x60 = (x59 + (x58 << 0x1)); + { u64 x61 = (x60 + x58); + { u64 x62 = (x47 + (x57 << 0x4)); + { u64 x63 = (x62 + (x57 << 0x1)); + { u64 x64 = (x63 + x57); + { u64 x65 = (x46 + (x56 << 0x4)); + { u64 x66 = (x65 + (x56 << 0x1)); + { u64 x67 = (x66 + x56); + { u64 x68 = (x45 + (x55 << 0x4)); + { u64 x69 = (x68 + (x55 << 0x1)); + { u64 x70 = (x69 + x55); + { u64 x71 = (x44 + (x54 << 0x4)); + { u64 x72 = (x71 + (x54 << 0x1)); + { u64 x73 = (x72 + x54); + { u64 x74 = (x43 + (x53 << 0x4)); + { u64 x75 = (x74 + (x53 << 0x1)); + { u64 x76 = (x75 + x53); + { u64 x77 = (x42 + (x52 << 0x4)); + { u64 x78 = (x77 + (x52 << 0x1)); + { u64 x79 = (x78 + x52); + { u64 x80 = (x41 + (x51 << 0x4)); + { u64 x81 = (x80 + (x51 << 0x1)); + { u64 x82 = (x81 + x51); + { u64 x83 = (x40 + (x50 << 0x4)); + { u64 x84 = (x83 + (x50 << 0x1)); + { u64 x85 = (x84 + x50); + { u64 x86 = (x85 >> 0x1a); + { u32 x87 = ((u32)x85 & 0x3ffffff); + { u64 x88 = (x86 + x82); + { u64 x89 = (x88 >> 0x19); + { u32 x90 = ((u32)x88 & 0x1ffffff); + { u64 x91 = (x89 + x79); + { u64 x92 = (x91 >> 0x1a); + { u32 x93 = ((u32)x91 & 0x3ffffff); + { u64 x94 = (x92 + x76); + { u64 x95 = (x94 >> 0x19); + { u32 x96 = ((u32)x94 & 0x1ffffff); + { u64 x97 = (x95 + x73); + { u64 x98 = (x97 >> 0x1a); + { u32 x99 = ((u32)x97 & 0x3ffffff); + { u64 x100 = (x98 + x70); + { u64 x101 = (x100 >> 0x19); + { u32 x102 = ((u32)x100 & 0x1ffffff); + { u64 x103 = (x101 + x67); + { u64 x104 = (x103 >> 0x1a); + { u32 x105 = ((u32)x103 & 0x3ffffff); + { u64 x106 = (x104 + x64); + { u64 x107 = (x106 >> 0x19); + { u32 x108 = ((u32)x106 & 0x1ffffff); + { u64 x109 = (x107 + x61); + { u64 x110 = (x109 >> 0x1a); + { u32 x111 = ((u32)x109 & 0x3ffffff); + { u64 x112 = (x110 + x49); + { u64 x113 = (x112 >> 0x19); + { u32 x114 = ((u32)x112 & 0x1ffffff); + { u64 x115 = (x87 + (0x13 * x113)); + { u32 x116 = (u32) (x115 >> 0x1a); + { u32 x117 = ((u32)x115 & 0x3ffffff); + { u32 x118 = (x116 + x90); + { u32 x119 = (x118 >> 0x19); + { u32 x120 = (x118 & 0x1ffffff); + out[0] = x117; + out[1] = x120; + out[2] = (x119 + x93); + out[3] = x96; + out[4] = x99; + out[5] = x102; + out[6] = x105; + out[7] = x108; + out[8] = x111; + out[9] = x114; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_mul_ttt(fe *h, const fe *f, const fe *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static __always_inline void fe_mul_tlt(fe *h, const fe_loose *f, const fe *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static __always_inline void +fe_mul_tll(fe *h, const fe_loose *f, const fe_loose *g) +{ + fe_mul_impl(h->v, f->v, g->v); +} + +static void fe_sqr_impl(u32 out[10], const u32 in1[10]) +{ + { const u32 x17 = in1[9]; + { const u32 x18 = in1[8]; + { const u32 x16 = in1[7]; + { const u32 x14 = in1[6]; + { const u32 x12 = in1[5]; + { const u32 x10 = in1[4]; + { const u32 x8 = in1[3]; + { const u32 x6 = in1[2]; + { const u32 x4 = in1[1]; + { const u32 x2 = in1[0]; + { u64 x19 = ((u64)x2 * x2); + { u64 x20 = ((u64)(0x2 * x2) * x4); + { u64 x21 = (0x2 * (((u64)x4 * x4) + ((u64)x2 * x6))); + { u64 x22 = (0x2 * (((u64)x4 * x6) + ((u64)x2 * x8))); + { u64 x23 = ((((u64)x6 * x6) + ((u64)(0x4 * x4) * x8)) + ((u64)(0x2 * x2) * x10)); + { u64 x24 = (0x2 * ((((u64)x6 * x8) + ((u64)x4 * x10)) + ((u64)x2 * x12))); + { u64 x25 = (0x2 * (((((u64)x8 * x8) + ((u64)x6 * x10)) + ((u64)x2 * x14)) + ((u64)(0x2 * x4) * x12))); + { u64 x26 = (0x2 * (((((u64)x8 * x10) + ((u64)x6 * x12)) + ((u64)x4 * x14)) + ((u64)x2 * x16))); + { u64 x27 = (((u64)x10 * x10) + (0x2 * ((((u64)x6 * x14) + ((u64)x2 * x18)) + (0x2 * (((u64)x4 * x16) + ((u64)x8 * x12)))))); + { u64 x28 = (0x2 * ((((((u64)x10 * x12) + ((u64)x8 * x14)) + ((u64)x6 * x16)) + ((u64)x4 * x18)) + ((u64)x2 * x17))); + { u64 x29 = (0x2 * (((((u64)x12 * x12) + ((u64)x10 * x14)) + ((u64)x6 * x18)) + (0x2 * (((u64)x8 * x16) + ((u64)x4 * x17))))); + { u64 x30 = (0x2 * (((((u64)x12 * x14) + ((u64)x10 * x16)) + ((u64)x8 * x18)) + ((u64)x6 * x17))); + { u64 x31 = (((u64)x14 * x14) + (0x2 * (((u64)x10 * x18) + (0x2 * (((u64)x12 * x16) + ((u64)x8 * x17)))))); + { u64 x32 = (0x2 * ((((u64)x14 * x16) + ((u64)x12 * x18)) + ((u64)x10 * x17))); + { u64 x33 = (0x2 * ((((u64)x16 * x16) + ((u64)x14 * x18)) + ((u64)(0x2 * x12) * x17))); + { u64 x34 = (0x2 * (((u64)x16 * x18) + ((u64)x14 * x17))); + { u64 x35 = (((u64)x18 * x18) + ((u64)(0x4 * x16) * x17)); + { u64 x36 = ((u64)(0x2 * x18) * x17); + { u64 x37 = ((u64)(0x2 * x17) * x17); + { u64 x38 = (x27 + (x37 << 0x4)); + { u64 x39 = (x38 + (x37 << 0x1)); + { u64 x40 = (x39 + x37); + { u64 x41 = (x26 + (x36 << 0x4)); + { u64 x42 = (x41 + (x36 << 0x1)); + { u64 x43 = (x42 + x36); + { u64 x44 = (x25 + (x35 << 0x4)); + { u64 x45 = (x44 + (x35 << 0x1)); + { u64 x46 = (x45 + x35); + { u64 x47 = (x24 + (x34 << 0x4)); + { u64 x48 = (x47 + (x34 << 0x1)); + { u64 x49 = (x48 + x34); + { u64 x50 = (x23 + (x33 << 0x4)); + { u64 x51 = (x50 + (x33 << 0x1)); + { u64 x52 = (x51 + x33); + { u64 x53 = (x22 + (x32 << 0x4)); + { u64 x54 = (x53 + (x32 << 0x1)); + { u64 x55 = (x54 + x32); + { u64 x56 = (x21 + (x31 << 0x4)); + { u64 x57 = (x56 + (x31 << 0x1)); + { u64 x58 = (x57 + x31); + { u64 x59 = (x20 + (x30 << 0x4)); + { u64 x60 = (x59 + (x30 << 0x1)); + { u64 x61 = (x60 + x30); + { u64 x62 = (x19 + (x29 << 0x4)); + { u64 x63 = (x62 + (x29 << 0x1)); + { u64 x64 = (x63 + x29); + { u64 x65 = (x64 >> 0x1a); + { u32 x66 = ((u32)x64 & 0x3ffffff); + { u64 x67 = (x65 + x61); + { u64 x68 = (x67 >> 0x19); + { u32 x69 = ((u32)x67 & 0x1ffffff); + { u64 x70 = (x68 + x58); + { u64 x71 = (x70 >> 0x1a); + { u32 x72 = ((u32)x70 & 0x3ffffff); + { u64 x73 = (x71 + x55); + { u64 x74 = (x73 >> 0x19); + { u32 x75 = ((u32)x73 & 0x1ffffff); + { u64 x76 = (x74 + x52); + { u64 x77 = (x76 >> 0x1a); + { u32 x78 = ((u32)x76 & 0x3ffffff); + { u64 x79 = (x77 + x49); + { u64 x80 = (x79 >> 0x19); + { u32 x81 = ((u32)x79 & 0x1ffffff); + { u64 x82 = (x80 + x46); + { u64 x83 = (x82 >> 0x1a); + { u32 x84 = ((u32)x82 & 0x3ffffff); + { u64 x85 = (x83 + x43); + { u64 x86 = (x85 >> 0x19); + { u32 x87 = ((u32)x85 & 0x1ffffff); + { u64 x88 = (x86 + x40); + { u64 x89 = (x88 >> 0x1a); + { u32 x90 = ((u32)x88 & 0x3ffffff); + { u64 x91 = (x89 + x28); + { u64 x92 = (x91 >> 0x19); + { u32 x93 = ((u32)x91 & 0x1ffffff); + { u64 x94 = (x66 + (0x13 * x92)); + { u32 x95 = (u32) (x94 >> 0x1a); + { u32 x96 = ((u32)x94 & 0x3ffffff); + { u32 x97 = (x95 + x69); + { u32 x98 = (x97 >> 0x19); + { u32 x99 = (x97 & 0x1ffffff); + out[0] = x96; + out[1] = x99; + out[2] = (x98 + x72); + out[3] = x75; + out[4] = x78; + out[5] = x81; + out[6] = x84; + out[7] = x87; + out[8] = x90; + out[9] = x93; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_sq_tl(fe *h, const fe_loose *f) +{ + fe_sqr_impl(h->v, f->v); +} + +static __always_inline void fe_sq_tt(fe *h, const fe *f) +{ + fe_sqr_impl(h->v, f->v); +} + +static __always_inline void fe_loose_invert(fe *out, const fe_loose *z) +{ + fe t0; + fe t1; + fe t2; + fe t3; + int i; + + fe_sq_tl(&t0, z); + fe_sq_tt(&t1, &t0); + for (i = 1; i < 2; ++i) + fe_sq_tt(&t1, &t1); + fe_mul_tlt(&t1, z, &t1); + fe_mul_ttt(&t0, &t0, &t1); + fe_sq_tt(&t2, &t0); + fe_mul_ttt(&t1, &t1, &t2); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 5; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 10; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t2, &t2, &t1); + fe_sq_tt(&t3, &t2); + for (i = 1; i < 20; ++i) + fe_sq_tt(&t3, &t3); + fe_mul_ttt(&t2, &t3, &t2); + fe_sq_tt(&t2, &t2); + for (i = 1; i < 10; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t2, &t1); + for (i = 1; i < 50; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t2, &t2, &t1); + fe_sq_tt(&t3, &t2); + for (i = 1; i < 100; ++i) + fe_sq_tt(&t3, &t3); + fe_mul_ttt(&t2, &t3, &t2); + fe_sq_tt(&t2, &t2); + for (i = 1; i < 50; ++i) + fe_sq_tt(&t2, &t2); + fe_mul_ttt(&t1, &t2, &t1); + fe_sq_tt(&t1, &t1); + for (i = 1; i < 5; ++i) + fe_sq_tt(&t1, &t1); + fe_mul_ttt(out, &t1, &t0); +} + +static __always_inline void fe_invert(fe *out, const fe *z) +{ + fe_loose l; + fe_copy_lt(&l, z); + fe_loose_invert(out, &l); +} + +/* Replace (f,g) with (g,f) if b == 1; + * replace (f,g) with (f,g) if b == 0. + * + * Preconditions: b in {0,1} + */ +static __always_inline void fe_cswap(fe *f, fe *g, unsigned int b) +{ + unsigned i; + b = 0 - b; + for (i = 0; i < 10; i++) { + u32 x = f->v[i] ^ g->v[i]; + x &= b; + f->v[i] ^= x; + g->v[i] ^= x; + } +} + +/* NOTE: based on fiat-crypto fe_mul, edited for in2=121666, 0, 0.*/ +static __always_inline void fe_mul_121666_impl(u32 out[10], const u32 in1[10]) +{ + { const u32 x20 = in1[9]; + { const u32 x21 = in1[8]; + { const u32 x19 = in1[7]; + { const u32 x17 = in1[6]; + { const u32 x15 = in1[5]; + { const u32 x13 = in1[4]; + { const u32 x11 = in1[3]; + { const u32 x9 = in1[2]; + { const u32 x7 = in1[1]; + { const u32 x5 = in1[0]; + { const u32 x38 = 0; + { const u32 x39 = 0; + { const u32 x37 = 0; + { const u32 x35 = 0; + { const u32 x33 = 0; + { const u32 x31 = 0; + { const u32 x29 = 0; + { const u32 x27 = 0; + { const u32 x25 = 0; + { const u32 x23 = 121666; + { u64 x40 = ((u64)x23 * x5); + { u64 x41 = (((u64)x23 * x7) + ((u64)x25 * x5)); + { u64 x42 = ((((u64)(0x2 * x25) * x7) + ((u64)x23 * x9)) + ((u64)x27 * x5)); + { u64 x43 = (((((u64)x25 * x9) + ((u64)x27 * x7)) + ((u64)x23 * x11)) + ((u64)x29 * x5)); + { u64 x44 = (((((u64)x27 * x9) + (0x2 * (((u64)x25 * x11) + ((u64)x29 * x7)))) + ((u64)x23 * x13)) + ((u64)x31 * x5)); + { u64 x45 = (((((((u64)x27 * x11) + ((u64)x29 * x9)) + ((u64)x25 * x13)) + ((u64)x31 * x7)) + ((u64)x23 * x15)) + ((u64)x33 * x5)); + { u64 x46 = (((((0x2 * ((((u64)x29 * x11) + ((u64)x25 * x15)) + ((u64)x33 * x7))) + ((u64)x27 * x13)) + ((u64)x31 * x9)) + ((u64)x23 * x17)) + ((u64)x35 * x5)); + { u64 x47 = (((((((((u64)x29 * x13) + ((u64)x31 * x11)) + ((u64)x27 * x15)) + ((u64)x33 * x9)) + ((u64)x25 * x17)) + ((u64)x35 * x7)) + ((u64)x23 * x19)) + ((u64)x37 * x5)); + { u64 x48 = (((((((u64)x31 * x13) + (0x2 * (((((u64)x29 * x15) + ((u64)x33 * x11)) + ((u64)x25 * x19)) + ((u64)x37 * x7)))) + ((u64)x27 * x17)) + ((u64)x35 * x9)) + ((u64)x23 * x21)) + ((u64)x39 * x5)); + { u64 x49 = (((((((((((u64)x31 * x15) + ((u64)x33 * x13)) + ((u64)x29 * x17)) + ((u64)x35 * x11)) + ((u64)x27 * x19)) + ((u64)x37 * x9)) + ((u64)x25 * x21)) + ((u64)x39 * x7)) + ((u64)x23 * x20)) + ((u64)x38 * x5)); + { u64 x50 = (((((0x2 * ((((((u64)x33 * x15) + ((u64)x29 * x19)) + ((u64)x37 * x11)) + ((u64)x25 * x20)) + ((u64)x38 * x7))) + ((u64)x31 * x17)) + ((u64)x35 * x13)) + ((u64)x27 * x21)) + ((u64)x39 * x9)); + { u64 x51 = (((((((((u64)x33 * x17) + ((u64)x35 * x15)) + ((u64)x31 * x19)) + ((u64)x37 * x13)) + ((u64)x29 * x21)) + ((u64)x39 * x11)) + ((u64)x27 * x20)) + ((u64)x38 * x9)); + { u64 x52 = (((((u64)x35 * x17) + (0x2 * (((((u64)x33 * x19) + ((u64)x37 * x15)) + ((u64)x29 * x20)) + ((u64)x38 * x11)))) + ((u64)x31 * x21)) + ((u64)x39 * x13)); + { u64 x53 = (((((((u64)x35 * x19) + ((u64)x37 * x17)) + ((u64)x33 * x21)) + ((u64)x39 * x15)) + ((u64)x31 * x20)) + ((u64)x38 * x13)); + { u64 x54 = (((0x2 * ((((u64)x37 * x19) + ((u64)x33 * x20)) + ((u64)x38 * x15))) + ((u64)x35 * x21)) + ((u64)x39 * x17)); + { u64 x55 = (((((u64)x37 * x21) + ((u64)x39 * x19)) + ((u64)x35 * x20)) + ((u64)x38 * x17)); + { u64 x56 = (((u64)x39 * x21) + (0x2 * (((u64)x37 * x20) + ((u64)x38 * x19)))); + { u64 x57 = (((u64)x39 * x20) + ((u64)x38 * x21)); + { u64 x58 = ((u64)(0x2 * x38) * x20); + { u64 x59 = (x48 + (x58 << 0x4)); + { u64 x60 = (x59 + (x58 << 0x1)); + { u64 x61 = (x60 + x58); + { u64 x62 = (x47 + (x57 << 0x4)); + { u64 x63 = (x62 + (x57 << 0x1)); + { u64 x64 = (x63 + x57); + { u64 x65 = (x46 + (x56 << 0x4)); + { u64 x66 = (x65 + (x56 << 0x1)); + { u64 x67 = (x66 + x56); + { u64 x68 = (x45 + (x55 << 0x4)); + { u64 x69 = (x68 + (x55 << 0x1)); + { u64 x70 = (x69 + x55); + { u64 x71 = (x44 + (x54 << 0x4)); + { u64 x72 = (x71 + (x54 << 0x1)); + { u64 x73 = (x72 + x54); + { u64 x74 = (x43 + (x53 << 0x4)); + { u64 x75 = (x74 + (x53 << 0x1)); + { u64 x76 = (x75 + x53); + { u64 x77 = (x42 + (x52 << 0x4)); + { u64 x78 = (x77 + (x52 << 0x1)); + { u64 x79 = (x78 + x52); + { u64 x80 = (x41 + (x51 << 0x4)); + { u64 x81 = (x80 + (x51 << 0x1)); + { u64 x82 = (x81 + x51); + { u64 x83 = (x40 + (x50 << 0x4)); + { u64 x84 = (x83 + (x50 << 0x1)); + { u64 x85 = (x84 + x50); + { u64 x86 = (x85 >> 0x1a); + { u32 x87 = ((u32)x85 & 0x3ffffff); + { u64 x88 = (x86 + x82); + { u64 x89 = (x88 >> 0x19); + { u32 x90 = ((u32)x88 & 0x1ffffff); + { u64 x91 = (x89 + x79); + { u64 x92 = (x91 >> 0x1a); + { u32 x93 = ((u32)x91 & 0x3ffffff); + { u64 x94 = (x92 + x76); + { u64 x95 = (x94 >> 0x19); + { u32 x96 = ((u32)x94 & 0x1ffffff); + { u64 x97 = (x95 + x73); + { u64 x98 = (x97 >> 0x1a); + { u32 x99 = ((u32)x97 & 0x3ffffff); + { u64 x100 = (x98 + x70); + { u64 x101 = (x100 >> 0x19); + { u32 x102 = ((u32)x100 & 0x1ffffff); + { u64 x103 = (x101 + x67); + { u64 x104 = (x103 >> 0x1a); + { u32 x105 = ((u32)x103 & 0x3ffffff); + { u64 x106 = (x104 + x64); + { u64 x107 = (x106 >> 0x19); + { u32 x108 = ((u32)x106 & 0x1ffffff); + { u64 x109 = (x107 + x61); + { u64 x110 = (x109 >> 0x1a); + { u32 x111 = ((u32)x109 & 0x3ffffff); + { u64 x112 = (x110 + x49); + { u64 x113 = (x112 >> 0x19); + { u32 x114 = ((u32)x112 & 0x1ffffff); + { u64 x115 = (x87 + (0x13 * x113)); + { u32 x116 = (u32) (x115 >> 0x1a); + { u32 x117 = ((u32)x115 & 0x3ffffff); + { u32 x118 = (x116 + x90); + { u32 x119 = (x118 >> 0x19); + { u32 x120 = (x118 & 0x1ffffff); + out[0] = x117; + out[1] = x120; + out[2] = (x119 + x93); + out[3] = x96; + out[4] = x99; + out[5] = x102; + out[6] = x105; + out[7] = x108; + out[8] = x111; + out[9] = x114; + }}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}} +} + +static __always_inline void fe_mul121666(fe *h, const fe_loose *f) +{ + fe_mul_121666_impl(h->v, f->v); +} + +static void curve25519_generic(u8 out[CURVE25519_KEY_SIZE], + const u8 scalar[CURVE25519_KEY_SIZE], + const u8 point[CURVE25519_KEY_SIZE]) +{ + fe x1, x2, z2, x3, z3; + fe_loose x2l, z2l, x3l; + unsigned swap = 0; + int pos; + u8 e[32]; + + memcpy(e, scalar, 32); + curve25519_clamp_secret(e); + + /* The following implementation was transcribed to Coq and proven to + * correspond to unary scalar multiplication in affine coordinates given + * that x1 != 0 is the x coordinate of some point on the curve. It was + * also checked in Coq that doing a ladderstep with x1 = x3 = 0 gives + * z2' = z3' = 0, and z2 = z3 = 0 gives z2' = z3' = 0. The statement was + * quantified over the underlying field, so it applies to Curve25519 + * itself and the quadratic twist of Curve25519. It was not proven in + * Coq that prime-field arithmetic correctly simulates extension-field + * arithmetic on prime-field values. The decoding of the byte array + * representation of e was not considered. + * + * Specification of Montgomery curves in affine coordinates: + * + * + * Proof that these form a group that is isomorphic to a Weierstrass + * curve: + * + * + * Coq transcription and correctness proof of the loop + * (where scalarbits=255): + * + * + * preconditions: 0 <= e < 2^255 (not necessarily e < order), + * fe_invert(0) = 0 + */ + fe_frombytes(&x1, point); + fe_1(&x2); + fe_0(&z2); + fe_copy(&x3, &x1); + fe_1(&z3); + + for (pos = 254; pos >= 0; --pos) { + fe tmp0, tmp1; + fe_loose tmp0l, tmp1l; + /* loop invariant as of right before the test, for the case + * where x1 != 0: + * pos >= -1; if z2 = 0 then x2 is nonzero; if z3 = 0 then x3 + * is nonzero + * let r := e >> (pos+1) in the following equalities of + * projective points: + * to_xz (r*P) === if swap then (x3, z3) else (x2, z2) + * to_xz ((r+1)*P) === if swap then (x2, z2) else (x3, z3) + * x1 is the nonzero x coordinate of the nonzero + * point (r*P-(r+1)*P) + */ + unsigned b = 1 & (e[pos / 8] >> (pos & 7)); + swap ^= b; + fe_cswap(&x2, &x3, swap); + fe_cswap(&z2, &z3, swap); + swap = b; + /* Coq transcription of ladderstep formula (called from + * transcribed loop): + * + * + * x1 != 0 + * x1 = 0 + */ + fe_sub(&tmp0l, &x3, &z3); + fe_sub(&tmp1l, &x2, &z2); + fe_add(&x2l, &x2, &z2); + fe_add(&z2l, &x3, &z3); + fe_mul_tll(&z3, &tmp0l, &x2l); + fe_mul_tll(&z2, &z2l, &tmp1l); + fe_sq_tl(&tmp0, &tmp1l); + fe_sq_tl(&tmp1, &x2l); + fe_add(&x3l, &z3, &z2); + fe_sub(&z2l, &z3, &z2); + fe_mul_ttt(&x2, &tmp1, &tmp0); + fe_sub(&tmp1l, &tmp1, &tmp0); + fe_sq_tl(&z2, &z2l); + fe_mul121666(&z3, &tmp1l); + fe_sq_tl(&x3, &x3l); + fe_add(&tmp0l, &tmp0, &z3); + fe_mul_ttt(&z3, &x1, &z2); + fe_mul_tll(&z2, &tmp1l, &tmp0l); + } + /* here pos=-1, so r=e, so to_xz (e*P) === if swap then (x3, z3) + * else (x2, z2) + */ + fe_cswap(&x2, &x3, swap); + fe_cswap(&z2, &z3, swap); + + fe_invert(&z2, &z2); + fe_mul_ttt(&x2, &x2, &z2); + fe_tobytes(out, &x2); + + memzero_explicit(&x1, sizeof(x1)); + memzero_explicit(&x2, sizeof(x2)); + memzero_explicit(&z2, sizeof(z2)); + memzero_explicit(&x3, sizeof(x3)); + memzero_explicit(&z3, sizeof(z3)); + memzero_explicit(&x2l, sizeof(x2l)); + memzero_explicit(&z2l, sizeof(z2l)); + memzero_explicit(&x3l, sizeof(x3l)); + memzero_explicit(&e, sizeof(e)); +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c b/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c new file mode 100644 index 0000000000000..0f729ec74ece8 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-hacl64.c @@ -0,0 +1,784 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2016-2017 INRIA and Microsoft Corporation. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is a machine-generated formally verified implementation of Curve25519 + * ECDH from: . Though originally machine + * generated, it has been tweaked to be suitable for use in the kernel. It is + * optimized for 64-bit machines that can efficiently work with 128-bit + * integer types. + */ + +typedef __uint128_t u128; + +static __always_inline u64 u64_eq_mask(u64 a, u64 b) +{ + u64 x = a ^ b; + u64 minus_x = ~x + (u64)1U; + u64 x_or_minus_x = x | minus_x; + u64 xnx = x_or_minus_x >> (u32)63U; + u64 c = xnx - (u64)1U; + return c; +} + +static __always_inline u64 u64_gte_mask(u64 a, u64 b) +{ + u64 x = a; + u64 y = b; + u64 x_xor_y = x ^ y; + u64 x_sub_y = x - y; + u64 x_sub_y_xor_y = x_sub_y ^ y; + u64 q = x_xor_y | x_sub_y_xor_y; + u64 x_xor_q = x ^ q; + u64 x_xor_q_ = x_xor_q >> (u32)63U; + u64 c = x_xor_q_ - (u64)1U; + return c; +} + +static __always_inline void modulo_carry_top(u64 *b) +{ + u64 b4 = b[4]; + u64 b0 = b[0]; + u64 b4_ = b4 & 0x7ffffffffffffLLU; + u64 b0_ = b0 + 19 * (b4 >> 51); + b[4] = b4_; + b[0] = b0_; +} + +static __always_inline void fproduct_copy_from_wide_(u64 *output, u128 *input) +{ + { + u128 xi = input[0]; + output[0] = ((u64)(xi)); + } + { + u128 xi = input[1]; + output[1] = ((u64)(xi)); + } + { + u128 xi = input[2]; + output[2] = ((u64)(xi)); + } + { + u128 xi = input[3]; + output[3] = ((u64)(xi)); + } + { + u128 xi = input[4]; + output[4] = ((u64)(xi)); + } +} + +static __always_inline void +fproduct_sum_scalar_multiplication_(u128 *output, u64 *input, u64 s) +{ + output[0] += (u128)input[0] * s; + output[1] += (u128)input[1] * s; + output[2] += (u128)input[2] * s; + output[3] += (u128)input[3] * s; + output[4] += (u128)input[4] * s; +} + +static __always_inline void fproduct_carry_wide_(u128 *tmp) +{ + { + u32 ctr = 0; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + { + u32 ctr = 1; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + + { + u32 ctr = 2; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } + { + u32 ctr = 3; + u128 tctr = tmp[ctr]; + u128 tctrp1 = tmp[ctr + 1]; + u64 r0 = ((u64)(tctr)) & 0x7ffffffffffffLLU; + u128 c = ((tctr) >> (51)); + tmp[ctr] = ((u128)(r0)); + tmp[ctr + 1] = ((tctrp1) + (c)); + } +} + +static __always_inline void fmul_shift_reduce(u64 *output) +{ + u64 tmp = output[4]; + u64 b0; + { + u32 ctr = 5 - 0 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 1 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 2 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + { + u32 ctr = 5 - 3 - 1; + u64 z = output[ctr - 1]; + output[ctr] = z; + } + output[0] = tmp; + b0 = output[0]; + output[0] = 19 * b0; +} + +static __always_inline void fmul_mul_shift_reduce_(u128 *output, u64 *input, + u64 *input21) +{ + u32 i; + u64 input2i; + { + u64 input2i = input21[0]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[1]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[2]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + { + u64 input2i = input21[3]; + fproduct_sum_scalar_multiplication_(output, input, input2i); + fmul_shift_reduce(input); + } + i = 4; + input2i = input21[i]; + fproduct_sum_scalar_multiplication_(output, input, input2i); +} + +static __always_inline void fmul_fmul(u64 *output, u64 *input, u64 *input21) +{ + u64 tmp[5] = { input[0], input[1], input[2], input[3], input[4] }; + { + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + u128 t[5] = { 0 }; + fmul_mul_shift_reduce_(t, tmp, input21); + fproduct_carry_wide_(t); + b4 = t[4]; + b0 = t[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + t[4] = b4_; + t[0] = b0_; + fproduct_copy_from_wide_(output, t); + i0 = output[0]; + i1 = output[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + output[0] = i0_; + output[1] = i1_; + } +} + +static __always_inline void fsquare_fsquare__(u128 *tmp, u64 *output) +{ + u64 r0 = output[0]; + u64 r1 = output[1]; + u64 r2 = output[2]; + u64 r3 = output[3]; + u64 r4 = output[4]; + u64 d0 = r0 * 2; + u64 d1 = r1 * 2; + u64 d2 = r2 * 2 * 19; + u64 d419 = r4 * 19; + u64 d4 = d419 * 2; + u128 s0 = ((((((u128)(r0) * (r0))) + (((u128)(d4) * (r1))))) + + (((u128)(d2) * (r3)))); + u128 s1 = ((((((u128)(d0) * (r1))) + (((u128)(d4) * (r2))))) + + (((u128)(r3 * 19) * (r3)))); + u128 s2 = ((((((u128)(d0) * (r2))) + (((u128)(r1) * (r1))))) + + (((u128)(d4) * (r3)))); + u128 s3 = ((((((u128)(d0) * (r3))) + (((u128)(d1) * (r2))))) + + (((u128)(r4) * (d419)))); + u128 s4 = ((((((u128)(d0) * (r4))) + (((u128)(d1) * (r3))))) + + (((u128)(r2) * (r2)))); + tmp[0] = s0; + tmp[1] = s1; + tmp[2] = s2; + tmp[3] = s3; + tmp[4] = s4; +} + +static __always_inline void fsquare_fsquare_(u128 *tmp, u64 *output) +{ + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + fsquare_fsquare__(tmp, output); + fproduct_carry_wide_(tmp); + b4 = tmp[4]; + b0 = tmp[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + tmp[4] = b4_; + tmp[0] = b0_; + fproduct_copy_from_wide_(output, tmp); + i0 = output[0]; + i1 = output[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + output[0] = i0_; + output[1] = i1_; +} + +static __always_inline void fsquare_fsquare_times_(u64 *output, u128 *tmp, + u32 count1) +{ + u32 i; + fsquare_fsquare_(tmp, output); + for (i = 1; i < count1; ++i) + fsquare_fsquare_(tmp, output); +} + +static __always_inline void fsquare_fsquare_times(u64 *output, u64 *input, + u32 count1) +{ + u128 t[5]; + memcpy(output, input, 5 * sizeof(*input)); + fsquare_fsquare_times_(output, t, count1); +} + +static __always_inline void fsquare_fsquare_times_inplace(u64 *output, + u32 count1) +{ + u128 t[5]; + fsquare_fsquare_times_(output, t, count1); +} + +static __always_inline void crecip_crecip(u64 *out, u64 *z) +{ + u64 buf[20] = { 0 }; + u64 *a0 = buf; + u64 *t00 = buf + 5; + u64 *b0 = buf + 10; + u64 *t01; + u64 *b1; + u64 *c0; + u64 *a; + u64 *t0; + u64 *b; + u64 *c; + fsquare_fsquare_times(a0, z, 1); + fsquare_fsquare_times(t00, a0, 2); + fmul_fmul(b0, t00, z); + fmul_fmul(a0, b0, a0); + fsquare_fsquare_times(t00, a0, 1); + fmul_fmul(b0, t00, b0); + fsquare_fsquare_times(t00, b0, 5); + t01 = buf + 5; + b1 = buf + 10; + c0 = buf + 15; + fmul_fmul(b1, t01, b1); + fsquare_fsquare_times(t01, b1, 10); + fmul_fmul(c0, t01, b1); + fsquare_fsquare_times(t01, c0, 20); + fmul_fmul(t01, t01, c0); + fsquare_fsquare_times_inplace(t01, 10); + fmul_fmul(b1, t01, b1); + fsquare_fsquare_times(t01, b1, 50); + a = buf; + t0 = buf + 5; + b = buf + 10; + c = buf + 15; + fmul_fmul(c, t0, b); + fsquare_fsquare_times(t0, c, 100); + fmul_fmul(t0, t0, c); + fsquare_fsquare_times_inplace(t0, 50); + fmul_fmul(t0, t0, b); + fsquare_fsquare_times_inplace(t0, 5); + fmul_fmul(out, t0, a); +} + +static __always_inline void fsum(u64 *a, u64 *b) +{ + a[0] += b[0]; + a[1] += b[1]; + a[2] += b[2]; + a[3] += b[3]; + a[4] += b[4]; +} + +static __always_inline void fdifference(u64 *a, u64 *b) +{ + u64 tmp[5] = { 0 }; + u64 b0; + u64 b1; + u64 b2; + u64 b3; + u64 b4; + memcpy(tmp, b, 5 * sizeof(*b)); + b0 = tmp[0]; + b1 = tmp[1]; + b2 = tmp[2]; + b3 = tmp[3]; + b4 = tmp[4]; + tmp[0] = b0 + 0x3fffffffffff68LLU; + tmp[1] = b1 + 0x3ffffffffffff8LLU; + tmp[2] = b2 + 0x3ffffffffffff8LLU; + tmp[3] = b3 + 0x3ffffffffffff8LLU; + tmp[4] = b4 + 0x3ffffffffffff8LLU; + { + u64 xi = a[0]; + u64 yi = tmp[0]; + a[0] = yi - xi; + } + { + u64 xi = a[1]; + u64 yi = tmp[1]; + a[1] = yi - xi; + } + { + u64 xi = a[2]; + u64 yi = tmp[2]; + a[2] = yi - xi; + } + { + u64 xi = a[3]; + u64 yi = tmp[3]; + a[3] = yi - xi; + } + { + u64 xi = a[4]; + u64 yi = tmp[4]; + a[4] = yi - xi; + } +} + +static __always_inline void fscalar(u64 *output, u64 *b, u64 s) +{ + u128 tmp[5]; + u128 b4; + u128 b0; + u128 b4_; + u128 b0_; + { + u64 xi = b[0]; + tmp[0] = ((u128)(xi) * (s)); + } + { + u64 xi = b[1]; + tmp[1] = ((u128)(xi) * (s)); + } + { + u64 xi = b[2]; + tmp[2] = ((u128)(xi) * (s)); + } + { + u64 xi = b[3]; + tmp[3] = ((u128)(xi) * (s)); + } + { + u64 xi = b[4]; + tmp[4] = ((u128)(xi) * (s)); + } + fproduct_carry_wide_(tmp); + b4 = tmp[4]; + b0 = tmp[0]; + b4_ = ((b4) & (((u128)(0x7ffffffffffffLLU)))); + b0_ = ((b0) + (((u128)(19) * (((u64)(((b4) >> (51)))))))); + tmp[4] = b4_; + tmp[0] = b0_; + fproduct_copy_from_wide_(output, tmp); +} + +static __always_inline void fmul(u64 *output, u64 *a, u64 *b) +{ + fmul_fmul(output, a, b); +} + +static __always_inline void crecip(u64 *output, u64 *input) +{ + crecip_crecip(output, input); +} + +static __always_inline void point_swap_conditional_step(u64 *a, u64 *b, + u64 swap1, u32 ctr) +{ + u32 i = ctr - 1; + u64 ai = a[i]; + u64 bi = b[i]; + u64 x = swap1 & (ai ^ bi); + u64 ai1 = ai ^ x; + u64 bi1 = bi ^ x; + a[i] = ai1; + b[i] = bi1; +} + +static __always_inline void point_swap_conditional5(u64 *a, u64 *b, u64 swap1) +{ + point_swap_conditional_step(a, b, swap1, 5); + point_swap_conditional_step(a, b, swap1, 4); + point_swap_conditional_step(a, b, swap1, 3); + point_swap_conditional_step(a, b, swap1, 2); + point_swap_conditional_step(a, b, swap1, 1); +} + +static __always_inline void point_swap_conditional(u64 *a, u64 *b, u64 iswap) +{ + u64 swap1 = 0 - iswap; + point_swap_conditional5(a, b, swap1); + point_swap_conditional5(a + 5, b + 5, swap1); +} + +static __always_inline void point_copy(u64 *output, u64 *input) +{ + memcpy(output, input, 5 * sizeof(*input)); + memcpy(output + 5, input + 5, 5 * sizeof(*input)); +} + +static __always_inline void addanddouble_fmonty(u64 *pp, u64 *ppq, u64 *p, + u64 *pq, u64 *qmqp) +{ + u64 *qx = qmqp; + u64 *x2 = pp; + u64 *z2 = pp + 5; + u64 *x3 = ppq; + u64 *z3 = ppq + 5; + u64 *x = p; + u64 *z = p + 5; + u64 *xprime = pq; + u64 *zprime = pq + 5; + u64 buf[40] = { 0 }; + u64 *origx = buf; + u64 *origxprime0 = buf + 5; + u64 *xxprime0; + u64 *zzprime0; + u64 *origxprime; + xxprime0 = buf + 25; + zzprime0 = buf + 30; + memcpy(origx, x, 5 * sizeof(*x)); + fsum(x, z); + fdifference(z, origx); + memcpy(origxprime0, xprime, 5 * sizeof(*xprime)); + fsum(xprime, zprime); + fdifference(zprime, origxprime0); + fmul(xxprime0, xprime, z); + fmul(zzprime0, x, zprime); + origxprime = buf + 5; + { + u64 *xx0; + u64 *zz0; + u64 *xxprime; + u64 *zzprime; + u64 *zzzprime; + xx0 = buf + 15; + zz0 = buf + 20; + xxprime = buf + 25; + zzprime = buf + 30; + zzzprime = buf + 35; + memcpy(origxprime, xxprime, 5 * sizeof(*xxprime)); + fsum(xxprime, zzprime); + fdifference(zzprime, origxprime); + fsquare_fsquare_times(x3, xxprime, 1); + fsquare_fsquare_times(zzzprime, zzprime, 1); + fmul(z3, zzzprime, qx); + fsquare_fsquare_times(xx0, x, 1); + fsquare_fsquare_times(zz0, z, 1); + { + u64 *zzz; + u64 *xx; + u64 *zz; + u64 scalar; + zzz = buf + 10; + xx = buf + 15; + zz = buf + 20; + fmul(x2, xx, zz); + fdifference(zz, xx); + scalar = 121665; + fscalar(zzz, zz, scalar); + fsum(zzz, xx); + fmul(z2, zzz, zz); + } + } +} + +static __always_inline void +ladder_smallloop_cmult_small_loop_step(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2, + u64 *q, u8 byt) +{ + u64 bit0 = (u64)(byt >> 7); + u64 bit; + point_swap_conditional(nq, nqpq, bit0); + addanddouble_fmonty(nq2, nqpq2, nq, nqpq, q); + bit = (u64)(byt >> 7); + point_swap_conditional(nq2, nqpq2, bit); +} + +static __always_inline void +ladder_smallloop_cmult_small_loop_double_step(u64 *nq, u64 *nqpq, u64 *nq2, + u64 *nqpq2, u64 *q, u8 byt) +{ + u8 byt1; + ladder_smallloop_cmult_small_loop_step(nq, nqpq, nq2, nqpq2, q, byt); + byt1 = byt << 1; + ladder_smallloop_cmult_small_loop_step(nq2, nqpq2, nq, nqpq, q, byt1); +} + +static __always_inline void +ladder_smallloop_cmult_small_loop(u64 *nq, u64 *nqpq, u64 *nq2, u64 *nqpq2, + u64 *q, u8 byt, u32 i) +{ + while (i--) { + ladder_smallloop_cmult_small_loop_double_step(nq, nqpq, nq2, + nqpq2, q, byt); + byt <<= 2; + } +} + +static __always_inline void ladder_bigloop_cmult_big_loop(u8 *n1, u64 *nq, + u64 *nqpq, u64 *nq2, + u64 *nqpq2, u64 *q, + u32 i) +{ + while (i--) { + u8 byte = n1[i]; + ladder_smallloop_cmult_small_loop(nq, nqpq, nq2, nqpq2, q, + byte, 4); + } +} + +static void ladder_cmult(u64 *result, u8 *n1, u64 *q) +{ + u64 point_buf[40] = { 0 }; + u64 *nq = point_buf; + u64 *nqpq = point_buf + 10; + u64 *nq2 = point_buf + 20; + u64 *nqpq2 = point_buf + 30; + point_copy(nqpq, q); + nq[0] = 1; + ladder_bigloop_cmult_big_loop(n1, nq, nqpq, nq2, nqpq2, q, 32); + point_copy(result, nq); +} + +static __always_inline void format_fexpand(u64 *output, const u8 *input) +{ + const u8 *x00 = input + 6; + const u8 *x01 = input + 12; + const u8 *x02 = input + 19; + const u8 *x0 = input + 24; + u64 i0, i1, i2, i3, i4, output0, output1, output2, output3, output4; + i0 = get_unaligned_le64(input); + i1 = get_unaligned_le64(x00); + i2 = get_unaligned_le64(x01); + i3 = get_unaligned_le64(x02); + i4 = get_unaligned_le64(x0); + output0 = i0 & 0x7ffffffffffffLLU; + output1 = i1 >> 3 & 0x7ffffffffffffLLU; + output2 = i2 >> 6 & 0x7ffffffffffffLLU; + output3 = i3 >> 1 & 0x7ffffffffffffLLU; + output4 = i4 >> 12 & 0x7ffffffffffffLLU; + output[0] = output0; + output[1] = output1; + output[2] = output2; + output[3] = output3; + output[4] = output4; +} + +static __always_inline void format_fcontract_first_carry_pass(u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 t1_ = t1 + (t0 >> 51); + u64 t0_ = t0 & 0x7ffffffffffffLLU; + u64 t2_ = t2 + (t1_ >> 51); + u64 t1__ = t1_ & 0x7ffffffffffffLLU; + u64 t3_ = t3 + (t2_ >> 51); + u64 t2__ = t2_ & 0x7ffffffffffffLLU; + u64 t4_ = t4 + (t3_ >> 51); + u64 t3__ = t3_ & 0x7ffffffffffffLLU; + input[0] = t0_; + input[1] = t1__; + input[2] = t2__; + input[3] = t3__; + input[4] = t4_; +} + +static __always_inline void format_fcontract_first_carry_full(u64 *input) +{ + format_fcontract_first_carry_pass(input); + modulo_carry_top(input); +} + +static __always_inline void format_fcontract_second_carry_pass(u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 t1_ = t1 + (t0 >> 51); + u64 t0_ = t0 & 0x7ffffffffffffLLU; + u64 t2_ = t2 + (t1_ >> 51); + u64 t1__ = t1_ & 0x7ffffffffffffLLU; + u64 t3_ = t3 + (t2_ >> 51); + u64 t2__ = t2_ & 0x7ffffffffffffLLU; + u64 t4_ = t4 + (t3_ >> 51); + u64 t3__ = t3_ & 0x7ffffffffffffLLU; + input[0] = t0_; + input[1] = t1__; + input[2] = t2__; + input[3] = t3__; + input[4] = t4_; +} + +static __always_inline void format_fcontract_second_carry_full(u64 *input) +{ + u64 i0; + u64 i1; + u64 i0_; + u64 i1_; + format_fcontract_second_carry_pass(input); + modulo_carry_top(input); + i0 = input[0]; + i1 = input[1]; + i0_ = i0 & 0x7ffffffffffffLLU; + i1_ = i1 + (i0 >> 51); + input[0] = i0_; + input[1] = i1_; +} + +static __always_inline void format_fcontract_trim(u64 *input) +{ + u64 a0 = input[0]; + u64 a1 = input[1]; + u64 a2 = input[2]; + u64 a3 = input[3]; + u64 a4 = input[4]; + u64 mask0 = u64_gte_mask(a0, 0x7ffffffffffedLLU); + u64 mask1 = u64_eq_mask(a1, 0x7ffffffffffffLLU); + u64 mask2 = u64_eq_mask(a2, 0x7ffffffffffffLLU); + u64 mask3 = u64_eq_mask(a3, 0x7ffffffffffffLLU); + u64 mask4 = u64_eq_mask(a4, 0x7ffffffffffffLLU); + u64 mask = (((mask0 & mask1) & mask2) & mask3) & mask4; + u64 a0_ = a0 - (0x7ffffffffffedLLU & mask); + u64 a1_ = a1 - (0x7ffffffffffffLLU & mask); + u64 a2_ = a2 - (0x7ffffffffffffLLU & mask); + u64 a3_ = a3 - (0x7ffffffffffffLLU & mask); + u64 a4_ = a4 - (0x7ffffffffffffLLU & mask); + input[0] = a0_; + input[1] = a1_; + input[2] = a2_; + input[3] = a3_; + input[4] = a4_; +} + +static __always_inline void format_fcontract_store(u8 *output, u64 *input) +{ + u64 t0 = input[0]; + u64 t1 = input[1]; + u64 t2 = input[2]; + u64 t3 = input[3]; + u64 t4 = input[4]; + u64 o0 = t1 << 51 | t0; + u64 o1 = t2 << 38 | t1 >> 13; + u64 o2 = t3 << 25 | t2 >> 26; + u64 o3 = t4 << 12 | t3 >> 39; + u8 *b0 = output; + u8 *b1 = output + 8; + u8 *b2 = output + 16; + u8 *b3 = output + 24; + put_unaligned_le64(o0, b0); + put_unaligned_le64(o1, b1); + put_unaligned_le64(o2, b2); + put_unaligned_le64(o3, b3); +} + +static __always_inline void format_fcontract(u8 *output, u64 *input) +{ + format_fcontract_first_carry_full(input); + format_fcontract_second_carry_full(input); + format_fcontract_trim(input); + format_fcontract_store(output, input); +} + +static __always_inline void format_scalar_of_point(u8 *scalar, u64 *point) +{ + u64 *x = point; + u64 *z = point + 5; + u64 buf[10] __aligned(32) = { 0 }; + u64 *zmone = buf; + u64 *sc = buf + 5; + crecip(zmone, z); + fmul(sc, x, zmone); + format_fcontract(scalar, sc); +} + +static void curve25519_generic(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + u64 buf0[10] __aligned(32) = { 0 }; + u64 *x0 = buf0; + u64 *z = buf0 + 5; + u64 *q; + format_fexpand(x0, basepoint); + z[0] = 1; + q = buf0; + { + u8 e[32] __aligned(32) = { 0 }; + u8 *scalar; + memcpy(e, secret, 32); + curve25519_clamp_secret(e); + scalar = e; + { + u64 buf[15] = { 0 }; + u64 *nq = buf; + u64 *x = nq; + x[0] = 1; + ladder_cmult(nq, scalar, q); + format_scalar_of_point(mypublic, nq); + memzero_explicit(buf, sizeof(buf)); + } + memzero_explicit(e, sizeof(e)); + } + memzero_explicit(buf0, sizeof(buf0)); +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c new file mode 100644 index 0000000000000..d62bd37b007f9 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64-glue.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include + +#include "curve25519-x86_64.c" + +static bool curve25519_use_bmi2 __ro_after_init; +static bool curve25519_use_adx __ro_after_init; +static bool *const curve25519_nobs[] __initconst = { + &curve25519_use_bmi2, &curve25519_use_adx }; + +static void __init curve25519_fpu_init(void) +{ + curve25519_use_bmi2 = IS_ENABLED(CONFIG_AS_BMI2) && + boot_cpu_has(X86_FEATURE_BMI2); + curve25519_use_adx = IS_ENABLED(CONFIG_AS_ADX) && + boot_cpu_has(X86_FEATURE_BMI2) && + boot_cpu_has(X86_FEATURE_ADX); +} + +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) { + curve25519_adx(mypublic, secret, basepoint); + return true; + } else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) { + curve25519_bmi2(mypublic, secret, basepoint); + return true; + } + return false; +} + +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + if (IS_ENABLED(CONFIG_AS_ADX) && curve25519_use_adx) { + curve25519_adx_base(pub, secret); + return true; + } else if (IS_ENABLED(CONFIG_AS_BMI2) && curve25519_use_bmi2) { + curve25519_bmi2_base(pub, secret); + return true; + } + return false; +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c new file mode 100644 index 0000000000000..e8af3e2e30f82 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519-x86_64.c @@ -0,0 +1,2366 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* + * Copyright (c) 2017 Armando Faz . All Rights Reserved. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * Copyright (C) 2018 Samuel Neves . All Rights Reserved. + */ + +enum { NUM_WORDS_ELTFP25519 = 4 }; +typedef __aligned(32) u64 eltfp25519_1w[NUM_WORDS_ELTFP25519]; +typedef __aligned(32) u64 eltfp25519_1w_buffer[2 * NUM_WORDS_ELTFP25519]; + +#define mul_eltfp25519_1w_adx(c, a, b) do { \ + mul_256x256_integer_adx(m.buffer, a, b); \ + red_eltfp25519_1w_adx(c, m.buffer); \ +} while (0) + +#define mul_eltfp25519_1w_bmi2(c, a, b) do { \ + mul_256x256_integer_bmi2(m.buffer, a, b); \ + red_eltfp25519_1w_bmi2(c, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_1w_adx(a) do { \ + sqr_256x256_integer_adx(m.buffer, a); \ + red_eltfp25519_1w_adx(a, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_1w_bmi2(a) do { \ + sqr_256x256_integer_bmi2(m.buffer, a); \ + red_eltfp25519_1w_bmi2(a, m.buffer); \ +} while (0) + +#define mul_eltfp25519_2w_adx(c, a, b) do { \ + mul2_256x256_integer_adx(m.buffer, a, b); \ + red_eltfp25519_2w_adx(c, m.buffer); \ +} while (0) + +#define mul_eltfp25519_2w_bmi2(c, a, b) do { \ + mul2_256x256_integer_bmi2(m.buffer, a, b); \ + red_eltfp25519_2w_bmi2(c, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_2w_adx(a) do { \ + sqr2_256x256_integer_adx(m.buffer, a); \ + red_eltfp25519_2w_adx(a, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_2w_bmi2(a) do { \ + sqr2_256x256_integer_bmi2(m.buffer, a); \ + red_eltfp25519_2w_bmi2(a, m.buffer); \ +} while (0) + +#define sqrn_eltfp25519_1w_adx(a, times) do { \ + int ____counter = (times); \ + while (____counter-- > 0) \ + sqr_eltfp25519_1w_adx(a); \ +} while (0) + +#define sqrn_eltfp25519_1w_bmi2(a, times) do { \ + int ____counter = (times); \ + while (____counter-- > 0) \ + sqr_eltfp25519_1w_bmi2(a); \ +} while (0) + +#define copy_eltfp25519_1w(C, A) do { \ + (C)[0] = (A)[0]; \ + (C)[1] = (A)[1]; \ + (C)[2] = (A)[2]; \ + (C)[3] = (A)[3]; \ +} while (0) + +#define setzero_eltfp25519_1w(C) do { \ + (C)[0] = 0; \ + (C)[1] = 0; \ + (C)[2] = 0; \ + (C)[3] = 0; \ +} while (0) + +__aligned(32) static const u64 table_ladder_8k[252 * NUM_WORDS_ELTFP25519] = { + /* 1 */ 0xfffffffffffffff3UL, 0xffffffffffffffffUL, + 0xffffffffffffffffUL, 0x5fffffffffffffffUL, + /* 2 */ 0x6b8220f416aafe96UL, 0x82ebeb2b4f566a34UL, + 0xd5a9a5b075a5950fUL, 0x5142b2cf4b2488f4UL, + /* 3 */ 0x6aaebc750069680cUL, 0x89cf7820a0f99c41UL, + 0x2a58d9183b56d0f4UL, 0x4b5aca80e36011a4UL, + /* 4 */ 0x329132348c29745dUL, 0xf4a2e616e1642fd7UL, + 0x1e45bb03ff67bc34UL, 0x306912d0f42a9b4aUL, + /* 5 */ 0xff886507e6af7154UL, 0x04f50e13dfeec82fUL, + 0xaa512fe82abab5ceUL, 0x174e251a68d5f222UL, + /* 6 */ 0xcf96700d82028898UL, 0x1743e3370a2c02c5UL, + 0x379eec98b4e86eaaUL, 0x0c59888a51e0482eUL, + /* 7 */ 0xfbcbf1d699b5d189UL, 0xacaef0d58e9fdc84UL, + 0xc1c20d06231f7614UL, 0x2938218da274f972UL, + /* 8 */ 0xf6af49beff1d7f18UL, 0xcc541c22387ac9c2UL, + 0x96fcc9ef4015c56bUL, 0x69c1627c690913a9UL, + /* 9 */ 0x7a86fd2f4733db0eUL, 0xfdb8c4f29e087de9UL, + 0x095e4b1a8ea2a229UL, 0x1ad7a7c829b37a79UL, + /* 10 */ 0x342d89cad17ea0c0UL, 0x67bedda6cced2051UL, + 0x19ca31bf2bb42f74UL, 0x3df7b4c84980acbbUL, + /* 11 */ 0xa8c6444dc80ad883UL, 0xb91e440366e3ab85UL, + 0xc215cda00164f6d8UL, 0x3d867c6ef247e668UL, + /* 12 */ 0xc7dd582bcc3e658cUL, 0xfd2c4748ee0e5528UL, + 0xa0fd9b95cc9f4f71UL, 0x7529d871b0675ddfUL, + /* 13 */ 0xb8f568b42d3cbd78UL, 0x1233011b91f3da82UL, + 0x2dce6ccd4a7c3b62UL, 0x75e7fc8e9e498603UL, + /* 14 */ 0x2f4f13f1fcd0b6ecUL, 0xf1a8ca1f29ff7a45UL, + 0xc249c1a72981e29bUL, 0x6ebe0dbb8c83b56aUL, + /* 15 */ 0x7114fa8d170bb222UL, 0x65a2dcd5bf93935fUL, + 0xbdc41f68b59c979aUL, 0x2f0eef79a2ce9289UL, + /* 16 */ 0x42ecbf0c083c37ceUL, 0x2930bc09ec496322UL, + 0xf294b0c19cfeac0dUL, 0x3780aa4bedfabb80UL, + /* 17 */ 0x56c17d3e7cead929UL, 0xe7cb4beb2e5722c5UL, + 0x0ce931732dbfe15aUL, 0x41b883c7621052f8UL, + /* 18 */ 0xdbf75ca0c3d25350UL, 0x2936be086eb1e351UL, + 0xc936e03cb4a9b212UL, 0x1d45bf82322225aaUL, + /* 19 */ 0xe81ab1036a024cc5UL, 0xe212201c304c9a72UL, + 0xc5d73fba6832b1fcUL, 0x20ffdb5a4d839581UL, + /* 20 */ 0xa283d367be5d0fadUL, 0x6c2b25ca8b164475UL, + 0x9d4935467caaf22eUL, 0x5166408eee85ff49UL, + /* 21 */ 0x3c67baa2fab4e361UL, 0xb3e433c67ef35cefUL, + 0x5259729241159b1cUL, 0x6a621892d5b0ab33UL, + /* 22 */ 0x20b74a387555cdcbUL, 0x532aa10e1208923fUL, + 0xeaa17b7762281dd1UL, 0x61ab3443f05c44bfUL, + /* 23 */ 0x257a6c422324def8UL, 0x131c6c1017e3cf7fUL, + 0x23758739f630a257UL, 0x295a407a01a78580UL, + /* 24 */ 0xf8c443246d5da8d9UL, 0x19d775450c52fa5dUL, + 0x2afcfc92731bf83dUL, 0x7d10c8e81b2b4700UL, + /* 25 */ 0xc8e0271f70baa20bUL, 0x993748867ca63957UL, + 0x5412efb3cb7ed4bbUL, 0x3196d36173e62975UL, + /* 26 */ 0xde5bcad141c7dffcUL, 0x47cc8cd2b395c848UL, + 0xa34cd942e11af3cbUL, 0x0256dbf2d04ecec2UL, + /* 27 */ 0x875ab7e94b0e667fUL, 0xcad4dd83c0850d10UL, + 0x47f12e8f4e72c79fUL, 0x5f1a87bb8c85b19bUL, + /* 28 */ 0x7ae9d0b6437f51b8UL, 0x12c7ce5518879065UL, + 0x2ade09fe5cf77aeeUL, 0x23a05a2f7d2c5627UL, + /* 29 */ 0x5908e128f17c169aUL, 0xf77498dd8ad0852dUL, + 0x74b4c4ceab102f64UL, 0x183abadd10139845UL, + /* 30 */ 0xb165ba8daa92aaacUL, 0xd5c5ef9599386705UL, + 0xbe2f8f0cf8fc40d1UL, 0x2701e635ee204514UL, + /* 31 */ 0x629fa80020156514UL, 0xf223868764a8c1ceUL, + 0x5b894fff0b3f060eUL, 0x60d9944cf708a3faUL, + /* 32 */ 0xaeea001a1c7a201fUL, 0xebf16a633ee2ce63UL, + 0x6f7709594c7a07e1UL, 0x79b958150d0208cbUL, + /* 33 */ 0x24b55e5301d410e7UL, 0xe3a34edff3fdc84dUL, + 0xd88768e4904032d8UL, 0x131384427b3aaeecUL, + /* 34 */ 0x8405e51286234f14UL, 0x14dc4739adb4c529UL, + 0xb8a2b5b250634ffdUL, 0x2fe2a94ad8a7ff93UL, + /* 35 */ 0xec5c57efe843faddUL, 0x2843ce40f0bb9918UL, + 0xa4b561d6cf3d6305UL, 0x743629bde8fb777eUL, + /* 36 */ 0x343edd46bbaf738fUL, 0xed981828b101a651UL, + 0xa401760b882c797aUL, 0x1fc223e28dc88730UL, + /* 37 */ 0x48604e91fc0fba0eUL, 0xb637f78f052c6fa4UL, + 0x91ccac3d09e9239cUL, 0x23f7eed4437a687cUL, + /* 38 */ 0x5173b1118d9bd800UL, 0x29d641b63189d4a7UL, + 0xfdbf177988bbc586UL, 0x2959894fcad81df5UL, + /* 39 */ 0xaebc8ef3b4bbc899UL, 0x4148995ab26992b9UL, + 0x24e20b0134f92cfbUL, 0x40d158894a05dee8UL, + /* 40 */ 0x46b00b1185af76f6UL, 0x26bac77873187a79UL, + 0x3dc0bf95ab8fff5fUL, 0x2a608bd8945524d7UL, + /* 41 */ 0x26449588bd446302UL, 0x7c4bc21c0388439cUL, + 0x8e98a4f383bd11b2UL, 0x26218d7bc9d876b9UL, + /* 42 */ 0xe3081542997c178aUL, 0x3c2d29a86fb6606fUL, + 0x5c217736fa279374UL, 0x7dde05734afeb1faUL, + /* 43 */ 0x3bf10e3906d42babUL, 0xe4f7803e1980649cUL, + 0xe6053bf89595bf7aUL, 0x394faf38da245530UL, + /* 44 */ 0x7a8efb58896928f4UL, 0xfbc778e9cc6a113cUL, + 0x72670ce330af596fUL, 0x48f222a81d3d6cf7UL, + /* 45 */ 0xf01fce410d72caa7UL, 0x5a20ecc7213b5595UL, + 0x7bc21165c1fa1483UL, 0x07f89ae31da8a741UL, + /* 46 */ 0x05d2c2b4c6830ff9UL, 0xd43e330fc6316293UL, + 0xa5a5590a96d3a904UL, 0x705edb91a65333b6UL, + /* 47 */ 0x048ee15e0bb9a5f7UL, 0x3240cfca9e0aaf5dUL, + 0x8f4b71ceedc4a40bUL, 0x621c0da3de544a6dUL, + /* 48 */ 0x92872836a08c4091UL, 0xce8375b010c91445UL, + 0x8a72eb524f276394UL, 0x2667fcfa7ec83635UL, + /* 49 */ 0x7f4c173345e8752aUL, 0x061b47feee7079a5UL, + 0x25dd9afa9f86ff34UL, 0x3780cef5425dc89cUL, + /* 50 */ 0x1a46035a513bb4e9UL, 0x3e1ef379ac575adaUL, + 0xc78c5f1c5fa24b50UL, 0x321a967634fd9f22UL, + /* 51 */ 0x946707b8826e27faUL, 0x3dca84d64c506fd0UL, + 0xc189218075e91436UL, 0x6d9284169b3b8484UL, + /* 52 */ 0x3a67e840383f2ddfUL, 0x33eec9a30c4f9b75UL, + 0x3ec7c86fa783ef47UL, 0x26ec449fbac9fbc4UL, + /* 53 */ 0x5c0f38cba09b9e7dUL, 0x81168cc762a3478cUL, + 0x3e23b0d306fc121cUL, 0x5a238aa0a5efdcddUL, + /* 54 */ 0x1ba26121c4ea43ffUL, 0x36f8c77f7c8832b5UL, + 0x88fbea0b0adcf99aUL, 0x5ca9938ec25bebf9UL, + /* 55 */ 0xd5436a5e51fccda0UL, 0x1dbc4797c2cd893bUL, + 0x19346a65d3224a08UL, 0x0f5034e49b9af466UL, + /* 56 */ 0xf23c3967a1e0b96eUL, 0xe58b08fa867a4d88UL, + 0xfb2fabc6a7341679UL, 0x2a75381eb6026946UL, + /* 57 */ 0xc80a3be4c19420acUL, 0x66b1f6c681f2b6dcUL, + 0x7cf7036761e93388UL, 0x25abbbd8a660a4c4UL, + /* 58 */ 0x91ea12ba14fd5198UL, 0x684950fc4a3cffa9UL, + 0xf826842130f5ad28UL, 0x3ea988f75301a441UL, + /* 59 */ 0xc978109a695f8c6fUL, 0x1746eb4a0530c3f3UL, + 0x444d6d77b4459995UL, 0x75952b8c054e5cc7UL, + /* 60 */ 0xa3703f7915f4d6aaUL, 0x66c346202f2647d8UL, + 0xd01469df811d644bUL, 0x77fea47d81a5d71fUL, + /* 61 */ 0xc5e9529ef57ca381UL, 0x6eeeb4b9ce2f881aUL, + 0xb6e91a28e8009bd6UL, 0x4b80be3e9afc3fecUL, + /* 62 */ 0x7e3773c526aed2c5UL, 0x1b4afcb453c9a49dUL, + 0xa920bdd7baffb24dUL, 0x7c54699f122d400eUL, + /* 63 */ 0xef46c8e14fa94bc8UL, 0xe0b074ce2952ed5eUL, + 0xbea450e1dbd885d5UL, 0x61b68649320f712cUL, + /* 64 */ 0x8a485f7309ccbdd1UL, 0xbd06320d7d4d1a2dUL, + 0x25232973322dbef4UL, 0x445dc4758c17f770UL, + /* 65 */ 0xdb0434177cc8933cUL, 0xed6fe82175ea059fUL, + 0x1efebefdc053db34UL, 0x4adbe867c65daf99UL, + /* 66 */ 0x3acd71a2a90609dfUL, 0xe5e991856dd04050UL, + 0x1ec69b688157c23cUL, 0x697427f6885cfe4dUL, + /* 67 */ 0xd7be7b9b65e1a851UL, 0xa03d28d522c536ddUL, + 0x28399d658fd2b645UL, 0x49e5b7e17c2641e1UL, + /* 68 */ 0x6f8c3a98700457a4UL, 0x5078f0a25ebb6778UL, + 0xd13c3ccbc382960fUL, 0x2e003258a7df84b1UL, + /* 69 */ 0x8ad1f39be6296a1cUL, 0xc1eeaa652a5fbfb2UL, + 0x33ee0673fd26f3cbUL, 0x59256173a69d2cccUL, + /* 70 */ 0x41ea07aa4e18fc41UL, 0xd9fc19527c87a51eUL, + 0xbdaacb805831ca6fUL, 0x445b652dc916694fUL, + /* 71 */ 0xce92a3a7f2172315UL, 0x1edc282de11b9964UL, + 0xa1823aafe04c314aUL, 0x790a2d94437cf586UL, + /* 72 */ 0x71c447fb93f6e009UL, 0x8922a56722845276UL, + 0xbf70903b204f5169UL, 0x2f7a89891ba319feUL, + /* 73 */ 0x02a08eb577e2140cUL, 0xed9a4ed4427bdcf4UL, + 0x5253ec44e4323cd1UL, 0x3e88363c14e9355bUL, + /* 74 */ 0xaa66c14277110b8cUL, 0x1ae0391610a23390UL, + 0x2030bd12c93fc2a2UL, 0x3ee141579555c7abUL, + /* 75 */ 0x9214de3a6d6e7d41UL, 0x3ccdd88607f17efeUL, + 0x674f1288f8e11217UL, 0x5682250f329f93d0UL, + /* 76 */ 0x6cf00b136d2e396eUL, 0x6e4cf86f1014debfUL, + 0x5930b1b5bfcc4e83UL, 0x047069b48aba16b6UL, + /* 77 */ 0x0d4ce4ab69b20793UL, 0xb24db91a97d0fb9eUL, + 0xcdfa50f54e00d01dUL, 0x221b1085368bddb5UL, + /* 78 */ 0xe7e59468b1e3d8d2UL, 0x53c56563bd122f93UL, + 0xeee8a903e0663f09UL, 0x61efa662cbbe3d42UL, + /* 79 */ 0x2cf8ddddde6eab2aUL, 0x9bf80ad51435f231UL, + 0x5deadacec9f04973UL, 0x29275b5d41d29b27UL, + /* 80 */ 0xcfde0f0895ebf14fUL, 0xb9aab96b054905a7UL, + 0xcae80dd9a1c420fdUL, 0x0a63bf2f1673bbc7UL, + /* 81 */ 0x092f6e11958fbc8cUL, 0x672a81e804822fadUL, + 0xcac8351560d52517UL, 0x6f3f7722c8f192f8UL, + /* 82 */ 0xf8ba90ccc2e894b7UL, 0x2c7557a438ff9f0dUL, + 0x894d1d855ae52359UL, 0x68e122157b743d69UL, + /* 83 */ 0xd87e5570cfb919f3UL, 0x3f2cdecd95798db9UL, + 0x2121154710c0a2ceUL, 0x3c66a115246dc5b2UL, + /* 84 */ 0xcbedc562294ecb72UL, 0xba7143c36a280b16UL, + 0x9610c2efd4078b67UL, 0x6144735d946a4b1eUL, + /* 85 */ 0x536f111ed75b3350UL, 0x0211db8c2041d81bUL, + 0xf93cb1000e10413cUL, 0x149dfd3c039e8876UL, + /* 86 */ 0xd479dde46b63155bUL, 0xb66e15e93c837976UL, + 0xdafde43b1f13e038UL, 0x5fafda1a2e4b0b35UL, + /* 87 */ 0x3600bbdf17197581UL, 0x3972050bbe3cd2c2UL, + 0x5938906dbdd5be86UL, 0x34fce5e43f9b860fUL, + /* 88 */ 0x75a8a4cd42d14d02UL, 0x828dabc53441df65UL, + 0x33dcabedd2e131d3UL, 0x3ebad76fb814d25fUL, + /* 89 */ 0xd4906f566f70e10fUL, 0x5d12f7aa51690f5aUL, + 0x45adb16e76cefcf2UL, 0x01f768aead232999UL, + /* 90 */ 0x2b6cc77b6248febdUL, 0x3cd30628ec3aaffdUL, + 0xce1c0b80d4ef486aUL, 0x4c3bff2ea6f66c23UL, + /* 91 */ 0x3f2ec4094aeaeb5fUL, 0x61b19b286e372ca7UL, + 0x5eefa966de2a701dUL, 0x23b20565de55e3efUL, + /* 92 */ 0xe301ca5279d58557UL, 0x07b2d4ce27c2874fUL, + 0xa532cd8a9dcf1d67UL, 0x2a52fee23f2bff56UL, + /* 93 */ 0x8624efb37cd8663dUL, 0xbbc7ac20ffbd7594UL, + 0x57b85e9c82d37445UL, 0x7b3052cb86a6ec66UL, + /* 94 */ 0x3482f0ad2525e91eUL, 0x2cb68043d28edca0UL, + 0xaf4f6d052e1b003aUL, 0x185f8c2529781b0aUL, + /* 95 */ 0xaa41de5bd80ce0d6UL, 0x9407b2416853e9d6UL, + 0x563ec36e357f4c3aUL, 0x4cc4b8dd0e297bceUL, + /* 96 */ 0xa2fc1a52ffb8730eUL, 0x1811f16e67058e37UL, + 0x10f9a366cddf4ee1UL, 0x72f4a0c4a0b9f099UL, + /* 97 */ 0x8c16c06f663f4ea7UL, 0x693b3af74e970fbaUL, + 0x2102e7f1d69ec345UL, 0x0ba53cbc968a8089UL, + /* 98 */ 0xca3d9dc7fea15537UL, 0x4c6824bb51536493UL, + 0xb9886314844006b1UL, 0x40d2a72ab454cc60UL, + /* 99 */ 0x5936a1b712570975UL, 0x91b9d648debda657UL, + 0x3344094bb64330eaUL, 0x006ba10d12ee51d0UL, + /* 100 */ 0x19228468f5de5d58UL, 0x0eb12f4c38cc05b0UL, + 0xa1039f9dd5601990UL, 0x4502d4ce4fff0e0bUL, + /* 101 */ 0xeb2054106837c189UL, 0xd0f6544c6dd3b93cUL, + 0x40727064c416d74fUL, 0x6e15c6114b502ef0UL, + /* 102 */ 0x4df2a398cfb1a76bUL, 0x11256c7419f2f6b1UL, + 0x4a497962066e6043UL, 0x705b3aab41355b44UL, + /* 103 */ 0x365ef536d797b1d8UL, 0x00076bd622ddf0dbUL, + 0x3bbf33b0e0575a88UL, 0x3777aa05c8e4ca4dUL, + /* 104 */ 0x392745c85578db5fUL, 0x6fda4149dbae5ae2UL, + 0xb1f0b00b8adc9867UL, 0x09963437d36f1da3UL, + /* 105 */ 0x7e824e90a5dc3853UL, 0xccb5f6641f135cbdUL, + 0x6736d86c87ce8fccUL, 0x625f3ce26604249fUL, + /* 106 */ 0xaf8ac8059502f63fUL, 0x0c05e70a2e351469UL, + 0x35292e9c764b6305UL, 0x1a394360c7e23ac3UL, + /* 107 */ 0xd5c6d53251183264UL, 0x62065abd43c2b74fUL, + 0xb5fbf5d03b973f9bUL, 0x13a3da3661206e5eUL, + /* 108 */ 0xc6bd5837725d94e5UL, 0x18e30912205016c5UL, + 0x2088ce1570033c68UL, 0x7fba1f495c837987UL, + /* 109 */ 0x5a8c7423f2f9079dUL, 0x1735157b34023fc5UL, + 0xe4f9b49ad2fab351UL, 0x6691ff72c878e33cUL, + /* 110 */ 0x122c2adedc5eff3eUL, 0xf8dd4bf1d8956cf4UL, + 0xeb86205d9e9e5bdaUL, 0x049b92b9d975c743UL, + /* 111 */ 0xa5379730b0f6c05aUL, 0x72a0ffacc6f3a553UL, + 0xb0032c34b20dcd6dUL, 0x470e9dbc88d5164aUL, + /* 112 */ 0xb19cf10ca237c047UL, 0xb65466711f6c81a2UL, + 0xb3321bd16dd80b43UL, 0x48c14f600c5fbe8eUL, + /* 113 */ 0x66451c264aa6c803UL, 0xb66e3904a4fa7da6UL, + 0xd45f19b0b3128395UL, 0x31602627c3c9bc10UL, + /* 114 */ 0x3120dc4832e4e10dUL, 0xeb20c46756c717f7UL, + 0x00f52e3f67280294UL, 0x566d4fc14730c509UL, + /* 115 */ 0x7e3a5d40fd837206UL, 0xc1e926dc7159547aUL, + 0x216730fba68d6095UL, 0x22e8c3843f69cea7UL, + /* 116 */ 0x33d074e8930e4b2bUL, 0xb6e4350e84d15816UL, + 0x5534c26ad6ba2365UL, 0x7773c12f89f1f3f3UL, + /* 117 */ 0x8cba404da57962aaUL, 0x5b9897a81999ce56UL, + 0x508e862f121692fcUL, 0x3a81907fa093c291UL, + /* 118 */ 0x0dded0ff4725a510UL, 0x10d8cc10673fc503UL, + 0x5b9d151c9f1f4e89UL, 0x32a5c1d5cb09a44cUL, + /* 119 */ 0x1e0aa442b90541fbUL, 0x5f85eb7cc1b485dbUL, + 0xbee595ce8a9df2e5UL, 0x25e496c722422236UL, + /* 120 */ 0x5edf3c46cd0fe5b9UL, 0x34e75a7ed2a43388UL, + 0xe488de11d761e352UL, 0x0e878a01a085545cUL, + /* 121 */ 0xba493c77e021bb04UL, 0x2b4d1843c7df899aUL, + 0x9ea37a487ae80d67UL, 0x67a9958011e41794UL, + /* 122 */ 0x4b58051a6697b065UL, 0x47e33f7d8d6ba6d4UL, + 0xbb4da8d483ca46c1UL, 0x68becaa181c2db0dUL, + /* 123 */ 0x8d8980e90b989aa5UL, 0xf95eb14a2c93c99bUL, + 0x51c6c7c4796e73a2UL, 0x6e228363b5efb569UL, + /* 124 */ 0xc6bbc0b02dd624c8UL, 0x777eb47dec8170eeUL, + 0x3cde15a004cfafa9UL, 0x1dc6bc087160bf9bUL, + /* 125 */ 0x2e07e043eec34002UL, 0x18e9fc677a68dc7fUL, + 0xd8da03188bd15b9aUL, 0x48fbc3bb00568253UL, + /* 126 */ 0x57547d4cfb654ce1UL, 0xd3565b82a058e2adUL, + 0xf63eaf0bbf154478UL, 0x47531ef114dfbb18UL, + /* 127 */ 0xe1ec630a4278c587UL, 0x5507d546ca8e83f3UL, + 0x85e135c63adc0c2bUL, 0x0aa7efa85682844eUL, + /* 128 */ 0x72691ba8b3e1f615UL, 0x32b4e9701fbe3ffaUL, + 0x97b6d92e39bb7868UL, 0x2cfe53dea02e39e8UL, + /* 129 */ 0x687392cd85cd52b0UL, 0x27ff66c910e29831UL, + 0x97134556a9832d06UL, 0x269bb0360a84f8a0UL, + /* 130 */ 0x706e55457643f85cUL, 0x3734a48c9b597d1bUL, + 0x7aee91e8c6efa472UL, 0x5cd6abc198a9d9e0UL, + /* 131 */ 0x0e04de06cb3ce41aUL, 0xd8c6eb893402e138UL, + 0x904659bb686e3772UL, 0x7215c371746ba8c8UL, + /* 132 */ 0xfd12a97eeae4a2d9UL, 0x9514b7516394f2c5UL, + 0x266fd5809208f294UL, 0x5c847085619a26b9UL, + /* 133 */ 0x52985410fed694eaUL, 0x3c905b934a2ed254UL, + 0x10bb47692d3be467UL, 0x063b3d2d69e5e9e1UL, + /* 134 */ 0x472726eedda57debUL, 0xefb6c4ae10f41891UL, + 0x2b1641917b307614UL, 0x117c554fc4f45b7cUL, + /* 135 */ 0xc07cf3118f9d8812UL, 0x01dbd82050017939UL, + 0xd7e803f4171b2827UL, 0x1015e87487d225eaUL, + /* 136 */ 0xc58de3fed23acc4dUL, 0x50db91c294a7be2dUL, + 0x0b94d43d1c9cf457UL, 0x6b1640fa6e37524aUL, + /* 137 */ 0x692f346c5fda0d09UL, 0x200b1c59fa4d3151UL, + 0xb8c46f760777a296UL, 0x4b38395f3ffdfbcfUL, + /* 138 */ 0x18d25e00be54d671UL, 0x60d50582bec8aba6UL, + 0x87ad8f263b78b982UL, 0x50fdf64e9cda0432UL, + /* 139 */ 0x90f567aac578dcf0UL, 0xef1e9b0ef2a3133bUL, + 0x0eebba9242d9de71UL, 0x15473c9bf03101c7UL, + /* 140 */ 0x7c77e8ae56b78095UL, 0xb678e7666e6f078eUL, + 0x2da0b9615348ba1fUL, 0x7cf931c1ff733f0bUL, + /* 141 */ 0x26b357f50a0a366cUL, 0xe9708cf42b87d732UL, + 0xc13aeea5f91cb2c0UL, 0x35d90c991143bb4cUL, + /* 142 */ 0x47c1c404a9a0d9dcUL, 0x659e58451972d251UL, + 0x3875a8c473b38c31UL, 0x1fbd9ed379561f24UL, + /* 143 */ 0x11fabc6fd41ec28dUL, 0x7ef8dfe3cd2a2dcaUL, + 0x72e73b5d8c404595UL, 0x6135fa4954b72f27UL, + /* 144 */ 0xccfc32a2de24b69cUL, 0x3f55698c1f095d88UL, + 0xbe3350ed5ac3f929UL, 0x5e9bf806ca477eebUL, + /* 145 */ 0xe9ce8fb63c309f68UL, 0x5376f63565e1f9f4UL, + 0xd1afcfb35a6393f1UL, 0x6632a1ede5623506UL, + /* 146 */ 0x0b7d6c390c2ded4cUL, 0x56cb3281df04cb1fUL, + 0x66305a1249ecc3c7UL, 0x5d588b60a38ca72aUL, + /* 147 */ 0xa6ecbf78e8e5f42dUL, 0x86eeb44b3c8a3eecUL, + 0xec219c48fbd21604UL, 0x1aaf1af517c36731UL, + /* 148 */ 0xc306a2836769bde7UL, 0x208280622b1e2adbUL, + 0x8027f51ffbff94a6UL, 0x76cfa1ce1124f26bUL, + /* 149 */ 0x18eb00562422abb6UL, 0xf377c4d58f8c29c3UL, + 0x4dbbc207f531561aUL, 0x0253b7f082128a27UL, + /* 150 */ 0x3d1f091cb62c17e0UL, 0x4860e1abd64628a9UL, + 0x52d17436309d4253UL, 0x356f97e13efae576UL, + /* 151 */ 0xd351e11aa150535bUL, 0x3e6b45bb1dd878ccUL, + 0x0c776128bed92c98UL, 0x1d34ae93032885b8UL, + /* 152 */ 0x4ba0488ca85ba4c3UL, 0x985348c33c9ce6ceUL, + 0x66124c6f97bda770UL, 0x0f81a0290654124aUL, + /* 153 */ 0x9ed09ca6569b86fdUL, 0x811009fd18af9a2dUL, + 0xff08d03f93d8c20aUL, 0x52a148199faef26bUL, + /* 154 */ 0x3e03f9dc2d8d1b73UL, 0x4205801873961a70UL, + 0xc0d987f041a35970UL, 0x07aa1f15a1c0d549UL, + /* 155 */ 0xdfd46ce08cd27224UL, 0x6d0a024f934e4239UL, + 0x808a7a6399897b59UL, 0x0a4556e9e13d95a2UL, + /* 156 */ 0xd21a991fe9c13045UL, 0x9b0e8548fe7751b8UL, + 0x5da643cb4bf30035UL, 0x77db28d63940f721UL, + /* 157 */ 0xfc5eeb614adc9011UL, 0x5229419ae8c411ebUL, + 0x9ec3e7787d1dcf74UL, 0x340d053e216e4cb5UL, + /* 158 */ 0xcac7af39b48df2b4UL, 0xc0faec2871a10a94UL, + 0x140a69245ca575edUL, 0x0cf1c37134273a4cUL, + /* 159 */ 0xc8ee306ac224b8a5UL, 0x57eaee7ccb4930b0UL, + 0xa1e806bdaacbe74fUL, 0x7d9a62742eeb657dUL, + /* 160 */ 0x9eb6b6ef546c4830UL, 0x885cca1fddb36e2eUL, + 0xe6b9f383ef0d7105UL, 0x58654fef9d2e0412UL, + /* 161 */ 0xa905c4ffbe0e8e26UL, 0x942de5df9b31816eUL, + 0x497d723f802e88e1UL, 0x30684dea602f408dUL, + /* 162 */ 0x21e5a278a3e6cb34UL, 0xaefb6e6f5b151dc4UL, + 0xb30b8e049d77ca15UL, 0x28c3c9cf53b98981UL, + /* 163 */ 0x287fb721556cdd2aUL, 0x0d317ca897022274UL, + 0x7468c7423a543258UL, 0x4a7f11464eb5642fUL, + /* 164 */ 0xa237a4774d193aa6UL, 0xd865986ea92129a1UL, + 0x24c515ecf87c1a88UL, 0x604003575f39f5ebUL, + /* 165 */ 0x47b9f189570a9b27UL, 0x2b98cede465e4b78UL, + 0x026df551dbb85c20UL, 0x74fcd91047e21901UL, + /* 166 */ 0x13e2a90a23c1bfa3UL, 0x0cb0074e478519f6UL, + 0x5ff1cbbe3af6cf44UL, 0x67fe5438be812dbeUL, + /* 167 */ 0xd13cf64fa40f05b0UL, 0x054dfb2f32283787UL, + 0x4173915b7f0d2aeaUL, 0x482f144f1f610d4eUL, + /* 168 */ 0xf6210201b47f8234UL, 0x5d0ae1929e70b990UL, + 0xdcd7f455b049567cUL, 0x7e93d0f1f0916f01UL, + /* 169 */ 0xdd79cbf18a7db4faUL, 0xbe8391bf6f74c62fUL, + 0x027145d14b8291bdUL, 0x585a73ea2cbf1705UL, + /* 170 */ 0x485ca03e928a0db2UL, 0x10fc01a5742857e7UL, + 0x2f482edbd6d551a7UL, 0x0f0433b5048fdb8aUL, + /* 171 */ 0x60da2e8dd7dc6247UL, 0x88b4c9d38cd4819aUL, + 0x13033ac001f66697UL, 0x273b24fe3b367d75UL, + /* 172 */ 0xc6e8f66a31b3b9d4UL, 0x281514a494df49d5UL, + 0xd1726fdfc8b23da7UL, 0x4b3ae7d103dee548UL, + /* 173 */ 0xc6256e19ce4b9d7eUL, 0xff5c5cf186e3c61cUL, + 0xacc63ca34b8ec145UL, 0x74621888fee66574UL, + /* 174 */ 0x956f409645290a1eUL, 0xef0bf8e3263a962eUL, + 0xed6a50eb5ec2647bUL, 0x0694283a9dca7502UL, + /* 175 */ 0x769b963643a2dcd1UL, 0x42b7c8ea09fc5353UL, + 0x4f002aee13397eabUL, 0x63005e2c19b7d63aUL, + /* 176 */ 0xca6736da63023beaUL, 0x966c7f6db12a99b7UL, + 0xace09390c537c5e1UL, 0x0b696063a1aa89eeUL, + /* 177 */ 0xebb03e97288c56e5UL, 0x432a9f9f938c8be8UL, + 0xa6a5a93d5b717f71UL, 0x1a5fb4c3e18f9d97UL, + /* 178 */ 0x1c94e7ad1c60cdceUL, 0xee202a43fc02c4a0UL, + 0x8dafe4d867c46a20UL, 0x0a10263c8ac27b58UL, + /* 179 */ 0xd0dea9dfe4432a4aUL, 0x856af87bbe9277c5UL, + 0xce8472acc212c71aUL, 0x6f151b6d9bbb1e91UL, + /* 180 */ 0x26776c527ceed56aUL, 0x7d211cb7fbf8faecUL, + 0x37ae66a6fd4609ccUL, 0x1f81b702d2770c42UL, + /* 181 */ 0x2fb0b057eac58392UL, 0xe1dd89fe29744e9dUL, + 0xc964f8eb17beb4f8UL, 0x29571073c9a2d41eUL, + /* 182 */ 0xa948a18981c0e254UL, 0x2df6369b65b22830UL, + 0xa33eb2d75fcfd3c6UL, 0x078cd6ec4199a01fUL, + /* 183 */ 0x4a584a41ad900d2fUL, 0x32142b78e2c74c52UL, + 0x68c4e8338431c978UL, 0x7f69ea9008689fc2UL, + /* 184 */ 0x52f2c81e46a38265UL, 0xfd78072d04a832fdUL, + 0x8cd7d5fa25359e94UL, 0x4de71b7454cc29d2UL, + /* 185 */ 0x42eb60ad1eda6ac9UL, 0x0aad37dfdbc09c3aUL, + 0x81004b71e33cc191UL, 0x44e6be345122803cUL, + /* 186 */ 0x03fe8388ba1920dbUL, 0xf5d57c32150db008UL, + 0x49c8c4281af60c29UL, 0x21edb518de701aeeUL, + /* 187 */ 0x7fb63e418f06dc99UL, 0xa4460d99c166d7b8UL, + 0x24dd5248ce520a83UL, 0x5ec3ad712b928358UL, + /* 188 */ 0x15022a5fbd17930fUL, 0xa4f64a77d82570e3UL, + 0x12bc8d6915783712UL, 0x498194c0fc620abbUL, + /* 189 */ 0x38a2d9d255686c82UL, 0x785c6bd9193e21f0UL, + 0xe4d5c81ab24a5484UL, 0x56307860b2e20989UL, + /* 190 */ 0x429d55f78b4d74c4UL, 0x22f1834643350131UL, + 0x1e60c24598c71fffUL, 0x59f2f014979983efUL, + /* 191 */ 0x46a47d56eb494a44UL, 0x3e22a854d636a18eUL, + 0xb346e15274491c3bUL, 0x2ceafd4e5390cde7UL, + /* 192 */ 0xba8a8538be0d6675UL, 0x4b9074bb50818e23UL, + 0xcbdab89085d304c3UL, 0x61a24fe0e56192c4UL, + /* 193 */ 0xcb7615e6db525bcbUL, 0xdd7d8c35a567e4caUL, + 0xe6b4153acafcdd69UL, 0x2d668e097f3c9766UL, + /* 194 */ 0xa57e7e265ce55ef0UL, 0x5d9f4e527cd4b967UL, + 0xfbc83606492fd1e5UL, 0x090d52beb7c3f7aeUL, + /* 195 */ 0x09b9515a1e7b4d7cUL, 0x1f266a2599da44c0UL, + 0xa1c49548e2c55504UL, 0x7ef04287126f15ccUL, + /* 196 */ 0xfed1659dbd30ef15UL, 0x8b4ab9eec4e0277bUL, + 0x884d6236a5df3291UL, 0x1fd96ea6bf5cf788UL, + /* 197 */ 0x42a161981f190d9aUL, 0x61d849507e6052c1UL, + 0x9fe113bf285a2cd5UL, 0x7c22d676dbad85d8UL, + /* 198 */ 0x82e770ed2bfbd27dUL, 0x4c05b2ece996f5a5UL, + 0xcd40a9c2b0900150UL, 0x5895319213d9bf64UL, + /* 199 */ 0xe7cc5d703fea2e08UL, 0xb50c491258e2188cUL, + 0xcce30baa48205bf0UL, 0x537c659ccfa32d62UL, + /* 200 */ 0x37b6623a98cfc088UL, 0xfe9bed1fa4d6aca4UL, + 0x04d29b8e56a8d1b0UL, 0x725f71c40b519575UL, + /* 201 */ 0x28c7f89cd0339ce6UL, 0x8367b14469ddc18bUL, + 0x883ada83a6a1652cUL, 0x585f1974034d6c17UL, + /* 202 */ 0x89cfb266f1b19188UL, 0xe63b4863e7c35217UL, + 0xd88c9da6b4c0526aUL, 0x3e035c9df0954635UL, + /* 203 */ 0xdd9d5412fb45de9dUL, 0xdd684532e4cff40dUL, + 0x4b5c999b151d671cUL, 0x2d8c2cc811e7f690UL, + /* 204 */ 0x7f54be1d90055d40UL, 0xa464c5df464aaf40UL, + 0x33979624f0e917beUL, 0x2c018dc527356b30UL, + /* 205 */ 0xa5415024e330b3d4UL, 0x73ff3d96691652d3UL, + 0x94ec42c4ef9b59f1UL, 0x0747201618d08e5aUL, + /* 206 */ 0x4d6ca48aca411c53UL, 0x66415f2fcfa66119UL, + 0x9c4dd40051e227ffUL, 0x59810bc09a02f7ebUL, + /* 207 */ 0x2a7eb171b3dc101dUL, 0x441c5ab99ffef68eUL, + 0x32025c9b93b359eaUL, 0x5e8ce0a71e9d112fUL, + /* 208 */ 0xbfcccb92429503fdUL, 0xd271ba752f095d55UL, + 0x345ead5e972d091eUL, 0x18c8df11a83103baUL, + /* 209 */ 0x90cd949a9aed0f4cUL, 0xc5d1f4cb6660e37eUL, + 0xb8cac52d56c52e0bUL, 0x6e42e400c5808e0dUL, + /* 210 */ 0xa3b46966eeaefd23UL, 0x0c4f1f0be39ecdcaUL, + 0x189dc8c9d683a51dUL, 0x51f27f054c09351bUL, + /* 211 */ 0x4c487ccd2a320682UL, 0x587ea95bb3df1c96UL, + 0xc8ccf79e555cb8e8UL, 0x547dc829a206d73dUL, + /* 212 */ 0xb822a6cd80c39b06UL, 0xe96d54732000d4c6UL, + 0x28535b6f91463b4dUL, 0x228f4660e2486e1dUL, + /* 213 */ 0x98799538de8d3abfUL, 0x8cd8330045ebca6eUL, + 0x79952a008221e738UL, 0x4322e1a7535cd2bbUL, + /* 214 */ 0xb114c11819d1801cUL, 0x2016e4d84f3f5ec7UL, + 0xdd0e2df409260f4cUL, 0x5ec362c0ae5f7266UL, + /* 215 */ 0xc0462b18b8b2b4eeUL, 0x7cc8d950274d1afbUL, + 0xf25f7105436b02d2UL, 0x43bbf8dcbff9ccd3UL, + /* 216 */ 0xb6ad1767a039e9dfUL, 0xb0714da8f69d3583UL, + 0x5e55fa18b42931f5UL, 0x4ed5558f33c60961UL, + /* 217 */ 0x1fe37901c647a5ddUL, 0x593ddf1f8081d357UL, + 0x0249a4fd813fd7a6UL, 0x69acca274e9caf61UL, + /* 218 */ 0x047ba3ea330721c9UL, 0x83423fc20e7e1ea0UL, + 0x1df4c0af01314a60UL, 0x09a62dab89289527UL, + /* 219 */ 0xa5b325a49cc6cb00UL, 0xe94b5dc654b56cb6UL, + 0x3be28779adc994a0UL, 0x4296e8f8ba3a4aadUL, + /* 220 */ 0x328689761e451eabUL, 0x2e4d598bff59594aUL, + 0x49b96853d7a7084aUL, 0x4980a319601420a8UL, + /* 221 */ 0x9565b9e12f552c42UL, 0x8a5318db7100fe96UL, + 0x05c90b4d43add0d7UL, 0x538b4cd66a5d4edaUL, + /* 222 */ 0xf4e94fc3e89f039fUL, 0x592c9af26f618045UL, + 0x08a36eb5fd4b9550UL, 0x25fffaf6c2ed1419UL, + /* 223 */ 0x34434459cc79d354UL, 0xeeecbfb4b1d5476bUL, + 0xddeb34a061615d99UL, 0x5129cecceb64b773UL, + /* 224 */ 0xee43215894993520UL, 0x772f9c7cf14c0b3bUL, + 0xd2e2fce306bedad5UL, 0x715f42b546f06a97UL, + /* 225 */ 0x434ecdceda5b5f1aUL, 0x0da17115a49741a9UL, + 0x680bd77c73edad2eUL, 0x487c02354edd9041UL, + /* 226 */ 0xb8efeff3a70ed9c4UL, 0x56a32aa3e857e302UL, + 0xdf3a68bd48a2a5a0UL, 0x07f650b73176c444UL, + /* 227 */ 0xe38b9b1626e0ccb1UL, 0x79e053c18b09fb36UL, + 0x56d90319c9f94964UL, 0x1ca941e7ac9ff5c4UL, + /* 228 */ 0x49c4df29162fa0bbUL, 0x8488cf3282b33305UL, + 0x95dfda14cabb437dUL, 0x3391f78264d5ad86UL, + /* 229 */ 0x729ae06ae2b5095dUL, 0xd58a58d73259a946UL, + 0xe9834262d13921edUL, 0x27fedafaa54bb592UL, + /* 230 */ 0xa99dc5b829ad48bbUL, 0x5f025742499ee260UL, + 0x802c8ecd5d7513fdUL, 0x78ceb3ef3f6dd938UL, + /* 231 */ 0xc342f44f8a135d94UL, 0x7b9edb44828cdda3UL, + 0x9436d11a0537cfe7UL, 0x5064b164ec1ab4c8UL, + /* 232 */ 0x7020eccfd37eb2fcUL, 0x1f31ea3ed90d25fcUL, + 0x1b930d7bdfa1bb34UL, 0x5344467a48113044UL, + /* 233 */ 0x70073170f25e6dfbUL, 0xe385dc1a50114cc8UL, + 0x2348698ac8fc4f00UL, 0x2a77a55284dd40d8UL, + /* 234 */ 0xfe06afe0c98c6ce4UL, 0xc235df96dddfd6e4UL, + 0x1428d01e33bf1ed3UL, 0x785768ec9300bdafUL, + /* 235 */ 0x9702e57a91deb63bUL, 0x61bdb8bfe5ce8b80UL, + 0x645b426f3d1d58acUL, 0x4804a82227a557bcUL, + /* 236 */ 0x8e57048ab44d2601UL, 0x68d6501a4b3a6935UL, + 0xc39c9ec3f9e1c293UL, 0x4172f257d4de63e2UL, + /* 237 */ 0xd368b450330c6401UL, 0x040d3017418f2391UL, + 0x2c34bb6090b7d90dUL, 0x16f649228fdfd51fUL, + /* 238 */ 0xbea6818e2b928ef5UL, 0xe28ccf91cdc11e72UL, + 0x594aaa68e77a36cdUL, 0x313034806c7ffd0fUL, + /* 239 */ 0x8a9d27ac2249bd65UL, 0x19a3b464018e9512UL, + 0xc26ccff352b37ec7UL, 0x056f68341d797b21UL, + /* 240 */ 0x5e79d6757efd2327UL, 0xfabdbcb6553afe15UL, + 0xd3e7222c6eaf5a60UL, 0x7046c76d4dae743bUL, + /* 241 */ 0x660be872b18d4a55UL, 0x19992518574e1496UL, + 0xc103053a302bdcbbUL, 0x3ed8e9800b218e8eUL, + /* 242 */ 0x7b0b9239fa75e03eUL, 0xefe9fb684633c083UL, + 0x98a35fbe391a7793UL, 0x6065510fe2d0fe34UL, + /* 243 */ 0x55cb668548abad0cUL, 0xb4584548da87e527UL, + 0x2c43ecea0107c1ddUL, 0x526028809372de35UL, + /* 244 */ 0x3415c56af9213b1fUL, 0x5bee1a4d017e98dbUL, + 0x13f6b105b5cf709bUL, 0x5ff20e3482b29ab6UL, + /* 245 */ 0x0aa29c75cc2e6c90UL, 0xfc7d73ca3a70e206UL, + 0x899fc38fc4b5c515UL, 0x250386b124ffc207UL, + /* 246 */ 0x54ea28d5ae3d2b56UL, 0x9913149dd6de60ceUL, + 0x16694fc58f06d6c1UL, 0x46b23975eb018fc7UL, + /* 247 */ 0x470a6a0fb4b7b4e2UL, 0x5d92475a8f7253deUL, + 0xabeee5b52fbd3adbUL, 0x7fa20801a0806968UL, + /* 248 */ 0x76f3faf19f7714d2UL, 0xb3e840c12f4660c3UL, + 0x0fb4cd8df212744eUL, 0x4b065a251d3a2dd2UL, + /* 249 */ 0x5cebde383d77cd4aUL, 0x6adf39df882c9cb1UL, + 0xa2dd242eb09af759UL, 0x3147c0e50e5f6422UL, + /* 250 */ 0x164ca5101d1350dbUL, 0xf8d13479c33fc962UL, + 0xe640ce4d13e5da08UL, 0x4bdee0c45061f8baUL, + /* 251 */ 0xd7c46dc1a4edb1c9UL, 0x5514d7b6437fd98aUL, + 0x58942f6bb2a1c00bUL, 0x2dffb2ab1d70710eUL, + /* 252 */ 0xccdfcf2fc18b6d68UL, 0xa8ebcba8b7806167UL, + 0x980697f95e2937e3UL, 0x02fbba1cd0126e8cUL +}; + +#ifdef CONFIG_AS_ADX +/* c is two 512-bit products: c0[0:7]=a0[0:3]*b0[0:3] and c1[8:15]=a1[4:7]*b1[4:7] + * a is two 256-bit integers: a0[0:3] and a1[4:7] + * b is two 256-bit integers: b0[0:3] and b1[4:7] + */ +static void mul2_256x256_integer_adx(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "xorl %%r14d, %%r14d ;" + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "adox %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adox %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adox %%r10, %%rbx ;" + /******************************************/ + "adox %%r14, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "adox %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rax ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rbx ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rcx ;" + /******************************************/ + "adox %%r14, %%r15 ;" + "adcx %%r14, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "xorl %%r10d, %%r10d ;" + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "adox %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rbx ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rcx ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%r15 ;" + /******************************************/ + "adox %%r14, %%rax ;" + "adcx %%r14, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "xorl %%r10d, %%r10d ;" + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "adox %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + /******************************************/ + "adox %%r14, %%rbx ;" + "adcx %%r14, %%rbx ;" + "movq %%rbx, 56(%0) ;" + + "movq 32(%1), %%rdx; " /* C[0] */ + "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, 64(%0);" + "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ + "adox %%r10, %%r15 ;" + "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ + "adox %%r8, %%rax ;" + "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ + "adox %%r10, %%rbx ;" + /******************************************/ + "adox %%r14, %%rcx ;" + + "movq 40(%1), %%rdx; " /* C[1] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ + "adox %%r15, %%r8 ;" + "movq %%r8, 72(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rax ;" + "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rbx ;" + "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rcx ;" + /******************************************/ + "adox %%r14, %%r15 ;" + "adcx %%r14, %%r15 ;" + + "movq 48(%1), %%rdx; " /* C[2] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ + "adox %%rax, %%r8 ;" + "movq %%r8, 80(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rbx ;" + "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rcx ;" + "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%r15 ;" + /******************************************/ + "adox %%r14, %%rax ;" + "adcx %%r14, %%rax ;" + + "movq 56(%1), %%rdx; " /* C[3] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ + "adox %%rbx, %%r8 ;" + "movq %%r8, 88(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rcx ;" + "movq %%rcx, 96(%0) ;" + "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%r15 ;" + "movq %%r15, 104(%0) ;" + "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rax ;" + "movq %%rax, 112(%0) ;" + /******************************************/ + "adox %%r14, %%rbx ;" + "adcx %%r14, %%rbx ;" + "movq %%rbx, 120(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "addq %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adcq %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 56(%0) ;" + + "movq 32(%1), %%rdx; " /* C[0] */ + "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ + "movq %%r8, 64(%0) ;" + "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ + "addq %%r10, %%r15 ;" + "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ + "adcq %%r8, %%rax ;" + "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 40(%1), %%rdx; " /* C[1] */ + "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 72(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 48(%1), %%rdx; " /* C[2] */ + "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 80(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 56(%1), %%rdx; " /* C[3] */ + "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 88(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 96(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 104(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 112(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 120(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ + "xorl %%r15d, %%r15d;" + "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ + "adcx %%r14, %%r9 ;" + "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ + "adcx %%rax, %%r10 ;" + "movq 24(%1), %%rdx ;" /* A[3] */ + "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ + "adcx %%rax, %%rbx ;" + "movq 8(%1), %%rdx ;" /* A[1] */ + "adcx %%r15, %%r13 ;" + "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq (%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + + + "movq 32(%1), %%rdx ;" /* B[0] */ + "mulx 40(%1), %%r8, %%r14 ;" /* B[1]*B[0] */ + "xorl %%r15d, %%r15d;" + "mulx 48(%1), %%r9, %%r10 ;" /* B[2]*B[0] */ + "adcx %%r14, %%r9 ;" + "mulx 56(%1), %%rax, %%rcx ;" /* B[3]*B[0] */ + "adcx %%rax, %%r10 ;" + "movq 56(%1), %%rdx ;" /* B[3] */ + "mulx 40(%1), %%r11, %%rbx ;" /* B[1]*B[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 48(%1), %%rax, %%r13 ;" /* B[2]*B[3] */ + "adcx %%rax, %%rbx ;" + "movq 40(%1), %%rdx ;" /* B[1] */ + "adcx %%r15, %%r13 ;" + "mulx 48(%1), %%rax, %%rcx ;" /* B[2]*B[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq 32(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[0]^2 */ + /*******************/ + "movq %%rax, 64(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 72(%0) ;" + "movq 40(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 80(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 88(%0) ;" + "movq 48(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 96(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 104(%0) ;" + "movq 56(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 112(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 120(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq 8(%1), %%rdx ;" /* A[1] */ + "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ + "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ + "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ + + "movq 16(%1), %%rdx ;" /* A[2] */ + "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ + "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ; " /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + + "movq 40(%1), %%rdx ;" /* B[1] */ + "mulx 32(%1), %%r8, %%r9 ;" /* B[0]*B[1] */ + "mulx 48(%1), %%r10, %%r11 ;" /* B[2]*B[1] */ + "mulx 56(%1), %%rcx, %%r14 ;" /* B[3]*B[1] */ + + "movq 48(%1), %%rdx ;" /* B[2] */ + "mulx 56(%1), %%r15, %%r13 ;" /* B[3]*B[2] */ + "mulx 32(%1), %%rax, %%rdx ;" /* B[0]*B[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq 32(%1), %%rdx ;" /* B[0] */ + "mulx 56(%1), %%rax, %%rcx ;" /* B[0]*B[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ; " /* B[0]^2 */ + /*******************/ + "movq %%rax, 64(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 72(%0) ;" + "movq 40(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 80(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 88(%0) ;" + "movq 48(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 96(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 104(%0) ;" + "movq 56(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 112(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 120(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx; " /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10; " /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox (%1), %%r8 ;" + "mulx 40(%1), %%r9, %%r11; " /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 8(%1), %%r9 ;" + "mulx 48(%1), %%r10, %%rax; " /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 16(%1), %%r10 ;" + "mulx 56(%1), %%r11, %%rcx; " /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 24(%1), %%r11 ;" + /***************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + + "mulx 96(%1), %%r8, %%r10; " /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox 64(%1), %%r8 ;" + "mulx 104(%1), %%r9, %%r11; " /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 72(%1), %%r9 ;" + "mulx 112(%1), %%r10, %%rax; " /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 80(%1), %%r10 ;" + "mulx 120(%1), %%r11, %%rcx; " /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 88(%1), %%r11 ;" + /****************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 40(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 48(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 56(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 32(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ; " /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /***************************************/ + "adcq $0, %%rcx ;" + "addq (%1), %%r8 ;" + "adcq 8(%1), %%r9 ;" + "adcq 16(%1), %%r10 ;" + "adcq 24(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + + "mulx 96(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 104(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 112(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 120(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /****************************************/ + "adcq $0, %%rcx ;" + "addq 64(%1), %%r8 ;" + "adcq 72(%1), %%r9 ;" + "adcq 80(%1), %%r10 ;" + "adcq 88(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 40(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 48(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 56(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 32(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void mul_256x256_integer_adx(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r9; " /* A[0]*B[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[0]*B[1] */ + "adox %%r9, %%r10 ;" + "movq %%r10, 8(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[0]*B[2] */ + "adox %%r11, %%r15 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[0]*B[3] */ + "adox %%r13, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 8(%0), %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 16(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[1]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[1]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 16(%0), %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 24(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[2]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[2]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 24(%0), %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 32(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[3]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[3]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq %%r14, 48(%0) ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + "movq %%rax, 56(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", + "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a, + const u64 *const b) +{ + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "addq %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adcq %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 56(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ + "xorl %%r15d, %%r15d;" + "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ + "adcx %%r14, %%r9 ;" + "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ + "adcx %%rax, %%r10 ;" + "movq 24(%1), %%rdx ;" /* A[3] */ + "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ + "adcx %%rax, %%rbx ;" + "movq 8(%1), %%rdx ;" /* A[1] */ + "adcx %%r15, %%r13 ;" + "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq (%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movq 8(%1), %%rdx ;" /* A[1] */ + "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ + "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ + "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ + + "movq 16(%1), %%rdx ;" /* A[2] */ + "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ + "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11", "%r13", "%r14", "%r15"); +} +#endif + +#ifdef CONFIG_AS_ADX +static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox (%1), %%r8 ;" + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 8(%1), %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 16(%1), %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 24(%1), %%r11 ;" + /***************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /***************************************/ + "adcq $0, %%rcx ;" + "addq (%1), %%r8 ;" + "adcq 8(%1), %%r9 ;" + "adcq 16(%1), %%r10 ;" + "adcq 24(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} +#endif + +#ifdef CONFIG_AS_ADX +static __always_inline void +add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "xorl %%ecx, %%ecx ;" + "movq (%2), %%r8 ;" + "adcx (%1), %%r8 ;" + "movq 8(%2), %%r9 ;" + "adcx 8(%1), %%r9 ;" + "movq 16(%2), %%r10 ;" + "adcx 16(%1), %%r10 ;" + "movq 24(%2), %%r11 ;" + "adcx 24(%1), %%r11 ;" + "cmovc %%eax, %%ecx ;" + "xorl %%eax, %%eax ;" + "adcx %%rcx, %%r8 ;" + "adcx %%rax, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rax, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rax, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $38, %%ecx ;" + "cmovc %%ecx, %%eax ;" + "addq %%rax, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static __always_inline void +add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "movq (%2), %%r8 ;" + "addq (%1), %%r8 ;" + "movq 8(%2), %%r9 ;" + "adcq 8(%1), %%r9 ;" + "movq 16(%2), %%r10 ;" + "adcq 16(%1), %%r10 ;" + "movq 24(%2), %%r11 ;" + "adcq 24(%1), %%r11 ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} +#endif + +static __always_inline void +sub_eltfp25519_1w(u64 *const c, const u64 *const a, const u64 *const b) +{ + asm volatile( + "mov $38, %%eax ;" + "movq (%1), %%r8 ;" + "subq (%2), %%r8 ;" + "movq 8(%1), %%r9 ;" + "sbbq 8(%2), %%r9 ;" + "movq 16(%1), %%r10 ;" + "sbbq 16(%2), %%r10 ;" + "movq 24(%1), %%r11 ;" + "sbbq 24(%2), %%r11 ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "subq %%rcx, %%r8 ;" + "sbbq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "sbbq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "sbbq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "subq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); +} + +/* Multiplication by a24 = (A+2)/4 = (486662+2)/4 = 121666 */ +static __always_inline void +mul_a24_eltfp25519_1w(u64 *const c, const u64 *const a) +{ + const u64 a24 = 121666; + asm volatile( + "movq %2, %%rdx ;" + "mulx (%1), %%r8, %%r10 ;" + "mulx 8(%1), %%r9, %%r11 ;" + "addq %%r10, %%r9 ;" + "mulx 16(%1), %%r10, %%rax ;" + "adcq %%r11, %%r10 ;" + "mulx 24(%1), %%r11, %%rcx ;" + "adcq %%rax, %%r11 ;" + /**************************/ + "adcq $0, %%rcx ;" + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 mod 2^255-19*/ + "imul %%rdx, %%rcx ;" + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(a24) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); +} + +#ifdef CONFIG_AS_ADX +static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +{ + struct { + eltfp25519_1w_buffer buffer; + eltfp25519_1w x0, x1, x2; + } __aligned(32) m; + u64 *T[4]; + + T[0] = m.x0; + T[1] = c; /* x^(-1) */ + T[2] = m.x1; + T[3] = m.x2; + + copy_eltfp25519_1w(T[1], a); + sqrn_eltfp25519_1w_adx(T[1], 1); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_adx(T[2], 2); + mul_eltfp25519_1w_adx(T[0], a, T[2]); + mul_eltfp25519_1w_adx(T[1], T[1], T[0]); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_adx(T[2], 1); + mul_eltfp25519_1w_adx(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 5); + mul_eltfp25519_1w_adx(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 10); + mul_eltfp25519_1w_adx(T[2], T[2], T[0]); + copy_eltfp25519_1w(T[3], T[2]); + sqrn_eltfp25519_1w_adx(T[3], 20); + mul_eltfp25519_1w_adx(T[3], T[3], T[2]); + sqrn_eltfp25519_1w_adx(T[3], 10); + mul_eltfp25519_1w_adx(T[3], T[3], T[0]); + copy_eltfp25519_1w(T[0], T[3]); + sqrn_eltfp25519_1w_adx(T[0], 50); + mul_eltfp25519_1w_adx(T[0], T[0], T[3]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 100); + mul_eltfp25519_1w_adx(T[2], T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 50); + mul_eltfp25519_1w_adx(T[2], T[2], T[3]); + sqrn_eltfp25519_1w_adx(T[2], 5); + mul_eltfp25519_1w_adx(T[1], T[1], T[2]); + + memzero_explicit(&m, sizeof(m)); +} +#endif + +#ifdef CONFIG_AS_BMI2 +static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) +{ + struct { + eltfp25519_1w_buffer buffer; + eltfp25519_1w x0, x1, x2; + } __aligned(32) m; + u64 *T[5]; + + T[0] = m.x0; + T[1] = c; /* x^(-1) */ + T[2] = m.x1; + T[3] = m.x2; + + copy_eltfp25519_1w(T[1], a); + sqrn_eltfp25519_1w_bmi2(T[1], 1); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_bmi2(T[2], 2); + mul_eltfp25519_1w_bmi2(T[0], a, T[2]); + mul_eltfp25519_1w_bmi2(T[1], T[1], T[0]); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_bmi2(T[2], 1); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 5); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 10); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); + copy_eltfp25519_1w(T[3], T[2]); + sqrn_eltfp25519_1w_bmi2(T[3], 20); + mul_eltfp25519_1w_bmi2(T[3], T[3], T[2]); + sqrn_eltfp25519_1w_bmi2(T[3], 10); + mul_eltfp25519_1w_bmi2(T[3], T[3], T[0]); + copy_eltfp25519_1w(T[0], T[3]); + sqrn_eltfp25519_1w_bmi2(T[0], 50); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[3]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 100); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 50); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[3]); + sqrn_eltfp25519_1w_bmi2(T[2], 5); + mul_eltfp25519_1w_bmi2(T[1], T[1], T[2]); + + memzero_explicit(&m, sizeof(m)); +} +#endif + +/* Given c, a 256-bit number, fred_eltfp25519_1w updates c + * with a number such that 0 <= C < 2**255-19. + */ +static __always_inline void fred_eltfp25519_1w(u64 *const c) +{ + u64 tmp0 = 38, tmp1 = 19; + asm volatile( + "btrq $63, %3 ;" /* Put bit 255 in carry flag and clear */ + "cmovncl %k5, %k4 ;" /* c[255] ? 38 : 19 */ + + /* Add either 19 or 38 to c */ + "addq %4, %0 ;" + "adcq $0, %1 ;" + "adcq $0, %2 ;" + "adcq $0, %3 ;" + + /* Test for bit 255 again; only triggered on overflow modulo 2^255-19 */ + "movl $0, %k4 ;" + "cmovnsl %k5, %k4 ;" /* c[255] ? 0 : 19 */ + "btrq $63, %3 ;" /* Clear bit 255 */ + + /* Subtract 19 if necessary */ + "subq %4, %0 ;" + "sbbq $0, %1 ;" + "sbbq $0, %2 ;" + "sbbq $0, %3 ;" + + : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3]), "+r"(tmp0), + "+r"(tmp1) + : + : "memory", "cc"); +} + +static __always_inline void cswap(u8 bit, u64 *const px, u64 *const py) +{ + u64 temp; + asm volatile( + "test %9, %9 ;" + "movq %0, %8 ;" + "cmovnzq %4, %0 ;" + "cmovnzq %8, %4 ;" + "movq %1, %8 ;" + "cmovnzq %5, %1 ;" + "cmovnzq %8, %5 ;" + "movq %2, %8 ;" + "cmovnzq %6, %2 ;" + "cmovnzq %8, %6 ;" + "movq %3, %8 ;" + "cmovnzq %7, %3 ;" + "cmovnzq %8, %7 ;" + : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]), + "+r"(py[0]), "+r"(py[1]), "+r"(py[2]), "+r"(py[3]), + "=r"(temp) + : "r"(bit) + : "cc" + ); +} + +static __always_inline void cselect(u8 bit, u64 *const px, const u64 *const py) +{ + asm volatile( + "test %4, %4 ;" + "cmovnzq %5, %0 ;" + "cmovnzq %6, %1 ;" + "cmovnzq %7, %2 ;" + "cmovnzq %8, %3 ;" + : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]) + : "r"(bit), "rm"(py[0]), "rm"(py[1]), "rm"(py[2]), "rm"(py[3]) + : "cc" + ); +} + +static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE], + const u8 session_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_ADX + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[6 * NUM_WORDS_ELTFP25519]; + u8 session[CURVE25519_KEY_SIZE]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + int i = 0, j = 0; + u64 prev = 0; + u64 *const X1 = (u64 *)m.session; + u64 *const key = (u64 *)m.private; + u64 *const Px = m.coordinates + 0; + u64 *const Pz = m.coordinates + 4; + u64 *const Qx = m.coordinates + 8; + u64 *const Qz = m.coordinates + 12; + u64 *const X2 = Qx; + u64 *const Z2 = Qz; + u64 *const X3 = Px; + u64 *const Z3 = Pz; + u64 *const X2Z2 = Qx; + u64 *const X3Z3 = Px; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const D = m.workspace + 8; + u64 *const C = m.workspace + 12; + u64 *const DA = m.workspace + 16; + u64 *const CB = m.workspace + 20; + u64 *const AB = A; + u64 *const DC = D; + u64 *const DACB = DA; + + memcpy(m.private, private_key, sizeof(m.private)); + memcpy(m.session, session_key, sizeof(m.session)); + + curve25519_clamp_secret(m.private); + + /* As in the draft: + * When receiving such an array, implementations of curve25519 + * MUST mask the most-significant bit in the final byte. This + * is done to preserve compatibility with point formats which + * reserve the sign bit for use in other protocols and to + * increase resistance to implementation fingerprinting + */ + m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; + + copy_eltfp25519_1w(Px, X1); + setzero_eltfp25519_1w(Pz); + setzero_eltfp25519_1w(Qx); + setzero_eltfp25519_1w(Qz); + + Pz[0] = 1; + Qx[0] = 1; + + /* main-loop */ + prev = 0; + j = 62; + for (i = 3; i >= 0; --i) { + while (j >= 0) { + u64 bit = (key[i] >> j) & 0x1; + u64 swap = bit ^ prev; + prev = bit; + + add_eltfp25519_1w_adx(A, X2, Z2); /* A = (X2+Z2) */ + sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ + add_eltfp25519_1w_adx(C, X3, Z3); /* C = (X3+Z3) */ + sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ + mul_eltfp25519_2w_adx(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ + + cselect(swap, A, C); + cselect(swap, B, D); + + sqr_eltfp25519_2w_adx(AB); /* [AA|BB] = [A^2|B^2] */ + add_eltfp25519_1w_adx(X3, DA, CB); /* X3 = (DA+CB) */ + sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ + sqr_eltfp25519_2w_adx(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ + + copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ + sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ + + mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ + add_eltfp25519_1w_adx(B, B, X2); /* B = a24*E+B */ + mul_eltfp25519_2w_adx(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ + mul_eltfp25519_1w_adx(Z3, Z3, X1); /* Z3 = Z3*X1 */ + --j; + } + j = 63; + } + + inv_eltfp25519_1w_adx(A, Qz); + mul_eltfp25519_1w_adx((u64 *)shared, Qx, A); + fred_eltfp25519_1w((u64 *)shared); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_ADX + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[4 * NUM_WORDS_ELTFP25519]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + const int ite[4] = { 64, 64, 64, 63 }; + const int q = 3; + u64 swap = 1; + + int i = 0, j = 0, k = 0; + u64 *const key = (u64 *)m.private; + u64 *const Ur1 = m.coordinates + 0; + u64 *const Zr1 = m.coordinates + 4; + u64 *const Ur2 = m.coordinates + 8; + u64 *const Zr2 = m.coordinates + 12; + + u64 *const UZr1 = m.coordinates + 0; + u64 *const ZUr2 = m.coordinates + 8; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const C = m.workspace + 8; + u64 *const D = m.workspace + 12; + + u64 *const AB = m.workspace + 0; + u64 *const CD = m.workspace + 8; + + const u64 *const P = table_ladder_8k; + + memcpy(m.private, private_key, sizeof(m.private)); + + curve25519_clamp_secret(m.private); + + setzero_eltfp25519_1w(Ur1); + setzero_eltfp25519_1w(Zr1); + setzero_eltfp25519_1w(Zr2); + Ur1[0] = 1; + Zr1[0] = 1; + Zr2[0] = 1; + + /* G-S */ + Ur2[3] = 0x1eaecdeee27cab34UL; + Ur2[2] = 0xadc7a0b9235d48e2UL; + Ur2[1] = 0xbbf095ae14b2edf8UL; + Ur2[0] = 0x7e94e1fec82faabdUL; + + /* main-loop */ + j = q; + for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { + while (j < ite[i]) { + u64 bit = (key[i] >> j) & 0x1; + k = (64 * i + j - q); + swap = swap ^ bit; + cswap(swap, Ur1, Ur2); + cswap(swap, Zr1, Zr2); + swap = bit; + /* Addition */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + mul_eltfp25519_1w_adx(C, &P[4 * k], B); /* C = M0-B */ + sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ + add_eltfp25519_1w_adx(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ + sqr_eltfp25519_2w_adx(AB); /* A = A^2 | B = B^2 */ + mul_eltfp25519_2w_adx(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ + ++j; + } + j = 0; + } + + /* Doubling */ + for (i = 0; i < q; ++i) { + add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + sqr_eltfp25519_2w_adx(AB); /* A = A**2 B = B**2 */ + copy_eltfp25519_1w(C, B); /* C = B */ + sub_eltfp25519_1w(B, A, B); /* B = A-B */ + mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ + add_eltfp25519_1w_adx(D, D, C); /* D = D+C */ + mul_eltfp25519_2w_adx(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ + } + + /* Convert to affine coordinates */ + inv_eltfp25519_1w_adx(A, Zr1); + mul_eltfp25519_1w_adx((u64 *)session_key, Ur1, A); + fred_eltfp25519_1w((u64 *)session_key); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE], + const u8 session_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_BMI2 + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[6 * NUM_WORDS_ELTFP25519]; + u8 session[CURVE25519_KEY_SIZE]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + int i = 0, j = 0; + u64 prev = 0; + u64 *const X1 = (u64 *)m.session; + u64 *const key = (u64 *)m.private; + u64 *const Px = m.coordinates + 0; + u64 *const Pz = m.coordinates + 4; + u64 *const Qx = m.coordinates + 8; + u64 *const Qz = m.coordinates + 12; + u64 *const X2 = Qx; + u64 *const Z2 = Qz; + u64 *const X3 = Px; + u64 *const Z3 = Pz; + u64 *const X2Z2 = Qx; + u64 *const X3Z3 = Px; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const D = m.workspace + 8; + u64 *const C = m.workspace + 12; + u64 *const DA = m.workspace + 16; + u64 *const CB = m.workspace + 20; + u64 *const AB = A; + u64 *const DC = D; + u64 *const DACB = DA; + + memcpy(m.private, private_key, sizeof(m.private)); + memcpy(m.session, session_key, sizeof(m.session)); + + curve25519_clamp_secret(m.private); + + /* As in the draft: + * When receiving such an array, implementations of curve25519 + * MUST mask the most-significant bit in the final byte. This + * is done to preserve compatibility with point formats which + * reserve the sign bit for use in other protocols and to + * increase resistance to implementation fingerprinting + */ + m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; + + copy_eltfp25519_1w(Px, X1); + setzero_eltfp25519_1w(Pz); + setzero_eltfp25519_1w(Qx); + setzero_eltfp25519_1w(Qz); + + Pz[0] = 1; + Qx[0] = 1; + + /* main-loop */ + prev = 0; + j = 62; + for (i = 3; i >= 0; --i) { + while (j >= 0) { + u64 bit = (key[i] >> j) & 0x1; + u64 swap = bit ^ prev; + prev = bit; + + add_eltfp25519_1w_bmi2(A, X2, Z2); /* A = (X2+Z2) */ + sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ + add_eltfp25519_1w_bmi2(C, X3, Z3); /* C = (X3+Z3) */ + sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ + mul_eltfp25519_2w_bmi2(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ + + cselect(swap, A, C); + cselect(swap, B, D); + + sqr_eltfp25519_2w_bmi2(AB); /* [AA|BB] = [A^2|B^2] */ + add_eltfp25519_1w_bmi2(X3, DA, CB); /* X3 = (DA+CB) */ + sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ + sqr_eltfp25519_2w_bmi2(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ + + copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ + sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ + + mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ + add_eltfp25519_1w_bmi2(B, B, X2); /* B = a24*E+B */ + mul_eltfp25519_2w_bmi2(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ + mul_eltfp25519_1w_bmi2(Z3, Z3, X1); /* Z3 = Z3*X1 */ + --j; + } + j = 63; + } + + inv_eltfp25519_1w_bmi2(A, Qz); + mul_eltfp25519_1w_bmi2((u64 *)shared, Qx, A); + fred_eltfp25519_1w((u64 *)shared); + + memzero_explicit(&m, sizeof(m)); +#endif +} + +static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE]) +{ +#ifdef CONFIG_AS_BMI2 + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[4 * NUM_WORDS_ELTFP25519]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + const int ite[4] = { 64, 64, 64, 63 }; + const int q = 3; + u64 swap = 1; + + int i = 0, j = 0, k = 0; + u64 *const key = (u64 *)m.private; + u64 *const Ur1 = m.coordinates + 0; + u64 *const Zr1 = m.coordinates + 4; + u64 *const Ur2 = m.coordinates + 8; + u64 *const Zr2 = m.coordinates + 12; + + u64 *const UZr1 = m.coordinates + 0; + u64 *const ZUr2 = m.coordinates + 8; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const C = m.workspace + 8; + u64 *const D = m.workspace + 12; + + u64 *const AB = m.workspace + 0; + u64 *const CD = m.workspace + 8; + + const u64 *const P = table_ladder_8k; + + memcpy(m.private, private_key, sizeof(m.private)); + + curve25519_clamp_secret(m.private); + + setzero_eltfp25519_1w(Ur1); + setzero_eltfp25519_1w(Zr1); + setzero_eltfp25519_1w(Zr2); + Ur1[0] = 1; + Zr1[0] = 1; + Zr2[0] = 1; + + /* G-S */ + Ur2[3] = 0x1eaecdeee27cab34UL; + Ur2[2] = 0xadc7a0b9235d48e2UL; + Ur2[1] = 0xbbf095ae14b2edf8UL; + Ur2[0] = 0x7e94e1fec82faabdUL; + + /* main-loop */ + j = q; + for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { + while (j < ite[i]) { + u64 bit = (key[i] >> j) & 0x1; + k = (64 * i + j - q); + swap = swap ^ bit; + cswap(swap, Ur1, Ur2); + cswap(swap, Zr1, Zr2); + swap = bit; + /* Addition */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + mul_eltfp25519_1w_bmi2(C, &P[4 * k], B);/* C = M0-B */ + sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ + add_eltfp25519_1w_bmi2(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ + sqr_eltfp25519_2w_bmi2(AB); /* A = A^2 | B = B^2 */ + mul_eltfp25519_2w_bmi2(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ + ++j; + } + j = 0; + } + + /* Doubling */ + for (i = 0; i < q; ++i) { + add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + sqr_eltfp25519_2w_bmi2(AB); /* A = A**2 B = B**2 */ + copy_eltfp25519_1w(C, B); /* C = B */ + sub_eltfp25519_1w(B, A, B); /* B = A-B */ + mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ + add_eltfp25519_1w_bmi2(D, D, C); /* D = D+C */ + mul_eltfp25519_2w_bmi2(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ + } + + /* Convert to affine coordinates */ + inv_eltfp25519_1w_bmi2(A, Zr1); + mul_eltfp25519_1w_bmi2((u64 *)session_key, Ur1, A); + fred_eltfp25519_1w((u64 *)session_key); + + memzero_explicit(&m, sizeof(m)); +#endif +} diff --git a/net/wireguard/crypto/zinc/curve25519/curve25519.c b/net/wireguard/crypto/zinc/curve25519/curve25519.c new file mode 100644 index 0000000000000..82c669e4123c4 --- /dev/null +++ b/net/wireguard/crypto/zinc/curve25519/curve25519.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the Curve25519 ECDH algorithm, using either + * a 32-bit implementation or a 64-bit implementation with 128-bit integers, + * depending on what is supported by the target compiler. + * + * Information: https://cr.yp.to/ecdh.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include +#include +#include // For crypto_memneq. + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "curve25519-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) +#include "curve25519-arm-glue.c" +#else +static bool *const curve25519_nobs[] __initconst = { }; +static void __init curve25519_fpu_init(void) +{ +} +static inline bool curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + return false; +} +static inline bool curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + return false; +} +#endif + +#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) +#include "curve25519-hacl64.c" +#else +#include "curve25519-fiat32.c" +#endif + +static const u8 null_point[CURVE25519_KEY_SIZE] = { 0 }; + +bool curve25519(u8 mypublic[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE], + const u8 basepoint[CURVE25519_KEY_SIZE]) +{ + if (!curve25519_arch(mypublic, secret, basepoint)) + curve25519_generic(mypublic, secret, basepoint); + return crypto_memneq(mypublic, null_point, CURVE25519_KEY_SIZE); +} +EXPORT_SYMBOL(curve25519); + +bool curve25519_generate_public(u8 pub[CURVE25519_KEY_SIZE], + const u8 secret[CURVE25519_KEY_SIZE]) +{ + static const u8 basepoint[CURVE25519_KEY_SIZE] __aligned(32) = { 9 }; + + if (unlikely(!crypto_memneq(secret, null_point, CURVE25519_KEY_SIZE))) + return false; + + if (curve25519_base_arch(pub, secret)) + return crypto_memneq(pub, null_point, CURVE25519_KEY_SIZE); + return curve25519(pub, secret, basepoint); +} +EXPORT_SYMBOL(curve25519_generate_public); + +void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]) +{ + get_random_bytes_wait(secret, CURVE25519_KEY_SIZE); + curve25519_clamp_secret(secret); +} +EXPORT_SYMBOL(curve25519_generate_secret); + +#include "../selftest/curve25519.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init curve25519_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + curve25519_fpu_init(); + if (!selftest_run("curve25519", curve25519_selftest, curve25519_nobs, + ARRAY_SIZE(curve25519_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Curve25519 scalar multiplication"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c new file mode 100644 index 0000000000000..291fe4ba98b0c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm-glue.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include + +asmlinkage void poly1305_init_arm(void *ctx, const u8 key[16]); +asmlinkage void poly1305_blocks_arm(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_arm(void *ctx, u8 mac[16], const u32 nonce[4]); +asmlinkage void poly1305_blocks_neon(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_neon(void *ctx, u8 mac[16], const u32 nonce[4]); + +static bool poly1305_use_neon __ro_after_init; +static bool *const poly1305_nobs[] __initconst = { &poly1305_use_neon }; + +static void __init poly1305_fpu_init(void) +{ +#if defined(CONFIG_ZINC_ARCH_ARM64) + poly1305_use_neon = cpu_have_named_feature(ASIMD); +#elif defined(CONFIG_ZINC_ARCH_ARM) + poly1305_use_neon = elf_hwcap & HWCAP_NEON; +#endif +} + +#if defined(CONFIG_ZINC_ARCH_ARM64) +struct poly1305_arch_internal { + union { + u32 h[5]; + struct { + u64 h0, h1, h2; + }; + }; + u64 is_base2_26; + u64 r[2]; +}; +#elif defined(CONFIG_ZINC_ARCH_ARM) +struct poly1305_arch_internal { + union { + u32 h[5]; + struct { + u64 h0, h1; + u32 h2; + } __packed; + }; + u32 r[4]; + u32 is_base2_26; +}; +#endif + +/* The NEON code uses base 2^26, while the scalar code uses base 2^64 on 64-bit + * and base 2^32 on 32-bit. If we hit the unfortunate situation of using NEON + * and then having to go back to scalar -- because the user is silly and has + * called the update function from two separate contexts -- then we need to + * convert back to the original base before proceeding. The below function is + * written for 64-bit integers, and so we have to swap words at the end on + * big-endian 32-bit. It is possible to reason that the initial reduction below + * is sufficient given the implementation invariants. However, for an avoidance + * of doubt and because this is not performance critical, we do the full + * reduction anyway. + */ +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->h2 = state->h[4] >> 24; + if (IS_ENABLED(CONFIG_ZINC_ARCH_ARM) && IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) { + state->h0 = rol64(state->h0, 32); + state->h1 = rol64(state->h1, 32); + } +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->h2 >> 2) + (state->h2 & ~3ULL); + state->h2 &= 3; + state->h0 += cy; + state->h1 += (cy = ULT(state->h0, cy)); + state->h2 += ULT(state->h1, cy); +#undef ULT + state->is_base2_26 = 0; +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_arm(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_blocks_arm(ctx, inp, len, padbit); + return true; + } + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + poly1305_blocks_neon(ctx, inp, bytes, padbit); + len -= bytes; + if (!len) + break; + inp += bytes; + simd_relax(simd_context); + } + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !poly1305_use_neon || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_emit_arm(ctx, mac, nonce); + } else + poly1305_emit_neon(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl new file mode 100644 index 0000000000000..468f41b76fbd2 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm.pl @@ -0,0 +1,1276 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# IALU(*)/gcc-4.4 NEON +# +# ARM11xx(ARMv6) 7.78/+100% - +# Cortex-A5 6.35/+130% 3.00 +# Cortex-A8 6.25/+115% 2.36 +# Cortex-A9 5.10/+95% 2.55 +# Cortex-A15 3.85/+85% 1.25(**) +# Snapdragon S4 5.70/+100% 1.48(**) +# +# (*) this is for -march=armv6, i.e. with bunch of ldrb loading data; +# (**) these are trade-off results, they can be improved by ~8% but at +# the cost of 15/12% regression on Cortex-A5/A7, it's even possible +# to improve Cortex-A9 result, but then A5/A7 loose more than 20%; + +$flavour = shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +($ctx,$inp,$len,$padbit)=map("r$_",(0..3)); + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ +# define poly1305_init poly1305_init_arm +# define poly1305_blocks poly1305_blocks_arm +# define poly1305_emit poly1305_emit_arm +#endif + +.text +#if defined(__thumb2__) +.syntax unified +.thumb +#else +.code 32 +#endif + +.globl poly1305_emit +.globl poly1305_blocks +.globl poly1305_init +.type poly1305_init,%function +.align 5 +poly1305_init: +.Lpoly1305_init: + stmdb sp!,{r4-r11} + + eor r3,r3,r3 + cmp $inp,#0 + str r3,[$ctx,#0] @ zero hash value + str r3,[$ctx,#4] + str r3,[$ctx,#8] + str r3,[$ctx,#12] + str r3,[$ctx,#16] + str r3,[$ctx,#36] @ is_base2_26 + add $ctx,$ctx,#20 + +#ifdef __thumb2__ + it eq +#endif + moveq r0,#0 + beq .Lno_key + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + adr r11,.Lpoly1305_init + ldr r12,.LOPENSSL_armcap +#endif + ldrb r4,[$inp,#0] + mov r10,#0x0fffffff + ldrb r5,[$inp,#1] + and r3,r10,#-4 @ 0x0ffffffc + ldrb r6,[$inp,#2] + ldrb r7,[$inp,#3] + orr r4,r4,r5,lsl#8 + ldrb r5,[$inp,#4] + orr r4,r4,r6,lsl#16 + ldrb r6,[$inp,#5] + orr r4,r4,r7,lsl#24 + ldrb r7,[$inp,#6] + and r4,r4,r10 + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + ldr r12,[r11,r12] @ OPENSSL_armcap_P +# ifdef __APPLE__ + ldr r12,[r12] +# endif +#endif + ldrb r8,[$inp,#7] + orr r5,r5,r6,lsl#8 + ldrb r6,[$inp,#8] + orr r5,r5,r7,lsl#16 + ldrb r7,[$inp,#9] + orr r5,r5,r8,lsl#24 + ldrb r8,[$inp,#10] + and r5,r5,r3 + +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + tst r12,#ARMV7_NEON @ check for NEON +# ifdef __APPLE__ + adr r9,poly1305_blocks_neon + adr r11,poly1305_blocks +# ifdef __thumb2__ + it ne +# endif + movne r11,r9 + adr r12,poly1305_emit + adr r10,poly1305_emit_neon +# ifdef __thumb2__ + it ne +# endif + movne r12,r10 +# else +# ifdef __thumb2__ + itete eq +# endif + addeq r12,r11,#(poly1305_emit-.Lpoly1305_init) + addne r12,r11,#(poly1305_emit_neon-.Lpoly1305_init) + addeq r11,r11,#(poly1305_blocks-.Lpoly1305_init) + addne r11,r11,#(poly1305_blocks_neon-.Lpoly1305_init) +# endif +# ifdef __thumb2__ + orr r12,r12,#1 @ thumb-ify address + orr r11,r11,#1 +# endif +#endif + ldrb r9,[$inp,#11] + orr r6,r6,r7,lsl#8 + ldrb r7,[$inp,#12] + orr r6,r6,r8,lsl#16 + ldrb r8,[$inp,#13] + orr r6,r6,r9,lsl#24 + ldrb r9,[$inp,#14] + and r6,r6,r3 + + ldrb r10,[$inp,#15] + orr r7,r7,r8,lsl#8 + str r4,[$ctx,#0] + orr r7,r7,r9,lsl#16 + str r5,[$ctx,#4] + orr r7,r7,r10,lsl#24 + str r6,[$ctx,#8] + and r7,r7,r3 + str r7,[$ctx,#12] +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + stmia r2,{r11,r12} @ fill functions table + mov r0,#1 +#else + mov r0,#0 +#endif +.Lno_key: + ldmia sp!,{r4-r11} +#if __ARM_ARCH__>=5 + ret @ bx lr +#else + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_init,.-poly1305_init +___ +{ +my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12)); +my ($s1,$s2,$s3)=($r1,$r2,$r3); + +$code.=<<___; +.type poly1305_blocks,%function +.align 5 +poly1305_blocks: +.Lpoly1305_blocks: + stmdb sp!,{r3-r11,lr} + + ands $len,$len,#-16 + beq .Lno_data + + cmp $padbit,#0 + add $len,$len,$inp @ end pointer + sub sp,sp,#32 + + ldmia $ctx,{$h0-$r3} @ load context + + str $ctx,[sp,#12] @ offload stuff + mov lr,$inp + str $len,[sp,#16] + str $r1,[sp,#20] + str $r2,[sp,#24] + str $r3,[sp,#28] + b .Loop + +.Loop: +#if __ARM_ARCH__<7 + ldrb r0,[lr],#16 @ load input +# ifdef __thumb2__ + it hi +# endif + addhi $h4,$h4,#1 @ 1<<128 + ldrb r1,[lr,#-15] + ldrb r2,[lr,#-14] + ldrb r3,[lr,#-13] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-12] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-11] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-10] + adds $h0,$h0,r3 @ accumulate input + + ldrb r3,[lr,#-9] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-8] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-7] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-6] + adcs $h1,$h1,r3 + + ldrb r3,[lr,#-5] + orr r1,r0,r1,lsl#8 + ldrb r0,[lr,#-4] + orr r2,r1,r2,lsl#16 + ldrb r1,[lr,#-3] + orr r3,r2,r3,lsl#24 + ldrb r2,[lr,#-2] + adcs $h2,$h2,r3 + + ldrb r3,[lr,#-1] + orr r1,r0,r1,lsl#8 + str lr,[sp,#8] @ offload input pointer + orr r2,r1,r2,lsl#16 + add $s1,$r1,$r1,lsr#2 + orr r3,r2,r3,lsl#24 +#else + ldr r0,[lr],#16 @ load input +# ifdef __thumb2__ + it hi +# endif + addhi $h4,$h4,#1 @ padbit + ldr r1,[lr,#-12] + ldr r2,[lr,#-8] + ldr r3,[lr,#-4] +# ifdef __ARMEB__ + rev r0,r0 + rev r1,r1 + rev r2,r2 + rev r3,r3 +# endif + adds $h0,$h0,r0 @ accumulate input + str lr,[sp,#8] @ offload input pointer + adcs $h1,$h1,r1 + add $s1,$r1,$r1,lsr#2 + adcs $h2,$h2,r2 +#endif + add $s2,$r2,$r2,lsr#2 + adcs $h3,$h3,r3 + add $s3,$r3,$r3,lsr#2 + + umull r2,r3,$h1,$r0 + adc $h4,$h4,#0 + umull r0,r1,$h0,$r0 + umlal r2,r3,$h4,$s1 + umlal r0,r1,$h3,$s1 + ldr $r1,[sp,#20] @ reload $r1 + umlal r2,r3,$h2,$s3 + umlal r0,r1,$h1,$s3 + umlal r2,r3,$h3,$s2 + umlal r0,r1,$h2,$s2 + umlal r2,r3,$h0,$r1 + str r0,[sp,#0] @ future $h0 + mul r0,$s2,$h4 + ldr $r2,[sp,#24] @ reload $r2 + adds r2,r2,r1 @ d1+=d0>>32 + eor r1,r1,r1 + adc lr,r3,#0 @ future $h2 + str r2,[sp,#4] @ future $h1 + + mul r2,$s3,$h4 + eor r3,r3,r3 + umlal r0,r1,$h3,$s3 + ldr $r3,[sp,#28] @ reload $r3 + umlal r2,r3,$h3,$r0 + umlal r0,r1,$h2,$r0 + umlal r2,r3,$h2,$r1 + umlal r0,r1,$h1,$r1 + umlal r2,r3,$h1,$r2 + umlal r0,r1,$h0,$r2 + umlal r2,r3,$h0,$r3 + ldr $h0,[sp,#0] + mul $h4,$r0,$h4 + ldr $h1,[sp,#4] + + adds $h2,lr,r0 @ d2+=d1>>32 + ldr lr,[sp,#8] @ reload input pointer + adc r1,r1,#0 + adds $h3,r2,r1 @ d3+=d2>>32 + ldr r0,[sp,#16] @ reload end pointer + adc r3,r3,#0 + add $h4,$h4,r3 @ h4+=d3>>32 + + and r1,$h4,#-4 + and $h4,$h4,#3 + add r1,r1,r1,lsr#2 @ *=5 + adds $h0,$h0,r1 + adcs $h1,$h1,#0 + adcs $h2,$h2,#0 + adcs $h3,$h3,#0 + adc $h4,$h4,#0 + + cmp r0,lr @ done yet? + bhi .Loop + + ldr $ctx,[sp,#12] + add sp,sp,#32 + stmia $ctx,{$h0-$h4} @ store the result + +.Lno_data: +#if __ARM_ARCH__>=5 + ldmia sp!,{r3-r11,pc} +#else + ldmia sp!,{r3-r11,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_blocks,.-poly1305_blocks +___ +} +{ +my ($ctx,$mac,$nonce)=map("r$_",(0..2)); +my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11)); +my $g4=$h4; + +$code.=<<___; +.type poly1305_emit,%function +.align 5 +poly1305_emit: + stmdb sp!,{r4-r11} +.Lpoly1305_emit_enter: + + ldmia $ctx,{$h0-$h4} + adds $g0,$h0,#5 @ compare to modulus + adcs $g1,$h1,#0 + adcs $g2,$h2,#0 + adcs $g3,$h3,#0 + adc $g4,$h4,#0 + tst $g4,#4 @ did it carry/borrow? + +#ifdef __thumb2__ + it ne +#endif + movne $h0,$g0 + ldr $g0,[$nonce,#0] +#ifdef __thumb2__ + it ne +#endif + movne $h1,$g1 + ldr $g1,[$nonce,#4] +#ifdef __thumb2__ + it ne +#endif + movne $h2,$g2 + ldr $g2,[$nonce,#8] +#ifdef __thumb2__ + it ne +#endif + movne $h3,$g3 + ldr $g3,[$nonce,#12] + + adds $h0,$h0,$g0 + adcs $h1,$h1,$g1 + adcs $h2,$h2,$g2 + adc $h3,$h3,$g3 + +#if __ARM_ARCH__>=7 +# ifdef __ARMEB__ + rev $h0,$h0 + rev $h1,$h1 + rev $h2,$h2 + rev $h3,$h3 +# endif + str $h0,[$mac,#0] + str $h1,[$mac,#4] + str $h2,[$mac,#8] + str $h3,[$mac,#12] +#else + strb $h0,[$mac,#0] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#4] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#8] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#12] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#1] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#5] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#9] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#13] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#2] + mov $h0,$h0,lsr#8 + strb $h1,[$mac,#6] + mov $h1,$h1,lsr#8 + strb $h2,[$mac,#10] + mov $h2,$h2,lsr#8 + strb $h3,[$mac,#14] + mov $h3,$h3,lsr#8 + + strb $h0,[$mac,#3] + strb $h1,[$mac,#7] + strb $h2,[$mac,#11] + strb $h3,[$mac,#15] +#endif + ldmia sp!,{r4-r11} +#if __ARM_ARCH__>=5 + ret @ bx lr +#else + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size poly1305_emit,.-poly1305_emit +___ +{ +my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9)); +my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14)); +my ($T0,$T1,$MASK) = map("q$_",(15,4,0)); + +my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7)); + +$code.=<<___; +#if (defined(__KERNEL__) && defined(CONFIG_KERNEL_MODE_NEON)) || (!defined(__KERNEL__) && __ARM_MAX_ARCH__>=7) +.fpu neon + +.type poly1305_init_neon,%function +.align 5 +poly1305_init_neon: +.Lpoly1305_init_neon: + ldr r4,[$ctx,#20] @ load key base 2^32 + ldr r5,[$ctx,#24] + ldr r6,[$ctx,#28] + ldr r7,[$ctx,#32] + + and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 + mov r3,r4,lsr#26 + mov r4,r5,lsr#20 + orr r3,r3,r5,lsl#6 + mov r5,r6,lsr#14 + orr r4,r4,r6,lsl#12 + mov r6,r7,lsr#8 + orr r5,r5,r7,lsl#18 + and r3,r3,#0x03ffffff + and r4,r4,#0x03ffffff + and r5,r5,#0x03ffffff + + vdup.32 $R0,r2 @ r^1 in both lanes + add r2,r3,r3,lsl#2 @ *5 + vdup.32 $R1,r3 + add r3,r4,r4,lsl#2 + vdup.32 $S1,r2 + vdup.32 $R2,r4 + add r4,r5,r5,lsl#2 + vdup.32 $S2,r3 + vdup.32 $R3,r5 + add r5,r6,r6,lsl#2 + vdup.32 $S3,r4 + vdup.32 $R4,r6 + vdup.32 $S4,r5 + + mov $zeros,#2 @ counter + +.Lsquare_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + + vmull.u32 $D0,$R0,${R0}[1] + vmull.u32 $D1,$R1,${R0}[1] + vmull.u32 $D2,$R2,${R0}[1] + vmull.u32 $D3,$R3,${R0}[1] + vmull.u32 $D4,$R4,${R0}[1] + + vmlal.u32 $D0,$R4,${S1}[1] + vmlal.u32 $D1,$R0,${R1}[1] + vmlal.u32 $D2,$R1,${R1}[1] + vmlal.u32 $D3,$R2,${R1}[1] + vmlal.u32 $D4,$R3,${R1}[1] + + vmlal.u32 $D0,$R3,${S2}[1] + vmlal.u32 $D1,$R4,${S2}[1] + vmlal.u32 $D3,$R1,${R2}[1] + vmlal.u32 $D2,$R0,${R2}[1] + vmlal.u32 $D4,$R2,${R2}[1] + + vmlal.u32 $D0,$R2,${S3}[1] + vmlal.u32 $D3,$R0,${R3}[1] + vmlal.u32 $D1,$R3,${S3}[1] + vmlal.u32 $D2,$R4,${S3}[1] + vmlal.u32 $D4,$R1,${R3}[1] + + vmlal.u32 $D3,$R4,${S4}[1] + vmlal.u32 $D0,$R1,${S4}[1] + vmlal.u32 $D1,$R2,${S4}[1] + vmlal.u32 $D2,$R3,${S4}[1] + vmlal.u32 $D4,$R0,${R4}[1] + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + @ and P. Schwabe + @ + @ H0>>+H1>>+H2>>+H3>>+H4 + @ H3>>+H4>>*5+H0>>+H1 + @ + @ Trivia. + @ + @ Result of multiplication of n-bit number by m-bit number is + @ n+m bits wide. However! Even though 2^n is a n+1-bit number, + @ m-bit number multiplied by 2^n is still n+m bits wide. + @ + @ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2, + @ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit + @ one is n+1 bits wide. + @ + @ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that + @ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4 + @ can be 27. However! In cases when their width exceeds 26 bits + @ they are limited by 2^26+2^6. This in turn means that *sum* + @ of the products with these values can still be viewed as sum + @ of 52-bit numbers as long as the amount of addends is not a + @ power of 2. For example, + @ + @ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4, + @ + @ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or + @ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than + @ 8 * (2^52) or 2^55. However, the value is then multiplied by + @ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12), + @ which is less than 32 * (2^52) or 2^57. And when processing + @ data we are looking at triple as many addends... + @ + @ In key setup procedure pre-reduced H0 is limited by 5*4+1 and + @ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the + @ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while + @ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32 + @ instruction accepts 2x32-bit input and writes 2x64-bit result. + @ This means that result of reduction have to be compressed upon + @ loop wrap-around. This can be done in the process of reduction + @ to minimize amount of instructions [as well as amount of + @ 128-bit instructions, which benefits low-end processors], but + @ one has to watch for H2 (which is narrower than H0) and 5*H4 + @ not being wider than 58 bits, so that result of right shift + @ by 26 bits fits in 32 bits. This is also useful on x86, + @ because it allows to use paddd in place for paddq, which + @ benefits Atom, where paddq is ridiculously slow. + + vshr.u64 $T0,$D3,#26 + vmovn.i64 $D3#lo,$D3 + vshr.u64 $T1,$D0,#26 + vmovn.i64 $D0#lo,$D0 + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vbic.i32 $D3#lo,#0xfc000000 @ &=0x03ffffff + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + vbic.i32 $D0#lo,#0xfc000000 + + vshrn.u64 $T0#lo,$D4,#26 + vmovn.i64 $D4#lo,$D4 + vshr.u64 $T1,$D1,#26 + vmovn.i64 $D1#lo,$D1 + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + vbic.i32 $D4#lo,#0xfc000000 + vbic.i32 $D1#lo,#0xfc000000 + + vadd.i32 $D0#lo,$D0#lo,$T0#lo + vshl.u32 $T0#lo,$T0#lo,#2 + vshrn.u64 $T1#lo,$D2,#26 + vmovn.i64 $D2#lo,$D2 + vadd.i32 $D0#lo,$D0#lo,$T0#lo @ h4 -> h0 + vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3 + vbic.i32 $D2#lo,#0xfc000000 + + vshr.u32 $T0#lo,$D0#lo,#26 + vbic.i32 $D0#lo,#0xfc000000 + vshr.u32 $T1#lo,$D3#lo,#26 + vbic.i32 $D3#lo,#0xfc000000 + vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1 + vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4 + + subs $zeros,$zeros,#1 + beq .Lsquare_break_neon + + add $tbl0,$ctx,#(48+0*9*4) + add $tbl1,$ctx,#(48+1*9*4) + + vtrn.32 $R0,$D0#lo @ r^2:r^1 + vtrn.32 $R2,$D2#lo + vtrn.32 $R3,$D3#lo + vtrn.32 $R1,$D1#lo + vtrn.32 $R4,$D4#lo + + vshl.u32 $S2,$R2,#2 @ *5 + vshl.u32 $S3,$R3,#2 + vshl.u32 $S1,$R1,#2 + vshl.u32 $S4,$R4,#2 + vadd.i32 $S2,$S2,$R2 + vadd.i32 $S1,$S1,$R1 + vadd.i32 $S3,$S3,$R3 + vadd.i32 $S4,$S4,$R4 + + vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! + vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! + vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vst1.32 {${S4}[0]},[$tbl0,:32] + vst1.32 {${S4}[1]},[$tbl1,:32] + + b .Lsquare_neon + +.align 4 +.Lsquare_break_neon: + add $tbl0,$ctx,#(48+2*4*9) + add $tbl1,$ctx,#(48+3*4*9) + + vmov $R0,$D0#lo @ r^4:r^3 + vshl.u32 $S1,$D1#lo,#2 @ *5 + vmov $R1,$D1#lo + vshl.u32 $S2,$D2#lo,#2 + vmov $R2,$D2#lo + vshl.u32 $S3,$D3#lo,#2 + vmov $R3,$D3#lo + vshl.u32 $S4,$D4#lo,#2 + vmov $R4,$D4#lo + vadd.i32 $S1,$S1,$D1#lo + vadd.i32 $S2,$S2,$D2#lo + vadd.i32 $S3,$S3,$D3#lo + vadd.i32 $S4,$S4,$D4#lo + + vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! + vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! + vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vst1.32 {${S4}[0]},[$tbl0] + vst1.32 {${S4}[1]},[$tbl1] + + ret @ bx lr +.size poly1305_init_neon,.-poly1305_init_neon + +#ifdef __KERNEL__ +.globl poly1305_blocks_neon +#endif +.type poly1305_blocks_neon,%function +.align 5 +poly1305_blocks_neon: + ldr ip,[$ctx,#36] @ is_base2_26 + ands $len,$len,#-16 + beq .Lno_data_neon + + cmp $len,#64 + bhs .Lenter_neon + tst ip,ip @ is_base2_26? + beq .Lpoly1305_blocks + +.Lenter_neon: + stmdb sp!,{r4-r7} + vstmdb sp!,{d8-d15} @ ABI specification says so + + tst ip,ip @ is_base2_26? + bne .Lbase2_26_neon + + stmdb sp!,{r1-r3,lr} + bl .Lpoly1305_init_neon + + ldr r4,[$ctx,#0] @ load hash value base 2^32 + ldr r5,[$ctx,#4] + ldr r6,[$ctx,#8] + ldr r7,[$ctx,#12] + ldr ip,[$ctx,#16] + + and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 + mov r3,r4,lsr#26 + veor $D0#lo,$D0#lo,$D0#lo + mov r4,r5,lsr#20 + orr r3,r3,r5,lsl#6 + veor $D1#lo,$D1#lo,$D1#lo + mov r5,r6,lsr#14 + orr r4,r4,r6,lsl#12 + veor $D2#lo,$D2#lo,$D2#lo + mov r6,r7,lsr#8 + orr r5,r5,r7,lsl#18 + veor $D3#lo,$D3#lo,$D3#lo + and r3,r3,#0x03ffffff + orr r6,r6,ip,lsl#24 + veor $D4#lo,$D4#lo,$D4#lo + and r4,r4,#0x03ffffff + mov r1,#1 + and r5,r5,#0x03ffffff + str r1,[$ctx,#36] @ is_base2_26 + + vmov.32 $D0#lo[0],r2 + vmov.32 $D1#lo[0],r3 + vmov.32 $D2#lo[0],r4 + vmov.32 $D3#lo[0],r5 + vmov.32 $D4#lo[0],r6 + adr $zeros,.Lzeros + + ldmia sp!,{r1-r3,lr} + b .Lbase2_32_neon + +.align 4 +.Lbase2_26_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ load hash value + + veor $D0#lo,$D0#lo,$D0#lo + veor $D1#lo,$D1#lo,$D1#lo + veor $D2#lo,$D2#lo,$D2#lo + veor $D3#lo,$D3#lo,$D3#lo + veor $D4#lo,$D4#lo,$D4#lo + vld4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]! + adr $zeros,.Lzeros + vld1.32 {$D4#lo[0]},[$ctx] + sub $ctx,$ctx,#16 @ rewind + +.Lbase2_32_neon: + add $in2,$inp,#32 + mov $padbit,$padbit,lsl#24 + tst $len,#31 + beq .Leven + + vld4.32 {$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]! + vmov.32 $H4#lo[0],$padbit + sub $len,$len,#16 + add $in2,$inp,#32 + +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H3,$H3 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 +# endif + vsri.u32 $H4#lo,$H3#lo,#8 @ base 2^32 -> base 2^26 + vshl.u32 $H3#lo,$H3#lo,#18 + + vsri.u32 $H3#lo,$H2#lo,#14 + vshl.u32 $H2#lo,$H2#lo,#12 + vadd.i32 $H4#hi,$H4#lo,$D4#lo @ add hash value and move to #hi + + vbic.i32 $H3#lo,#0xfc000000 + vsri.u32 $H2#lo,$H1#lo,#20 + vshl.u32 $H1#lo,$H1#lo,#6 + + vbic.i32 $H2#lo,#0xfc000000 + vsri.u32 $H1#lo,$H0#lo,#26 + vadd.i32 $H3#hi,$H3#lo,$D3#lo + + vbic.i32 $H0#lo,#0xfc000000 + vbic.i32 $H1#lo,#0xfc000000 + vadd.i32 $H2#hi,$H2#lo,$D2#lo + + vadd.i32 $H0#hi,$H0#lo,$D0#lo + vadd.i32 $H1#hi,$H1#lo,$D1#lo + + mov $tbl1,$zeros + add $tbl0,$ctx,#48 + + cmp $len,$len + b .Long_tail + +.align 4 +.Leven: + subs $len,$len,#64 + it lo + movlo $in2,$zeros + + vmov.i32 $H4,#1<<24 @ padbit, yes, always + vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1] + add $inp,$inp,#64 + vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0) + add $in2,$in2,#64 + itt hi + addhi $tbl1,$ctx,#(48+1*9*4) + addhi $tbl0,$ctx,#(48+3*9*4) + +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H3,$H3 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 +# endif + vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26 + vshl.u32 $H3,$H3,#18 + + vsri.u32 $H3,$H2,#14 + vshl.u32 $H2,$H2,#12 + + vbic.i32 $H3,#0xfc000000 + vsri.u32 $H2,$H1,#20 + vshl.u32 $H1,$H1,#6 + + vbic.i32 $H2,#0xfc000000 + vsri.u32 $H1,$H0,#26 + + vbic.i32 $H0,#0xfc000000 + vbic.i32 $H1,#0xfc000000 + + bls .Lskip_loop + + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^2 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + b .Loop_neon + +.align 5 +.Loop_neon: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + @ \___________________/ + @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + @ \___________________/ \____________________/ + @ + @ Note that we start with inp[2:3]*r^2. This is because it + @ doesn't depend on reduction in previous iteration. + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ inp[2:3]*r^2 + + vadd.i32 $H2#lo,$H2#lo,$D2#lo @ accumulate inp[0:1] + vmull.u32 $D2,$H2#hi,${R0}[1] + vadd.i32 $H0#lo,$H0#lo,$D0#lo + vmull.u32 $D0,$H0#hi,${R0}[1] + vadd.i32 $H3#lo,$H3#lo,$D3#lo + vmull.u32 $D3,$H3#hi,${R0}[1] + vmlal.u32 $D2,$H1#hi,${R1}[1] + vadd.i32 $H1#lo,$H1#lo,$D1#lo + vmull.u32 $D1,$H1#hi,${R0}[1] + + vadd.i32 $H4#lo,$H4#lo,$D4#lo + vmull.u32 $D4,$H4#hi,${R0}[1] + subs $len,$len,#64 + vmlal.u32 $D0,$H4#hi,${S1}[1] + it lo + movlo $in2,$zeros + vmlal.u32 $D3,$H2#hi,${R1}[1] + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D1,$H0#hi,${R1}[1] + vmlal.u32 $D4,$H3#hi,${R1}[1] + + vmlal.u32 $D0,$H3#hi,${S2}[1] + vmlal.u32 $D3,$H1#hi,${R2}[1] + vmlal.u32 $D4,$H2#hi,${R2}[1] + vmlal.u32 $D1,$H4#hi,${S2}[1] + vmlal.u32 $D2,$H0#hi,${R2}[1] + + vmlal.u32 $D3,$H0#hi,${R3}[1] + vmlal.u32 $D0,$H2#hi,${S3}[1] + vmlal.u32 $D4,$H1#hi,${R3}[1] + vmlal.u32 $D1,$H3#hi,${S3}[1] + vmlal.u32 $D2,$H4#hi,${S3}[1] + + vmlal.u32 $D3,$H4#hi,${S4}[1] + vmlal.u32 $D0,$H1#hi,${S4}[1] + vmlal.u32 $D4,$H0#hi,${R4}[1] + vmlal.u32 $D1,$H2#hi,${S4}[1] + vmlal.u32 $D2,$H3#hi,${S4}[1] + + vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0) + add $in2,$in2,#64 + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ (hash+inp[0:1])*r^4 and accumulate + + vmlal.u32 $D3,$H3#lo,${R0}[0] + vmlal.u32 $D0,$H0#lo,${R0}[0] + vmlal.u32 $D4,$H4#lo,${R0}[0] + vmlal.u32 $D1,$H1#lo,${R0}[0] + vmlal.u32 $D2,$H2#lo,${R0}[0] + vld1.32 ${S4}[0],[$tbl0,:32] + + vmlal.u32 $D3,$H2#lo,${R1}[0] + vmlal.u32 $D0,$H4#lo,${S1}[0] + vmlal.u32 $D4,$H3#lo,${R1}[0] + vmlal.u32 $D1,$H0#lo,${R1}[0] + vmlal.u32 $D2,$H1#lo,${R1}[0] + + vmlal.u32 $D3,$H1#lo,${R2}[0] + vmlal.u32 $D0,$H3#lo,${S2}[0] + vmlal.u32 $D4,$H2#lo,${R2}[0] + vmlal.u32 $D1,$H4#lo,${S2}[0] + vmlal.u32 $D2,$H0#lo,${R2}[0] + + vmlal.u32 $D3,$H0#lo,${R3}[0] + vmlal.u32 $D0,$H2#lo,${S3}[0] + vmlal.u32 $D4,$H1#lo,${R3}[0] + vmlal.u32 $D1,$H3#lo,${S3}[0] + vmlal.u32 $D3,$H4#lo,${S4}[0] + + vmlal.u32 $D2,$H4#lo,${S3}[0] + vmlal.u32 $D0,$H1#lo,${S4}[0] + vmlal.u32 $D4,$H0#lo,${R4}[0] + vmov.i32 $H4,#1<<24 @ padbit, yes, always + vmlal.u32 $D1,$H2#lo,${S4}[0] + vmlal.u32 $D2,$H3#lo,${S4}[0] + + vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1] + add $inp,$inp,#64 +# ifdef __ARMEB__ + vrev32.8 $H0,$H0 + vrev32.8 $H1,$H1 + vrev32.8 $H2,$H2 + vrev32.8 $H3,$H3 +# endif + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction interleaved with base 2^32 -> base 2^26 of + @ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4. + + vshr.u64 $T0,$D3,#26 + vmovn.i64 $D3#lo,$D3 + vshr.u64 $T1,$D0,#26 + vmovn.i64 $D0#lo,$D0 + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vbic.i32 $D3#lo,#0xfc000000 + vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26 + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + vshl.u32 $H3,$H3,#18 + vbic.i32 $D0#lo,#0xfc000000 + + vshrn.u64 $T0#lo,$D4,#26 + vmovn.i64 $D4#lo,$D4 + vshr.u64 $T1,$D1,#26 + vmovn.i64 $D1#lo,$D1 + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + vsri.u32 $H3,$H2,#14 + vbic.i32 $D4#lo,#0xfc000000 + vshl.u32 $H2,$H2,#12 + vbic.i32 $D1#lo,#0xfc000000 + + vadd.i32 $D0#lo,$D0#lo,$T0#lo + vshl.u32 $T0#lo,$T0#lo,#2 + vbic.i32 $H3,#0xfc000000 + vshrn.u64 $T1#lo,$D2,#26 + vmovn.i64 $D2#lo,$D2 + vaddl.u32 $D0,$D0#lo,$T0#lo @ h4 -> h0 [widen for a sec] + vsri.u32 $H2,$H1,#20 + vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3 + vshl.u32 $H1,$H1,#6 + vbic.i32 $D2#lo,#0xfc000000 + vbic.i32 $H2,#0xfc000000 + + vshrn.u64 $T0#lo,$D0,#26 @ re-narrow + vmovn.i64 $D0#lo,$D0 + vsri.u32 $H1,$H0,#26 + vbic.i32 $H0,#0xfc000000 + vshr.u32 $T1#lo,$D3#lo,#26 + vbic.i32 $D3#lo,#0xfc000000 + vbic.i32 $D0#lo,#0xfc000000 + vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1 + vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4 + vbic.i32 $H1,#0xfc000000 + + bhi .Loop_neon + +.Lskip_loop: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + add $tbl1,$ctx,#(48+0*9*4) + add $tbl0,$ctx,#(48+1*9*4) + adds $len,$len,#32 + it ne + movne $len,#0 + bne .Long_tail + + vadd.i32 $H2#hi,$H2#lo,$D2#lo @ add hash value and move to #hi + vadd.i32 $H0#hi,$H0#lo,$D0#lo + vadd.i32 $H3#hi,$H3#lo,$D3#lo + vadd.i32 $H1#hi,$H1#lo,$D1#lo + vadd.i32 $H4#hi,$H4#lo,$D4#lo + +.Long_tail: + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^1 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^2 + + vadd.i32 $H2#lo,$H2#lo,$D2#lo @ can be redundant + vmull.u32 $D2,$H2#hi,$R0 + vadd.i32 $H0#lo,$H0#lo,$D0#lo + vmull.u32 $D0,$H0#hi,$R0 + vadd.i32 $H3#lo,$H3#lo,$D3#lo + vmull.u32 $D3,$H3#hi,$R0 + vadd.i32 $H1#lo,$H1#lo,$D1#lo + vmull.u32 $D1,$H1#hi,$R0 + vadd.i32 $H4#lo,$H4#lo,$D4#lo + vmull.u32 $D4,$H4#hi,$R0 + + vmlal.u32 $D0,$H4#hi,$S1 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vmlal.u32 $D3,$H2#hi,$R1 + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vmlal.u32 $D1,$H0#hi,$R1 + vmlal.u32 $D4,$H3#hi,$R1 + vmlal.u32 $D2,$H1#hi,$R1 + + vmlal.u32 $D3,$H1#hi,$R2 + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D0,$H3#hi,$S2 + vld1.32 ${S4}[0],[$tbl0,:32] + vmlal.u32 $D4,$H2#hi,$R2 + vmlal.u32 $D1,$H4#hi,$S2 + vmlal.u32 $D2,$H0#hi,$R2 + + vmlal.u32 $D3,$H0#hi,$R3 + it ne + addne $tbl1,$ctx,#(48+2*9*4) + vmlal.u32 $D0,$H2#hi,$S3 + it ne + addne $tbl0,$ctx,#(48+3*9*4) + vmlal.u32 $D4,$H1#hi,$R3 + vmlal.u32 $D1,$H3#hi,$S3 + vmlal.u32 $D2,$H4#hi,$S3 + + vmlal.u32 $D3,$H4#hi,$S4 + vorn $MASK,$MASK,$MASK @ all-ones, can be redundant + vmlal.u32 $D0,$H1#hi,$S4 + vshr.u64 $MASK,$MASK,#38 + vmlal.u32 $D4,$H0#hi,$R4 + vmlal.u32 $D1,$H2#hi,$S4 + vmlal.u32 $D2,$H3#hi,$S4 + + beq .Lshort_tail + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ (hash+inp[0:1])*r^4:r^3 and accumulate + + vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^3 + vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4 + + vmlal.u32 $D2,$H2#lo,$R0 + vmlal.u32 $D0,$H0#lo,$R0 + vmlal.u32 $D3,$H3#lo,$R0 + vmlal.u32 $D1,$H1#lo,$R0 + vmlal.u32 $D4,$H4#lo,$R0 + + vmlal.u32 $D0,$H4#lo,$S1 + vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]! + vmlal.u32 $D3,$H2#lo,$R1 + vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]! + vmlal.u32 $D1,$H0#lo,$R1 + vmlal.u32 $D4,$H3#lo,$R1 + vmlal.u32 $D2,$H1#lo,$R1 + + vmlal.u32 $D3,$H1#lo,$R2 + vld1.32 ${S4}[1],[$tbl1,:32] + vmlal.u32 $D0,$H3#lo,$S2 + vld1.32 ${S4}[0],[$tbl0,:32] + vmlal.u32 $D4,$H2#lo,$R2 + vmlal.u32 $D1,$H4#lo,$S2 + vmlal.u32 $D2,$H0#lo,$R2 + + vmlal.u32 $D3,$H0#lo,$R3 + vmlal.u32 $D0,$H2#lo,$S3 + vmlal.u32 $D4,$H1#lo,$R3 + vmlal.u32 $D1,$H3#lo,$S3 + vmlal.u32 $D2,$H4#lo,$S3 + + vmlal.u32 $D3,$H4#lo,$S4 + vorn $MASK,$MASK,$MASK @ all-ones + vmlal.u32 $D0,$H1#lo,$S4 + vshr.u64 $MASK,$MASK,#38 + vmlal.u32 $D4,$H0#lo,$R4 + vmlal.u32 $D1,$H2#lo,$S4 + vmlal.u32 $D2,$H3#lo,$S4 + +.Lshort_tail: + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ horizontal addition + + vadd.i64 $D3#lo,$D3#lo,$D3#hi + vadd.i64 $D0#lo,$D0#lo,$D0#hi + vadd.i64 $D4#lo,$D4#lo,$D4#hi + vadd.i64 $D1#lo,$D1#lo,$D1#hi + vadd.i64 $D2#lo,$D2#lo,$D2#hi + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ lazy reduction, but without narrowing + + vshr.u64 $T0,$D3,#26 + vand.i64 $D3,$D3,$MASK + vshr.u64 $T1,$D0,#26 + vand.i64 $D0,$D0,$MASK + vadd.i64 $D4,$D4,$T0 @ h3 -> h4 + vadd.i64 $D1,$D1,$T1 @ h0 -> h1 + + vshr.u64 $T0,$D4,#26 + vand.i64 $D4,$D4,$MASK + vshr.u64 $T1,$D1,#26 + vand.i64 $D1,$D1,$MASK + vadd.i64 $D2,$D2,$T1 @ h1 -> h2 + + vadd.i64 $D0,$D0,$T0 + vshl.u64 $T0,$T0,#2 + vshr.u64 $T1,$D2,#26 + vand.i64 $D2,$D2,$MASK + vadd.i64 $D0,$D0,$T0 @ h4 -> h0 + vadd.i64 $D3,$D3,$T1 @ h2 -> h3 + + vshr.u64 $T0,$D0,#26 + vand.i64 $D0,$D0,$MASK + vshr.u64 $T1,$D3,#26 + vand.i64 $D3,$D3,$MASK + vadd.i64 $D1,$D1,$T0 @ h0 -> h1 + vadd.i64 $D4,$D4,$T1 @ h3 -> h4 + + cmp $len,#0 + bne .Leven + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + @ store hash value + + vst4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]! + vst1.32 {$D4#lo[0]},[$ctx] + + vldmia sp!,{d8-d15} @ epilogue + ldmia sp!,{r4-r7} +.Lno_data_neon: + ret @ bx lr +.size poly1305_blocks_neon,.-poly1305_blocks_neon + +#ifdef __KERNEL__ +.globl poly1305_emit_neon +#endif +.type poly1305_emit_neon,%function +.align 5 +poly1305_emit_neon: + ldr ip,[$ctx,#36] @ is_base2_26 + + stmdb sp!,{r4-r11} + + tst ip,ip + beq .Lpoly1305_emit_enter + + ldmia $ctx,{$h0-$h4} + eor $g0,$g0,$g0 + + adds $h0,$h0,$h1,lsl#26 @ base 2^26 -> base 2^32 + mov $h1,$h1,lsr#6 + adcs $h1,$h1,$h2,lsl#20 + mov $h2,$h2,lsr#12 + adcs $h2,$h2,$h3,lsl#14 + mov $h3,$h3,lsr#18 + adcs $h3,$h3,$h4,lsl#8 + adc $h4,$g0,$h4,lsr#24 @ can be partially reduced ... + + and $g0,$h4,#-4 @ ... so reduce + and $h4,$h3,#3 + add $g0,$g0,$g0,lsr#2 @ *= 5 + adds $h0,$h0,$g0 + adcs $h1,$h1,#0 + adcs $h2,$h2,#0 + adcs $h3,$h3,#0 + adc $h4,$h4,#0 + + adds $g0,$h0,#5 @ compare to modulus + adcs $g1,$h1,#0 + adcs $g2,$h2,#0 + adcs $g3,$h3,#0 + adc $g4,$h4,#0 + tst $g4,#4 @ did it carry/borrow? + + it ne + movne $h0,$g0 + ldr $g0,[$nonce,#0] + it ne + movne $h1,$g1 + ldr $g1,[$nonce,#4] + it ne + movne $h2,$g2 + ldr $g2,[$nonce,#8] + it ne + movne $h3,$g3 + ldr $g3,[$nonce,#12] + + adds $h0,$h0,$g0 @ accumulate nonce + adcs $h1,$h1,$g1 + adcs $h2,$h2,$g2 + adc $h3,$h3,$g3 + +# ifdef __ARMEB__ + rev $h0,$h0 + rev $h1,$h1 + rev $h2,$h2 + rev $h3,$h3 +# endif + str $h0,[$mac,#0] @ store the result + str $h1,[$mac,#4] + str $h2,[$mac,#8] + str $h3,[$mac,#12] + + ldmia sp!,{r4-r11} + ret @ bx lr +.size poly1305_emit_neon,.-poly1305_emit_neon + +.align 5 +.Lzeros: +.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +# ifndef __KERNEL__ +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-.Lpoly1305_init +# endif +#endif +___ +} } +$code.=<<___; +.align 2 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.comm OPENSSL_armcap_P,4,4 +#endif +___ + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\`([^\`]*)\`/eval $1/geo; + + s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or + s/\bret\b/bx lr/go or + s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4 + + print $_,"\n"; +} +close STDOUT; # enforce flush diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl new file mode 100644 index 0000000000000..d513b45a149bf --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-arm64.pl @@ -0,0 +1,974 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements Poly1305 hash for ARMv8. +# +# June 2015 +# +# Numbers are cycles per processed byte with poly1305_blocks alone. +# +# IALU/gcc-4.9 NEON +# +# Apple A7 1.86/+5% 0.72 +# Cortex-A53 2.69/+58% 1.47 +# Cortex-A57 2.70/+7% 1.14 +# Denver 1.64/+50% 1.18(*) +# X-Gene 2.13/+68% 2.27 +# Mongoose 1.77/+75% 1.12 +# Kryo 2.70/+55% 1.13 +# +# (*) estimate based on resources availability is less than 1.0, +# i.e. measured result is worse than expected, presumably binary +# translator is not almighty; + +$flavour=shift; +if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; } +else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} } + +if ($flavour && $flavour ne "void") { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or + die "can't locate arm-xlate.pl"; + + open STDOUT,"| \"$^X\" $xlate $flavour $output"; +} else { + open STDOUT,">$output"; +} + +my ($ctx,$inp,$len,$padbit) = map("x$_",(0..3)); +my ($mac,$nonce)=($inp,$len); + +my ($h0,$h1,$h2,$r0,$r1,$s1,$t0,$t1,$d0,$d1,$d2) = map("x$_",(4..14)); + +$code.=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +.extern OPENSSL_armcap_P +#else +# define poly1305_init poly1305_init_arm +# define poly1305_blocks poly1305_blocks_arm +# define poly1305_emit poly1305_emit_arm +#endif + +.text + +// forward "declarations" are required for Apple +.globl poly1305_blocks +.globl poly1305_emit +.globl poly1305_init +.type poly1305_init,%function +.align 5 +poly1305_init: + cmp $inp,xzr + stp xzr,xzr,[$ctx] // zero hash value + stp xzr,xzr,[$ctx,#16] // [along with is_base2_26] + + csel x0,xzr,x0,eq + b.eq .Lno_key + +#ifndef __KERNEL__ +# ifdef __ILP32__ + ldrsw $t1,.LOPENSSL_armcap_P +# else + ldr $t1,.LOPENSSL_armcap_P +# endif + adr $t0,.LOPENSSL_armcap_P + ldr w17,[$t0,$t1] +#endif + + ldp $r0,$r1,[$inp] // load key + mov $s1,#0xfffffffc0fffffff + movk $s1,#0x0fff,lsl#48 +#ifdef __AARCH64EB__ + rev $r0,$r0 // flip bytes + rev $r1,$r1 +#endif + and $r0,$r0,$s1 // &=0ffffffc0fffffff + and $s1,$s1,#-4 + and $r1,$r1,$s1 // &=0ffffffc0ffffffc + stp $r0,$r1,[$ctx,#32] // save key value + +#ifndef __KERNEL__ + tst w17,#ARMV7_NEON + + adr $d0,poly1305_blocks + adr $r0,poly1305_blocks_neon + adr $d1,poly1305_emit + adr $r1,poly1305_emit_neon + + csel $d0,$d0,$r0,eq + csel $d1,$d1,$r1,eq + +# ifdef __ILP32__ + stp w12,w13,[$len] +# else + stp $d0,$d1,[$len] +# endif + + mov x0,#1 +#else + mov x0,#0 +#endif +.Lno_key: + ret +.size poly1305_init,.-poly1305_init + +.type poly1305_blocks,%function +.align 5 +poly1305_blocks: + ands $len,$len,#-16 + b.eq .Lno_data + + ldp $h0,$h1,[$ctx] // load hash value + ldp $r0,$r1,[$ctx,#32] // load key value + ldr $h2,[$ctx,#16] + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + b .Loop + +.align 5 +.Loop: + ldp $t0,$t1,[$inp],#16 // load input + sub $len,$len,#16 +#ifdef __AARCH64EB__ + rev $t0,$t0 + rev $t1,$t1 +#endif + adds $h0,$h0,$t0 // accumulate input + adcs $h1,$h1,$t1 + + mul $d0,$h0,$r0 // h0*r0 + adc $h2,$h2,$padbit + umulh $d1,$h0,$r0 + + mul $t0,$h1,$s1 // h1*5*r1 + umulh $t1,$h1,$s1 + + adds $d0,$d0,$t0 + mul $t0,$h0,$r1 // h0*r1 + adc $d1,$d1,$t1 + umulh $d2,$h0,$r1 + + adds $d1,$d1,$t0 + mul $t0,$h1,$r0 // h1*r0 + adc $d2,$d2,xzr + umulh $t1,$h1,$r0 + + adds $d1,$d1,$t0 + mul $t0,$h2,$s1 // h2*5*r1 + adc $d2,$d2,$t1 + mul $t1,$h2,$r0 // h2*r0 + + adds $d1,$d1,$t0 + adc $d2,$d2,$t1 + + and $t0,$d2,#-4 // final reduction + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$d0,$t0 + adcs $h1,$d1,xzr + adc $h2,$h2,xzr + + cbnz $len,.Loop + + stp $h0,$h1,[$ctx] // store hash value + str $h2,[$ctx,#16] + +.Lno_data: + ret +.size poly1305_blocks,.-poly1305_blocks + +.type poly1305_emit,%function +.align 5 +poly1305_emit: + ldp $h0,$h1,[$ctx] // load hash base 2^64 + ldr $h2,[$ctx,#16] + ldp $t0,$t1,[$nonce] // load nonce + + adds $d0,$h0,#5 // compare to modulus + adcs $d1,$h1,xzr + adc $d2,$h2,xzr + + tst $d2,#-4 // see if it's carried/borrowed + + csel $h0,$h0,$d0,eq + csel $h1,$h1,$d1,eq + +#ifdef __AARCH64EB__ + ror $t0,$t0,#32 // flip nonce words + ror $t1,$t1,#32 +#endif + adds $h0,$h0,$t0 // accumulate nonce + adc $h1,$h1,$t1 +#ifdef __AARCH64EB__ + rev $h0,$h0 // flip output bytes + rev $h1,$h1 +#endif + stp $h0,$h1,[$mac] // write result + + ret +.size poly1305_emit,.-poly1305_emit +___ +my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("v$_.4s",(0..8)); +my ($IN01_0,$IN01_1,$IN01_2,$IN01_3,$IN01_4) = map("v$_.2s",(9..13)); +my ($IN23_0,$IN23_1,$IN23_2,$IN23_3,$IN23_4) = map("v$_.2s",(14..18)); +my ($ACC0,$ACC1,$ACC2,$ACC3,$ACC4) = map("v$_.2d",(19..23)); +my ($H0,$H1,$H2,$H3,$H4) = map("v$_.2s",(24..28)); +my ($T0,$T1,$MASK) = map("v$_",(29..31)); + +my ($in2,$zeros)=("x16","x17"); +my $is_base2_26 = $zeros; # borrow + +$code.=<<___; +.type __poly1305_mult,%function +.align 5 +__poly1305_mult: + mul $d0,$h0,$r0 // h0*r0 + umulh $d1,$h0,$r0 + + mul $t0,$h1,$s1 // h1*5*r1 + umulh $t1,$h1,$s1 + + adds $d0,$d0,$t0 + mul $t0,$h0,$r1 // h0*r1 + adc $d1,$d1,$t1 + umulh $d2,$h0,$r1 + + adds $d1,$d1,$t0 + mul $t0,$h1,$r0 // h1*r0 + adc $d2,$d2,xzr + umulh $t1,$h1,$r0 + + adds $d1,$d1,$t0 + mul $t0,$h2,$s1 // h2*5*r1 + adc $d2,$d2,$t1 + mul $t1,$h2,$r0 // h2*r0 + + adds $d1,$d1,$t0 + adc $d2,$d2,$t1 + + and $t0,$d2,#-4 // final reduction + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$d0,$t0 + adcs $h1,$d1,xzr + adc $h2,$h2,xzr + + ret +.size __poly1305_mult,.-__poly1305_mult + +.type __poly1305_splat,%function +.align 5 +__poly1305_splat: + and x12,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x13,$h0,#26,#26 + extr x14,$h1,$h0,#52 + and x14,x14,#0x03ffffff + ubfx x15,$h1,#14,#26 + extr x16,$h2,$h1,#40 + + str w12,[$ctx,#16*0] // r0 + add w12,w13,w13,lsl#2 // r1*5 + str w13,[$ctx,#16*1] // r1 + add w13,w14,w14,lsl#2 // r2*5 + str w12,[$ctx,#16*2] // s1 + str w14,[$ctx,#16*3] // r2 + add w14,w15,w15,lsl#2 // r3*5 + str w13,[$ctx,#16*4] // s2 + str w15,[$ctx,#16*5] // r3 + add w15,w16,w16,lsl#2 // r4*5 + str w14,[$ctx,#16*6] // s3 + str w16,[$ctx,#16*7] // r4 + str w15,[$ctx,#16*8] // s4 + + ret +.size __poly1305_splat,.-__poly1305_splat + +#if !defined(__KERNEL__) || defined(CONFIG_KERNEL_MODE_NEON) +#ifdef __KERNEL__ +.globl poly1305_blocks_neon +.globl poly1305_emit_neon +#endif + +.type poly1305_blocks_neon,%function +.align 5 +poly1305_blocks_neon: + ldr $is_base2_26,[$ctx,#24] + cmp $len,#128 + b.hs .Lblocks_neon + cbz $is_base2_26,poly1305_blocks + +.Lblocks_neon: + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + + ands $len,$len,#-16 + b.eq .Lno_data_neon + + cbz $is_base2_26,.Lbase2_64_neon + + ldp w10,w11,[$ctx] // load hash value base 2^26 + ldp w12,w13,[$ctx,#8] + ldr w14,[$ctx,#16] + + tst $len,#31 + b.eq .Leven_neon + + ldp $r0,$r1,[$ctx,#32] // load key value + + add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64 + lsr $h1,x12,#12 + adds $h0,$h0,x12,lsl#52 + add $h1,$h1,x13,lsl#14 + adc $h1,$h1,xzr + lsr $h2,x14,#24 + adds $h1,$h1,x14,lsl#40 + adc $d2,$h2,xzr // can be partially reduced... + + ldp $d0,$d1,[$inp],#16 // load input + sub $len,$len,#16 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + + and $t0,$d2,#-4 // ... so reduce + and $h2,$d2,#3 + add $t0,$t0,$d2,lsr#2 + adds $h0,$h0,$t0 + adcs $h1,$h1,xzr + adc $h2,$h2,xzr + +#ifdef __AARCH64EB__ + rev $d0,$d0 + rev $d1,$d1 +#endif + adds $h0,$h0,$d0 // accumulate input + adcs $h1,$h1,$d1 + adc $h2,$h2,$padbit + + bl __poly1305_mult + ldr x30,[sp,#8] + + cbz $padbit,.Lstore_base2_64_neon + + and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x11,$h0,#26,#26 + extr x12,$h1,$h0,#52 + and x12,x12,#0x03ffffff + ubfx x13,$h1,#14,#26 + extr x14,$h2,$h1,#40 + + cbnz $len,.Leven_neon + + stp w10,w11,[$ctx] // store hash value base 2^26 + stp w12,w13,[$ctx,#8] + str w14,[$ctx,#16] + b .Lno_data_neon + +.align 4 +.Lstore_base2_64_neon: + stp $h0,$h1,[$ctx] // store hash value base 2^64 + stp $h2,xzr,[$ctx,#16] // note that is_base2_26 is zeroed + b .Lno_data_neon + +.align 4 +.Lbase2_64_neon: + ldp $r0,$r1,[$ctx,#32] // load key value + + ldp $h0,$h1,[$ctx] // load hash value base 2^64 + ldr $h2,[$ctx,#16] + + tst $len,#31 + b.eq .Linit_neon + + ldp $d0,$d1,[$inp],#16 // load input + sub $len,$len,#16 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) +#ifdef __AARCH64EB__ + rev $d0,$d0 + rev $d1,$d1 +#endif + adds $h0,$h0,$d0 // accumulate input + adcs $h1,$h1,$d1 + adc $h2,$h2,$padbit + + bl __poly1305_mult + +.Linit_neon: + and x10,$h0,#0x03ffffff // base 2^64 -> base 2^26 + ubfx x11,$h0,#26,#26 + extr x12,$h1,$h0,#52 + and x12,x12,#0x03ffffff + ubfx x13,$h1,#14,#26 + extr x14,$h2,$h1,#40 + + stp d8,d9,[sp,#16] // meet ABI requirements + stp d10,d11,[sp,#32] + stp d12,d13,[sp,#48] + stp d14,d15,[sp,#64] + + fmov ${H0},x10 + fmov ${H1},x11 + fmov ${H2},x12 + fmov ${H3},x13 + fmov ${H4},x14 + + ////////////////////////////////// initialize r^n table + mov $h0,$r0 // r^1 + add $s1,$r1,$r1,lsr#2 // s1 = r1 + (r1 >> 2) + mov $h1,$r1 + mov $h2,xzr + add $ctx,$ctx,#48+12 + bl __poly1305_splat + + bl __poly1305_mult // r^2 + sub $ctx,$ctx,#4 + bl __poly1305_splat + + bl __poly1305_mult // r^3 + sub $ctx,$ctx,#4 + bl __poly1305_splat + + bl __poly1305_mult // r^4 + sub $ctx,$ctx,#4 + bl __poly1305_splat + ldr x30,[sp,#8] + + add $in2,$inp,#32 + adr $zeros,.Lzeros + subs $len,$len,#64 + csel $in2,$zeros,$in2,lo + + mov x4,#1 + str x4,[$ctx,#-24] // set is_base2_26 + sub $ctx,$ctx,#48 // restore original $ctx + b .Ldo_neon + +.align 4 +.Leven_neon: + add $in2,$inp,#32 + adr $zeros,.Lzeros + subs $len,$len,#64 + csel $in2,$zeros,$in2,lo + + stp d8,d9,[sp,#16] // meet ABI requirements + stp d10,d11,[sp,#32] + stp d12,d13,[sp,#48] + stp d14,d15,[sp,#64] + + fmov ${H0},x10 + fmov ${H1},x11 + fmov ${H2},x12 + fmov ${H3},x13 + fmov ${H4},x14 + +.Ldo_neon: + ldp x8,x12,[$in2],#16 // inp[2:3] (or zero) + ldp x9,x13,[$in2],#48 + + lsl $padbit,$padbit,#24 + add x15,$ctx,#48 + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + and x5,x9,#0x03ffffff + ubfx x6,x8,#26,#26 + ubfx x7,x9,#26,#26 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + extr x8,x12,x8,#52 + extr x9,x13,x9,#52 + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + fmov $IN23_0,x4 + and x8,x8,#0x03ffffff + and x9,x9,#0x03ffffff + ubfx x10,x12,#14,#26 + ubfx x11,x13,#14,#26 + add x12,$padbit,x12,lsr#40 + add x13,$padbit,x13,lsr#40 + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + fmov $IN23_1,x6 + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + fmov $IN23_2,x8 + fmov $IN23_3,x10 + fmov $IN23_4,x12 + + ldp x8,x12,[$inp],#16 // inp[0:1] + ldp x9,x13,[$inp],#48 + + ld1 {$R0,$R1,$S1,$R2},[x15],#64 + ld1 {$S2,$R3,$S3,$R4},[x15],#64 + ld1 {$S4},[x15] + +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + and x5,x9,#0x03ffffff + ubfx x6,x8,#26,#26 + ubfx x7,x9,#26,#26 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + extr x8,x12,x8,#52 + extr x9,x13,x9,#52 + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + fmov $IN01_0,x4 + and x8,x8,#0x03ffffff + and x9,x9,#0x03ffffff + ubfx x10,x12,#14,#26 + ubfx x11,x13,#14,#26 + add x12,$padbit,x12,lsr#40 + add x13,$padbit,x13,lsr#40 + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + fmov $IN01_1,x6 + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + movi $MASK.2d,#-1 + fmov $IN01_2,x8 + fmov $IN01_3,x10 + fmov $IN01_4,x12 + ushr $MASK.2d,$MASK.2d,#38 + + b.ls .Lskip_loop + +.align 4 +.Loop_neon: + //////////////////////////////////////////////////////////////// + // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + // \___________________/ + // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + // \___________________/ \____________________/ + // + // Note that we start with inp[2:3]*r^2. This is because it + // doesn't depend on reduction in previous iteration. + //////////////////////////////////////////////////////////////// + // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0 + // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4 + // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3 + // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2 + // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1 + + subs $len,$len,#64 + umull $ACC4,$IN23_0,${R4}[2] + csel $in2,$zeros,$in2,lo + umull $ACC3,$IN23_0,${R3}[2] + umull $ACC2,$IN23_0,${R2}[2] + ldp x8,x12,[$in2],#16 // inp[2:3] (or zero) + umull $ACC1,$IN23_0,${R1}[2] + ldp x9,x13,[$in2],#48 + umull $ACC0,$IN23_0,${R0}[2] +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + + umlal $ACC4,$IN23_1,${R3}[2] + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + umlal $ACC3,$IN23_1,${R2}[2] + and x5,x9,#0x03ffffff + umlal $ACC2,$IN23_1,${R1}[2] + ubfx x6,x8,#26,#26 + umlal $ACC1,$IN23_1,${R0}[2] + ubfx x7,x9,#26,#26 + umlal $ACC0,$IN23_1,${S4}[2] + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + + umlal $ACC4,$IN23_2,${R2}[2] + extr x8,x12,x8,#52 + umlal $ACC3,$IN23_2,${R1}[2] + extr x9,x13,x9,#52 + umlal $ACC2,$IN23_2,${R0}[2] + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + umlal $ACC1,$IN23_2,${S4}[2] + fmov $IN23_0,x4 + umlal $ACC0,$IN23_2,${S3}[2] + and x8,x8,#0x03ffffff + + umlal $ACC4,$IN23_3,${R1}[2] + and x9,x9,#0x03ffffff + umlal $ACC3,$IN23_3,${R0}[2] + ubfx x10,x12,#14,#26 + umlal $ACC2,$IN23_3,${S4}[2] + ubfx x11,x13,#14,#26 + umlal $ACC1,$IN23_3,${S3}[2] + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + umlal $ACC0,$IN23_3,${S2}[2] + fmov $IN23_1,x6 + + add $IN01_2,$IN01_2,$H2 + add x12,$padbit,x12,lsr#40 + umlal $ACC4,$IN23_4,${R0}[2] + add x13,$padbit,x13,lsr#40 + umlal $ACC3,$IN23_4,${S4}[2] + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + umlal $ACC2,$IN23_4,${S3}[2] + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + umlal $ACC1,$IN23_4,${S2}[2] + fmov $IN23_2,x8 + umlal $ACC0,$IN23_4,${S1}[2] + fmov $IN23_3,x10 + + //////////////////////////////////////////////////////////////// + // (hash+inp[0:1])*r^4 and accumulate + + add $IN01_0,$IN01_0,$H0 + fmov $IN23_4,x12 + umlal $ACC3,$IN01_2,${R1}[0] + ldp x8,x12,[$inp],#16 // inp[0:1] + umlal $ACC0,$IN01_2,${S3}[0] + ldp x9,x13,[$inp],#48 + umlal $ACC4,$IN01_2,${R2}[0] + umlal $ACC1,$IN01_2,${S4}[0] + umlal $ACC2,$IN01_2,${R0}[0] +#ifdef __AARCH64EB__ + rev x8,x8 + rev x12,x12 + rev x9,x9 + rev x13,x13 +#endif + + add $IN01_1,$IN01_1,$H1 + umlal $ACC3,$IN01_0,${R3}[0] + umlal $ACC4,$IN01_0,${R4}[0] + and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 + umlal $ACC2,$IN01_0,${R2}[0] + and x5,x9,#0x03ffffff + umlal $ACC0,$IN01_0,${R0}[0] + ubfx x6,x8,#26,#26 + umlal $ACC1,$IN01_0,${R1}[0] + ubfx x7,x9,#26,#26 + + add $IN01_3,$IN01_3,$H3 + add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 + umlal $ACC3,$IN01_1,${R2}[0] + extr x8,x12,x8,#52 + umlal $ACC4,$IN01_1,${R3}[0] + extr x9,x13,x9,#52 + umlal $ACC0,$IN01_1,${S4}[0] + add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 + umlal $ACC2,$IN01_1,${R1}[0] + fmov $IN01_0,x4 + umlal $ACC1,$IN01_1,${R0}[0] + and x8,x8,#0x03ffffff + + add $IN01_4,$IN01_4,$H4 + and x9,x9,#0x03ffffff + umlal $ACC3,$IN01_3,${R0}[0] + ubfx x10,x12,#14,#26 + umlal $ACC0,$IN01_3,${S2}[0] + ubfx x11,x13,#14,#26 + umlal $ACC4,$IN01_3,${R1}[0] + add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 + umlal $ACC1,$IN01_3,${S3}[0] + fmov $IN01_1,x6 + umlal $ACC2,$IN01_3,${S4}[0] + add x12,$padbit,x12,lsr#40 + + umlal $ACC3,$IN01_4,${S4}[0] + add x13,$padbit,x13,lsr#40 + umlal $ACC0,$IN01_4,${S1}[0] + add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 + umlal $ACC4,$IN01_4,${R0}[0] + add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 + umlal $ACC1,$IN01_4,${S2}[0] + fmov $IN01_2,x8 + umlal $ACC2,$IN01_4,${S3}[0] + fmov $IN01_3,x10 + fmov $IN01_4,x12 + + ///////////////////////////////////////////////////////////////// + // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + // and P. Schwabe + // + // [see discussion in poly1305-armv4 module] + + ushr $T0.2d,$ACC3,#26 + xtn $H3,$ACC3 + ushr $T1.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + add $ACC4,$ACC4,$T0.2d // h3 -> h4 + bic $H3,#0xfc,lsl#24 // &=0x03ffffff + add $ACC1,$ACC1,$T1.2d // h0 -> h1 + + ushr $T0.2d,$ACC4,#26 + xtn $H4,$ACC4 + ushr $T1.2d,$ACC1,#26 + xtn $H1,$ACC1 + bic $H4,#0xfc,lsl#24 + add $ACC2,$ACC2,$T1.2d // h1 -> h2 + + add $ACC0,$ACC0,$T0.2d + shl $T0.2d,$T0.2d,#2 + shrn $T1.2s,$ACC2,#26 + xtn $H2,$ACC2 + add $ACC0,$ACC0,$T0.2d // h4 -> h0 + bic $H1,#0xfc,lsl#24 + add $H3,$H3,$T1.2s // h2 -> h3 + bic $H2,#0xfc,lsl#24 + + shrn $T0.2s,$ACC0,#26 + xtn $H0,$ACC0 + ushr $T1.2s,$H3,#26 + bic $H3,#0xfc,lsl#24 + bic $H0,#0xfc,lsl#24 + add $H1,$H1,$T0.2s // h0 -> h1 + add $H4,$H4,$T1.2s // h3 -> h4 + + b.hi .Loop_neon + +.Lskip_loop: + dup $IN23_2,${IN23_2}[0] + add $IN01_2,$IN01_2,$H2 + + //////////////////////////////////////////////////////////////// + // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + adds $len,$len,#32 + b.ne .Long_tail + + dup $IN23_2,${IN01_2}[0] + add $IN23_0,$IN01_0,$H0 + add $IN23_3,$IN01_3,$H3 + add $IN23_1,$IN01_1,$H1 + add $IN23_4,$IN01_4,$H4 + +.Long_tail: + dup $IN23_0,${IN23_0}[0] + umull2 $ACC0,$IN23_2,${S3} + umull2 $ACC3,$IN23_2,${R1} + umull2 $ACC4,$IN23_2,${R2} + umull2 $ACC2,$IN23_2,${R0} + umull2 $ACC1,$IN23_2,${S4} + + dup $IN23_1,${IN23_1}[0] + umlal2 $ACC0,$IN23_0,${R0} + umlal2 $ACC2,$IN23_0,${R2} + umlal2 $ACC3,$IN23_0,${R3} + umlal2 $ACC4,$IN23_0,${R4} + umlal2 $ACC1,$IN23_0,${R1} + + dup $IN23_3,${IN23_3}[0] + umlal2 $ACC0,$IN23_1,${S4} + umlal2 $ACC3,$IN23_1,${R2} + umlal2 $ACC2,$IN23_1,${R1} + umlal2 $ACC4,$IN23_1,${R3} + umlal2 $ACC1,$IN23_1,${R0} + + dup $IN23_4,${IN23_4}[0] + umlal2 $ACC3,$IN23_3,${R0} + umlal2 $ACC4,$IN23_3,${R1} + umlal2 $ACC0,$IN23_3,${S2} + umlal2 $ACC1,$IN23_3,${S3} + umlal2 $ACC2,$IN23_3,${S4} + + umlal2 $ACC3,$IN23_4,${S4} + umlal2 $ACC0,$IN23_4,${S1} + umlal2 $ACC4,$IN23_4,${R0} + umlal2 $ACC1,$IN23_4,${S2} + umlal2 $ACC2,$IN23_4,${S3} + + b.eq .Lshort_tail + + //////////////////////////////////////////////////////////////// + // (hash+inp[0:1])*r^4:r^3 and accumulate + + add $IN01_0,$IN01_0,$H0 + umlal $ACC3,$IN01_2,${R1} + umlal $ACC0,$IN01_2,${S3} + umlal $ACC4,$IN01_2,${R2} + umlal $ACC1,$IN01_2,${S4} + umlal $ACC2,$IN01_2,${R0} + + add $IN01_1,$IN01_1,$H1 + umlal $ACC3,$IN01_0,${R3} + umlal $ACC0,$IN01_0,${R0} + umlal $ACC4,$IN01_0,${R4} + umlal $ACC1,$IN01_0,${R1} + umlal $ACC2,$IN01_0,${R2} + + add $IN01_3,$IN01_3,$H3 + umlal $ACC3,$IN01_1,${R2} + umlal $ACC0,$IN01_1,${S4} + umlal $ACC4,$IN01_1,${R3} + umlal $ACC1,$IN01_1,${R0} + umlal $ACC2,$IN01_1,${R1} + + add $IN01_4,$IN01_4,$H4 + umlal $ACC3,$IN01_3,${R0} + umlal $ACC0,$IN01_3,${S2} + umlal $ACC4,$IN01_3,${R1} + umlal $ACC1,$IN01_3,${S3} + umlal $ACC2,$IN01_3,${S4} + + umlal $ACC3,$IN01_4,${S4} + umlal $ACC0,$IN01_4,${S1} + umlal $ACC4,$IN01_4,${R0} + umlal $ACC1,$IN01_4,${S2} + umlal $ACC2,$IN01_4,${S3} + +.Lshort_tail: + //////////////////////////////////////////////////////////////// + // horizontal add + + addp $ACC3,$ACC3,$ACC3 + ldp d8,d9,[sp,#16] // meet ABI requirements + addp $ACC0,$ACC0,$ACC0 + ldp d10,d11,[sp,#32] + addp $ACC4,$ACC4,$ACC4 + ldp d12,d13,[sp,#48] + addp $ACC1,$ACC1,$ACC1 + ldp d14,d15,[sp,#64] + addp $ACC2,$ACC2,$ACC2 + + //////////////////////////////////////////////////////////////// + // lazy reduction, but without narrowing + + ushr $T0.2d,$ACC3,#26 + and $ACC3,$ACC3,$MASK.2d + ushr $T1.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + + add $ACC4,$ACC4,$T0.2d // h3 -> h4 + add $ACC1,$ACC1,$T1.2d // h0 -> h1 + + ushr $T0.2d,$ACC4,#26 + and $ACC4,$ACC4,$MASK.2d + ushr $T1.2d,$ACC1,#26 + and $ACC1,$ACC1,$MASK.2d + add $ACC2,$ACC2,$T1.2d // h1 -> h2 + + add $ACC0,$ACC0,$T0.2d + shl $T0.2d,$T0.2d,#2 + ushr $T1.2d,$ACC2,#26 + and $ACC2,$ACC2,$MASK.2d + add $ACC0,$ACC0,$T0.2d // h4 -> h0 + add $ACC3,$ACC3,$T1.2d // h2 -> h3 + + ushr $T0.2d,$ACC0,#26 + and $ACC0,$ACC0,$MASK.2d + ushr $T1.2d,$ACC3,#26 + and $ACC3,$ACC3,$MASK.2d + add $ACC1,$ACC1,$T0.2d // h0 -> h1 + add $ACC4,$ACC4,$T1.2d // h3 -> h4 + + //////////////////////////////////////////////////////////////// + // write the result, can be partially reduced + + st4 {$ACC0,$ACC1,$ACC2,$ACC3}[0],[$ctx],#16 + st1 {$ACC4}[0],[$ctx] + +.Lno_data_neon: + ldr x29,[sp],#80 + ret +.size poly1305_blocks_neon,.-poly1305_blocks_neon + +.type poly1305_emit_neon,%function +.align 5 +poly1305_emit_neon: + ldr $is_base2_26,[$ctx,#24] + cbz $is_base2_26,poly1305_emit + + ldp w10,w11,[$ctx] // load hash value base 2^26 + ldp w12,w13,[$ctx,#8] + ldr w14,[$ctx,#16] + + add $h0,x10,x11,lsl#26 // base 2^26 -> base 2^64 + lsr $h1,x12,#12 + adds $h0,$h0,x12,lsl#52 + add $h1,$h1,x13,lsl#14 + adc $h1,$h1,xzr + lsr $h2,x14,#24 + adds $h1,$h1,x14,lsl#40 + adc $h2,$h2,xzr // can be partially reduced... + + ldp $t0,$t1,[$nonce] // load nonce + + and $d0,$h2,#-4 // ... so reduce + add $d0,$d0,$h2,lsr#2 + and $h2,$h2,#3 + adds $h0,$h0,$d0 + adcs $h1,$h1,xzr + adc $h2,$h2,xzr + + adds $d0,$h0,#5 // compare to modulus + adcs $d1,$h1,xzr + adc $d2,$h2,xzr + + tst $d2,#-4 // see if it's carried/borrowed + + csel $h0,$h0,$d0,eq + csel $h1,$h1,$d1,eq + +#ifdef __AARCH64EB__ + ror $t0,$t0,#32 // flip nonce words + ror $t1,$t1,#32 +#endif + adds $h0,$h0,$t0 // accumulate nonce + adc $h1,$h1,$t1 +#ifdef __AARCH64EB__ + rev $h0,$h0 // flip output bytes + rev $h1,$h1 +#endif + stp $h0,$h1,[$mac] // write result + + ret +.size poly1305_emit_neon,.-poly1305_emit_neon +#endif + +.align 5 +.Lzeros: +.long 0,0,0,0,0,0,0,0 +#ifndef __KERNEL__ +.LOPENSSL_armcap_P: +#ifdef __ILP32__ +.long OPENSSL_armcap_P-. +#else +.quad OPENSSL_armcap_P-. +#endif +#endif +.align 2 +___ + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split("\n",$code)) { + s/\b(shrn\s+v[0-9]+)\.[24]d/$1.2s/ or + s/\b(fmov\s+)v([0-9]+)[^,]*,\s*x([0-9]+)/$1d$2,x$3/ or + (m/\bdup\b/ and (s/\.[24]s/.2d/g or 1)) or + (m/\b(eor|and)/ and (s/\.[248][sdh]/.16b/g or 1)) or + (m/\bum(ul|la)l\b/ and (s/\.4s/.2s/g or 1)) or + (m/\bum(ul|la)l2\b/ and (s/\.2s/.4s/g or 1)) or + (m/\bst[1-4]\s+{[^}]+}\[/ and (s/\.[24]d/.s/g or 1)); + + s/\.[124]([sd])\[/.$1\[/; + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c b/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c new file mode 100644 index 0000000000000..527ccc3b59cc8 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-donna32.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +struct poly1305_internal { + u32 h[5]; + u32 r[5]; + u32 s[4]; +}; + +static void poly1305_init_generic(void *ctx, const u8 key[16]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + st->r[0] = (get_unaligned_le32(&key[0])) & 0x3ffffff; + st->r[1] = (get_unaligned_le32(&key[3]) >> 2) & 0x3ffff03; + st->r[2] = (get_unaligned_le32(&key[6]) >> 4) & 0x3ffc0ff; + st->r[3] = (get_unaligned_le32(&key[9]) >> 6) & 0x3f03fff; + st->r[4] = (get_unaligned_le32(&key[12]) >> 8) & 0x00fffff; + + /* s = 5*r */ + st->s[0] = st->r[1] * 5; + st->s[1] = st->r[2] * 5; + st->s[2] = st->r[3] * 5; + st->s[3] = st->r[4] * 5; + + /* h = 0 */ + st->h[0] = 0; + st->h[1] = 0; + st->h[2] = 0; + st->h[3] = 0; + st->h[4] = 0; +} + +static void poly1305_blocks_generic(void *ctx, const u8 *input, size_t len, + const u32 padbit) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + const u32 hibit = padbit << 24; + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + u32 c; + + r0 = st->r[0]; + r1 = st->r[1]; + r2 = st->r[2]; + r3 = st->r[3]; + r4 = st->r[4]; + + s1 = st->s[0]; + s2 = st->s[1]; + s3 = st->s[2]; + s4 = st->s[3]; + + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + h3 = st->h[3]; + h4 = st->h[4]; + + while (len >= POLY1305_BLOCK_SIZE) { + /* h += m[i] */ + h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff; + h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff; + h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff; + h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff; + h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit; + + /* h *= r */ + d0 = ((u64)h0 * r0) + ((u64)h1 * s4) + + ((u64)h2 * s3) + ((u64)h3 * s2) + + ((u64)h4 * s1); + d1 = ((u64)h0 * r1) + ((u64)h1 * r0) + + ((u64)h2 * s4) + ((u64)h3 * s3) + + ((u64)h4 * s2); + d2 = ((u64)h0 * r2) + ((u64)h1 * r1) + + ((u64)h2 * r0) + ((u64)h3 * s4) + + ((u64)h4 * s3); + d3 = ((u64)h0 * r3) + ((u64)h1 * r2) + + ((u64)h2 * r1) + ((u64)h3 * r0) + + ((u64)h4 * s4); + d4 = ((u64)h0 * r4) + ((u64)h1 * r3) + + ((u64)h2 * r2) + ((u64)h3 * r1) + + ((u64)h4 * r0); + + /* (partial) h %= p */ + c = (u32)(d0 >> 26); + h0 = (u32)d0 & 0x3ffffff; + d1 += c; + c = (u32)(d1 >> 26); + h1 = (u32)d1 & 0x3ffffff; + d2 += c; + c = (u32)(d2 >> 26); + h2 = (u32)d2 & 0x3ffffff; + d3 += c; + c = (u32)(d3 >> 26); + h3 = (u32)d3 & 0x3ffffff; + d4 += c; + c = (u32)(d4 >> 26); + h4 = (u32)d4 & 0x3ffffff; + h0 += c * 5; + c = (h0 >> 26); + h0 = h0 & 0x3ffffff; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + } + + st->h[0] = h0; + st->h[1] = h1; + st->h[2] = h2; + st->h[3] = h3; + st->h[4] = h4; +} + +static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u32 h0, h1, h2, h3, h4, c; + u32 g0, g1, g2, g3, g4; + u64 f; + u32 mask; + + /* fully carry h */ + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + h3 = st->h[3]; + h4 = st->h[4]; + + c = h1 >> 26; + h1 = h1 & 0x3ffffff; + h2 += c; + c = h2 >> 26; + h2 = h2 & 0x3ffffff; + h3 += c; + c = h3 >> 26; + h3 = h3 & 0x3ffffff; + h4 += c; + c = h4 >> 26; + h4 = h4 & 0x3ffffff; + h0 += c * 5; + c = h0 >> 26; + h0 = h0 & 0x3ffffff; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 26; + g0 &= 0x3ffffff; + g1 = h1 + c; + c = g1 >> 26; + g1 &= 0x3ffffff; + g2 = h2 + c; + c = g2 >> 26; + g2 &= 0x3ffffff; + g3 = h3 + c; + c = g3 >> 26; + g3 &= 0x3ffffff; + g4 = h4 + c - (1UL << 26); + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + h0 = ((h0) | (h1 << 26)) & 0xffffffff; + h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff; + h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff; + h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff; + + /* mac = (h + nonce) % (2^128) */ + f = (u64)h0 + nonce[0]; + h0 = (u32)f; + f = (u64)h1 + nonce[1] + (f >> 32); + h1 = (u32)f; + f = (u64)h2 + nonce[2] + (f >> 32); + h2 = (u32)f; + f = (u64)h3 + nonce[3] + (f >> 32); + h3 = (u32)f; + + put_unaligned_le32(h0, &mac[0]); + put_unaligned_le32(h1, &mac[4]); + put_unaligned_le32(h2, &mac[8]); + put_unaligned_le32(h3, &mac[12]); +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c b/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c new file mode 100644 index 0000000000000..131f1dda1b1db --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-donna64.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +typedef __uint128_t u128; + +struct poly1305_internal { + u64 r[3]; + u64 h[3]; + u64 s[2]; +}; + +static void poly1305_init_generic(void *ctx, const u8 key[16]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u64 t0, t1; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + t0 = get_unaligned_le64(&key[0]); + t1 = get_unaligned_le64(&key[8]); + + st->r[0] = t0 & 0xffc0fffffffULL; + st->r[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; + st->r[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; + + /* s = 20*r */ + st->s[0] = st->r[1] * 20; + st->s[1] = st->r[2] * 20; + + /* h = 0 */ + st->h[0] = 0; + st->h[1] = 0; + st->h[2] = 0; +} + +static void poly1305_blocks_generic(void *ctx, const u8 *input, size_t len, + const u32 padbit) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + const u64 hibit = ((u64)padbit) << 40; + u64 r0, r1, r2; + u64 s1, s2; + u64 h0, h1, h2; + u64 c; + u128 d0, d1, d2, d; + + r0 = st->r[0]; + r1 = st->r[1]; + r2 = st->r[2]; + + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + + s1 = st->s[0]; + s2 = st->s[1]; + + while (len >= POLY1305_BLOCK_SIZE) { + u64 t0, t1; + + /* h += m[i] */ + t0 = get_unaligned_le64(&input[0]); + t1 = get_unaligned_le64(&input[8]); + + h0 += t0 & 0xfffffffffffULL; + h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit; + + /* h *= r */ + d0 = (u128)h0 * r0; + d = (u128)h1 * s2; + d0 += d; + d = (u128)h2 * s1; + d0 += d; + d1 = (u128)h0 * r1; + d = (u128)h1 * r0; + d1 += d; + d = (u128)h2 * s2; + d1 += d; + d2 = (u128)h0 * r2; + d = (u128)h1 * r1; + d2 += d; + d = (u128)h2 * r0; + d2 += d; + + /* (partial) h %= p */ + c = (u64)(d0 >> 44); + h0 = (u64)d0 & 0xfffffffffffULL; + d1 += c; + c = (u64)(d1 >> 44); + h1 = (u64)d1 & 0xfffffffffffULL; + d2 += c; + c = (u64)(d2 >> 42); + h2 = (u64)d2 & 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 = h0 & 0xfffffffffffULL; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + } + + st->h[0] = h0; + st->h[1] = h1; + st->h[2] = h2; +} + +static void poly1305_emit_generic(void *ctx, u8 mac[16], const u32 nonce[4]) +{ + struct poly1305_internal *st = (struct poly1305_internal *)ctx; + u64 h0, h1, h2, c; + u64 g0, g1, g2; + u64 t0, t1; + + /* fully carry h */ + h0 = st->h[0]; + h1 = st->h[1]; + h2 = st->h[2]; + + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 44; + g0 &= 0xfffffffffffULL; + g1 = h1 + c; + c = g1 >> 44; + g1 &= 0xfffffffffffULL; + g2 = h2 + c - (1ULL << 42); + + /* select h if h < p, or h + -p if h >= p */ + c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; + g0 &= c; + g1 &= c; + g2 &= c; + c = ~c; + h0 = (h0 & c) | g0; + h1 = (h1 & c) | g1; + h2 = (h2 & c) | g2; + + /* h = (h + nonce) */ + t0 = ((u64)nonce[1] << 32) | nonce[0]; + t1 = ((u64)nonce[3] << 32) | nonce[2]; + + h0 += t0 & 0xfffffffffffULL; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; + h2 &= 0x3ffffffffffULL; + + /* mac = h % (2^128) */ + h0 = h0 | (h1 << 44); + h1 = (h1 >> 20) | (h2 << 24); + + put_unaligned_le64(h0, &mac[0]); + put_unaligned_le64(h1, &mac[8]); +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c new file mode 100644 index 0000000000000..a540e9c4eee8b --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips-glue.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +asmlinkage void poly1305_init_mips(void *ctx, const u8 key[16]); +asmlinkage void poly1305_blocks_mips(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_emit_mips(void *ctx, u8 mac[16], const u32 nonce[4]); + +static bool *const poly1305_nobs[] __initconst = { }; +static void __init poly1305_fpu_init(void) +{ +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_mips(ctx, key); + return true; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + poly1305_blocks_mips(ctx, inp, len, padbit); + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + poly1305_emit_mips(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S b/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S new file mode 100644 index 0000000000000..4291c156815b3 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips.S @@ -0,0 +1,407 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2016-2018 René van Dorst All Rights Reserved. + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define MSB 0 +#define LSB 3 +#else +#define MSB 3 +#define LSB 0 +#endif + +#define POLY1305_BLOCK_SIZE 16 +.text +#define H0 $t0 +#define H1 $t1 +#define H2 $t2 +#define H3 $t3 +#define H4 $t4 + +#define R0 $t5 +#define R1 $t6 +#define R2 $t7 +#define R3 $t8 + +#define O0 $s0 +#define O1 $s4 +#define O2 $v1 +#define O3 $t9 +#define O4 $s5 + +#define S1 $s1 +#define S2 $s2 +#define S3 $s3 + +#define SC $at +#define CA $v0 + +/* Input arguments */ +#define poly $a0 +#define src $a1 +#define srclen $a2 +#define hibit $a3 + +/* Location in the opaque buffer + * R[0..3], CA, H[0..4] + */ +#define PTR_POLY1305_R(n) ( 0 + (n*4)) ## ($a0) +#define PTR_POLY1305_CA (16 ) ## ($a0) +#define PTR_POLY1305_H(n) (20 + (n*4)) ## ($a0) + +#define POLY1305_BLOCK_SIZE 16 +#define POLY1305_STACK_SIZE 32 + +.set noat +.align 4 +.globl poly1305_blocks_mips +.ent poly1305_blocks_mips +poly1305_blocks_mips: + .frame $sp, POLY1305_STACK_SIZE, $ra + /* srclen &= 0xFFFFFFF0 */ + ins srclen, $zero, 0, 4 + + addiu $sp, -(POLY1305_STACK_SIZE) + + /* check srclen >= 16 bytes */ + beqz srclen, .Lpoly1305_blocks_mips_end + + /* Calculate last round based on src address pointer. + * last round src ptr (srclen) = src + (srclen & 0xFFFFFFF0) + */ + addu srclen, src + + lw R0, PTR_POLY1305_R(0) + lw R1, PTR_POLY1305_R(1) + lw R2, PTR_POLY1305_R(2) + lw R3, PTR_POLY1305_R(3) + + /* store the used save registers. */ + sw $s0, 0($sp) + sw $s1, 4($sp) + sw $s2, 8($sp) + sw $s3, 12($sp) + sw $s4, 16($sp) + sw $s5, 20($sp) + + /* load Hx and Carry */ + lw CA, PTR_POLY1305_CA + lw H0, PTR_POLY1305_H(0) + lw H1, PTR_POLY1305_H(1) + lw H2, PTR_POLY1305_H(2) + lw H3, PTR_POLY1305_H(3) + lw H4, PTR_POLY1305_H(4) + + /* Sx = Rx + (Rx >> 2) */ + srl S1, R1, 2 + srl S2, R2, 2 + srl S3, R3, 2 + addu S1, R1 + addu S2, R2 + addu S3, R3 + + addiu SC, $zero, 1 + +.Lpoly1305_loop: + lwl O0, 0+MSB(src) + lwl O1, 4+MSB(src) + lwl O2, 8+MSB(src) + lwl O3,12+MSB(src) + lwr O0, 0+LSB(src) + lwr O1, 4+LSB(src) + lwr O2, 8+LSB(src) + lwr O3,12+LSB(src) + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh O0 + wsbh O1 + wsbh O2 + wsbh O3 + rotr O0, 16 + rotr O1, 16 + rotr O2, 16 + rotr O3, 16 +#endif + + /* h0 = (u32)(d0 = (u64)h0 + inp[0] + c 'Carry_previous cycle'); */ + addu H0, CA + sltu CA, H0, CA + addu O0, H0 + sltu H0, O0, H0 + addu CA, H0 + + /* h1 = (u32)(d1 = (u64)h1 + (d0 >> 32) + inp[4]); */ + addu H1, CA + sltu CA, H1, CA + addu O1, H1 + sltu H1, O1, H1 + addu CA, H1 + + /* h2 = (u32)(d2 = (u64)h2 + (d1 >> 32) + inp[8]); */ + addu H2, CA + sltu CA, H2, CA + addu O2, H2 + sltu H2, O2, H2 + addu CA, H2 + + /* h3 = (u32)(d3 = (u64)h3 + (d2 >> 32) + inp[12]); */ + addu H3, CA + sltu CA, H3, CA + addu O3, H3 + sltu H3, O3, H3 + addu CA, H3 + + /* h4 += (u32)(d3 >> 32) + padbit; */ + addu H4, hibit + addu O4, H4, CA + + /* D0 */ + multu O0, R0 + maddu O1, S3 + maddu O2, S2 + maddu O3, S1 + mfhi CA + mflo H0 + + /* D1 */ + multu O0, R1 + maddu O1, R0 + maddu O2, S3 + maddu O3, S2 + maddu O4, S1 + maddu CA, SC + mfhi CA + mflo H1 + + /* D2 */ + multu O0, R2 + maddu O1, R1 + maddu O2, R0 + maddu O3, S3 + maddu O4, S2 + maddu CA, SC + mfhi CA + mflo H2 + + /* D4 */ + mul H4, O4, R0 + + /* D3 */ + multu O0, R3 + maddu O1, R2 + maddu O2, R1 + maddu O3, R0 + maddu O4, S3 + maddu CA, SC + mfhi CA + mflo H3 + + addiu src, POLY1305_BLOCK_SIZE + + /* h4 += (u32)(d3 >> 32); */ + addu O4, H4, CA + /* h4 &= 3 */ + andi H4, O4, 3 + /* c = (h4 >> 2) + (h4 & ~3U); */ + srl CA, O4, 2 + ins O4, $zero, 0, 2 + + addu CA, O4 + + /* able to do a 16 byte block. */ + bne src, srclen, .Lpoly1305_loop + + /* restore the used save registers. */ + lw $s0, 0($sp) + lw $s1, 4($sp) + lw $s2, 8($sp) + lw $s3, 12($sp) + lw $s4, 16($sp) + lw $s5, 20($sp) + + /* store Hx and Carry */ + sw CA, PTR_POLY1305_CA + sw H0, PTR_POLY1305_H(0) + sw H1, PTR_POLY1305_H(1) + sw H2, PTR_POLY1305_H(2) + sw H3, PTR_POLY1305_H(3) + sw H4, PTR_POLY1305_H(4) + +.Lpoly1305_blocks_mips_end: + addiu $sp, POLY1305_STACK_SIZE + + /* Jump Back */ + jr $ra +.end poly1305_blocks_mips +.set at + +/* Input arguments CTX=$a0, MAC=$a1, NONCE=$a2 */ +#define MAC $a1 +#define NONCE $a2 + +#define G0 $t5 +#define G1 $t6 +#define G2 $t7 +#define G3 $t8 +#define G4 $t9 + +.set noat +.align 4 +.globl poly1305_emit_mips +.ent poly1305_emit_mips +poly1305_emit_mips: + /* load Hx and Carry */ + lw CA, PTR_POLY1305_CA + lw H0, PTR_POLY1305_H(0) + lw H1, PTR_POLY1305_H(1) + lw H2, PTR_POLY1305_H(2) + lw H3, PTR_POLY1305_H(3) + lw H4, PTR_POLY1305_H(4) + + /* Add left over carry */ + addu H0, CA + sltu CA, H0, CA + addu H1, CA + sltu CA, H1, CA + addu H2, CA + sltu CA, H2, CA + addu H3, CA + sltu CA, H3, CA + addu H4, CA + + /* compare to modulus by computing h + -p */ + addiu G0, H0, 5 + sltu CA, G0, H0 + addu G1, H1, CA + sltu CA, G1, H1 + addu G2, H2, CA + sltu CA, G2, H2 + addu G3, H3, CA + sltu CA, G3, H3 + addu G4, H4, CA + + srl SC, G4, 2 + + /* if there was carry into 131st bit, h3:h0 = g3:g0 */ + movn H0, G0, SC + movn H1, G1, SC + movn H2, G2, SC + movn H3, G3, SC + + lwl G0, 0+MSB(NONCE) + lwl G1, 4+MSB(NONCE) + lwl G2, 8+MSB(NONCE) + lwl G3,12+MSB(NONCE) + lwr G0, 0+LSB(NONCE) + lwr G1, 4+LSB(NONCE) + lwr G2, 8+LSB(NONCE) + lwr G3,12+LSB(NONCE) + + /* mac = (h + nonce) % (2^128) */ + addu H0, G0 + sltu CA, H0, G0 + + /* H1 */ + addu H1, CA + sltu CA, H1, CA + addu H1, G1 + sltu G1, H1, G1 + addu CA, G1 + + /* H2 */ + addu H2, CA + sltu CA, H2, CA + addu H2, G2 + sltu G2, H2, G2 + addu CA, G2 + + /* H3 */ + addu H3, CA + addu H3, G3 + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh H0 + wsbh H1 + wsbh H2 + wsbh H3 + rotr H0, 16 + rotr H1, 16 + rotr H2, 16 + rotr H3, 16 +#endif + + /* store MAC */ + swl H0, 0+MSB(MAC) + swl H1, 4+MSB(MAC) + swl H2, 8+MSB(MAC) + swl H3,12+MSB(MAC) + swr H0, 0+LSB(MAC) + swr H1, 4+LSB(MAC) + swr H2, 8+LSB(MAC) + swr H3,12+LSB(MAC) + + jr $ra +.end poly1305_emit_mips + +#define PR0 $t0 +#define PR1 $t1 +#define PR2 $t2 +#define PR3 $t3 +#define PT0 $t4 + +/* Input arguments CTX=$a0, KEY=$a1 */ + +.align 4 +.globl poly1305_init_mips +.ent poly1305_init_mips +poly1305_init_mips: + lwl PR0, 0+MSB($a1) + lwl PR1, 4+MSB($a1) + lwl PR2, 8+MSB($a1) + lwl PR3,12+MSB($a1) + lwr PR0, 0+LSB($a1) + lwr PR1, 4+LSB($a1) + lwr PR2, 8+LSB($a1) + lwr PR3,12+LSB($a1) + + /* store Hx and Carry */ + sw $zero, PTR_POLY1305_CA + sw $zero, PTR_POLY1305_H(0) + sw $zero, PTR_POLY1305_H(1) + sw $zero, PTR_POLY1305_H(2) + sw $zero, PTR_POLY1305_H(3) + sw $zero, PTR_POLY1305_H(4) + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + wsbh PR0 + wsbh PR1 + wsbh PR2 + wsbh PR3 + rotr PR0, 16 + rotr PR1, 16 + rotr PR2, 16 + rotr PR3, 16 +#endif + + lui PT0, 0x0FFF + ori PT0, 0xFFFC + + /* AND 0x0fffffff; */ + ext PR0, PR0, 0, (32-4) + + /* AND 0x0ffffffc; */ + and PR1, PT0 + and PR2, PT0 + and PR3, PT0 + + /* store Rx */ + sw PR0, PTR_POLY1305_R(0) + sw PR1, PTR_POLY1305_R(1) + sw PR2, PTR_POLY1305_R(2) + sw PR3, PTR_POLY1305_R(3) + + /* Jump Back */ + jr $ra +.end poly1305_init_mips diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl new file mode 100644 index 0000000000000..4adb9d4007ce6 --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-mips64.pl @@ -0,0 +1,466 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# Poly1305 hash for MIPS64. +# +# May 2016 +# +# Numbers are cycles per processed byte with poly1305_blocks alone. +# +# IALU/gcc +# R1x000 5.64/+120% (big-endian) +# Octeon II 3.80/+280% (little-endian) + +###################################################################### +# There is a number of MIPS ABI in use, O32 and N32/64 are most +# widely used. Then there is a new contender: NUBI. It appears that if +# one picks the latter, it's possible to arrange code in ABI neutral +# manner. Therefore let's stick to NUBI register layout: +# +($zero,$at,$t0,$t1,$t2)=map("\$$_",(0..2,24,25)); +($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7,$s8,$s9,$s10,$s11)=map("\$$_",(12..23)); +($gp,$tp,$sp,$fp,$ra)=map("\$$_",(3,28..31)); +# +# The return value is placed in $a0. Following coding rules facilitate +# interoperability: +# +# - never ever touch $tp, "thread pointer", former $gp [o32 can be +# excluded from the rule, because it's specified volatile]; +# - copy return value to $t0, former $v0 [or to $a0 if you're adapting +# old code]; +# - on O32 populate $a4-$a7 with 'lw $aN,4*N($sp)' if necessary; +# +# For reference here is register layout for N32/64 MIPS ABIs: +# +# ($zero,$at,$v0,$v1)=map("\$$_",(0..3)); +# ($a0,$a1,$a2,$a3,$a4,$a5,$a6,$a7)=map("\$$_",(4..11)); +# ($t0,$t1,$t2,$t3,$t8,$t9)=map("\$$_",(12..15,24,25)); +# ($s0,$s1,$s2,$s3,$s4,$s5,$s6,$s7)=map("\$$_",(16..23)); +# ($gp,$sp,$fp,$ra)=map("\$$_",(28..31)); +# +# +# +###################################################################### + +$flavour = shift || "64"; # supported flavours are o32,n32,64,nubi32,nubi64 + +die "MIPS64 only" unless ($flavour =~ /64|n32/i); + +$v0 = ($flavour =~ /nubi/i) ? $a0 : $t0; +$SAVED_REGS_MASK = ($flavour =~ /nubi/i) ? "0x0003f000" : "0x00030000"; + +($ctx,$inp,$len,$padbit) = ($a0,$a1,$a2,$a3); +($in0,$in1,$tmp0,$tmp1,$tmp2,$tmp3,$tmp4) = ($a4,$a5,$a6,$a7,$at,$t0,$t1); + +$code.=<<___; +#if (defined(_MIPS_ARCH_MIPS64R3) || defined(_MIPS_ARCH_MIPS64R5) || \\ + defined(_MIPS_ARCH_MIPS64R6)) \\ + && !defined(_MIPS_ARCH_MIPS64R2) +# define _MIPS_ARCH_MIPS64R2 +#endif + +#if defined(_MIPS_ARCH_MIPS64R6) +# define dmultu(rs,rt) +# define mflo(rd,rs,rt) dmulu rd,rs,rt +# define mfhi(rd,rs,rt) dmuhu rd,rs,rt +#else +# define dmultu(rs,rt) dmultu rs,rt +# define mflo(rd,rs,rt) mflo rd +# define mfhi(rd,rs,rt) mfhi rd +#endif + +#ifdef __KERNEL__ +# define poly1305_init poly1305_init_mips +# define poly1305_blocks poly1305_blocks_mips +# define poly1305_emit poly1305_emit_mips +#endif + +#if defined(__MIPSEB__) && !defined(MIPSEB) +# define MIPSEB +#endif + +#ifdef MIPSEB +# define MSB 0 +# define LSB 7 +#else +# define MSB 7 +# define LSB 0 +#endif + +.text +.set noat +.set noreorder + +.align 5 +.globl poly1305_init +.ent poly1305_init +poly1305_init: + .frame $sp,0,$ra + .set reorder + + sd $zero,0($ctx) + sd $zero,8($ctx) + sd $zero,16($ctx) + + beqz $inp,.Lno_key + +#if defined(_MIPS_ARCH_MIPS64R6) + ld $in0,0($inp) + ld $in1,8($inp) +#else + ldl $in0,0+MSB($inp) + ldl $in1,8+MSB($inp) + ldr $in0,0+LSB($inp) + ldr $in1,8+LSB($inp) +#endif +#ifdef MIPSEB +# if defined(_MIPS_ARCH_MIPS64R2) + dsbh $in0,$in0 # byte swap + dsbh $in1,$in1 + dshd $in0,$in0 + dshd $in1,$in1 +# else + ori $tmp0,$zero,0xFF + dsll $tmp2,$tmp0,32 + or $tmp0,$tmp2 # 0x000000FF000000FF + + and $tmp1,$in0,$tmp0 # byte swap + and $tmp3,$in1,$tmp0 + dsrl $tmp2,$in0,24 + dsrl $tmp4,$in1,24 + dsll $tmp1,24 + dsll $tmp3,24 + and $tmp2,$tmp0 + and $tmp4,$tmp0 + dsll $tmp0,8 # 0x0000FF000000FF00 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + and $tmp2,$in0,$tmp0 + and $tmp4,$in1,$tmp0 + dsrl $in0,8 + dsrl $in1,8 + dsll $tmp2,8 + dsll $tmp4,8 + and $in0,$tmp0 + and $in1,$tmp0 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + or $in0,$tmp1 + or $in1,$tmp3 + dsrl $tmp1,$in0,32 + dsrl $tmp3,$in1,32 + dsll $in0,32 + dsll $in1,32 + or $in0,$tmp1 + or $in1,$tmp3 +# endif +#endif + li $tmp0,1 + dsll $tmp0,32 + daddiu $tmp0,-63 + dsll $tmp0,28 + daddiu $tmp0,-1 # 0ffffffc0fffffff + + and $in0,$tmp0 + daddiu $tmp0,-3 # 0ffffffc0ffffffc + and $in1,$tmp0 + + sd $in0,24($ctx) + dsrl $tmp0,$in1,2 + sd $in1,32($ctx) + daddu $tmp0,$in1 # s1 = r1 + (r1 >> 2) + sd $tmp0,40($ctx) + +.Lno_key: + li $v0,0 # return 0 + jr $ra +.end poly1305_init +___ +{ +my ($h0,$h1,$h2,$r0,$r1,$s1,$d0,$d1,$d2) = + ($s0,$s1,$s2,$s3,$s4,$s5,$in0,$in1,$t2); + +$code.=<<___; +.align 5 +.globl poly1305_blocks +.ent poly1305_blocks +poly1305_blocks: + .set noreorder + dsrl $len,4 # number of complete blocks + bnez $len,poly1305_blocks_internal + nop + jr $ra + nop +.end poly1305_blocks + +.align 5 +.ent poly1305_blocks_internal +poly1305_blocks_internal: + .frame $sp,6*8,$ra + .mask $SAVED_REGS_MASK,-8 + .set noreorder + dsubu $sp,6*8 + sd $s5,40($sp) + sd $s4,32($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); # optimize non-nubi prologue + sd $s3,24($sp) + sd $s2,16($sp) + sd $s1,8($sp) + sd $s0,0($sp) +___ +$code.=<<___; + .set reorder + + ld $h0,0($ctx) # load hash value + ld $h1,8($ctx) + ld $h2,16($ctx) + + ld $r0,24($ctx) # load key + ld $r1,32($ctx) + ld $s1,40($ctx) + +.Loop: +#if defined(_MIPS_ARCH_MIPS64R6) + ld $in0,0($inp) # load input + ld $in1,8($inp) +#else + ldl $in0,0+MSB($inp) # load input + ldl $in1,8+MSB($inp) + ldr $in0,0+LSB($inp) + ldr $in1,8+LSB($inp) +#endif + daddiu $len,-1 + daddiu $inp,16 +#ifdef MIPSEB +# if defined(_MIPS_ARCH_MIPS64R2) + dsbh $in0,$in0 # byte swap + dsbh $in1,$in1 + dshd $in0,$in0 + dshd $in1,$in1 +# else + ori $tmp0,$zero,0xFF + dsll $tmp2,$tmp0,32 + or $tmp0,$tmp2 # 0x000000FF000000FF + + and $tmp1,$in0,$tmp0 # byte swap + and $tmp3,$in1,$tmp0 + dsrl $tmp2,$in0,24 + dsrl $tmp4,$in1,24 + dsll $tmp1,24 + dsll $tmp3,24 + and $tmp2,$tmp0 + and $tmp4,$tmp0 + dsll $tmp0,8 # 0x0000FF000000FF00 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + and $tmp2,$in0,$tmp0 + and $tmp4,$in1,$tmp0 + dsrl $in0,8 + dsrl $in1,8 + dsll $tmp2,8 + dsll $tmp4,8 + and $in0,$tmp0 + and $in1,$tmp0 + or $tmp1,$tmp2 + or $tmp3,$tmp4 + or $in0,$tmp1 + or $in1,$tmp3 + dsrl $tmp1,$in0,32 + dsrl $tmp3,$in1,32 + dsll $in0,32 + dsll $in1,32 + or $in0,$tmp1 + or $in1,$tmp3 +# endif +#endif + daddu $h0,$in0 # accumulate input + daddu $h1,$in1 + sltu $tmp0,$h0,$in0 + sltu $tmp1,$h1,$in1 + daddu $h1,$tmp0 + + dmultu ($r0,$h0) # h0*r0 + daddu $h2,$padbit + sltu $tmp0,$h1,$tmp0 + mflo ($d0,$r0,$h0) + mfhi ($d1,$r0,$h0) + + dmultu ($s1,$h1) # h1*5*r1 + daddu $tmp0,$tmp1 + daddu $h2,$tmp0 + mflo ($tmp0,$s1,$h1) + mfhi ($tmp1,$s1,$h1) + + dmultu ($r1,$h0) # h0*r1 + daddu $d0,$tmp0 + daddu $d1,$tmp1 + mflo ($tmp2,$r1,$h0) + mfhi ($d2,$r1,$h0) + sltu $tmp0,$d0,$tmp0 + daddu $d1,$tmp0 + + dmultu ($r0,$h1) # h1*r0 + daddu $d1,$tmp2 + sltu $tmp2,$d1,$tmp2 + mflo ($tmp0,$r0,$h1) + mfhi ($tmp1,$r0,$h1) + daddu $d2,$tmp2 + + dmultu ($s1,$h2) # h2*5*r1 + daddu $d1,$tmp0 + daddu $d2,$tmp1 + mflo ($tmp2,$s1,$h2) + + dmultu ($r0,$h2) # h2*r0 + sltu $tmp0,$d1,$tmp0 + daddu $d2,$tmp0 + mflo ($tmp3,$r0,$h2) + + daddu $d1,$tmp2 + daddu $d2,$tmp3 + sltu $tmp2,$d1,$tmp2 + daddu $d2,$tmp2 + + li $tmp0,-4 # final reduction + and $tmp0,$d2 + dsrl $tmp1,$d2,2 + andi $h2,$d2,3 + daddu $tmp0,$tmp1 + daddu $h0,$d0,$tmp0 + sltu $tmp0,$h0,$tmp0 + daddu $h1,$d1,$tmp0 + sltu $tmp0,$h1,$tmp0 + daddu $h2,$h2,$tmp0 + + bnez $len,.Loop + + sd $h0,0($ctx) # store hash value + sd $h1,8($ctx) + sd $h2,16($ctx) + + .set noreorder + ld $s5,40($sp) # epilogue + ld $s4,32($sp) +___ +$code.=<<___ if ($flavour =~ /nubi/i); # optimize non-nubi epilogue + ld $s3,24($sp) + ld $s2,16($sp) + ld $s1,8($sp) + ld $s0,0($sp) +___ +$code.=<<___; + jr $ra + daddu $sp,6*8 +.end poly1305_blocks_internal +___ +} +{ +my ($ctx,$mac,$nonce) = ($a0,$a1,$a2); + +$code.=<<___; +.align 5 +.globl poly1305_emit +.ent poly1305_emit +poly1305_emit: + .frame $sp,0,$ra + .set reorder + + ld $tmp0,0($ctx) + ld $tmp1,8($ctx) + ld $tmp2,16($ctx) + + daddiu $in0,$tmp0,5 # compare to modulus + sltiu $tmp3,$in0,5 + daddu $in1,$tmp1,$tmp3 + sltu $tmp3,$in1,$tmp3 + daddu $tmp2,$tmp2,$tmp3 + + dsrl $tmp2,2 # see if it carried/borrowed + dsubu $tmp2,$zero,$tmp2 + nor $tmp3,$zero,$tmp2 + + and $in0,$tmp2 + and $tmp0,$tmp3 + and $in1,$tmp2 + and $tmp1,$tmp3 + or $in0,$tmp0 + or $in1,$tmp1 + + lwu $tmp0,0($nonce) # load nonce + lwu $tmp1,4($nonce) + lwu $tmp2,8($nonce) + lwu $tmp3,12($nonce) + dsll $tmp1,32 + dsll $tmp3,32 + or $tmp0,$tmp1 + or $tmp2,$tmp3 + + daddu $in0,$tmp0 # accumulate nonce + daddu $in1,$tmp2 + sltu $tmp0,$in0,$tmp0 + daddu $in1,$tmp0 + + dsrl $tmp0,$in0,8 # write mac value + dsrl $tmp1,$in0,16 + dsrl $tmp2,$in0,24 + sb $in0,0($mac) + dsrl $tmp3,$in0,32 + sb $tmp0,1($mac) + dsrl $tmp0,$in0,40 + sb $tmp1,2($mac) + dsrl $tmp1,$in0,48 + sb $tmp2,3($mac) + dsrl $tmp2,$in0,56 + sb $tmp3,4($mac) + dsrl $tmp3,$in1,8 + sb $tmp0,5($mac) + dsrl $tmp0,$in1,16 + sb $tmp1,6($mac) + dsrl $tmp1,$in1,24 + sb $tmp2,7($mac) + + sb $in1,8($mac) + dsrl $tmp2,$in1,32 + sb $tmp3,9($mac) + dsrl $tmp3,$in1,40 + sb $tmp0,10($mac) + dsrl $tmp0,$in1,48 + sb $tmp1,11($mac) + dsrl $tmp1,$in1,56 + sb $tmp2,12($mac) + sb $tmp3,13($mac) + sb $tmp0,14($mac) + sb $tmp1,15($mac) + + jr $ra +.end poly1305_emit +.rdata +.align 2 +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +$output=pop and open STDOUT,">$output"; +print $code; +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c new file mode 100644 index 0000000000000..ce48a42f7654c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64-glue.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include +#include +#include + +asmlinkage void poly1305_init_x86_64(void *ctx, + const u8 key[POLY1305_KEY_SIZE]); +asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); +asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); + +static bool poly1305_use_avx __ro_after_init; +static bool poly1305_use_avx2 __ro_after_init; +static bool poly1305_use_avx512 __ro_after_init; +static bool *const poly1305_nobs[] __initconst = { + &poly1305_use_avx, &poly1305_use_avx2, &poly1305_use_avx512 }; + +static void __init poly1305_fpu_init(void) +{ + poly1305_use_avx = + boot_cpu_has(X86_FEATURE_AVX) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); + poly1305_use_avx2 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL); +#ifndef COMPAT_CANNOT_USE_AVX512 + poly1305_use_avx512 = + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | + XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X; +#endif +} + +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + poly1305_init_x86_64(ctx, key); + return true; +} + +struct poly1305_arch_internal { + union { + struct { + u32 h[5]; + u32 is_base2_26; + }; + u64 hs[3]; + }; + u64 r[2]; + u64 pad; + struct { u32 r2, r1, r4, r3; } rn[9]; +}; + +/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit + * the unfortunate situation of using AVX and then having to go back to scalar + * -- because the user is silly and has called the update function from two + * separate contexts -- then we need to convert back to the original base before + * proceeding. It is possible to reason that the initial reduction below is + * sufficient given the implementation invariants. However, for an avoidance of + * doubt and because this is not performance critical, we do the full reduction + * anyway. + */ +static void convert_to_base2_64(void *ctx) +{ + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->hs[2] = state->h[4] >> 24; +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); + state->hs[2] &= 3; + state->hs[0] += cy; + state->hs[1] += (cy = ULT(state->hs[0], cy)); + state->hs[2] += ULT(state->hs[1], cy); +#undef ULT + state->is_base2_26 = 0; +} + +static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + struct poly1305_arch_internal *state = ctx; + + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx || + (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || + !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_blocks_x86_64(ctx, inp, len, padbit); + return true; + } + + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + if (IS_ENABLED(CONFIG_AS_AVX512) && poly1305_use_avx512) + poly1305_blocks_avx512(ctx, inp, bytes, padbit); + else if (IS_ENABLED(CONFIG_AS_AVX2) && poly1305_use_avx2) + poly1305_blocks_avx2(ctx, inp, bytes, padbit); + else + poly1305_blocks_avx(ctx, inp, bytes, padbit); + len -= bytes; + if (!len) + break; + inp += bytes; + simd_relax(simd_context); + } + + return true; +} + +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + struct poly1305_arch_internal *state = ctx; + + if (!IS_ENABLED(CONFIG_AS_AVX) || !poly1305_use_avx || + !state->is_base2_26 || !simd_use(simd_context)) { + convert_to_base2_64(ctx); + poly1305_emit_x86_64(ctx, mac, nonce); + } else + poly1305_emit_avx(ctx, mac, nonce); + return true; +} diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl new file mode 100644 index 0000000000000..c9a8e0037bdbd --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305-x86_64.pl @@ -0,0 +1,4266 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# Copyright (C) 2017-2018 Samuel Neves . All Rights Reserved. +# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. +# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements Poly1305 hash for x86_64. +# +# March 2015 +# +# Initial release. +# +# December 2016 +# +# Add AVX512F+VL+BW code path. +# +# November 2017 +# +# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be +# executed even on Knights Landing. Trigger for modification was +# observation that AVX512 code paths can negatively affect overall +# Skylake-X system performance. Since we are likely to suppress +# AVX512F capability flag [at least on Skylake-X], conversion serves +# as kind of "investment protection". Note that next *lake processor, +# Cannonlake, has AVX512IFMA code path to execute... +# +# Numbers are cycles per processed byte with poly1305_blocks alone, +# measured with rdtsc at fixed clock frequency. +# +# IALU/gcc-4.8(*) AVX(**) AVX2 AVX-512 +# P4 4.46/+120% - +# Core 2 2.41/+90% - +# Westmere 1.88/+120% - +# Sandy Bridge 1.39/+140% 1.10 +# Haswell 1.14/+175% 1.11 0.65 +# Skylake[-X] 1.13/+120% 0.96 0.51 [0.35] +# Silvermont 2.83/+95% - +# Knights L 3.60/? 1.65 1.10 0.41(***) +# Goldmont 1.70/+180% - +# VIA Nano 1.82/+150% - +# Sledgehammer 1.38/+160% - +# Bulldozer 2.30/+130% 0.97 +# Ryzen 1.15/+200% 1.08 1.18 +# +# (*) improvement coefficients relative to clang are more modest and +# are ~50% on most processors, in both cases we are comparing to +# __int128 code; +# (**) SSE2 implementation was attempted, but among non-AVX processors +# it was faster than integer-only code only on older Intel P4 and +# Core processors, 50-30%, less newer processor is, but slower on +# contemporary ones, for example almost 2x slower on Atom, and as +# former are naturally disappearing, SSE2 is deemed unnecessary; +# (***) strangely enough performance seems to vary from core to core, +# listed result is best case; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$kernel=0; $kernel=1 if (!$flavour && !$output); + +if (!$kernel) { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or + die "can't locate x86_64-xlate.pl"; + + open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; + *STDOUT=*OUT; + + if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); + } + + if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); + $avx += 1 if ($1==2.11 && $2>=8); + } + + if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + } + + if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); + } +} else { + $avx = 4; # The kernel uses ifdefs for this. +} + +sub declare_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= ".align $align\n"; + $code .= "SYM_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } +} + +sub end_function() { + my ($name) = @_; + if($kernel) { + $code .= "SYM_FUNC_END($name)\n"; + } else { + $code .= ".size $name,.-$name\n"; + } +} + +$code.=<<___ if $kernel; +#include +___ + +if ($avx) { +$code.=<<___ if $kernel; +.section .rodata +___ +$code.=<<___; +.align 64 +.Lconst: +.Lmask24: +.long 0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0 +.L129: +.long `1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0 +.Lmask26: +.long 0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0 +.Lpermd_avx2: +.long 2,2,2,3,2,0,2,1 +.Lpermd_avx512: +.long 0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7 + +.L2_44_inp_permd: +.long 0,1,1,2,2,3,7,7 +.L2_44_inp_shift: +.quad 0,12,24,64 +.L2_44_mask: +.quad 0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff +.L2_44_shift_rgt: +.quad 44,44,42,64 +.L2_44_shift_lft: +.quad 8,8,10,64 + +.align 64 +.Lx_mask44: +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.Lx_mask42: +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +___ +} +$code.=<<___ if (!$kernel); +.asciz "Poly1305 for x86_64, CRYPTOGAMS by " +.align 16 +___ + +my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx"); +my ($mac,$nonce)=($inp,$len); # *_emit arguments +my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13"); +my ($h0,$h1,$h2)=("%r14","%rbx","%r10"); + +sub poly1305_iteration { +# input: copy of $r1 in %rax, $h0-$h2, $r0-$r1 +# output: $h0-$h2 *= $r0-$r1 +$code.=<<___; + mulq $h0 # h0*r1 + mov %rax,$d2 + mov $r0,%rax + mov %rdx,$d3 + + mulq $h0 # h0*r0 + mov %rax,$h0 # future $h0 + mov $r0,%rax + mov %rdx,$d1 + + mulq $h1 # h1*r0 + add %rax,$d2 + mov $s1,%rax + adc %rdx,$d3 + + mulq $h1 # h1*s1 + mov $h2,$h1 # borrow $h1 + add %rax,$h0 + adc %rdx,$d1 + + imulq $s1,$h1 # h2*s1 + add $h1,$d2 + mov $d1,$h1 + adc \$0,$d3 + + imulq $r0,$h2 # h2*r0 + add $d2,$h1 + mov \$-4,%rax # mask value + adc $h2,$d3 + + and $d3,%rax # last reduction step + mov $d3,$h2 + shr \$2,$d3 + and \$3,$h2 + add $d3,%rax + add %rax,$h0 + adc \$0,$h1 + adc \$0,$h2 +___ +} + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int64 h[3]; # current hash value base 2^64 +# unsigned __int64 r[2]; # key value base 2^64 + +$code.=<<___; +.text +___ +$code.=<<___ if (!$kernel); +.extern OPENSSL_ia32cap_P + +.globl poly1305_init_x86_64 +.hidden poly1305_init_x86_64 +.globl poly1305_blocks_x86_64 +.hidden poly1305_blocks_x86_64 +.globl poly1305_emit_x86_64 +.hidden poly1305_emit_x86_64 +___ +&declare_function("poly1305_init_x86_64", 32, 3); +$code.=<<___; + xor %rax,%rax + mov %rax,0($ctx) # initialize hash value + mov %rax,8($ctx) + mov %rax,16($ctx) + + cmp \$0,$inp + je .Lno_key +___ +$code.=<<___ if (!$kernel); + lea poly1305_blocks_x86_64(%rip),%r10 + lea poly1305_emit_x86_64(%rip),%r11 +___ +$code.=<<___ if (!$kernel && $avx); + mov OPENSSL_ia32cap_P+4(%rip),%r9 + lea poly1305_blocks_avx(%rip),%rax + lea poly1305_emit_avx(%rip),%rcx + bt \$`60-32`,%r9 # AVX? + cmovc %rax,%r10 + cmovc %rcx,%r11 +___ +$code.=<<___ if (!$kernel && $avx>1); + lea poly1305_blocks_avx2(%rip),%rax + bt \$`5+32`,%r9 # AVX2? + cmovc %rax,%r10 +___ +$code.=<<___ if (!$kernel && $avx>3); + mov \$`(1<<31|1<<21|1<<16)`,%rax + shr \$32,%r9 + and %rax,%r9 + cmp %rax,%r9 + je .Linit_base2_44 +___ +$code.=<<___; + mov \$0x0ffffffc0fffffff,%rax + mov \$0x0ffffffc0ffffffc,%rcx + and 0($inp),%rax + and 8($inp),%rcx + mov %rax,24($ctx) + mov %rcx,32($ctx) +___ +$code.=<<___ if (!$kernel && $flavour !~ /elf32/); + mov %r10,0(%rdx) + mov %r11,8(%rdx) +___ +$code.=<<___ if (!$kernel && $flavour =~ /elf32/); + mov %r10d,0(%rdx) + mov %r11d,4(%rdx) +___ +$code.=<<___; + mov \$1,%eax +.Lno_key: + ret +___ +&end_function("poly1305_init_x86_64"); + +&declare_function("poly1305_blocks_x86_64", 32, 4); +$code.=<<___; +.cfi_startproc +.Lblocks: + shr \$4,$len + jz .Lno_data # too short + + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + push $ctx +.cfi_push $ctx +.Lblocks_body: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2 + + mov $s1,$r1 + shr \$2,$s1 + mov $r1,%rax + add $r1,$s1 # s1 = r1 + (r1 >> 2) + jmp .Loop + +.align 32 +.Loop: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 +___ + + &poly1305_iteration(); + +$code.=<<___; + mov $r1,%rax + dec %r15 # len-=16 + jnz .Loop + + mov 0(%rsp),$ctx +.cfi_restore $ctx + + mov $h0,0($ctx) # store hash value + mov $h1,8($ctx) + mov $h2,16($ctx) + + mov 8(%rsp),%r15 +.cfi_restore %r15 + mov 16(%rsp),%r14 +.cfi_restore %r14 + mov 24(%rsp),%r13 +.cfi_restore %r13 + mov 32(%rsp),%r12 +.cfi_restore %r12 + mov 40(%rsp),%rbx +.cfi_restore %rbx + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lno_data: +.Lblocks_epilogue: + ret +.cfi_endproc +___ +&end_function("poly1305_blocks_x86_64"); + +&declare_function("poly1305_emit_x86_64", 32, 3); +$code.=<<___; +.Lemit: + mov 0($ctx),%r8 # load hash value + mov 8($ctx),%r9 + mov 16($ctx),%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +___ +&end_function("poly1305_emit_x86_64"); +if ($avx) { + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX\n"; +} + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int32 h[5]; # current hash value base 2^26 +# unsigned __int32 is_base2_26; +# unsigned __int64 r[2]; # key value base 2^64 +# unsigned __int64 pad; +# struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9]; +# +# where r^n are base 2^26 digits of degrees of multiplier key. There are +# 5 digits, but last four are interleaved with multiples of 5, totalling +# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. + +my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) = + map("%xmm$_",(0..15)); + +$code.=<<___; +.type __poly1305_block,\@abi-omnipotent +.align 32 +__poly1305_block: + push $ctx +___ + &poly1305_iteration(); +$code.=<<___; + pop $ctx + ret +.size __poly1305_block,.-__poly1305_block + +.type __poly1305_init_avx,\@abi-omnipotent +.align 32 +__poly1305_init_avx: + push %rbp + mov %rsp,%rbp + mov $r0,$h0 + mov $r1,$h1 + xor $h2,$h2 + + lea 48+64($ctx),$ctx # size optimization + + mov $r1,%rax + call __poly1305_block # r^2 + + mov \$0x3ffffff,%eax # save interleaved r^2 and r base 2^26 + mov \$0x3ffffff,%edx + mov $h0,$d1 + and $h0#d,%eax + mov $r0,$d2 + and $r0#d,%edx + mov %eax,`16*0+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*0+4-64`($ctx) + shr \$26,$d2 + + mov \$0x3ffffff,%eax + mov \$0x3ffffff,%edx + and $d1#d,%eax + and $d2#d,%edx + mov %eax,`16*1+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*1+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*2+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*2+4-64`($ctx) + shr \$26,$d2 + + mov $h1,%rax + mov $r1,%rdx + shl \$12,%rax + shl \$12,%rdx + or $d1,%rax + or $d2,%rdx + and \$0x3ffffff,%eax + and \$0x3ffffff,%edx + mov %eax,`16*3+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*3+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*4+0-64`($ctx) + mov $h1,$d1 + mov %edx,`16*4+4-64`($ctx) + mov $r1,$d2 + + mov \$0x3ffffff,%eax + mov \$0x3ffffff,%edx + shr \$14,$d1 + shr \$14,$d2 + and $d1#d,%eax + and $d2#d,%edx + mov %eax,`16*5+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*5+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*6+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*6+4-64`($ctx) + shr \$26,$d2 + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+0-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d2#d,`16*7+4-64`($ctx) + lea ($d2,$d2,4),$d2 # *5 + mov $d1#d,`16*8+0-64`($ctx) + mov $d2#d,`16*8+4-64`($ctx) + + mov $r1,%rax + call __poly1305_block # r^3 + + mov \$0x3ffffff,%eax # save r^3 base 2^26 + mov $h0,$d1 + and $h0#d,%eax + shr \$26,$d1 + mov %eax,`16*0+12-64`($ctx) + + mov \$0x3ffffff,%edx + and $d1#d,%edx + mov %edx,`16*1+12-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*2+12-64`($ctx) + + mov $h1,%rax + shl \$12,%rax + or $d1,%rax + and \$0x3ffffff,%eax + mov %eax,`16*3+12-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov $h1,$d1 + mov %eax,`16*4+12-64`($ctx) + + mov \$0x3ffffff,%edx + shr \$14,$d1 + and $d1#d,%edx + mov %edx,`16*5+12-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*6+12-64`($ctx) + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+12-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d1#d,`16*8+12-64`($ctx) + + mov $r1,%rax + call __poly1305_block # r^4 + + mov \$0x3ffffff,%eax # save r^4 base 2^26 + mov $h0,$d1 + and $h0#d,%eax + shr \$26,$d1 + mov %eax,`16*0+8-64`($ctx) + + mov \$0x3ffffff,%edx + and $d1#d,%edx + mov %edx,`16*1+8-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*2+8-64`($ctx) + + mov $h1,%rax + shl \$12,%rax + or $d1,%rax + and \$0x3ffffff,%eax + mov %eax,`16*3+8-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov $h1,$d1 + mov %eax,`16*4+8-64`($ctx) + + mov \$0x3ffffff,%edx + shr \$14,$d1 + and $d1#d,%edx + mov %edx,`16*5+8-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*6+8-64`($ctx) + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+8-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d1#d,`16*8+8-64`($ctx) + + lea -48-64($ctx),$ctx # size [de-]optimization + pop %rbp + ret +.size __poly1305_init_avx,.-__poly1305_init_avx +___ + +&declare_function("poly1305_blocks_avx", 32, 4); +$code.=<<___; +.cfi_startproc + mov 20($ctx),%r8d # is_base2_26 + cmp \$128,$len + jae .Lblocks_avx + test %r8d,%r8d + jz .Lblocks + +.Lblocks_avx: + and \$-16,$len + jz .Lno_data_avx + + vzeroupper + + test %r8d,%r8d + jz .Lbase2_64_avx + + test \$31,$len + jz .Leven_avx + + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lblocks_avx_body: + + mov $len,%r15 # reassign $len + + mov 0($ctx),$d1 # load hash value + mov 8($ctx),$d2 + mov 16($ctx),$h2#d + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + ################################# base 2^26 -> base 2^64 + mov $d1#d,$h0#d + and \$`-1*(1<<31)`,$d1 + mov $d2,$r1 # borrow $r1 + mov $d2#d,$h1#d + and \$`-1*(1<<31)`,$d2 + + shr \$6,$d1 + shl \$52,$r1 + add $d1,$h0 + shr \$12,$h1 + shr \$18,$d2 + add $r1,$h0 + adc $d2,$h1 + + mov $h2,$d1 + shl \$40,$d1 + shr \$24,$h2 + add $d1,$h1 + adc \$0,$h2 # can be partially reduced... + + mov \$-4,$d2 # ... so reduce + mov $h2,$d1 + and $h2,$d2 + shr \$2,$d1 + and \$3,$h2 + add $d2,$d1 # =*5 + add $d1,$h0 + adc \$0,$h1 + adc \$0,$h2 + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + + call __poly1305_block + + test $padbit,$padbit # if $padbit is zero, + jz .Lstore_base2_64_avx # store hash in base 2^64 format + + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$r0 + mov $h1,$r1 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$r0 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $r0,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$r1 + and \$0x3ffffff,$h1 # h[3] + or $r1,$h2 # h[4] + + sub \$16,%r15 + jz .Lstore_base2_26_avx + + vmovd %rax#d,$H0 + vmovd %rdx#d,$H1 + vmovd $h0#d,$H2 + vmovd $h1#d,$H3 + vmovd $h2#d,$H4 + jmp .Lproceed_avx + +.align 32 +.Lstore_base2_64_avx: + mov $h0,0($ctx) + mov $h1,8($ctx) + mov $h2,16($ctx) # note that is_base2_26 is zeroed + jmp .Ldone_avx + +.align 16 +.Lstore_base2_26_avx: + mov %rax#d,0($ctx) # store hash value base 2^26 + mov %rdx#d,4($ctx) + mov $h0#d,8($ctx) + mov $h1#d,12($ctx) + mov $h2#d,16($ctx) +.align 16 +.Ldone_avx: + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lno_data_avx: +.Lblocks_avx_epilogue: + ret +.cfi_endproc + +.align 32 +.Lbase2_64_avx: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lbase2_64_avx_body: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2#d + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + test \$31,$len + jz .Linit_avx + + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + +.Linit_avx: + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$d1 + mov $h1,$d2 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$d1 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $d1,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$d2 + and \$0x3ffffff,$h1 # h[3] + or $d2,$h2 # h[4] + + vmovd %rax#d,$H0 + vmovd %rdx#d,$H1 + vmovd $h0#d,$H2 + vmovd $h1#d,$H3 + vmovd $h2#d,$H4 + movl \$1,20($ctx) # set is_base2_26 + + call __poly1305_init_avx + +.Lproceed_avx: + mov %r15,$len + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx_epilogue: + jmp .Ldo_avx +.cfi_endproc + +.align 32 +.Leven_avx: +.cfi_startproc + vmovd 4*0($ctx),$H0 # load hash value + vmovd 4*1($ctx),$H1 + vmovd 4*2($ctx),$H2 + vmovd 4*3($ctx),$H3 + vmovd 4*4($ctx),$H4 + +.Ldo_avx: +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + and \$-32,%rsp + sub \$-8,%rsp + lea -0x58(%rsp),%r11 + sub \$0x178,%rsp + +___ +$code.=<<___ if ($win64); + lea -0xf8(%rsp),%r11 + sub \$0x218,%rsp + vmovdqa %xmm6,0x50(%r11) + vmovdqa %xmm7,0x60(%r11) + vmovdqa %xmm8,0x70(%r11) + vmovdqa %xmm9,0x80(%r11) + vmovdqa %xmm10,0x90(%r11) + vmovdqa %xmm11,0xa0(%r11) + vmovdqa %xmm12,0xb0(%r11) + vmovdqa %xmm13,0xc0(%r11) + vmovdqa %xmm14,0xd0(%r11) + vmovdqa %xmm15,0xe0(%r11) +.Ldo_avx_body: +___ +$code.=<<___; + sub \$64,$len + lea -32($inp),%rax + cmovc %rax,$inp + + vmovdqu `16*3`($ctx),$D4 # preload r0^2 + lea `16*3+64`($ctx),$ctx # size optimization + lea .Lconst(%rip),%rcx + + ################################################################ + # load input + vmovdqu 16*2($inp),$T0 + vmovdqu 16*3($inp),$T1 + vmovdqa 64(%rcx),$MASK # .Lmask26 + + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + + vpsrlq \$40,$T4,$T4 # 4 + vpsrlq \$26,$T0,$T1 + vpand $MASK,$T0,$T0 # 0 + vpsrlq \$4,$T3,$T2 + vpand $MASK,$T1,$T1 # 1 + vpsrlq \$30,$T3,$T3 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + jbe .Lskip_loop_avx + + # expand and copy pre-calculated table to stack + vmovdqu `16*1-64`($ctx),$D1 + vmovdqu `16*2-64`($ctx),$D2 + vpshufd \$0xEE,$D4,$D3 # 34xx -> 3434 + vpshufd \$0x44,$D4,$D0 # xx12 -> 1212 + vmovdqa $D3,-0x90(%r11) + vmovdqa $D0,0x00(%rsp) + vpshufd \$0xEE,$D1,$D4 + vmovdqu `16*3-64`($ctx),$D0 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D4,-0x80(%r11) + vmovdqa $D1,0x10(%rsp) + vpshufd \$0xEE,$D2,$D3 + vmovdqu `16*4-64`($ctx),$D1 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D3,-0x70(%r11) + vmovdqa $D2,0x20(%rsp) + vpshufd \$0xEE,$D0,$D4 + vmovdqu `16*5-64`($ctx),$D2 + vpshufd \$0x44,$D0,$D0 + vmovdqa $D4,-0x60(%r11) + vmovdqa $D0,0x30(%rsp) + vpshufd \$0xEE,$D1,$D3 + vmovdqu `16*6-64`($ctx),$D0 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D3,-0x50(%r11) + vmovdqa $D1,0x40(%rsp) + vpshufd \$0xEE,$D2,$D4 + vmovdqu `16*7-64`($ctx),$D1 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D4,-0x40(%r11) + vmovdqa $D2,0x50(%rsp) + vpshufd \$0xEE,$D0,$D3 + vmovdqu `16*8-64`($ctx),$D2 + vpshufd \$0x44,$D0,$D0 + vmovdqa $D3,-0x30(%r11) + vmovdqa $D0,0x60(%rsp) + vpshufd \$0xEE,$D1,$D4 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D4,-0x20(%r11) + vmovdqa $D1,0x70(%rsp) + vpshufd \$0xEE,$D2,$D3 + vmovdqa 0x00(%rsp),$D4 # preload r0^2 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D3,-0x10(%r11) + vmovdqa $D2,0x80(%rsp) + + jmp .Loop_avx + +.align 32 +.Loop_avx: + ################################################################ + # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + # \___________________/ + # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + # \___________________/ \____________________/ + # + # Note that we start with inp[2:3]*r^2. This is because it + # doesn't depend on reduction in previous iteration. + ################################################################ + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # though note that $Tx and $Hx are "reversed" in this section, + # and $D4 is preloaded with r0^2... + + vpmuludq $T0,$D4,$D0 # d0 = h0*r0 + vpmuludq $T1,$D4,$D1 # d1 = h1*r0 + vmovdqa $H2,0x20(%r11) # offload hash + vpmuludq $T2,$D4,$D2 # d3 = h2*r0 + vmovdqa 0x10(%rsp),$H2 # r1^2 + vpmuludq $T3,$D4,$D3 # d3 = h3*r0 + vpmuludq $T4,$D4,$D4 # d4 = h4*r0 + + vmovdqa $H0,0x00(%r11) # + vpmuludq 0x20(%rsp),$T4,$H0 # h4*s1 + vmovdqa $H1,0x10(%r11) # + vpmuludq $T3,$H2,$H1 # h3*r1 + vpaddq $H0,$D0,$D0 # d0 += h4*s1 + vpaddq $H1,$D4,$D4 # d4 += h3*r1 + vmovdqa $H3,0x30(%r11) # + vpmuludq $T2,$H2,$H0 # h2*r1 + vpmuludq $T1,$H2,$H1 # h1*r1 + vpaddq $H0,$D3,$D3 # d3 += h2*r1 + vmovdqa 0x30(%rsp),$H3 # r2^2 + vpaddq $H1,$D2,$D2 # d2 += h1*r1 + vmovdqa $H4,0x40(%r11) # + vpmuludq $T0,$H2,$H2 # h0*r1 + vpmuludq $T2,$H3,$H0 # h2*r2 + vpaddq $H2,$D1,$D1 # d1 += h0*r1 + + vmovdqa 0x40(%rsp),$H4 # s2^2 + vpaddq $H0,$D4,$D4 # d4 += h2*r2 + vpmuludq $T1,$H3,$H1 # h1*r2 + vpmuludq $T0,$H3,$H3 # h0*r2 + vpaddq $H1,$D3,$D3 # d3 += h1*r2 + vmovdqa 0x50(%rsp),$H2 # r3^2 + vpaddq $H3,$D2,$D2 # d2 += h0*r2 + vpmuludq $T4,$H4,$H0 # h4*s2 + vpmuludq $T3,$H4,$H4 # h3*s2 + vpaddq $H0,$D1,$D1 # d1 += h4*s2 + vmovdqa 0x60(%rsp),$H3 # s3^2 + vpaddq $H4,$D0,$D0 # d0 += h3*s2 + + vmovdqa 0x80(%rsp),$H4 # s4^2 + vpmuludq $T1,$H2,$H1 # h1*r3 + vpmuludq $T0,$H2,$H2 # h0*r3 + vpaddq $H1,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $T4,$H3,$H0 # h4*s3 + vpmuludq $T3,$H3,$H1 # h3*s3 + vpaddq $H0,$D2,$D2 # d2 += h4*s3 + vmovdqu 16*0($inp),$H0 # load input + vpaddq $H1,$D1,$D1 # d1 += h3*s3 + vpmuludq $T2,$H3,$H3 # h2*s3 + vpmuludq $T2,$H4,$T2 # h2*s4 + vpaddq $H3,$D0,$D0 # d0 += h2*s3 + + vmovdqu 16*1($inp),$H1 # + vpaddq $T2,$D1,$D1 # d1 += h2*s4 + vpmuludq $T3,$H4,$T3 # h3*s4 + vpmuludq $T4,$H4,$T4 # h4*s4 + vpsrldq \$6,$H0,$H2 # splat input + vpaddq $T3,$D2,$D2 # d2 += h3*s4 + vpaddq $T4,$D3,$D3 # d3 += h4*s4 + vpsrldq \$6,$H1,$H3 # + vpmuludq 0x70(%rsp),$T0,$T4 # h0*r4 + vpmuludq $T1,$H4,$T0 # h1*s4 + vpunpckhqdq $H1,$H0,$H4 # 4 + vpaddq $T4,$D4,$D4 # d4 += h0*r4 + vmovdqa -0x90(%r11),$T4 # r0^4 + vpaddq $T0,$D0,$D0 # d0 += h1*s4 + + vpunpcklqdq $H1,$H0,$H0 # 0:1 + vpunpcklqdq $H3,$H2,$H3 # 2:3 + + #vpsrlq \$40,$H4,$H4 # 4 + vpsrldq \$`40/8`,$H4,$H4 # 4 + vpsrlq \$26,$H0,$H1 + vpand $MASK,$H0,$H0 # 0 + vpsrlq \$4,$H3,$H2 + vpand $MASK,$H1,$H1 # 1 + vpand 0(%rcx),$H4,$H4 # .Lmask24 + vpsrlq \$30,$H3,$H3 + vpand $MASK,$H2,$H2 # 2 + vpand $MASK,$H3,$H3 # 3 + vpor 32(%rcx),$H4,$H4 # padbit, yes, always + + vpaddq 0x00(%r11),$H0,$H0 # add hash value + vpaddq 0x10(%r11),$H1,$H1 + vpaddq 0x20(%r11),$H2,$H2 + vpaddq 0x30(%r11),$H3,$H3 + vpaddq 0x40(%r11),$H4,$H4 + + lea 16*2($inp),%rax + lea 16*4($inp),$inp + sub \$64,$len + cmovc %rax,$inp + + ################################################################ + # Now we accumulate (inp[0:1]+hash)*r^4 + ################################################################ + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + vpmuludq $H0,$T4,$T0 # h0*r0 + vpmuludq $H1,$T4,$T1 # h1*r0 + vpaddq $T0,$D0,$D0 + vpaddq $T1,$D1,$D1 + vmovdqa -0x80(%r11),$T2 # r1^4 + vpmuludq $H2,$T4,$T0 # h2*r0 + vpmuludq $H3,$T4,$T1 # h3*r0 + vpaddq $T0,$D2,$D2 + vpaddq $T1,$D3,$D3 + vpmuludq $H4,$T4,$T4 # h4*r0 + vpmuludq -0x70(%r11),$H4,$T0 # h4*s1 + vpaddq $T4,$D4,$D4 + + vpaddq $T0,$D0,$D0 # d0 += h4*s1 + vpmuludq $H2,$T2,$T1 # h2*r1 + vpmuludq $H3,$T2,$T0 # h3*r1 + vpaddq $T1,$D3,$D3 # d3 += h2*r1 + vmovdqa -0x60(%r11),$T3 # r2^4 + vpaddq $T0,$D4,$D4 # d4 += h3*r1 + vpmuludq $H1,$T2,$T1 # h1*r1 + vpmuludq $H0,$T2,$T2 # h0*r1 + vpaddq $T1,$D2,$D2 # d2 += h1*r1 + vpaddq $T2,$D1,$D1 # d1 += h0*r1 + + vmovdqa -0x50(%r11),$T4 # s2^4 + vpmuludq $H2,$T3,$T0 # h2*r2 + vpmuludq $H1,$T3,$T1 # h1*r2 + vpaddq $T0,$D4,$D4 # d4 += h2*r2 + vpaddq $T1,$D3,$D3 # d3 += h1*r2 + vmovdqa -0x40(%r11),$T2 # r3^4 + vpmuludq $H0,$T3,$T3 # h0*r2 + vpmuludq $H4,$T4,$T0 # h4*s2 + vpaddq $T3,$D2,$D2 # d2 += h0*r2 + vpaddq $T0,$D1,$D1 # d1 += h4*s2 + vmovdqa -0x30(%r11),$T3 # s3^4 + vpmuludq $H3,$T4,$T4 # h3*s2 + vpmuludq $H1,$T2,$T1 # h1*r3 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + + vmovdqa -0x10(%r11),$T4 # s4^4 + vpaddq $T1,$D4,$D4 # d4 += h1*r3 + vpmuludq $H0,$T2,$T2 # h0*r3 + vpmuludq $H4,$T3,$T0 # h4*s3 + vpaddq $T2,$D3,$D3 # d3 += h0*r3 + vpaddq $T0,$D2,$D2 # d2 += h4*s3 + vmovdqu 16*2($inp),$T0 # load input + vpmuludq $H3,$T3,$T2 # h3*s3 + vpmuludq $H2,$T3,$T3 # h2*s3 + vpaddq $T2,$D1,$D1 # d1 += h3*s3 + vmovdqu 16*3($inp),$T1 # + vpaddq $T3,$D0,$D0 # d0 += h2*s3 + + vpmuludq $H2,$T4,$H2 # h2*s4 + vpmuludq $H3,$T4,$H3 # h3*s4 + vpsrldq \$6,$T0,$T2 # splat input + vpaddq $H2,$D1,$D1 # d1 += h2*s4 + vpmuludq $H4,$T4,$H4 # h4*s4 + vpsrldq \$6,$T1,$T3 # + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*s4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*s4 + vpmuludq -0x20(%r11),$H0,$H4 # h0*r4 + vpmuludq $H1,$T4,$H0 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + + #vpsrlq \$40,$T4,$T4 # 4 + vpsrldq \$`40/8`,$T4,$T4 # 4 + vpsrlq \$26,$T0,$T1 + vmovdqa 0x00(%rsp),$D4 # preload r0^2 + vpand $MASK,$T0,$T0 # 0 + vpsrlq \$4,$T3,$T2 + vpand $MASK,$T1,$T1 # 1 + vpand 0(%rcx),$T4,$T4 # .Lmask24 + vpsrlq \$30,$T3,$T3 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + ################################################################ + # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + # and P. Schwabe + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D0 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D0,$H0,$H0 + vpsllq \$2,$D0,$D0 + vpaddq $D0,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + ja .Loop_avx + +.Lskip_loop_avx: + ################################################################ + # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + vpshufd \$0x10,$D4,$D4 # r0^n, xx12 -> x1x2 + add \$32,$len + jnz .Long_tail_avx + + vpaddq $H2,$T2,$T2 + vpaddq $H0,$T0,$T0 + vpaddq $H1,$T1,$T1 + vpaddq $H3,$T3,$T3 + vpaddq $H4,$T4,$T4 + +.Long_tail_avx: + vmovdqa $H2,0x20(%r11) + vmovdqa $H0,0x00(%r11) + vmovdqa $H1,0x10(%r11) + vmovdqa $H3,0x30(%r11) + vmovdqa $H4,0x40(%r11) + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + vpmuludq $T2,$D4,$D2 # d2 = h2*r0 + vpmuludq $T0,$D4,$D0 # d0 = h0*r0 + vpshufd \$0x10,`16*1-64`($ctx),$H2 # r1^n + vpmuludq $T1,$D4,$D1 # d1 = h1*r0 + vpmuludq $T3,$D4,$D3 # d3 = h3*r0 + vpmuludq $T4,$D4,$D4 # d4 = h4*r0 + + vpmuludq $T3,$H2,$H0 # h3*r1 + vpaddq $H0,$D4,$D4 # d4 += h3*r1 + vpshufd \$0x10,`16*2-64`($ctx),$H3 # s1^n + vpmuludq $T2,$H2,$H1 # h2*r1 + vpaddq $H1,$D3,$D3 # d3 += h2*r1 + vpshufd \$0x10,`16*3-64`($ctx),$H4 # r2^n + vpmuludq $T1,$H2,$H0 # h1*r1 + vpaddq $H0,$D2,$D2 # d2 += h1*r1 + vpmuludq $T0,$H2,$H2 # h0*r1 + vpaddq $H2,$D1,$D1 # d1 += h0*r1 + vpmuludq $T4,$H3,$H3 # h4*s1 + vpaddq $H3,$D0,$D0 # d0 += h4*s1 + + vpshufd \$0x10,`16*4-64`($ctx),$H2 # s2^n + vpmuludq $T2,$H4,$H1 # h2*r2 + vpaddq $H1,$D4,$D4 # d4 += h2*r2 + vpmuludq $T1,$H4,$H0 # h1*r2 + vpaddq $H0,$D3,$D3 # d3 += h1*r2 + vpshufd \$0x10,`16*5-64`($ctx),$H3 # r3^n + vpmuludq $T0,$H4,$H4 # h0*r2 + vpaddq $H4,$D2,$D2 # d2 += h0*r2 + vpmuludq $T4,$H2,$H1 # h4*s2 + vpaddq $H1,$D1,$D1 # d1 += h4*s2 + vpshufd \$0x10,`16*6-64`($ctx),$H4 # s3^n + vpmuludq $T3,$H2,$H2 # h3*s2 + vpaddq $H2,$D0,$D0 # d0 += h3*s2 + + vpmuludq $T1,$H3,$H0 # h1*r3 + vpaddq $H0,$D4,$D4 # d4 += h1*r3 + vpmuludq $T0,$H3,$H3 # h0*r3 + vpaddq $H3,$D3,$D3 # d3 += h0*r3 + vpshufd \$0x10,`16*7-64`($ctx),$H2 # r4^n + vpmuludq $T4,$H4,$H1 # h4*s3 + vpaddq $H1,$D2,$D2 # d2 += h4*s3 + vpshufd \$0x10,`16*8-64`($ctx),$H3 # s4^n + vpmuludq $T3,$H4,$H0 # h3*s3 + vpaddq $H0,$D1,$D1 # d1 += h3*s3 + vpmuludq $T2,$H4,$H4 # h2*s3 + vpaddq $H4,$D0,$D0 # d0 += h2*s3 + + vpmuludq $T0,$H2,$H2 # h0*r4 + vpaddq $H2,$D4,$D4 # h4 = d4 + h0*r4 + vpmuludq $T4,$H3,$H1 # h4*s4 + vpaddq $H1,$D3,$D3 # h3 = d3 + h4*s4 + vpmuludq $T3,$H3,$H0 # h3*s4 + vpaddq $H0,$D2,$D2 # h2 = d2 + h3*s4 + vpmuludq $T2,$H3,$H1 # h2*s4 + vpaddq $H1,$D1,$D1 # h1 = d1 + h2*s4 + vpmuludq $T1,$H3,$H3 # h1*s4 + vpaddq $H3,$D0,$D0 # h0 = d0 + h1*s4 + + jz .Lshort_tail_avx + + vmovdqu 16*0($inp),$H0 # load input + vmovdqu 16*1($inp),$H1 + + vpsrldq \$6,$H0,$H2 # splat input + vpsrldq \$6,$H1,$H3 + vpunpckhqdq $H1,$H0,$H4 # 4 + vpunpcklqdq $H1,$H0,$H0 # 0:1 + vpunpcklqdq $H3,$H2,$H3 # 2:3 + + vpsrlq \$40,$H4,$H4 # 4 + vpsrlq \$26,$H0,$H1 + vpand $MASK,$H0,$H0 # 0 + vpsrlq \$4,$H3,$H2 + vpand $MASK,$H1,$H1 # 1 + vpsrlq \$30,$H3,$H3 + vpand $MASK,$H2,$H2 # 2 + vpand $MASK,$H3,$H3 # 3 + vpor 32(%rcx),$H4,$H4 # padbit, yes, always + + vpshufd \$0x32,`16*0-64`($ctx),$T4 # r0^n, 34xx -> x3x4 + vpaddq 0x00(%r11),$H0,$H0 + vpaddq 0x10(%r11),$H1,$H1 + vpaddq 0x20(%r11),$H2,$H2 + vpaddq 0x30(%r11),$H3,$H3 + vpaddq 0x40(%r11),$H4,$H4 + + ################################################################ + # multiply (inp[0:1]+hash) by r^4:r^3 and accumulate + + vpmuludq $H0,$T4,$T0 # h0*r0 + vpaddq $T0,$D0,$D0 # d0 += h0*r0 + vpmuludq $H1,$T4,$T1 # h1*r0 + vpaddq $T1,$D1,$D1 # d1 += h1*r0 + vpmuludq $H2,$T4,$T0 # h2*r0 + vpaddq $T0,$D2,$D2 # d2 += h2*r0 + vpshufd \$0x32,`16*1-64`($ctx),$T2 # r1^n + vpmuludq $H3,$T4,$T1 # h3*r0 + vpaddq $T1,$D3,$D3 # d3 += h3*r0 + vpmuludq $H4,$T4,$T4 # h4*r0 + vpaddq $T4,$D4,$D4 # d4 += h4*r0 + + vpmuludq $H3,$T2,$T0 # h3*r1 + vpaddq $T0,$D4,$D4 # d4 += h3*r1 + vpshufd \$0x32,`16*2-64`($ctx),$T3 # s1 + vpmuludq $H2,$T2,$T1 # h2*r1 + vpaddq $T1,$D3,$D3 # d3 += h2*r1 + vpshufd \$0x32,`16*3-64`($ctx),$T4 # r2 + vpmuludq $H1,$T2,$T0 # h1*r1 + vpaddq $T0,$D2,$D2 # d2 += h1*r1 + vpmuludq $H0,$T2,$T2 # h0*r1 + vpaddq $T2,$D1,$D1 # d1 += h0*r1 + vpmuludq $H4,$T3,$T3 # h4*s1 + vpaddq $T3,$D0,$D0 # d0 += h4*s1 + + vpshufd \$0x32,`16*4-64`($ctx),$T2 # s2 + vpmuludq $H2,$T4,$T1 # h2*r2 + vpaddq $T1,$D4,$D4 # d4 += h2*r2 + vpmuludq $H1,$T4,$T0 # h1*r2 + vpaddq $T0,$D3,$D3 # d3 += h1*r2 + vpshufd \$0x32,`16*5-64`($ctx),$T3 # r3 + vpmuludq $H0,$T4,$T4 # h0*r2 + vpaddq $T4,$D2,$D2 # d2 += h0*r2 + vpmuludq $H4,$T2,$T1 # h4*s2 + vpaddq $T1,$D1,$D1 # d1 += h4*s2 + vpshufd \$0x32,`16*6-64`($ctx),$T4 # s3 + vpmuludq $H3,$T2,$T2 # h3*s2 + vpaddq $T2,$D0,$D0 # d0 += h3*s2 + + vpmuludq $H1,$T3,$T0 # h1*r3 + vpaddq $T0,$D4,$D4 # d4 += h1*r3 + vpmuludq $H0,$T3,$T3 # h0*r3 + vpaddq $T3,$D3,$D3 # d3 += h0*r3 + vpshufd \$0x32,`16*7-64`($ctx),$T2 # r4 + vpmuludq $H4,$T4,$T1 # h4*s3 + vpaddq $T1,$D2,$D2 # d2 += h4*s3 + vpshufd \$0x32,`16*8-64`($ctx),$T3 # s4 + vpmuludq $H3,$T4,$T0 # h3*s3 + vpaddq $T0,$D1,$D1 # d1 += h3*s3 + vpmuludq $H2,$T4,$T4 # h2*s3 + vpaddq $T4,$D0,$D0 # d0 += h2*s3 + + vpmuludq $H0,$T2,$T2 # h0*r4 + vpaddq $T2,$D4,$D4 # d4 += h0*r4 + vpmuludq $H4,$T3,$T1 # h4*s4 + vpaddq $T1,$D3,$D3 # d3 += h4*s4 + vpmuludq $H3,$T3,$T0 # h3*s4 + vpaddq $T0,$D2,$D2 # d2 += h3*s4 + vpmuludq $H2,$T3,$T1 # h2*s4 + vpaddq $T1,$D1,$D1 # d1 += h2*s4 + vpmuludq $H1,$T3,$T3 # h1*s4 + vpaddq $T3,$D0,$D0 # d0 += h1*s4 + +.Lshort_tail_avx: + ################################################################ + # horizontal addition + + vpsrldq \$8,$D4,$T4 + vpsrldq \$8,$D3,$T3 + vpsrldq \$8,$D1,$T1 + vpsrldq \$8,$D0,$T0 + vpsrldq \$8,$D2,$T2 + vpaddq $T3,$D3,$D3 + vpaddq $T4,$D4,$D4 + vpaddq $T0,$D0,$D0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$D2,$D2 + + ################################################################ + # lazy reduction + + vpsrlq \$26,$D3,$H3 + vpand $MASK,$D3,$D3 + vpaddq $H3,$D4,$D4 # h3 -> h4 + + vpsrlq \$26,$D0,$H0 + vpand $MASK,$D0,$D0 + vpaddq $H0,$D1,$D1 # h0 -> h1 + + vpsrlq \$26,$D4,$H4 + vpand $MASK,$D4,$D4 + + vpsrlq \$26,$D1,$H1 + vpand $MASK,$D1,$D1 + vpaddq $H1,$D2,$D2 # h1 -> h2 + + vpaddq $H4,$D0,$D0 + vpsllq \$2,$H4,$H4 + vpaddq $H4,$D0,$D0 # h4 -> h0 + + vpsrlq \$26,$D2,$H2 + vpand $MASK,$D2,$D2 + vpaddq $H2,$D3,$D3 # h2 -> h3 + + vpsrlq \$26,$D0,$H0 + vpand $MASK,$D0,$D0 + vpaddq $H0,$D1,$D1 # h0 -> h1 + + vpsrlq \$26,$D3,$H3 + vpand $MASK,$D3,$D3 + vpaddq $H3,$D4,$D4 # h3 -> h4 + + vmovd $D0,`4*0-48-64`($ctx) # save partially reduced + vmovd $D1,`4*1-48-64`($ctx) + vmovd $D2,`4*2-48-64`($ctx) + vmovd $D3,`4*3-48-64`($ctx) + vmovd $D4,`4*4-48-64`($ctx) +___ +$code.=<<___ if ($win64); + vmovdqa 0x50(%r11),%xmm6 + vmovdqa 0x60(%r11),%xmm7 + vmovdqa 0x70(%r11),%xmm8 + vmovdqa 0x80(%r11),%xmm9 + vmovdqa 0x90(%r11),%xmm10 + vmovdqa 0xa0(%r11),%xmm11 + vmovdqa 0xb0(%r11),%xmm12 + vmovdqa 0xc0(%r11),%xmm13 + vmovdqa 0xd0(%r11),%xmm14 + vmovdqa 0xe0(%r11),%xmm15 + lea 0xf8(%r11),%rsp +.Ldo_avx_epilogue: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + vzeroupper + ret +.cfi_endproc +___ +&end_function("poly1305_blocks_avx"); + +&declare_function("poly1305_emit_avx", 32, 3); +$code.=<<___; + cmpl \$0,20($ctx) # is_base2_26? + je .Lemit + + mov 0($ctx),%eax # load hash value base 2^26 + mov 4($ctx),%ecx + mov 8($ctx),%r8d + mov 12($ctx),%r11d + mov 16($ctx),%r10d + + shl \$26,%rcx # base 2^26 -> base 2^64 + mov %r8,%r9 + shl \$52,%r8 + add %rcx,%rax + shr \$12,%r9 + add %rax,%r8 # h0 + adc \$0,%r9 + + shl \$14,%r11 + mov %r10,%rax + shr \$24,%r10 + add %r11,%r9 + shl \$40,%rax + add %rax,%r9 # h1 + adc \$0,%r10 # h2 + + mov %r10,%rax # could be partially reduced, so reduce + mov %r10,%rcx + and \$3,%r10 + shr \$2,%rax + and \$-4,%rcx + add %rcx,%rax + add %rax,%r8 + adc \$0,%r9 + adc \$0,%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +___ +&end_function("poly1305_emit_avx"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if ($avx>1) { + +if ($kernel) { + $code .= "#ifdef CONFIG_AS_AVX2\n"; +} + +my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) = + map("%ymm$_",(0..15)); +my $S4=$MASK; + +sub poly1305_blocks_avxN { + my ($avx512) = @_; + my $suffix = $avx512 ? "_avx512" : ""; +$code.=<<___; +.cfi_startproc + mov 20($ctx),%r8d # is_base2_26 + cmp \$128,$len + jae .Lblocks_avx2$suffix + test %r8d,%r8d + jz .Lblocks + +.Lblocks_avx2$suffix: + and \$-16,$len + jz .Lno_data_avx2$suffix + + vzeroupper + + test %r8d,%r8d + jz .Lbase2_64_avx2$suffix + + test \$63,$len + jz .Leven_avx2$suffix + + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lblocks_avx2_body$suffix: + + mov $len,%r15 # reassign $len + + mov 0($ctx),$d1 # load hash value + mov 8($ctx),$d2 + mov 16($ctx),$h2#d + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + ################################# base 2^26 -> base 2^64 + mov $d1#d,$h0#d + and \$`-1*(1<<31)`,$d1 + mov $d2,$r1 # borrow $r1 + mov $d2#d,$h1#d + and \$`-1*(1<<31)`,$d2 + + shr \$6,$d1 + shl \$52,$r1 + add $d1,$h0 + shr \$12,$h1 + shr \$18,$d2 + add $r1,$h0 + adc $d2,$h1 + + mov $h2,$d1 + shl \$40,$d1 + shr \$24,$h2 + add $d1,$h1 + adc \$0,$h2 # can be partially reduced... + + mov \$-4,$d2 # ... so reduce + mov $h2,$d1 + and $h2,$d2 + shr \$2,$d1 + and \$3,$h2 + add $d2,$d1 # =*5 + add $d1,$h0 + adc \$0,$h1 + adc \$0,$h2 + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + +.Lbase2_26_pre_avx2$suffix: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + mov $r1,%rax + + test \$63,%r15 + jnz .Lbase2_26_pre_avx2$suffix + + test $padbit,$padbit # if $padbit is zero, + jz .Lstore_base2_64_avx2$suffix # store hash in base 2^64 format + + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$r0 + mov $h1,$r1 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$r0 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $r0,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$r1 + and \$0x3ffffff,$h1 # h[3] + or $r1,$h2 # h[4] + + test %r15,%r15 + jz .Lstore_base2_26_avx2$suffix + + vmovd %rax#d,%x#$H0 + vmovd %rdx#d,%x#$H1 + vmovd $h0#d,%x#$H2 + vmovd $h1#d,%x#$H3 + vmovd $h2#d,%x#$H4 + jmp .Lproceed_avx2$suffix + +.align 32 +.Lstore_base2_64_avx2$suffix: + mov $h0,0($ctx) + mov $h1,8($ctx) + mov $h2,16($ctx) # note that is_base2_26 is zeroed + jmp .Ldone_avx2$suffix + +.align 16 +.Lstore_base2_26_avx2$suffix: + mov %rax#d,0($ctx) # store hash value base 2^26 + mov %rdx#d,4($ctx) + mov $h0#d,8($ctx) + mov $h1#d,12($ctx) + mov $h2#d,16($ctx) +.align 16 +.Ldone_avx2$suffix: + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lno_data_avx2$suffix: +.Lblocks_avx2_epilogue$suffix: + ret +.cfi_endproc + +.align 32 +.Lbase2_64_avx2$suffix: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lbase2_64_avx2_body$suffix: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2#d + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + test \$63,$len + jz .Linit_avx2$suffix + +.Lbase2_64_pre_avx2$suffix: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + mov $r1,%rax + + test \$63,%r15 + jnz .Lbase2_64_pre_avx2$suffix + +.Linit_avx2$suffix: + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$d1 + mov $h1,$d2 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$d1 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $d1,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$d2 + and \$0x3ffffff,$h1 # h[3] + or $d2,$h2 # h[4] + + vmovd %rax#d,%x#$H0 + vmovd %rdx#d,%x#$H1 + vmovd $h0#d,%x#$H2 + vmovd $h1#d,%x#$H3 + vmovd $h2#d,%x#$H4 + movl \$1,20($ctx) # set is_base2_26 + + call __poly1305_init_avx + +.Lproceed_avx2$suffix: + mov %r15,$len # restore $len +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d + mov \$`(1<<31|1<<30|1<<16)`,%r11d +___ +$code.=<<___; + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx2_epilogue$suffix: + jmp .Ldo_avx2$suffix +.cfi_endproc + +.align 32 +.Leven_avx2$suffix: +.cfi_startproc +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d +___ +$code.=<<___; + vmovd 4*0($ctx),%x#$H0 # load hash value base 2^26 + vmovd 4*1($ctx),%x#$H1 + vmovd 4*2($ctx),%x#$H2 + vmovd 4*3($ctx),%x#$H3 + vmovd 4*4($ctx),%x#$H4 + +.Ldo_avx2$suffix: +___ +$code.=<<___ if (!$kernel && $avx>2); + cmp \$512,$len + jb .Lskip_avx512 + and %r11d,%r9d + test \$`1<<16`,%r9d # check for AVX512F + jnz .Lblocks_avx512 +.Lskip_avx512$suffix: +___ +$code.=<<___ if ($avx > 2 && $avx512 && $kernel); + cmp \$512,$len + jae .Lblocks_avx512 +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + sub \$0x128,%rsp +___ +$code.=<<___ if ($win64); + lea 8(%rsp),%r10 + sub \$0x1c8,%rsp + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx2_body$suffix: +___ +$code.=<<___; + lea .Lconst(%rip),%rcx + lea 48+64($ctx),$ctx # size optimization + vmovdqa 96(%rcx),$T0 # .Lpermd_avx2 + + # expand and copy pre-calculated table to stack + vmovdqu `16*0-64`($ctx),%x#$T2 + and \$-512,%rsp + vmovdqu `16*1-64`($ctx),%x#$T3 + vmovdqu `16*2-64`($ctx),%x#$T4 + vmovdqu `16*3-64`($ctx),%x#$D0 + vmovdqu `16*4-64`($ctx),%x#$D1 + vmovdqu `16*5-64`($ctx),%x#$D2 + lea 0x90(%rsp),%rax # size optimization + vmovdqu `16*6-64`($ctx),%x#$D3 + vpermd $T2,$T0,$T2 # 00003412 -> 14243444 + vmovdqu `16*7-64`($ctx),%x#$D4 + vpermd $T3,$T0,$T3 + vmovdqu `16*8-64`($ctx),%x#$MASK + vpermd $T4,$T0,$T4 + vmovdqa $T2,0x00(%rsp) + vpermd $D0,$T0,$D0 + vmovdqa $T3,0x20-0x90(%rax) + vpermd $D1,$T0,$D1 + vmovdqa $T4,0x40-0x90(%rax) + vpermd $D2,$T0,$D2 + vmovdqa $D0,0x60-0x90(%rax) + vpermd $D3,$T0,$D3 + vmovdqa $D1,0x80-0x90(%rax) + vpermd $D4,$T0,$D4 + vmovdqa $D2,0xa0-0x90(%rax) + vpermd $MASK,$T0,$MASK + vmovdqa $D3,0xc0-0x90(%rax) + vmovdqa $D4,0xe0-0x90(%rax) + vmovdqa $MASK,0x100-0x90(%rax) + vmovdqa 64(%rcx),$MASK # .Lmask26 + + ################################################################ + # load input + vmovdqu 16*0($inp),%x#$T0 + vmovdqu 16*1($inp),%x#$T1 + vinserti128 \$1,16*2($inp),$T0,$T0 + vinserti128 \$1,16*3($inp),$T1,$T1 + lea 16*4($inp),$inp + + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpunpcklqdq $T3,$T2,$T2 # 2:3 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + + vpsrlq \$30,$T2,$T3 + vpsrlq \$4,$T2,$T2 + vpsrlq \$26,$T0,$T1 + vpsrlq \$40,$T4,$T4 # 4 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T0,$T0 # 0 + vpand $MASK,$T1,$T1 # 1 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + vpaddq $H2,$T2,$H2 # accumulate input + sub \$64,$len + jz .Ltail_avx2$suffix + jmp .Loop_avx2$suffix + +.align 32 +.Loop_avx2$suffix: + ################################################################ + # ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4 + # ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3 + # ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2 + # ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1 + # \________/\__________/ + ################################################################ + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + vmovdqa `32*0`(%rsp),$T0 # r0^4 + vpaddq $H1,$T1,$H1 + vmovdqa `32*1`(%rsp),$T1 # r1^4 + vpaddq $H3,$T3,$H3 + vmovdqa `32*3`(%rsp),$T2 # r2^4 + vpaddq $H4,$T4,$H4 + vmovdqa `32*6-0x90`(%rax),$T3 # s3^4 + vmovdqa `32*8-0x90`(%rax),$S4 # s4^4 + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # however, as h2 is "chronologically" first one available pull + # corresponding operations up, so it's + # + # d4 = h2*r2 + h4*r0 + h3*r1 + h1*r3 + h0*r4 + # d3 = h2*r1 + h3*r0 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h2*5*r4 + h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + # d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2 + h1*5*r4 + + vpmuludq $H2,$T0,$D2 # d2 = h2*r0 + vpmuludq $H2,$T1,$D3 # d3 = h2*r1 + vpmuludq $H2,$T2,$D4 # d4 = h2*r2 + vpmuludq $H2,$T3,$D0 # d0 = h2*s3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + + vpmuludq $H0,$T1,$T4 # h0*r1 + vpmuludq $H1,$T1,$H2 # h1*r1, borrow $H2 as temp + vpaddq $T4,$D1,$D1 # d1 += h0*r1 + vpaddq $H2,$D2,$D2 # d2 += h1*r1 + vpmuludq $H3,$T1,$T4 # h3*r1 + vpmuludq `32*2`(%rsp),$H4,$H2 # h4*s1 + vpaddq $T4,$D4,$D4 # d4 += h3*r1 + vpaddq $H2,$D0,$D0 # d0 += h4*s1 + vmovdqa `32*4-0x90`(%rax),$T1 # s2 + + vpmuludq $H0,$T0,$T4 # h0*r0 + vpmuludq $H1,$T0,$H2 # h1*r0 + vpaddq $T4,$D0,$D0 # d0 += h0*r0 + vpaddq $H2,$D1,$D1 # d1 += h1*r0 + vpmuludq $H3,$T0,$T4 # h3*r0 + vpmuludq $H4,$T0,$H2 # h4*r0 + vmovdqu 16*0($inp),%x#$T0 # load input + vpaddq $T4,$D3,$D3 # d3 += h3*r0 + vpaddq $H2,$D4,$D4 # d4 += h4*r0 + vinserti128 \$1,16*2($inp),$T0,$T0 + + vpmuludq $H3,$T1,$T4 # h3*s2 + vpmuludq $H4,$T1,$H2 # h4*s2 + vmovdqu 16*1($inp),%x#$T1 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + vpaddq $H2,$D1,$D1 # d1 += h4*s2 + vmovdqa `32*5-0x90`(%rax),$H2 # r3 + vpmuludq $H1,$T2,$T4 # h1*r2 + vpmuludq $H0,$T2,$T2 # h0*r2 + vpaddq $T4,$D3,$D3 # d3 += h1*r2 + vpaddq $T2,$D2,$D2 # d2 += h0*r2 + vinserti128 \$1,16*3($inp),$T1,$T1 + lea 16*4($inp),$inp + + vpmuludq $H1,$H2,$T4 # h1*r3 + vpmuludq $H0,$H2,$H2 # h0*r3 + vpsrldq \$6,$T0,$T2 # splat input + vpaddq $T4,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $H3,$T3,$T4 # h3*s3 + vpmuludq $H4,$T3,$H2 # h4*s3 + vpsrldq \$6,$T1,$T3 + vpaddq $T4,$D1,$D1 # d1 += h3*s3 + vpaddq $H2,$D2,$D2 # d2 += h4*s3 + vpunpckhqdq $T1,$T0,$T4 # 4 + + vpmuludq $H3,$S4,$H3 # h3*s4 + vpmuludq $H4,$S4,$H4 # h4*s4 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + vpmuludq `32*7-0x90`(%rax),$H0,$H4 # h0*r4 + vpmuludq $H1,$S4,$H0 # h1*s4 + vmovdqa 64(%rcx),$MASK # .Lmask26 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + ################################################################ + # lazy reduction (interleaved with tail of input splat) + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$4,$T3,$T2 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpand $MASK,$T2,$T2 # 2 + vpsrlq \$26,$T0,$T1 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpaddq $T2,$H2,$H2 # modulo-scheduled + vpsrlq \$30,$T3,$T3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$40,$T4,$T4 # 4 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpand $MASK,$T0,$T0 # 0 + vpand $MASK,$T1,$T1 # 1 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + sub \$64,$len + jnz .Loop_avx2$suffix + + .byte 0x66,0x90 +.Ltail_avx2$suffix: + ################################################################ + # while above multiplications were by r^4 in all lanes, in last + # iteration we multiply least significant lane by r^4 and most + # significant one by r, so copy of above except that references + # to the precomputed table are displaced by 4... + + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + vmovdqu `32*0+4`(%rsp),$T0 # r0^4 + vpaddq $H1,$T1,$H1 + vmovdqu `32*1+4`(%rsp),$T1 # r1^4 + vpaddq $H3,$T3,$H3 + vmovdqu `32*3+4`(%rsp),$T2 # r2^4 + vpaddq $H4,$T4,$H4 + vmovdqu `32*6+4-0x90`(%rax),$T3 # s3^4 + vmovdqu `32*8+4-0x90`(%rax),$S4 # s4^4 + + vpmuludq $H2,$T0,$D2 # d2 = h2*r0 + vpmuludq $H2,$T1,$D3 # d3 = h2*r1 + vpmuludq $H2,$T2,$D4 # d4 = h2*r2 + vpmuludq $H2,$T3,$D0 # d0 = h2*s3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + + vpmuludq $H0,$T1,$T4 # h0*r1 + vpmuludq $H1,$T1,$H2 # h1*r1 + vpaddq $T4,$D1,$D1 # d1 += h0*r1 + vpaddq $H2,$D2,$D2 # d2 += h1*r1 + vpmuludq $H3,$T1,$T4 # h3*r1 + vpmuludq `32*2+4`(%rsp),$H4,$H2 # h4*s1 + vpaddq $T4,$D4,$D4 # d4 += h3*r1 + vpaddq $H2,$D0,$D0 # d0 += h4*s1 + + vpmuludq $H0,$T0,$T4 # h0*r0 + vpmuludq $H1,$T0,$H2 # h1*r0 + vpaddq $T4,$D0,$D0 # d0 += h0*r0 + vmovdqu `32*4+4-0x90`(%rax),$T1 # s2 + vpaddq $H2,$D1,$D1 # d1 += h1*r0 + vpmuludq $H3,$T0,$T4 # h3*r0 + vpmuludq $H4,$T0,$H2 # h4*r0 + vpaddq $T4,$D3,$D3 # d3 += h3*r0 + vpaddq $H2,$D4,$D4 # d4 += h4*r0 + + vpmuludq $H3,$T1,$T4 # h3*s2 + vpmuludq $H4,$T1,$H2 # h4*s2 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + vpaddq $H2,$D1,$D1 # d1 += h4*s2 + vmovdqu `32*5+4-0x90`(%rax),$H2 # r3 + vpmuludq $H1,$T2,$T4 # h1*r2 + vpmuludq $H0,$T2,$T2 # h0*r2 + vpaddq $T4,$D3,$D3 # d3 += h1*r2 + vpaddq $T2,$D2,$D2 # d2 += h0*r2 + + vpmuludq $H1,$H2,$T4 # h1*r3 + vpmuludq $H0,$H2,$H2 # h0*r3 + vpaddq $T4,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $H3,$T3,$T4 # h3*s3 + vpmuludq $H4,$T3,$H2 # h4*s3 + vpaddq $T4,$D1,$D1 # d1 += h3*s3 + vpaddq $H2,$D2,$D2 # d2 += h4*s3 + + vpmuludq $H3,$S4,$H3 # h3*s4 + vpmuludq $H4,$S4,$H4 # h4*s4 + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 + vpmuludq `32*7+4-0x90`(%rax),$H0,$H4 # h0*r4 + vpmuludq $H1,$S4,$H0 # h1*s4 + vmovdqa 64(%rcx),$MASK # .Lmask26 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + ################################################################ + # horizontal addition + + vpsrldq \$8,$D1,$T1 + vpsrldq \$8,$H2,$T2 + vpsrldq \$8,$H3,$T3 + vpsrldq \$8,$H4,$T4 + vpsrldq \$8,$H0,$T0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$H2,$H2 + vpaddq $T3,$H3,$H3 + vpaddq $T4,$H4,$H4 + vpaddq $T0,$H0,$H0 + + vpermq \$0x2,$H3,$T3 + vpermq \$0x2,$H4,$T4 + vpermq \$0x2,$H0,$T0 + vpermq \$0x2,$D1,$T1 + vpermq \$0x2,$H2,$T2 + vpaddq $T3,$H3,$H3 + vpaddq $T4,$H4,$H4 + vpaddq $T0,$H0,$H0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$H2,$H2 + + ################################################################ + # lazy reduction + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced + vmovd %x#$H1,`4*1-48-64`($ctx) + vmovd %x#$H2,`4*2-48-64`($ctx) + vmovd %x#$H3,`4*3-48-64`($ctx) + vmovd %x#$H4,`4*4-48-64`($ctx) +___ +$code.=<<___ if ($win64); + vmovdqa -0xb0(%r10),%xmm6 + vmovdqa -0xa0(%r10),%xmm7 + vmovdqa -0x90(%r10),%xmm8 + vmovdqa -0x80(%r10),%xmm9 + vmovdqa -0x70(%r10),%xmm10 + vmovdqa -0x60(%r10),%xmm11 + vmovdqa -0x50(%r10),%xmm12 + vmovdqa -0x40(%r10),%xmm13 + vmovdqa -0x30(%r10),%xmm14 + vmovdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx2_epilogue$suffix: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + vzeroupper + ret +.cfi_endproc +___ +if($avx > 2 && $avx512) { +my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24)); +my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29)); +my $PADBIT="%zmm30"; + +map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3)); # switch to %zmm domain +map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4)); +map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4)); +map(s/%y/%z/,($MASK)); + +$code.=<<___; +.cfi_startproc +.Lblocks_avx512: + mov \$15,%eax + kmovw %eax,%k2 +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + sub \$0x128,%rsp +___ +$code.=<<___ if ($win64); + lea 8(%rsp),%r10 + sub \$0x1c8,%rsp + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx512_body: +___ +$code.=<<___; + lea .Lconst(%rip),%rcx + lea 48+64($ctx),$ctx # size optimization + vmovdqa 96(%rcx),%y#$T2 # .Lpermd_avx2 + + # expand pre-calculated table + vmovdqu `16*0-64`($ctx),%x#$D0 # will become expanded ${R0} + and \$-512,%rsp + vmovdqu `16*1-64`($ctx),%x#$D1 # will become ... ${R1} + mov \$0x20,%rax + vmovdqu `16*2-64`($ctx),%x#$T0 # ... ${S1} + vmovdqu `16*3-64`($ctx),%x#$D2 # ... ${R2} + vmovdqu `16*4-64`($ctx),%x#$T1 # ... ${S2} + vmovdqu `16*5-64`($ctx),%x#$D3 # ... ${R3} + vmovdqu `16*6-64`($ctx),%x#$T3 # ... ${S3} + vmovdqu `16*7-64`($ctx),%x#$D4 # ... ${R4} + vmovdqu `16*8-64`($ctx),%x#$T4 # ... ${S4} + vpermd $D0,$T2,$R0 # 00003412 -> 14243444 + vpbroadcastq 64(%rcx),$MASK # .Lmask26 + vpermd $D1,$T2,$R1 + vpermd $T0,$T2,$S1 + vpermd $D2,$T2,$R2 + vmovdqa64 $R0,0x00(%rsp){%k2} # save in case $len%128 != 0 + vpsrlq \$32,$R0,$T0 # 14243444 -> 01020304 + vpermd $T1,$T2,$S2 + vmovdqu64 $R1,0x00(%rsp,%rax){%k2} + vpsrlq \$32,$R1,$T1 + vpermd $D3,$T2,$R3 + vmovdqa64 $S1,0x40(%rsp){%k2} + vpermd $T3,$T2,$S3 + vpermd $D4,$T2,$R4 + vmovdqu64 $R2,0x40(%rsp,%rax){%k2} + vpermd $T4,$T2,$S4 + vmovdqa64 $S2,0x80(%rsp){%k2} + vmovdqu64 $R3,0x80(%rsp,%rax){%k2} + vmovdqa64 $S3,0xc0(%rsp){%k2} + vmovdqu64 $R4,0xc0(%rsp,%rax){%k2} + vmovdqa64 $S4,0x100(%rsp){%k2} + + ################################################################ + # calculate 5th through 8th powers of the key + # + # d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1 + # d1 = r0'*r1 + r1'*r0 + r2'*5*r4 + r3'*5*r3 + r4'*5*r2 + # d2 = r0'*r2 + r1'*r1 + r2'*r0 + r3'*5*r4 + r4'*5*r3 + # d3 = r0'*r3 + r1'*r2 + r2'*r1 + r3'*r0 + r4'*5*r4 + # d4 = r0'*r4 + r1'*r3 + r2'*r2 + r3'*r1 + r4'*r0 + + vpmuludq $T0,$R0,$D0 # d0 = r0'*r0 + vpmuludq $T0,$R1,$D1 # d1 = r0'*r1 + vpmuludq $T0,$R2,$D2 # d2 = r0'*r2 + vpmuludq $T0,$R3,$D3 # d3 = r0'*r3 + vpmuludq $T0,$R4,$D4 # d4 = r0'*r4 + vpsrlq \$32,$R2,$T2 + + vpmuludq $T1,$S4,$M0 + vpmuludq $T1,$R0,$M1 + vpmuludq $T1,$R1,$M2 + vpmuludq $T1,$R2,$M3 + vpmuludq $T1,$R3,$M4 + vpsrlq \$32,$R3,$T3 + vpaddq $M0,$D0,$D0 # d0 += r1'*5*r4 + vpaddq $M1,$D1,$D1 # d1 += r1'*r0 + vpaddq $M2,$D2,$D2 # d2 += r1'*r1 + vpaddq $M3,$D3,$D3 # d3 += r1'*r2 + vpaddq $M4,$D4,$D4 # d4 += r1'*r3 + + vpmuludq $T2,$S3,$M0 + vpmuludq $T2,$S4,$M1 + vpmuludq $T2,$R1,$M3 + vpmuludq $T2,$R2,$M4 + vpmuludq $T2,$R0,$M2 + vpsrlq \$32,$R4,$T4 + vpaddq $M0,$D0,$D0 # d0 += r2'*5*r3 + vpaddq $M1,$D1,$D1 # d1 += r2'*5*r4 + vpaddq $M3,$D3,$D3 # d3 += r2'*r1 + vpaddq $M4,$D4,$D4 # d4 += r2'*r2 + vpaddq $M2,$D2,$D2 # d2 += r2'*r0 + + vpmuludq $T3,$S2,$M0 + vpmuludq $T3,$R0,$M3 + vpmuludq $T3,$R1,$M4 + vpmuludq $T3,$S3,$M1 + vpmuludq $T3,$S4,$M2 + vpaddq $M0,$D0,$D0 # d0 += r3'*5*r2 + vpaddq $M3,$D3,$D3 # d3 += r3'*r0 + vpaddq $M4,$D4,$D4 # d4 += r3'*r1 + vpaddq $M1,$D1,$D1 # d1 += r3'*5*r3 + vpaddq $M2,$D2,$D2 # d2 += r3'*5*r4 + + vpmuludq $T4,$S4,$M3 + vpmuludq $T4,$R0,$M4 + vpmuludq $T4,$S1,$M0 + vpmuludq $T4,$S2,$M1 + vpmuludq $T4,$S3,$M2 + vpaddq $M3,$D3,$D3 # d3 += r2'*5*r4 + vpaddq $M4,$D4,$D4 # d4 += r2'*r0 + vpaddq $M0,$D0,$D0 # d0 += r2'*5*r1 + vpaddq $M1,$D1,$D1 # d1 += r2'*5*r2 + vpaddq $M2,$D2,$D2 # d2 += r2'*5*r3 + + ################################################################ + # load input + vmovdqu64 16*0($inp),%z#$T3 + vmovdqu64 16*4($inp),%z#$T4 + lea 16*8($inp),$inp + + ################################################################ + # lazy reduction + + vpsrlq \$26,$D3,$M3 + vpandq $MASK,$D3,$D3 + vpaddq $M3,$D4,$D4 # d3 -> d4 + + vpsrlq \$26,$D0,$M0 + vpandq $MASK,$D0,$D0 + vpaddq $M0,$D1,$D1 # d0 -> d1 + + vpsrlq \$26,$D4,$M4 + vpandq $MASK,$D4,$D4 + + vpsrlq \$26,$D1,$M1 + vpandq $MASK,$D1,$D1 + vpaddq $M1,$D2,$D2 # d1 -> d2 + + vpaddq $M4,$D0,$D0 + vpsllq \$2,$M4,$M4 + vpaddq $M4,$D0,$D0 # d4 -> d0 + + vpsrlq \$26,$D2,$M2 + vpandq $MASK,$D2,$D2 + vpaddq $M2,$D3,$D3 # d2 -> d3 + + vpsrlq \$26,$D0,$M0 + vpandq $MASK,$D0,$D0 + vpaddq $M0,$D1,$D1 # d0 -> d1 + + vpsrlq \$26,$D3,$M3 + vpandq $MASK,$D3,$D3 + vpaddq $M3,$D4,$D4 # d3 -> d4 + + ################################################################ + # at this point we have 14243444 in $R0-$S4 and 05060708 in + # $D0-$D4, ... + + vpunpcklqdq $T4,$T3,$T0 # transpose input + vpunpckhqdq $T4,$T3,$T4 + + # ... since input 64-bit lanes are ordered as 73625140, we could + # "vperm" it to 76543210 (here and in each loop iteration), *or* + # we could just flow along, hence the goal for $R0-$S4 is + # 1858286838784888 ... + + vmovdqa32 128(%rcx),$M0 # .Lpermd_avx512: + mov \$0x7777,%eax + kmovw %eax,%k1 + + vpermd $R0,$M0,$R0 # 14243444 -> 1---2---3---4--- + vpermd $R1,$M0,$R1 + vpermd $R2,$M0,$R2 + vpermd $R3,$M0,$R3 + vpermd $R4,$M0,$R4 + + vpermd $D0,$M0,${R0}{%k1} # 05060708 -> 1858286838784888 + vpermd $D1,$M0,${R1}{%k1} + vpermd $D2,$M0,${R2}{%k1} + vpermd $D3,$M0,${R3}{%k1} + vpermd $D4,$M0,${R4}{%k1} + + vpslld \$2,$R1,$S1 # *5 + vpslld \$2,$R2,$S2 + vpslld \$2,$R3,$S3 + vpslld \$2,$R4,$S4 + vpaddd $R1,$S1,$S1 + vpaddd $R2,$S2,$S2 + vpaddd $R3,$S3,$S3 + vpaddd $R4,$S4,$S4 + + vpbroadcastq 32(%rcx),$PADBIT # .L129 + + vpsrlq \$52,$T0,$T2 # splat input + vpsllq \$12,$T4,$T3 + vporq $T3,$T2,$T2 + vpsrlq \$26,$T0,$T1 + vpsrlq \$14,$T4,$T3 + vpsrlq \$40,$T4,$T4 # 4 + vpandq $MASK,$T2,$T2 # 2 + vpandq $MASK,$T0,$T0 # 0 + #vpandq $MASK,$T1,$T1 # 1 + #vpandq $MASK,$T3,$T3 # 3 + #vporq $PADBIT,$T4,$T4 # padbit, yes, always + + vpaddq $H2,$T2,$H2 # accumulate input + sub \$192,$len + jbe .Ltail_avx512 + jmp .Loop_avx512 + +.align 32 +.Loop_avx512: + ################################################################ + # ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8 + # ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7 + # ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6 + # ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5 + # ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4 + # ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3 + # ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2 + # ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1 + # \________/\___________/ + ################################################################ + #vpaddq $H2,$T2,$H2 # accumulate input + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # however, as h2 is "chronologically" first one available pull + # corresponding operations up, so it's + # + # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4 + # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0 + # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1 + # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2 + # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3 + + vpmuludq $H2,$R1,$D3 # d3 = h2*r1 + vpaddq $H0,$T0,$H0 + vpmuludq $H2,$R2,$D4 # d4 = h2*r2 + vpandq $MASK,$T1,$T1 # 1 + vpmuludq $H2,$S3,$D0 # d0 = h2*s3 + vpandq $MASK,$T3,$T3 # 3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + vporq $PADBIT,$T4,$T4 # padbit, yes, always + vpmuludq $H2,$R0,$D2 # d2 = h2*r0 + vpaddq $H1,$T1,$H1 # accumulate input + vpaddq $H3,$T3,$H3 + vpaddq $H4,$T4,$H4 + + vmovdqu64 16*0($inp),$T3 # load input + vmovdqu64 16*4($inp),$T4 + lea 16*8($inp),$inp + vpmuludq $H0,$R3,$M3 + vpmuludq $H0,$R4,$M4 + vpmuludq $H0,$R0,$M0 + vpmuludq $H0,$R1,$M1 + vpaddq $M3,$D3,$D3 # d3 += h0*r3 + vpaddq $M4,$D4,$D4 # d4 += h0*r4 + vpaddq $M0,$D0,$D0 # d0 += h0*r0 + vpaddq $M1,$D1,$D1 # d1 += h0*r1 + + vpmuludq $H1,$R2,$M3 + vpmuludq $H1,$R3,$M4 + vpmuludq $H1,$S4,$M0 + vpmuludq $H0,$R2,$M2 + vpaddq $M3,$D3,$D3 # d3 += h1*r2 + vpaddq $M4,$D4,$D4 # d4 += h1*r3 + vpaddq $M0,$D0,$D0 # d0 += h1*s4 + vpaddq $M2,$D2,$D2 # d2 += h0*r2 + + vpunpcklqdq $T4,$T3,$T0 # transpose input + vpunpckhqdq $T4,$T3,$T4 + + vpmuludq $H3,$R0,$M3 + vpmuludq $H3,$R1,$M4 + vpmuludq $H1,$R0,$M1 + vpmuludq $H1,$R1,$M2 + vpaddq $M3,$D3,$D3 # d3 += h3*r0 + vpaddq $M4,$D4,$D4 # d4 += h3*r1 + vpaddq $M1,$D1,$D1 # d1 += h1*r0 + vpaddq $M2,$D2,$D2 # d2 += h1*r1 + + vpmuludq $H4,$S4,$M3 + vpmuludq $H4,$R0,$M4 + vpmuludq $H3,$S2,$M0 + vpmuludq $H3,$S3,$M1 + vpaddq $M3,$D3,$D3 # d3 += h4*s4 + vpmuludq $H3,$S4,$M2 + vpaddq $M4,$D4,$D4 # d4 += h4*r0 + vpaddq $M0,$D0,$D0 # d0 += h3*s2 + vpaddq $M1,$D1,$D1 # d1 += h3*s3 + vpaddq $M2,$D2,$D2 # d2 += h3*s4 + + vpmuludq $H4,$S1,$M0 + vpmuludq $H4,$S2,$M1 + vpmuludq $H4,$S3,$M2 + vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 + vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 + vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 + + ################################################################ + # lazy reduction (interleaved with input splat) + + vpsrlq \$52,$T0,$T2 # splat input + vpsllq \$12,$T4,$T3 + + vpsrlq \$26,$D3,$H3 + vpandq $MASK,$D3,$D3 + vpaddq $H3,$D4,$H4 # h3 -> h4 + + vporq $T3,$T2,$T2 + + vpsrlq \$26,$H0,$D0 + vpandq $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpandq $MASK,$T2,$T2 # 2 + + vpsrlq \$26,$H4,$D4 + vpandq $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpandq $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpaddq $T2,$H2,$H2 # modulo-scheduled + vpsrlq \$26,$T0,$T1 + + vpsrlq \$26,$H2,$D2 + vpandq $MASK,$H2,$H2 + vpaddq $D2,$D3,$H3 # h2 -> h3 + + vpsrlq \$14,$T4,$T3 + + vpsrlq \$26,$H0,$D0 + vpandq $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$40,$T4,$T4 # 4 + + vpsrlq \$26,$H3,$D3 + vpandq $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpandq $MASK,$T0,$T0 # 0 + #vpandq $MASK,$T1,$T1 # 1 + #vpandq $MASK,$T3,$T3 # 3 + #vporq $PADBIT,$T4,$T4 # padbit, yes, always + + sub \$128,$len + ja .Loop_avx512 + +.Ltail_avx512: + ################################################################ + # while above multiplications were by r^8 in all lanes, in last + # iteration we multiply least significant lane by r^8 and most + # significant one by r, that's why table gets shifted... + + vpsrlq \$32,$R0,$R0 # 0105020603070408 + vpsrlq \$32,$R1,$R1 + vpsrlq \$32,$R2,$R2 + vpsrlq \$32,$S3,$S3 + vpsrlq \$32,$S4,$S4 + vpsrlq \$32,$R3,$R3 + vpsrlq \$32,$R4,$R4 + vpsrlq \$32,$S1,$S1 + vpsrlq \$32,$S2,$S2 + + ################################################################ + # load either next or last 64 byte of input + lea ($inp,$len),$inp + + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + + vpmuludq $H2,$R1,$D3 # d3 = h2*r1 + vpmuludq $H2,$R2,$D4 # d4 = h2*r2 + vpmuludq $H2,$S3,$D0 # d0 = h2*s3 + vpandq $MASK,$T1,$T1 # 1 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + vpandq $MASK,$T3,$T3 # 3 + vpmuludq $H2,$R0,$D2 # d2 = h2*r0 + vporq $PADBIT,$T4,$T4 # padbit, yes, always + vpaddq $H1,$T1,$H1 # accumulate input + vpaddq $H3,$T3,$H3 + vpaddq $H4,$T4,$H4 + + vmovdqu 16*0($inp),%x#$T0 + vpmuludq $H0,$R3,$M3 + vpmuludq $H0,$R4,$M4 + vpmuludq $H0,$R0,$M0 + vpmuludq $H0,$R1,$M1 + vpaddq $M3,$D3,$D3 # d3 += h0*r3 + vpaddq $M4,$D4,$D4 # d4 += h0*r4 + vpaddq $M0,$D0,$D0 # d0 += h0*r0 + vpaddq $M1,$D1,$D1 # d1 += h0*r1 + + vmovdqu 16*1($inp),%x#$T1 + vpmuludq $H1,$R2,$M3 + vpmuludq $H1,$R3,$M4 + vpmuludq $H1,$S4,$M0 + vpmuludq $H0,$R2,$M2 + vpaddq $M3,$D3,$D3 # d3 += h1*r2 + vpaddq $M4,$D4,$D4 # d4 += h1*r3 + vpaddq $M0,$D0,$D0 # d0 += h1*s4 + vpaddq $M2,$D2,$D2 # d2 += h0*r2 + + vinserti128 \$1,16*2($inp),%y#$T0,%y#$T0 + vpmuludq $H3,$R0,$M3 + vpmuludq $H3,$R1,$M4 + vpmuludq $H1,$R0,$M1 + vpmuludq $H1,$R1,$M2 + vpaddq $M3,$D3,$D3 # d3 += h3*r0 + vpaddq $M4,$D4,$D4 # d4 += h3*r1 + vpaddq $M1,$D1,$D1 # d1 += h1*r0 + vpaddq $M2,$D2,$D2 # d2 += h1*r1 + + vinserti128 \$1,16*3($inp),%y#$T1,%y#$T1 + vpmuludq $H4,$S4,$M3 + vpmuludq $H4,$R0,$M4 + vpmuludq $H3,$S2,$M0 + vpmuludq $H3,$S3,$M1 + vpmuludq $H3,$S4,$M2 + vpaddq $M3,$D3,$H3 # h3 = d3 + h4*s4 + vpaddq $M4,$D4,$D4 # d4 += h4*r0 + vpaddq $M0,$D0,$D0 # d0 += h3*s2 + vpaddq $M1,$D1,$D1 # d1 += h3*s3 + vpaddq $M2,$D2,$D2 # d2 += h3*s4 + + vpmuludq $H4,$S1,$M0 + vpmuludq $H4,$S2,$M1 + vpmuludq $H4,$S3,$M2 + vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 + vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 + vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 + + ################################################################ + # horizontal addition + + mov \$1,%eax + vpermq \$0xb1,$H3,$D3 + vpermq \$0xb1,$D4,$H4 + vpermq \$0xb1,$H0,$D0 + vpermq \$0xb1,$H1,$D1 + vpermq \$0xb1,$H2,$D2 + vpaddq $D3,$H3,$H3 + vpaddq $D4,$H4,$H4 + vpaddq $D0,$H0,$H0 + vpaddq $D1,$H1,$H1 + vpaddq $D2,$H2,$H2 + + kmovw %eax,%k3 + vpermq \$0x2,$H3,$D3 + vpermq \$0x2,$H4,$D4 + vpermq \$0x2,$H0,$D0 + vpermq \$0x2,$H1,$D1 + vpermq \$0x2,$H2,$D2 + vpaddq $D3,$H3,$H3 + vpaddq $D4,$H4,$H4 + vpaddq $D0,$H0,$H0 + vpaddq $D1,$H1,$H1 + vpaddq $D2,$H2,$H2 + + vextracti64x4 \$0x1,$H3,%y#$D3 + vextracti64x4 \$0x1,$H4,%y#$D4 + vextracti64x4 \$0x1,$H0,%y#$D0 + vextracti64x4 \$0x1,$H1,%y#$D1 + vextracti64x4 \$0x1,$H2,%y#$D2 + vpaddq $D3,$H3,${H3}{%k3}{z} # keep single qword in case + vpaddq $D4,$H4,${H4}{%k3}{z} # it's passed to .Ltail_avx2 + vpaddq $D0,$H0,${H0}{%k3}{z} + vpaddq $D1,$H1,${H1}{%k3}{z} + vpaddq $D2,$H2,${H2}{%k3}{z} +___ +map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT)); +map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK)); +$code.=<<___; + ################################################################ + # lazy reduction (interleaved with input splat) + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpunpcklqdq $T3,$T2,$T2 # 2:3 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpsrlq \$30,$T2,$T3 + vpsrlq \$4,$T2,$T2 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpsrlq \$26,$T0,$T1 + vpsrlq \$40,$T4,$T4 # 4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T0,$T0 # 0 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $H2,$T2,$H2 # accumulate input for .Ltail_avx2 + vpand $MASK,$T1,$T1 # 1 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + vpaddq $D3,$H4,$H4 # h3 -> h4 + + lea 0x90(%rsp),%rax # size optimization for .Ltail_avx2 + add \$64,$len + jnz .Ltail_avx2$suffix + + vpsubq $T2,$H2,$H2 # undo input accumulation + vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced + vmovd %x#$H1,`4*1-48-64`($ctx) + vmovd %x#$H2,`4*2-48-64`($ctx) + vmovd %x#$H3,`4*3-48-64`($ctx) + vmovd %x#$H4,`4*4-48-64`($ctx) + vzeroall +___ +$code.=<<___ if ($win64); + movdqa -0xb0(%r10),%xmm6 + movdqa -0xa0(%r10),%xmm7 + movdqa -0x90(%r10),%xmm8 + movdqa -0x80(%r10),%xmm9 + movdqa -0x70(%r10),%xmm10 + movdqa -0x60(%r10),%xmm11 + movdqa -0x50(%r10),%xmm12 + movdqa -0x40(%r10),%xmm13 + movdqa -0x30(%r10),%xmm14 + movdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx512_epilogue: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + ret +.cfi_endproc +___ + +} + +} + +&declare_function("poly1305_blocks_avx2", 32, 4); +poly1305_blocks_avxN(0); +&end_function("poly1305_blocks_avx2"); + +if($kernel) { + $code .= "#endif\n"; +} + +####################################################################### +if ($avx>2) { +# On entry we have input length divisible by 64. But since inner loop +# processes 128 bytes per iteration, cases when length is not divisible +# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this +# reason stack layout is kept identical to poly1305_blocks_avx2. If not +# for this tail, we wouldn't have to even allocate stack frame... + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX512\n"; +} + +&declare_function("poly1305_blocks_avx512", 32, 4); +poly1305_blocks_avxN(1); +&end_function("poly1305_blocks_avx512"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if (!$kernel && $avx>3) { +######################################################################## +# VPMADD52 version using 2^44 radix. +# +# One can argue that base 2^52 would be more natural. Well, even though +# some operations would be more natural, one has to recognize couple of +# things. Base 2^52 doesn't provide advantage over base 2^44 if you look +# at amount of multiply-n-accumulate operations. Secondly, it makes it +# impossible to pre-compute multiples of 5 [referred to as s[]/sN in +# reference implementations], which means that more such operations +# would have to be performed in inner loop, which in turn makes critical +# path longer. In other words, even though base 2^44 reduction might +# look less elegant, overall critical path is actually shorter... + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int64 h[3]; # current hash value base 2^44 +# unsigned __int64 s[2]; # key value*20 base 2^44 +# unsigned __int64 r[3]; # key value base 2^44 +# struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4]; +# # r^n positions reflect +# # placement in register, not +# # memory, R[3] is R[1]*20 + +$code.=<<___; +.type poly1305_init_base2_44,\@function,3 +.align 32 +poly1305_init_base2_44: + xor %rax,%rax + mov %rax,0($ctx) # initialize hash value + mov %rax,8($ctx) + mov %rax,16($ctx) + +.Linit_base2_44: + lea poly1305_blocks_vpmadd52(%rip),%r10 + lea poly1305_emit_base2_44(%rip),%r11 + + mov \$0x0ffffffc0fffffff,%rax + mov \$0x0ffffffc0ffffffc,%rcx + and 0($inp),%rax + mov \$0x00000fffffffffff,%r8 + and 8($inp),%rcx + mov \$0x00000fffffffffff,%r9 + and %rax,%r8 + shrd \$44,%rcx,%rax + mov %r8,40($ctx) # r0 + and %r9,%rax + shr \$24,%rcx + mov %rax,48($ctx) # r1 + lea (%rax,%rax,4),%rax # *5 + mov %rcx,56($ctx) # r2 + shl \$2,%rax # magic <<2 + lea (%rcx,%rcx,4),%rcx # *5 + shl \$2,%rcx # magic <<2 + mov %rax,24($ctx) # s1 + mov %rcx,32($ctx) # s2 + movq \$-1,64($ctx) # write impossible value +___ +$code.=<<___ if ($flavour !~ /elf32/); + mov %r10,0(%rdx) + mov %r11,8(%rdx) +___ +$code.=<<___ if ($flavour =~ /elf32/); + mov %r10d,0(%rdx) + mov %r11d,4(%rdx) +___ +$code.=<<___; + mov \$1,%eax + ret +.size poly1305_init_base2_44,.-poly1305_init_base2_44 +___ +{ +my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17)); +my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21)); +my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52,\@function,4 +.align 32 +poly1305_blocks_vpmadd52: + shr \$4,$len + jz .Lno_data_vpmadd52 # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + + # if powers of the key are not calculated yet, process up to 3 + # blocks with this single-block subroutine, otherwise ensure that + # length is divisible by 2 blocks and pass the rest down to next + # subroutine... + + mov \$3,%rax + mov \$1,%r10 + cmp \$4,$len # is input long + cmovae %r10,%rax + test %r8,%r8 # is power value impossible? + cmovns %r10,%rax + + and $len,%rax # is input of favourable length? + jz .Lblocks_vpmadd52_4x + + sub %rax,$len + mov \$7,%r10d + mov \$1,%r11d + kmovw %r10d,%k7 + lea .L2_44_inp_permd(%rip),%r10 + kmovw %r11d,%k1 + + vmovq $padbit,%x#$PAD + vmovdqa64 0(%r10),$inp_permd # .L2_44_inp_permd + vmovdqa64 32(%r10),$inp_shift # .L2_44_inp_shift + vpermq \$0xcf,$PAD,$PAD + vmovdqa64 64(%r10),$reduc_mask # .L2_44_mask + + vmovdqu64 0($ctx),${Dlo}{%k7}{z} # load hash value + vmovdqu64 40($ctx),${r2r1r0}{%k7}{z} # load keys + vmovdqu64 32($ctx),${r1r0s2}{%k7}{z} + vmovdqu64 24($ctx),${r0s2s1}{%k7}{z} + + vmovdqa64 96(%r10),$reduc_rght # .L2_44_shift_rgt + vmovdqa64 128(%r10),$reduc_left # .L2_44_shift_lft + + jmp .Loop_vpmadd52 + +.align 32 +.Loop_vpmadd52: + vmovdqu32 0($inp),%x#$T0 # load input as ----3210 + lea 16($inp),$inp + + vpermd $T0,$inp_permd,$T0 # ----3210 -> --322110 + vpsrlvq $inp_shift,$T0,$T0 + vpandq $reduc_mask,$T0,$T0 + vporq $PAD,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo # accumulate input + + vpermq \$0,$Dlo,${H0}{%k7}{z} # smash hash value + vpermq \$0b01010101,$Dlo,${H1}{%k7}{z} + vpermq \$0b10101010,$Dlo,${H2}{%k7}{z} + + vpxord $Dlo,$Dlo,$Dlo + vpxord $Dhi,$Dhi,$Dhi + + vpmadd52luq $r2r1r0,$H0,$Dlo + vpmadd52huq $r2r1r0,$H0,$Dhi + + vpmadd52luq $r1r0s2,$H1,$Dlo + vpmadd52huq $r1r0s2,$H1,$Dhi + + vpmadd52luq $r0s2s1,$H2,$Dlo + vpmadd52huq $r0s2s1,$H2,$Dhi + + vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost qword + vpsllvq $reduc_left,$Dhi,$Dhi # 0 in topmost qword + vpandq $reduc_mask,$Dlo,$Dlo + + vpaddq $T0,$Dhi,$Dhi + + vpermq \$0b10010011,$Dhi,$Dhi # 0 in lowest qword + + vpaddq $Dhi,$Dlo,$Dlo # note topmost qword :-) + + vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost word + vpandq $reduc_mask,$Dlo,$Dlo + + vpermq \$0b10010011,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo + + vpermq \$0b10010011,$Dlo,${T0}{%k1}{z} + + vpaddq $T0,$Dlo,$Dlo + vpsllq \$2,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo + + dec %rax # len-=16 + jnz .Loop_vpmadd52 + + vmovdqu64 $Dlo,0($ctx){%k7} # store hash value + + test $len,$len + jnz .Lblocks_vpmadd52_4x + +.Lno_data_vpmadd52: + ret +.size poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52 +___ +} +{ +######################################################################## +# As implied by its name 4x subroutine processes 4 blocks in parallel +# (but handles even 4*n+2 blocks lengths). It takes up to 4th key power +# and is handled in 256-bit %ymm registers. + +my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); +my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); +my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52_4x,\@function,4 +.align 32 +poly1305_blocks_vpmadd52_4x: + shr \$4,$len + jz .Lno_data_vpmadd52_4x # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + +.Lblocks_vpmadd52_4x: + vpbroadcastq $padbit,$PAD + + vmovdqa64 .Lx_mask44(%rip),$mask44 + mov \$5,%eax + vmovdqa64 .Lx_mask42(%rip),$mask42 + kmovw %eax,%k1 # used in 2x path + + test %r8,%r8 # is power value impossible? + js .Linit_vpmadd52 # if it is, then init R[4] + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + + test \$3,$len # is length 4*n+2? + jnz .Lblocks_vpmadd52_2x_do + +.Lblocks_vpmadd52_4x_do: + vpbroadcastq 64($ctx),$R0 # load 4th power of the key + vpbroadcastq 96($ctx),$R1 + vpbroadcastq 128($ctx),$R2 + vpbroadcastq 160($ctx),$S1 + +.Lblocks_vpmadd52_4x_key_loaded: + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + test \$7,$len # is len 8*n? + jz .Lblocks_vpmadd52_8x + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*2($inp),$T3 + lea 16*4($inp),$inp + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as 3-1-2-0 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + sub \$4,$len + jz .Ltail_vpmadd52_4x + jmp .Loop_vpmadd52_4x + ud2 + +.align 32 +.Linit_vpmadd52: + vmovq 24($ctx),%x#$S1 # load key + vmovq 56($ctx),%x#$H2 + vmovq 32($ctx),%x#$S2 + vmovq 40($ctx),%x#$R0 + vmovq 48($ctx),%x#$R1 + + vmovdqa $R0,$H0 + vmovdqa $R1,$H1 + vmovdqa $H2,$R2 + + mov \$2,%eax + +.Lmul_init_vpmadd52: + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + dec %eax + jz .Ldone_init_vpmadd52 + + vpunpcklqdq $R1,$H1,$R1 # 1,2 + vpbroadcastq %x#$H1,%x#$H1 # 2,2 + vpunpcklqdq $R2,$H2,$R2 + vpbroadcastq %x#$H2,%x#$H2 + vpunpcklqdq $R0,$H0,$R0 + vpbroadcastq %x#$H0,%x#$H0 + + vpsllq \$2,$R1,$S1 # S1 = R1*5*4 + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R1,$S1,$S1 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S1,$S1 + vpsllq \$2,$S2,$S2 + + jmp .Lmul_init_vpmadd52 + ud2 + +.align 32 +.Ldone_init_vpmadd52: + vinserti128 \$1,%x#$R1,$H1,$R1 # 1,2,3,4 + vinserti128 \$1,%x#$R2,$H2,$R2 + vinserti128 \$1,%x#$R0,$H0,$R0 + + vpermq \$0b11011000,$R1,$R1 # 1,3,2,4 + vpermq \$0b11011000,$R2,$R2 + vpermq \$0b11011000,$R0,$R0 + + vpsllq \$2,$R1,$S1 # S1 = R1*5*4 + vpaddq $R1,$S1,$S1 + vpsllq \$2,$S1,$S1 + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + + test \$3,$len # is length 4*n+2? + jnz .Ldone_init_vpmadd52_2x + + vmovdqu64 $R0,64($ctx) # save key powers + vpbroadcastq %x#$R0,$R0 # broadcast 4th power + vmovdqu64 $R1,96($ctx) + vpbroadcastq %x#$R1,$R1 + vmovdqu64 $R2,128($ctx) + vpbroadcastq %x#$R2,$R2 + vmovdqu64 $S1,160($ctx) + vpbroadcastq %x#$S1,$S1 + + jmp .Lblocks_vpmadd52_4x_key_loaded + ud2 + +.align 32 +.Ldone_init_vpmadd52_2x: + vmovdqu64 $R0,64($ctx) # save key powers + vpsrldq \$8,$R0,$R0 # 0-1-0-2 + vmovdqu64 $R1,96($ctx) + vpsrldq \$8,$R1,$R1 + vmovdqu64 $R2,128($ctx) + vpsrldq \$8,$R2,$R2 + vmovdqu64 $S1,160($ctx) + vpsrldq \$8,$S1,$S1 + jmp .Lblocks_vpmadd52_2x_key_loaded + ud2 + +.align 32 +.Lblocks_vpmadd52_2x_do: + vmovdqu64 128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers + vmovdqu64 160+8($ctx),${S1}{%k1}{z} + vmovdqu64 64+8($ctx),${R0}{%k1}{z} + vmovdqu64 96+8($ctx),${R1}{%k1}{z} + +.Lblocks_vpmadd52_2x_key_loaded: + vmovdqu64 16*0($inp),$T2 # load data + vpxorq $T3,$T3,$T3 + lea 16*2($inp),$inp + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as x-1-x-0 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + jmp .Ltail_vpmadd52_2x + ud2 + +.align 32 +.Loop_vpmadd52_4x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*2($inp),$T3 + lea 16*4($inp),$inp + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction (interleaved with data splat) + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpsrlq \$24,$T3,$T2 + vporq $PAD,$T2,$T2 + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + sub \$4,$len # len-=64 + jnz .Loop_vpmadd52_4x + +.Ltail_vpmadd52_4x: + vmovdqu64 128($ctx),$R2 # load all key powers + vmovdqu64 160($ctx),$S1 + vmovdqu64 64($ctx),$R0 + vmovdqu64 96($ctx),$R1 + +.Ltail_vpmadd52_2x: + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # horizontal addition + + mov \$1,%eax + kmovw %eax,%k1 + vpsrldq \$8,$D0lo,$T0 + vpsrldq \$8,$D0hi,$H0 + vpsrldq \$8,$D1lo,$T1 + vpsrldq \$8,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpsrldq \$8,$D2lo,$T2 + vpsrldq \$8,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vpermq \$0x2,$D0lo,$T0 + vpermq \$0x2,$D0hi,$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vpermq \$0x2,$D1lo,$T1 + vpermq \$0x2,$D1hi,$H1 + vpaddq $T0,$D0lo,${D0lo}{%k1}{z} + vpaddq $H0,$D0hi,${D0hi}{%k1}{z} + vpermq \$0x2,$D2lo,$T2 + vpermq \$0x2,$D2hi,$H2 + vpaddq $T1,$D1lo,${D1lo}{%k1}{z} + vpaddq $H1,$D1hi,${D1hi}{%k1}{z} + vpaddq $T2,$D2lo,${D2lo}{%k1}{z} + vpaddq $H2,$D2hi,${D2hi}{%k1}{z} + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + # at this point $len is + # either 4*n+2 or 0... + sub \$2,$len # len-=32 + ja .Lblocks_vpmadd52_4x_do + + vmovq %x#$H0,0($ctx) + vmovq %x#$H1,8($ctx) + vmovq %x#$H2,16($ctx) + vzeroall + +.Lno_data_vpmadd52_4x: + ret +.size poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x +___ +} +{ +######################################################################## +# As implied by its name 8x subroutine processes 8 blocks in parallel... +# This is intermediate version, as it's used only in cases when input +# length is either 8*n, 8*n+1 or 8*n+2... + +my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); +my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); +my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); +my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52_8x,\@function,4 +.align 32 +poly1305_blocks_vpmadd52_8x: + shr \$4,$len + jz .Lno_data_vpmadd52_8x # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + + vmovdqa64 .Lx_mask44(%rip),$mask44 + vmovdqa64 .Lx_mask42(%rip),$mask42 + + test %r8,%r8 # is power value impossible? + js .Linit_vpmadd52 # if it is, then init R[4] + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + +.Lblocks_vpmadd52_8x: + ################################################################ + # fist we calculate more key powers + + vmovdqu64 128($ctx),$R2 # load 1-3-2-4 powers + vmovdqu64 160($ctx),$S1 + vmovdqu64 64($ctx),$R0 + vmovdqu64 96($ctx),$R1 + + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + vpbroadcastq %x#$R2,$RR2 # broadcast 4th power + vpbroadcastq %x#$R0,$RR0 + vpbroadcastq %x#$R1,$RR1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $RR2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $RR2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $RR2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $RR2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $RR2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $RR2,$R0,$D2hi + + vpmadd52luq $RR0,$R0,$D0lo + vpmadd52huq $RR0,$R0,$D0hi + vpmadd52luq $RR0,$R1,$D1lo + vpmadd52huq $RR0,$R1,$D1hi + vpmadd52luq $RR0,$R2,$D2lo + vpmadd52huq $RR0,$R2,$D2hi + + vpmadd52luq $RR1,$S2,$D0lo + vpmadd52huq $RR1,$S2,$D0hi + vpmadd52luq $RR1,$R0,$D1lo + vpmadd52huq $RR1,$R0,$D1hi + vpmadd52luq $RR1,$R1,$D2lo + vpmadd52huq $RR1,$R1,$D2hi + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$RR0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$RR1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$RR2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$RR0,$RR0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$RR0,$RR0 + + vpsrlq \$44,$RR0,$tmp # additional step + vpandq $mask44,$RR0,$RR0 + + vpaddq $tmp,$RR1,$RR1 + + ################################################################ + # At this point Rx holds 1324 powers, RRx - 5768, and the goal + # is 15263748, which reflects how data is loaded... + + vpunpcklqdq $R2,$RR2,$T2 # 3748 + vpunpckhqdq $R2,$RR2,$R2 # 1526 + vpunpcklqdq $R0,$RR0,$T0 + vpunpckhqdq $R0,$RR0,$R0 + vpunpcklqdq $R1,$RR1,$T1 + vpunpckhqdq $R1,$RR1,$R1 +___ +######## switch to %zmm +map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); +map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); +map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); +map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2); + +$code.=<<___; + vshufi64x2 \$0x44,$R2,$T2,$RR2 # 15263748 + vshufi64x2 \$0x44,$R0,$T0,$RR0 + vshufi64x2 \$0x44,$R1,$T1,$RR1 + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*4($inp),$T3 + lea 16*8($inp),$inp + + vpsllq \$2,$RR2,$SS2 # S2 = R2*5*4 + vpsllq \$2,$RR1,$SS1 # S1 = R1*5*4 + vpaddq $RR2,$SS2,$SS2 + vpaddq $RR1,$SS1,$SS1 + vpsllq \$2,$SS2,$SS2 + vpsllq \$2,$SS1,$SS1 + + vpbroadcastq $padbit,$PAD + vpbroadcastq %x#$mask44,$mask44 + vpbroadcastq %x#$mask42,$mask42 + + vpbroadcastq %x#$SS1,$S1 # broadcast 8th power + vpbroadcastq %x#$SS2,$S2 + vpbroadcastq %x#$RR0,$R0 + vpbroadcastq %x#$RR1,$R1 + vpbroadcastq %x#$RR2,$R2 + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as 73625140 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + sub \$8,$len + jz .Ltail_vpmadd52_8x + jmp .Loop_vpmadd52_8x + +.align 32 +.Loop_vpmadd52_8x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*4($inp),$T3 + lea 16*8($inp),$inp + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction (interleaved with data splat) + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpsrlq \$24,$T3,$T2 + vporq $PAD,$T2,$T2 + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + sub \$8,$len # len-=128 + jnz .Loop_vpmadd52_8x + +.Ltail_vpmadd52_8x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$SS1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$SS1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$SS2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$SS2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$RR0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$RR0,$D2hi + + vpmadd52luq $H0,$RR0,$D0lo + vpmadd52huq $H0,$RR0,$D0hi + vpmadd52luq $H0,$RR1,$D1lo + vpmadd52huq $H0,$RR1,$D1hi + vpmadd52luq $H0,$RR2,$D2lo + vpmadd52huq $H0,$RR2,$D2hi + + vpmadd52luq $H1,$SS2,$D0lo + vpmadd52huq $H1,$SS2,$D0hi + vpmadd52luq $H1,$RR0,$D1lo + vpmadd52huq $H1,$RR0,$D1hi + vpmadd52luq $H1,$RR1,$D2lo + vpmadd52huq $H1,$RR1,$D2hi + + ################################################################ + # horizontal addition + + mov \$1,%eax + kmovw %eax,%k1 + vpsrldq \$8,$D0lo,$T0 + vpsrldq \$8,$D0hi,$H0 + vpsrldq \$8,$D1lo,$T1 + vpsrldq \$8,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpsrldq \$8,$D2lo,$T2 + vpsrldq \$8,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vpermq \$0x2,$D0lo,$T0 + vpermq \$0x2,$D0hi,$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vpermq \$0x2,$D1lo,$T1 + vpermq \$0x2,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpermq \$0x2,$D2lo,$T2 + vpermq \$0x2,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vextracti64x4 \$1,$D0lo,%y#$T0 + vextracti64x4 \$1,$D0hi,%y#$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vextracti64x4 \$1,$D1lo,%y#$T1 + vextracti64x4 \$1,$D1hi,%y#$H1 + vextracti64x4 \$1,$D2lo,%y#$T2 + vextracti64x4 \$1,$D2hi,%y#$H2 +___ +######## switch back to %ymm +map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); +map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); +map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); + +$code.=<<___; + vpaddq $T0,$D0lo,${D0lo}{%k1}{z} + vpaddq $H0,$D0hi,${D0hi}{%k1}{z} + vpaddq $T1,$D1lo,${D1lo}{%k1}{z} + vpaddq $H1,$D1hi,${D1hi}{%k1}{z} + vpaddq $T2,$D2lo,${D2lo}{%k1}{z} + vpaddq $H2,$D2hi,${D2hi}{%k1}{z} + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + ################################################################ + + vmovq %x#$H0,0($ctx) + vmovq %x#$H1,8($ctx) + vmovq %x#$H2,16($ctx) + vzeroall + +.Lno_data_vpmadd52_8x: + ret +.size poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x +___ +} +$code.=<<___; +.type poly1305_emit_base2_44,\@function,3 +.align 32 +poly1305_emit_base2_44: + mov 0($ctx),%r8 # load hash value + mov 8($ctx),%r9 + mov 16($ctx),%r10 + + mov %r9,%rax + shr \$20,%r9 + shl \$44,%rax + mov %r10,%rcx + shr \$40,%r10 + shl \$24,%rcx + + add %rax,%r8 + adc %rcx,%r9 + adc \$0,%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +.size poly1305_emit_base2_44,.-poly1305_emit_base2_44 +___ +} } } +} + +if (!$kernel) +{ # chacha20-poly1305 helpers +my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order +$code.=<<___; +.globl xor128_encrypt_n_pad +.type xor128_encrypt_n_pad,\@abi-omnipotent +.align 16 +xor128_encrypt_n_pad: + sub $otp,$inp + sub $otp,$out + mov $len,%r10 # put len aside + shr \$4,$len # len / 16 + jz .Ltail_enc + nop +.Loop_enc_xmm: + movdqu ($inp,$otp),%xmm0 + pxor ($otp),%xmm0 + movdqu %xmm0,($out,$otp) + movdqa %xmm0,($otp) + lea 16($otp),$otp + dec $len + jnz .Loop_enc_xmm + + and \$15,%r10 # len % 16 + jz .Ldone_enc + +.Ltail_enc: + mov \$16,$len + sub %r10,$len + xor %eax,%eax +.Loop_enc_byte: + mov ($inp,$otp),%al + xor ($otp),%al + mov %al,($out,$otp) + mov %al,($otp) + lea 1($otp),$otp + dec %r10 + jnz .Loop_enc_byte + + xor %eax,%eax +.Loop_enc_pad: + mov %al,($otp) + lea 1($otp),$otp + dec $len + jnz .Loop_enc_pad + +.Ldone_enc: + mov $otp,%rax + ret +.size xor128_encrypt_n_pad,.-xor128_encrypt_n_pad + +.globl xor128_decrypt_n_pad +.type xor128_decrypt_n_pad,\@abi-omnipotent +.align 16 +xor128_decrypt_n_pad: + sub $otp,$inp + sub $otp,$out + mov $len,%r10 # put len aside + shr \$4,$len # len / 16 + jz .Ltail_dec + nop +.Loop_dec_xmm: + movdqu ($inp,$otp),%xmm0 + movdqa ($otp),%xmm1 + pxor %xmm0,%xmm1 + movdqu %xmm1,($out,$otp) + movdqa %xmm0,($otp) + lea 16($otp),$otp + dec $len + jnz .Loop_dec_xmm + + pxor %xmm1,%xmm1 + and \$15,%r10 # len % 16 + jz .Ldone_dec + +.Ltail_dec: + mov \$16,$len + sub %r10,$len + xor %eax,%eax + xor %r11,%r11 +.Loop_dec_byte: + mov ($inp,$otp),%r11b + mov ($otp),%al + xor %r11b,%al + mov %al,($out,$otp) + mov %r11b,($otp) + lea 1($otp),$otp + dec %r10 + jnz .Loop_dec_byte + + xor %eax,%eax +.Loop_dec_pad: + mov %al,($otp) + lea 1($otp),$otp + dec $len + jnz .Loop_dec_pad + +.Ldone_dec: + mov $otp,%rax + ret +.size xor128_decrypt_n_pad,.-xor128_decrypt_n_pad +___ +} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea 48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + + jmp .Lcommon_seh_tail +.size se_handler,.-se_handler + +.type avx_handler,\@abi-omnipotent +.align 16 +avx_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + mov 208($context),%rax # pull context->R11 + + lea 0x50(%rax),%rsi + lea 0xf8(%rax),%rax + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size avx_handler,.-avx_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_poly1305_init_x86_64 + .rva .LSEH_end_poly1305_init_x86_64 + .rva .LSEH_info_poly1305_init_x86_64 + + .rva .LSEH_begin_poly1305_blocks_x86_64 + .rva .LSEH_end_poly1305_blocks_x86_64 + .rva .LSEH_info_poly1305_blocks_x86_64 + + .rva .LSEH_begin_poly1305_emit_x86_64 + .rva .LSEH_end_poly1305_emit_x86_64 + .rva .LSEH_info_poly1305_emit_x86_64 +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_poly1305_blocks_avx + .rva .Lbase2_64_avx + .rva .LSEH_info_poly1305_blocks_avx_1 + + .rva .Lbase2_64_avx + .rva .Leven_avx + .rva .LSEH_info_poly1305_blocks_avx_2 + + .rva .Leven_avx + .rva .LSEH_end_poly1305_blocks_avx + .rva .LSEH_info_poly1305_blocks_avx_3 + + .rva .LSEH_begin_poly1305_emit_avx + .rva .LSEH_end_poly1305_emit_avx + .rva .LSEH_info_poly1305_emit_avx +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_poly1305_blocks_avx2 + .rva .Lbase2_64_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_1 + + .rva .Lbase2_64_avx2 + .rva .Leven_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_2 + + .rva .Leven_avx2 + .rva .LSEH_end_poly1305_blocks_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_3 +___ +$code.=<<___ if ($avx>2); + .rva .LSEH_begin_poly1305_blocks_avx512 + .rva .LSEH_end_poly1305_blocks_avx512 + .rva .LSEH_info_poly1305_blocks_avx512 +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_poly1305_init_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64 + +.LSEH_info_poly1305_blocks_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_body,.Lblocks_epilogue + +.LSEH_info_poly1305_emit_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64 +___ +$code.=<<___ if ($avx); +.LSEH_info_poly1305_blocks_avx_1: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_avx_body,.Lblocks_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx_2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbase2_64_avx_body,.Lbase2_64_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx_3: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx_body,.Ldo_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_emit_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx +___ +$code.=<<___ if ($avx>1); +.LSEH_info_poly1305_blocks_avx2_1: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_avx2_body,.Lblocks_avx2_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx2_2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx2_3: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx2_body,.Ldo_avx2_epilogue # HandlerData[] +___ +$code.=<<___ if ($avx>2); +.LSEH_info_poly1305_blocks_avx512: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx512_body,.Ldo_avx512_epilogue # HandlerData[] +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split('\n',$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + s/%r([a-z]+)#d/%e$1/g; + s/%r([0-9]+)#d/%r$1d/g; + s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g; + + if ($kernel) { + s/(^\.type.*),[0-9]+$/\1/; + s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/; + next if /^\.cfi.*/; + } + + print $_,"\n"; +} +close STDOUT; diff --git a/net/wireguard/crypto/zinc/poly1305/poly1305.c b/net/wireguard/crypto/zinc/poly1305/poly1305.c new file mode 100644 index 0000000000000..7d373b90e011c --- /dev/null +++ b/net/wireguard/crypto/zinc/poly1305/poly1305.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Implementation of the Poly1305 message authenticator. + * + * Information: https://cr.yp.to/mac.html + */ + +#include +#include "../selftest/run.h" + +#include +#include +#include +#include +#include + +#if defined(CONFIG_ZINC_ARCH_X86_64) +#include "poly1305-x86_64-glue.c" +#elif defined(CONFIG_ZINC_ARCH_ARM) || defined(CONFIG_ZINC_ARCH_ARM64) +#include "poly1305-arm-glue.c" +#elif defined(CONFIG_ZINC_ARCH_MIPS) || defined(CONFIG_ZINC_ARCH_MIPS64) +#include "poly1305-mips-glue.c" +#else +static inline bool poly1305_init_arch(void *ctx, + const u8 key[POLY1305_KEY_SIZE]) +{ + return false; +} +static inline bool poly1305_blocks_arch(void *ctx, const u8 *input, + size_t len, const u32 padbit, + simd_context_t *simd_context) +{ + return false; +} +static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + return false; +} +static bool *const poly1305_nobs[] __initconst = { }; +static void __init poly1305_fpu_init(void) +{ +} +#endif + +#if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) +#include "poly1305-donna64.c" +#else +#include "poly1305-donna32.c" +#endif + +void poly1305_init(struct poly1305_ctx *ctx, const u8 key[POLY1305_KEY_SIZE]) +{ + ctx->nonce[0] = get_unaligned_le32(&key[16]); + ctx->nonce[1] = get_unaligned_le32(&key[20]); + ctx->nonce[2] = get_unaligned_le32(&key[24]); + ctx->nonce[3] = get_unaligned_le32(&key[28]); + + if (!poly1305_init_arch(ctx->opaque, key)) + poly1305_init_generic(ctx->opaque, key); + + ctx->num = 0; +} +EXPORT_SYMBOL(poly1305_init); + +static inline void poly1305_blocks(void *ctx, const u8 *input, const size_t len, + const u32 padbit, + simd_context_t *simd_context) +{ + if (!poly1305_blocks_arch(ctx, input, len, padbit, simd_context)) + poly1305_blocks_generic(ctx, input, len, padbit); +} + +static inline void poly1305_emit(void *ctx, u8 mac[POLY1305_KEY_SIZE], + const u32 nonce[4], + simd_context_t *simd_context) +{ + if (!poly1305_emit_arch(ctx, mac, nonce, simd_context)) + poly1305_emit_generic(ctx, mac, nonce); +} + +void poly1305_update(struct poly1305_ctx *ctx, const u8 *input, size_t len, + simd_context_t *simd_context) +{ + const size_t num = ctx->num; + size_t rem; + + if (num) { + rem = POLY1305_BLOCK_SIZE - num; + if (len < rem) { + memcpy(ctx->data + num, input, len); + ctx->num = num + len; + return; + } + memcpy(ctx->data + num, input, rem); + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 1, + simd_context); + input += rem; + len -= rem; + } + + rem = len % POLY1305_BLOCK_SIZE; + len -= rem; + + if (len >= POLY1305_BLOCK_SIZE) { + poly1305_blocks(ctx->opaque, input, len, 1, simd_context); + input += len; + } + + if (rem) + memcpy(ctx->data, input, rem); + + ctx->num = rem; +} +EXPORT_SYMBOL(poly1305_update); + +void poly1305_final(struct poly1305_ctx *ctx, u8 mac[POLY1305_MAC_SIZE], + simd_context_t *simd_context) +{ + size_t num = ctx->num; + + if (num) { + ctx->data[num++] = 1; + while (num < POLY1305_BLOCK_SIZE) + ctx->data[num++] = 0; + poly1305_blocks(ctx->opaque, ctx->data, POLY1305_BLOCK_SIZE, 0, + simd_context); + } + + poly1305_emit(ctx->opaque, mac, ctx->nonce, simd_context); + + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL(poly1305_final); + +#include "../selftest/poly1305.c" + +static bool nosimd __initdata = false; + +#ifndef COMPAT_ZINC_IS_A_MODULE +int __init poly1305_mod_init(void) +#else +static int __init mod_init(void) +#endif +{ + if (!nosimd) + poly1305_fpu_init(); + if (!selftest_run("poly1305", poly1305_selftest, poly1305_nobs, + ARRAY_SIZE(poly1305_nobs))) + return -ENOTRECOVERABLE; + return 0; +} + +#ifdef COMPAT_ZINC_IS_A_MODULE +static void __exit mod_exit(void) +{ +} + +module_param(nosimd, bool, 0); +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Poly1305 one-time authenticator"); +MODULE_AUTHOR("Jason A. Donenfeld "); +#endif diff --git a/net/wireguard/crypto/zinc/selftest/blake2s.c b/net/wireguard/crypto/zinc/selftest/blake2s.c new file mode 100644 index 0000000000000..1b5c210dc7a84 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/blake2s.c @@ -0,0 +1,2090 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = { + { 0x69, 0x21, 0x7a, 0x30, 0x79, 0x90, 0x80, 0x94, + 0xe1, 0x11, 0x21, 0xd0, 0x42, 0x35, 0x4a, 0x7c, + 0x1f, 0x55, 0xb6, 0x48, 0x2c, 0xa1, 0xa5, 0x1e, + 0x1b, 0x25, 0x0d, 0xfd, 0x1e, 0xd0, 0xee, 0xf9 }, + { 0xe3, 0x4d, 0x74, 0xdb, 0xaf, 0x4f, 0xf4, 0xc6, + 0xab, 0xd8, 0x71, 0xcc, 0x22, 0x04, 0x51, 0xd2, + 0xea, 0x26, 0x48, 0x84, 0x6c, 0x77, 0x57, 0xfb, + 0xaa, 0xc8, 0x2f, 0xe5, 0x1a, 0xd6, 0x4b, 0xea }, + { 0xdd, 0xad, 0x9a, 0xb1, 0x5d, 0xac, 0x45, 0x49, + 0xba, 0x42, 0xf4, 0x9d, 0x26, 0x24, 0x96, 0xbe, + 0xf6, 0xc0, 0xba, 0xe1, 0xdd, 0x34, 0x2a, 0x88, + 0x08, 0xf8, 0xea, 0x26, 0x7c, 0x6e, 0x21, 0x0c }, + { 0xe8, 0xf9, 0x1c, 0x6e, 0xf2, 0x32, 0xa0, 0x41, + 0x45, 0x2a, 0xb0, 0xe1, 0x49, 0x07, 0x0c, 0xdd, + 0x7d, 0xd1, 0x76, 0x9e, 0x75, 0xb3, 0xa5, 0x92, + 0x1b, 0xe3, 0x78, 0x76, 0xc4, 0x5c, 0x99, 0x00 }, + { 0x0c, 0xc7, 0x0e, 0x00, 0x34, 0x8b, 0x86, 0xba, + 0x29, 0x44, 0xd0, 0xc3, 0x20, 0x38, 0xb2, 0x5c, + 0x55, 0x58, 0x4f, 0x90, 0xdf, 0x23, 0x04, 0xf5, + 0x5f, 0xa3, 0x32, 0xaf, 0x5f, 0xb0, 0x1e, 0x20 }, + { 0xec, 0x19, 0x64, 0x19, 0x10, 0x87, 0xa4, 0xfe, + 0x9d, 0xf1, 0xc7, 0x95, 0x34, 0x2a, 0x02, 0xff, + 0xc1, 0x91, 0xa5, 0xb2, 0x51, 0x76, 0x48, 0x56, + 0xae, 0x5b, 0x8b, 0x57, 0x69, 0xf0, 0xc6, 0xcd }, + { 0xe1, 0xfa, 0x51, 0x61, 0x8d, 0x7d, 0xf4, 0xeb, + 0x70, 0xcf, 0x0d, 0x5a, 0x9e, 0x90, 0x6f, 0x80, + 0x6e, 0x9d, 0x19, 0xf7, 0xf4, 0xf0, 0x1e, 0x3b, + 0x62, 0x12, 0x88, 0xe4, 0x12, 0x04, 0x05, 0xd6 }, + { 0x59, 0x80, 0x01, 0xfa, 0xfb, 0xe8, 0xf9, 0x4e, + 0xc6, 0x6d, 0xc8, 0x27, 0xd0, 0x12, 0xcf, 0xcb, + 0xba, 0x22, 0x28, 0x56, 0x9f, 0x44, 0x8e, 0x89, + 0xea, 0x22, 0x08, 0xc8, 0xbf, 0x76, 0x92, 0x93 }, + { 0xc7, 0xe8, 0x87, 0xb5, 0x46, 0x62, 0x36, 0x35, + 0xe9, 0x3e, 0x04, 0x95, 0x59, 0x8f, 0x17, 0x26, + 0x82, 0x19, 0x96, 0xc2, 0x37, 0x77, 0x05, 0xb9, + 0x3a, 0x1f, 0x63, 0x6f, 0x87, 0x2b, 0xfa, 0x2d }, + { 0xc3, 0x15, 0xa4, 0x37, 0xdd, 0x28, 0x06, 0x2a, + 0x77, 0x0d, 0x48, 0x19, 0x67, 0x13, 0x6b, 0x1b, + 0x5e, 0xb8, 0x8b, 0x21, 0xee, 0x53, 0xd0, 0x32, + 0x9c, 0x58, 0x97, 0x12, 0x6e, 0x9d, 0xb0, 0x2c }, + { 0xbb, 0x47, 0x3d, 0xed, 0xdc, 0x05, 0x5f, 0xea, + 0x62, 0x28, 0xf2, 0x07, 0xda, 0x57, 0x53, 0x47, + 0xbb, 0x00, 0x40, 0x4c, 0xd3, 0x49, 0xd3, 0x8c, + 0x18, 0x02, 0x63, 0x07, 0xa2, 0x24, 0xcb, 0xff }, + { 0x68, 0x7e, 0x18, 0x73, 0xa8, 0x27, 0x75, 0x91, + 0xbb, 0x33, 0xd9, 0xad, 0xf9, 0xa1, 0x39, 0x12, + 0xef, 0xef, 0xe5, 0x57, 0xca, 0xfc, 0x39, 0xa7, + 0x95, 0x26, 0x23, 0xe4, 0x72, 0x55, 0xf1, 0x6d }, + { 0x1a, 0xc7, 0xba, 0x75, 0x4d, 0x6e, 0x2f, 0x94, + 0xe0, 0xe8, 0x6c, 0x46, 0xbf, 0xb2, 0x62, 0xab, + 0xbb, 0x74, 0xf4, 0x50, 0xef, 0x45, 0x6d, 0x6b, + 0x4d, 0x97, 0xaa, 0x80, 0xce, 0x6d, 0xa7, 0x67 }, + { 0x01, 0x2c, 0x97, 0x80, 0x96, 0x14, 0x81, 0x6b, + 0x5d, 0x94, 0x94, 0x47, 0x7d, 0x4b, 0x68, 0x7d, + 0x15, 0xb9, 0x6e, 0xb6, 0x9c, 0x0e, 0x80, 0x74, + 0xa8, 0x51, 0x6f, 0x31, 0x22, 0x4b, 0x5c, 0x98 }, + { 0x91, 0xff, 0xd2, 0x6c, 0xfa, 0x4d, 0xa5, 0x13, + 0x4c, 0x7e, 0xa2, 0x62, 0xf7, 0x88, 0x9c, 0x32, + 0x9f, 0x61, 0xf6, 0xa6, 0x57, 0x22, 0x5c, 0xc2, + 0x12, 0xf4, 0x00, 0x56, 0xd9, 0x86, 0xb3, 0xf4 }, + { 0xd9, 0x7c, 0x82, 0x8d, 0x81, 0x82, 0xa7, 0x21, + 0x80, 0xa0, 0x6a, 0x78, 0x26, 0x83, 0x30, 0x67, + 0x3f, 0x7c, 0x4e, 0x06, 0x35, 0x94, 0x7c, 0x04, + 0xc0, 0x23, 0x23, 0xfd, 0x45, 0xc0, 0xa5, 0x2d }, + { 0xef, 0xc0, 0x4c, 0xdc, 0x39, 0x1c, 0x7e, 0x91, + 0x19, 0xbd, 0x38, 0x66, 0x8a, 0x53, 0x4e, 0x65, + 0xfe, 0x31, 0x03, 0x6d, 0x6a, 0x62, 0x11, 0x2e, + 0x44, 0xeb, 0xeb, 0x11, 0xf9, 0xc5, 0x70, 0x80 }, + { 0x99, 0x2c, 0xf5, 0xc0, 0x53, 0x44, 0x2a, 0x5f, + 0xbc, 0x4f, 0xaf, 0x58, 0x3e, 0x04, 0xe5, 0x0b, + 0xb7, 0x0d, 0x2f, 0x39, 0xfb, 0xb6, 0xa5, 0x03, + 0xf8, 0x9e, 0x56, 0xa6, 0x3e, 0x18, 0x57, 0x8a }, + { 0x38, 0x64, 0x0e, 0x9f, 0x21, 0x98, 0x3e, 0x67, + 0xb5, 0x39, 0xca, 0xcc, 0xae, 0x5e, 0xcf, 0x61, + 0x5a, 0xe2, 0x76, 0x4f, 0x75, 0xa0, 0x9c, 0x9c, + 0x59, 0xb7, 0x64, 0x83, 0xc1, 0xfb, 0xc7, 0x35 }, + { 0x21, 0x3d, 0xd3, 0x4c, 0x7e, 0xfe, 0x4f, 0xb2, + 0x7a, 0x6b, 0x35, 0xf6, 0xb4, 0x00, 0x0d, 0x1f, + 0xe0, 0x32, 0x81, 0xaf, 0x3c, 0x72, 0x3e, 0x5c, + 0x9f, 0x94, 0x74, 0x7a, 0x5f, 0x31, 0xcd, 0x3b }, + { 0xec, 0x24, 0x6e, 0xee, 0xb9, 0xce, 0xd3, 0xf7, + 0xad, 0x33, 0xed, 0x28, 0x66, 0x0d, 0xd9, 0xbb, + 0x07, 0x32, 0x51, 0x3d, 0xb4, 0xe2, 0xfa, 0x27, + 0x8b, 0x60, 0xcd, 0xe3, 0x68, 0x2a, 0x4c, 0xcd }, + { 0xac, 0x9b, 0x61, 0xd4, 0x46, 0x64, 0x8c, 0x30, + 0x05, 0xd7, 0x89, 0x2b, 0xf3, 0xa8, 0x71, 0x9f, + 0x4c, 0x81, 0x81, 0xcf, 0xdc, 0xbc, 0x2b, 0x79, + 0xfe, 0xf1, 0x0a, 0x27, 0x9b, 0x91, 0x10, 0x95 }, + { 0x7b, 0xf8, 0xb2, 0x29, 0x59, 0xe3, 0x4e, 0x3a, + 0x43, 0xf7, 0x07, 0x92, 0x23, 0xe8, 0x3a, 0x97, + 0x54, 0x61, 0x7d, 0x39, 0x1e, 0x21, 0x3d, 0xfd, + 0x80, 0x8e, 0x41, 0xb9, 0xbe, 0xad, 0x4c, 0xe7 }, + { 0x68, 0xd4, 0xb5, 0xd4, 0xfa, 0x0e, 0x30, 0x2b, + 0x64, 0xcc, 0xc5, 0xaf, 0x79, 0x29, 0x13, 0xac, + 0x4c, 0x88, 0xec, 0x95, 0xc0, 0x7d, 0xdf, 0x40, + 0x69, 0x42, 0x56, 0xeb, 0x88, 0xce, 0x9f, 0x3d }, + { 0xb2, 0xc2, 0x42, 0x0f, 0x05, 0xf9, 0xab, 0xe3, + 0x63, 0x15, 0x91, 0x93, 0x36, 0xb3, 0x7e, 0x4e, + 0x0f, 0xa3, 0x3f, 0xf7, 0xe7, 0x6a, 0x49, 0x27, + 0x67, 0x00, 0x6f, 0xdb, 0x5d, 0x93, 0x54, 0x62 }, + { 0x13, 0x4f, 0x61, 0xbb, 0xd0, 0xbb, 0xb6, 0x9a, + 0xed, 0x53, 0x43, 0x90, 0x45, 0x51, 0xa3, 0xe6, + 0xc1, 0xaa, 0x7d, 0xcd, 0xd7, 0x7e, 0x90, 0x3e, + 0x70, 0x23, 0xeb, 0x7c, 0x60, 0x32, 0x0a, 0xa7 }, + { 0x46, 0x93, 0xf9, 0xbf, 0xf7, 0xd4, 0xf3, 0x98, + 0x6a, 0x7d, 0x17, 0x6e, 0x6e, 0x06, 0xf7, 0x2a, + 0xd1, 0x49, 0x0d, 0x80, 0x5c, 0x99, 0xe2, 0x53, + 0x47, 0xb8, 0xde, 0x77, 0xb4, 0xdb, 0x6d, 0x9b }, + { 0x85, 0x3e, 0x26, 0xf7, 0x41, 0x95, 0x3b, 0x0f, + 0xd5, 0xbd, 0xb4, 0x24, 0xe8, 0xab, 0x9e, 0x8b, + 0x37, 0x50, 0xea, 0xa8, 0xef, 0x61, 0xe4, 0x79, + 0x02, 0xc9, 0x1e, 0x55, 0x4e, 0x9c, 0x73, 0xb9 }, + { 0xf7, 0xde, 0x53, 0x63, 0x61, 0xab, 0xaa, 0x0e, + 0x15, 0x81, 0x56, 0xcf, 0x0e, 0xa4, 0xf6, 0x3a, + 0x99, 0xb5, 0xe4, 0x05, 0x4f, 0x8f, 0xa4, 0xc9, + 0xd4, 0x5f, 0x62, 0x85, 0xca, 0xd5, 0x56, 0x94 }, + { 0x4c, 0x23, 0x06, 0x08, 0x86, 0x0a, 0x99, 0xae, + 0x8d, 0x7b, 0xd5, 0xc2, 0xcc, 0x17, 0xfa, 0x52, + 0x09, 0x6b, 0x9a, 0x61, 0xbe, 0xdb, 0x17, 0xcb, + 0x76, 0x17, 0x86, 0x4a, 0xd2, 0x9c, 0xa7, 0xa6 }, + { 0xae, 0xb9, 0x20, 0xea, 0x87, 0x95, 0x2d, 0xad, + 0xb1, 0xfb, 0x75, 0x92, 0x91, 0xe3, 0x38, 0x81, + 0x39, 0xa8, 0x72, 0x86, 0x50, 0x01, 0x88, 0x6e, + 0xd8, 0x47, 0x52, 0xe9, 0x3c, 0x25, 0x0c, 0x2a }, + { 0xab, 0xa4, 0xad, 0x9b, 0x48, 0x0b, 0x9d, 0xf3, + 0xd0, 0x8c, 0xa5, 0xe8, 0x7b, 0x0c, 0x24, 0x40, + 0xd4, 0xe4, 0xea, 0x21, 0x22, 0x4c, 0x2e, 0xb4, + 0x2c, 0xba, 0xe4, 0x69, 0xd0, 0x89, 0xb9, 0x31 }, + { 0x05, 0x82, 0x56, 0x07, 0xd7, 0xfd, 0xf2, 0xd8, + 0x2e, 0xf4, 0xc3, 0xc8, 0xc2, 0xae, 0xa9, 0x61, + 0xad, 0x98, 0xd6, 0x0e, 0xdf, 0xf7, 0xd0, 0x18, + 0x98, 0x3e, 0x21, 0x20, 0x4c, 0x0d, 0x93, 0xd1 }, + { 0xa7, 0x42, 0xf8, 0xb6, 0xaf, 0x82, 0xd8, 0xa6, + 0xca, 0x23, 0x57, 0xc5, 0xf1, 0xcf, 0x91, 0xde, + 0xfb, 0xd0, 0x66, 0x26, 0x7d, 0x75, 0xc0, 0x48, + 0xb3, 0x52, 0x36, 0x65, 0x85, 0x02, 0x59, 0x62 }, + { 0x2b, 0xca, 0xc8, 0x95, 0x99, 0x00, 0x0b, 0x42, + 0xc9, 0x5a, 0xe2, 0x38, 0x35, 0xa7, 0x13, 0x70, + 0x4e, 0xd7, 0x97, 0x89, 0xc8, 0x4f, 0xef, 0x14, + 0x9a, 0x87, 0x4f, 0xf7, 0x33, 0xf0, 0x17, 0xa2 }, + { 0xac, 0x1e, 0xd0, 0x7d, 0x04, 0x8f, 0x10, 0x5a, + 0x9e, 0x5b, 0x7a, 0xb8, 0x5b, 0x09, 0xa4, 0x92, + 0xd5, 0xba, 0xff, 0x14, 0xb8, 0xbf, 0xb0, 0xe9, + 0xfd, 0x78, 0x94, 0x86, 0xee, 0xa2, 0xb9, 0x74 }, + { 0xe4, 0x8d, 0x0e, 0xcf, 0xaf, 0x49, 0x7d, 0x5b, + 0x27, 0xc2, 0x5d, 0x99, 0xe1, 0x56, 0xcb, 0x05, + 0x79, 0xd4, 0x40, 0xd6, 0xe3, 0x1f, 0xb6, 0x24, + 0x73, 0x69, 0x6d, 0xbf, 0x95, 0xe0, 0x10, 0xe4 }, + { 0x12, 0xa9, 0x1f, 0xad, 0xf8, 0xb2, 0x16, 0x44, + 0xfd, 0x0f, 0x93, 0x4f, 0x3c, 0x4a, 0x8f, 0x62, + 0xba, 0x86, 0x2f, 0xfd, 0x20, 0xe8, 0xe9, 0x61, + 0x15, 0x4c, 0x15, 0xc1, 0x38, 0x84, 0xed, 0x3d }, + { 0x7c, 0xbe, 0xe9, 0x6e, 0x13, 0x98, 0x97, 0xdc, + 0x98, 0xfb, 0xef, 0x3b, 0xe8, 0x1a, 0xd4, 0xd9, + 0x64, 0xd2, 0x35, 0xcb, 0x12, 0x14, 0x1f, 0xb6, + 0x67, 0x27, 0xe6, 0xe5, 0xdf, 0x73, 0xa8, 0x78 }, + { 0xeb, 0xf6, 0x6a, 0xbb, 0x59, 0x7a, 0xe5, 0x72, + 0xa7, 0x29, 0x7c, 0xb0, 0x87, 0x1e, 0x35, 0x5a, + 0xcc, 0xaf, 0xad, 0x83, 0x77, 0xb8, 0xe7, 0x8b, + 0xf1, 0x64, 0xce, 0x2a, 0x18, 0xde, 0x4b, 0xaf }, + { 0x71, 0xb9, 0x33, 0xb0, 0x7e, 0x4f, 0xf7, 0x81, + 0x8c, 0xe0, 0x59, 0xd0, 0x08, 0x82, 0x9e, 0x45, + 0x3c, 0x6f, 0xf0, 0x2e, 0xc0, 0xa7, 0xdb, 0x39, + 0x3f, 0xc2, 0xd8, 0x70, 0xf3, 0x7a, 0x72, 0x86 }, + { 0x7c, 0xf7, 0xc5, 0x13, 0x31, 0x22, 0x0b, 0x8d, + 0x3e, 0xba, 0xed, 0x9c, 0x29, 0x39, 0x8a, 0x16, + 0xd9, 0x81, 0x56, 0xe2, 0x61, 0x3c, 0xb0, 0x88, + 0xf2, 0xb0, 0xe0, 0x8a, 0x1b, 0xe4, 0xcf, 0x4f }, + { 0x3e, 0x41, 0xa1, 0x08, 0xe0, 0xf6, 0x4a, 0xd2, + 0x76, 0xb9, 0x79, 0xe1, 0xce, 0x06, 0x82, 0x79, + 0xe1, 0x6f, 0x7b, 0xc7, 0xe4, 0xaa, 0x1d, 0x21, + 0x1e, 0x17, 0xb8, 0x11, 0x61, 0xdf, 0x16, 0x02 }, + { 0x88, 0x65, 0x02, 0xa8, 0x2a, 0xb4, 0x7b, 0xa8, + 0xd8, 0x67, 0x10, 0xaa, 0x9d, 0xe3, 0xd4, 0x6e, + 0xa6, 0x5c, 0x47, 0xaf, 0x6e, 0xe8, 0xde, 0x45, + 0x0c, 0xce, 0xb8, 0xb1, 0x1b, 0x04, 0x5f, 0x50 }, + { 0xc0, 0x21, 0xbc, 0x5f, 0x09, 0x54, 0xfe, 0xe9, + 0x4f, 0x46, 0xea, 0x09, 0x48, 0x7e, 0x10, 0xa8, + 0x48, 0x40, 0xd0, 0x2f, 0x64, 0x81, 0x0b, 0xc0, + 0x8d, 0x9e, 0x55, 0x1f, 0x7d, 0x41, 0x68, 0x14 }, + { 0x20, 0x30, 0x51, 0x6e, 0x8a, 0x5f, 0xe1, 0x9a, + 0xe7, 0x9c, 0x33, 0x6f, 0xce, 0x26, 0x38, 0x2a, + 0x74, 0x9d, 0x3f, 0xd0, 0xec, 0x91, 0xe5, 0x37, + 0xd4, 0xbd, 0x23, 0x58, 0xc1, 0x2d, 0xfb, 0x22 }, + { 0x55, 0x66, 0x98, 0xda, 0xc8, 0x31, 0x7f, 0xd3, + 0x6d, 0xfb, 0xdf, 0x25, 0xa7, 0x9c, 0xb1, 0x12, + 0xd5, 0x42, 0x58, 0x60, 0x60, 0x5c, 0xba, 0xf5, + 0x07, 0xf2, 0x3b, 0xf7, 0xe9, 0xf4, 0x2a, 0xfe }, + { 0x2f, 0x86, 0x7b, 0xa6, 0x77, 0x73, 0xfd, 0xc3, + 0xe9, 0x2f, 0xce, 0xd9, 0x9a, 0x64, 0x09, 0xad, + 0x39, 0xd0, 0xb8, 0x80, 0xfd, 0xe8, 0xf1, 0x09, + 0xa8, 0x17, 0x30, 0xc4, 0x45, 0x1d, 0x01, 0x78 }, + { 0x17, 0x2e, 0xc2, 0x18, 0xf1, 0x19, 0xdf, 0xae, + 0x98, 0x89, 0x6d, 0xff, 0x29, 0xdd, 0x98, 0x76, + 0xc9, 0x4a, 0xf8, 0x74, 0x17, 0xf9, 0xae, 0x4c, + 0x70, 0x14, 0xbb, 0x4e, 0x4b, 0x96, 0xaf, 0xc7 }, + { 0x3f, 0x85, 0x81, 0x4a, 0x18, 0x19, 0x5f, 0x87, + 0x9a, 0xa9, 0x62, 0xf9, 0x5d, 0x26, 0xbd, 0x82, + 0xa2, 0x78, 0xf2, 0xb8, 0x23, 0x20, 0x21, 0x8f, + 0x6b, 0x3b, 0xd6, 0xf7, 0xf6, 0x67, 0xa6, 0xd9 }, + { 0x1b, 0x61, 0x8f, 0xba, 0xa5, 0x66, 0xb3, 0xd4, + 0x98, 0xc1, 0x2e, 0x98, 0x2c, 0x9e, 0xc5, 0x2e, + 0x4d, 0xa8, 0x5a, 0x8c, 0x54, 0xf3, 0x8f, 0x34, + 0xc0, 0x90, 0x39, 0x4f, 0x23, 0xc1, 0x84, 0xc1 }, + { 0x0c, 0x75, 0x8f, 0xb5, 0x69, 0x2f, 0xfd, 0x41, + 0xa3, 0x57, 0x5d, 0x0a, 0xf0, 0x0c, 0xc7, 0xfb, + 0xf2, 0xcb, 0xe5, 0x90, 0x5a, 0x58, 0x32, 0x3a, + 0x88, 0xae, 0x42, 0x44, 0xf6, 0xe4, 0xc9, 0x93 }, + { 0xa9, 0x31, 0x36, 0x0c, 0xad, 0x62, 0x8c, 0x7f, + 0x12, 0xa6, 0xc1, 0xc4, 0xb7, 0x53, 0xb0, 0xf4, + 0x06, 0x2a, 0xef, 0x3c, 0xe6, 0x5a, 0x1a, 0xe3, + 0xf1, 0x93, 0x69, 0xda, 0xdf, 0x3a, 0xe2, 0x3d }, + { 0xcb, 0xac, 0x7d, 0x77, 0x3b, 0x1e, 0x3b, 0x3c, + 0x66, 0x91, 0xd7, 0xab, 0xb7, 0xe9, 0xdf, 0x04, + 0x5c, 0x8b, 0xa1, 0x92, 0x68, 0xde, 0xd1, 0x53, + 0x20, 0x7f, 0x5e, 0x80, 0x43, 0x52, 0xec, 0x5d }, + { 0x23, 0xa1, 0x96, 0xd3, 0x80, 0x2e, 0xd3, 0xc1, + 0xb3, 0x84, 0x01, 0x9a, 0x82, 0x32, 0x58, 0x40, + 0xd3, 0x2f, 0x71, 0x95, 0x0c, 0x45, 0x80, 0xb0, + 0x34, 0x45, 0xe0, 0x89, 0x8e, 0x14, 0x05, 0x3c }, + { 0xf4, 0x49, 0x54, 0x70, 0xf2, 0x26, 0xc8, 0xc2, + 0x14, 0xbe, 0x08, 0xfd, 0xfa, 0xd4, 0xbc, 0x4a, + 0x2a, 0x9d, 0xbe, 0xa9, 0x13, 0x6a, 0x21, 0x0d, + 0xf0, 0xd4, 0xb6, 0x49, 0x29, 0xe6, 0xfc, 0x14 }, + { 0xe2, 0x90, 0xdd, 0x27, 0x0b, 0x46, 0x7f, 0x34, + 0xab, 0x1c, 0x00, 0x2d, 0x34, 0x0f, 0xa0, 0x16, + 0x25, 0x7f, 0xf1, 0x9e, 0x58, 0x33, 0xfd, 0xbb, + 0xf2, 0xcb, 0x40, 0x1c, 0x3b, 0x28, 0x17, 0xde }, + { 0x9f, 0xc7, 0xb5, 0xde, 0xd3, 0xc1, 0x50, 0x42, + 0xb2, 0xa6, 0x58, 0x2d, 0xc3, 0x9b, 0xe0, 0x16, + 0xd2, 0x4a, 0x68, 0x2d, 0x5e, 0x61, 0xad, 0x1e, + 0xff, 0x9c, 0x63, 0x30, 0x98, 0x48, 0xf7, 0x06 }, + { 0x8c, 0xca, 0x67, 0xa3, 0x6d, 0x17, 0xd5, 0xe6, + 0x34, 0x1c, 0xb5, 0x92, 0xfd, 0x7b, 0xef, 0x99, + 0x26, 0xc9, 0xe3, 0xaa, 0x10, 0x27, 0xea, 0x11, + 0xa7, 0xd8, 0xbd, 0x26, 0x0b, 0x57, 0x6e, 0x04 }, + { 0x40, 0x93, 0x92, 0xf5, 0x60, 0xf8, 0x68, 0x31, + 0xda, 0x43, 0x73, 0xee, 0x5e, 0x00, 0x74, 0x26, + 0x05, 0x95, 0xd7, 0xbc, 0x24, 0x18, 0x3b, 0x60, + 0xed, 0x70, 0x0d, 0x45, 0x83, 0xd3, 0xf6, 0xf0 }, + { 0x28, 0x02, 0x16, 0x5d, 0xe0, 0x90, 0x91, 0x55, + 0x46, 0xf3, 0x39, 0x8c, 0xd8, 0x49, 0x16, 0x4a, + 0x19, 0xf9, 0x2a, 0xdb, 0xc3, 0x61, 0xad, 0xc9, + 0x9b, 0x0f, 0x20, 0xc8, 0xea, 0x07, 0x10, 0x54 }, + { 0xad, 0x83, 0x91, 0x68, 0xd9, 0xf8, 0xa4, 0xbe, + 0x95, 0xba, 0x9e, 0xf9, 0xa6, 0x92, 0xf0, 0x72, + 0x56, 0xae, 0x43, 0xfe, 0x6f, 0x98, 0x64, 0xe2, + 0x90, 0x69, 0x1b, 0x02, 0x56, 0xce, 0x50, 0xa9 }, + { 0x75, 0xfd, 0xaa, 0x50, 0x38, 0xc2, 0x84, 0xb8, + 0x6d, 0x6e, 0x8a, 0xff, 0xe8, 0xb2, 0x80, 0x7e, + 0x46, 0x7b, 0x86, 0x60, 0x0e, 0x79, 0xaf, 0x36, + 0x89, 0xfb, 0xc0, 0x63, 0x28, 0xcb, 0xf8, 0x94 }, + { 0xe5, 0x7c, 0xb7, 0x94, 0x87, 0xdd, 0x57, 0x90, + 0x24, 0x32, 0xb2, 0x50, 0x73, 0x38, 0x13, 0xbd, + 0x96, 0xa8, 0x4e, 0xfc, 0xe5, 0x9f, 0x65, 0x0f, + 0xac, 0x26, 0xe6, 0x69, 0x6a, 0xef, 0xaf, 0xc3 }, + { 0x56, 0xf3, 0x4e, 0x8b, 0x96, 0x55, 0x7e, 0x90, + 0xc1, 0xf2, 0x4b, 0x52, 0xd0, 0xc8, 0x9d, 0x51, + 0x08, 0x6a, 0xcf, 0x1b, 0x00, 0xf6, 0x34, 0xcf, + 0x1d, 0xde, 0x92, 0x33, 0xb8, 0xea, 0xaa, 0x3e }, + { 0x1b, 0x53, 0xee, 0x94, 0xaa, 0xf3, 0x4e, 0x4b, + 0x15, 0x9d, 0x48, 0xde, 0x35, 0x2c, 0x7f, 0x06, + 0x61, 0xd0, 0xa4, 0x0e, 0xdf, 0xf9, 0x5a, 0x0b, + 0x16, 0x39, 0xb4, 0x09, 0x0e, 0x97, 0x44, 0x72 }, + { 0x05, 0x70, 0x5e, 0x2a, 0x81, 0x75, 0x7c, 0x14, + 0xbd, 0x38, 0x3e, 0xa9, 0x8d, 0xda, 0x54, 0x4e, + 0xb1, 0x0e, 0x6b, 0xc0, 0x7b, 0xae, 0x43, 0x5e, + 0x25, 0x18, 0xdb, 0xe1, 0x33, 0x52, 0x53, 0x75 }, + { 0xd8, 0xb2, 0x86, 0x6e, 0x8a, 0x30, 0x9d, 0xb5, + 0x3e, 0x52, 0x9e, 0xc3, 0x29, 0x11, 0xd8, 0x2f, + 0x5c, 0xa1, 0x6c, 0xff, 0x76, 0x21, 0x68, 0x91, + 0xa9, 0x67, 0x6a, 0xa3, 0x1a, 0xaa, 0x6c, 0x42 }, + { 0xf5, 0x04, 0x1c, 0x24, 0x12, 0x70, 0xeb, 0x04, + 0xc7, 0x1e, 0xc2, 0xc9, 0x5d, 0x4c, 0x38, 0xd8, + 0x03, 0xb1, 0x23, 0x7b, 0x0f, 0x29, 0xfd, 0x4d, + 0xb3, 0xeb, 0x39, 0x76, 0x69, 0xe8, 0x86, 0x99 }, + { 0x9a, 0x4c, 0xe0, 0x77, 0xc3, 0x49, 0x32, 0x2f, + 0x59, 0x5e, 0x0e, 0xe7, 0x9e, 0xd0, 0xda, 0x5f, + 0xab, 0x66, 0x75, 0x2c, 0xbf, 0xef, 0x8f, 0x87, + 0xd0, 0xe9, 0xd0, 0x72, 0x3c, 0x75, 0x30, 0xdd }, + { 0x65, 0x7b, 0x09, 0xf3, 0xd0, 0xf5, 0x2b, 0x5b, + 0x8f, 0x2f, 0x97, 0x16, 0x3a, 0x0e, 0xdf, 0x0c, + 0x04, 0xf0, 0x75, 0x40, 0x8a, 0x07, 0xbb, 0xeb, + 0x3a, 0x41, 0x01, 0xa8, 0x91, 0x99, 0x0d, 0x62 }, + { 0x1e, 0x3f, 0x7b, 0xd5, 0xa5, 0x8f, 0xa5, 0x33, + 0x34, 0x4a, 0xa8, 0xed, 0x3a, 0xc1, 0x22, 0xbb, + 0x9e, 0x70, 0xd4, 0xef, 0x50, 0xd0, 0x04, 0x53, + 0x08, 0x21, 0x94, 0x8f, 0x5f, 0xe6, 0x31, 0x5a }, + { 0x80, 0xdc, 0xcf, 0x3f, 0xd8, 0x3d, 0xfd, 0x0d, + 0x35, 0xaa, 0x28, 0x58, 0x59, 0x22, 0xab, 0x89, + 0xd5, 0x31, 0x39, 0x97, 0x67, 0x3e, 0xaf, 0x90, + 0x5c, 0xea, 0x9c, 0x0b, 0x22, 0x5c, 0x7b, 0x5f }, + { 0x8a, 0x0d, 0x0f, 0xbf, 0x63, 0x77, 0xd8, 0x3b, + 0xb0, 0x8b, 0x51, 0x4b, 0x4b, 0x1c, 0x43, 0xac, + 0xc9, 0x5d, 0x75, 0x17, 0x14, 0xf8, 0x92, 0x56, + 0x45, 0xcb, 0x6b, 0xc8, 0x56, 0xca, 0x15, 0x0a }, + { 0x9f, 0xa5, 0xb4, 0x87, 0x73, 0x8a, 0xd2, 0x84, + 0x4c, 0xc6, 0x34, 0x8a, 0x90, 0x19, 0x18, 0xf6, + 0x59, 0xa3, 0xb8, 0x9e, 0x9c, 0x0d, 0xfe, 0xea, + 0xd3, 0x0d, 0xd9, 0x4b, 0xcf, 0x42, 0xef, 0x8e }, + { 0x80, 0x83, 0x2c, 0x4a, 0x16, 0x77, 0xf5, 0xea, + 0x25, 0x60, 0xf6, 0x68, 0xe9, 0x35, 0x4d, 0xd3, + 0x69, 0x97, 0xf0, 0x37, 0x28, 0xcf, 0xa5, 0x5e, + 0x1b, 0x38, 0x33, 0x7c, 0x0c, 0x9e, 0xf8, 0x18 }, + { 0xab, 0x37, 0xdd, 0xb6, 0x83, 0x13, 0x7e, 0x74, + 0x08, 0x0d, 0x02, 0x6b, 0x59, 0x0b, 0x96, 0xae, + 0x9b, 0xb4, 0x47, 0x72, 0x2f, 0x30, 0x5a, 0x5a, + 0xc5, 0x70, 0xec, 0x1d, 0xf9, 0xb1, 0x74, 0x3c }, + { 0x3e, 0xe7, 0x35, 0xa6, 0x94, 0xc2, 0x55, 0x9b, + 0x69, 0x3a, 0xa6, 0x86, 0x29, 0x36, 0x1e, 0x15, + 0xd1, 0x22, 0x65, 0xad, 0x6a, 0x3d, 0xed, 0xf4, + 0x88, 0xb0, 0xb0, 0x0f, 0xac, 0x97, 0x54, 0xba }, + { 0xd6, 0xfc, 0xd2, 0x32, 0x19, 0xb6, 0x47, 0xe4, + 0xcb, 0xd5, 0xeb, 0x2d, 0x0a, 0xd0, 0x1e, 0xc8, + 0x83, 0x8a, 0x4b, 0x29, 0x01, 0xfc, 0x32, 0x5c, + 0xc3, 0x70, 0x19, 0x81, 0xca, 0x6c, 0x88, 0x8b }, + { 0x05, 0x20, 0xec, 0x2f, 0x5b, 0xf7, 0xa7, 0x55, + 0xda, 0xcb, 0x50, 0xc6, 0xbf, 0x23, 0x3e, 0x35, + 0x15, 0x43, 0x47, 0x63, 0xdb, 0x01, 0x39, 0xcc, + 0xd9, 0xfa, 0xef, 0xbb, 0x82, 0x07, 0x61, 0x2d }, + { 0xaf, 0xf3, 0xb7, 0x5f, 0x3f, 0x58, 0x12, 0x64, + 0xd7, 0x66, 0x16, 0x62, 0xb9, 0x2f, 0x5a, 0xd3, + 0x7c, 0x1d, 0x32, 0xbd, 0x45, 0xff, 0x81, 0xa4, + 0xed, 0x8a, 0xdc, 0x9e, 0xf3, 0x0d, 0xd9, 0x89 }, + { 0xd0, 0xdd, 0x65, 0x0b, 0xef, 0xd3, 0xba, 0x63, + 0xdc, 0x25, 0x10, 0x2c, 0x62, 0x7c, 0x92, 0x1b, + 0x9c, 0xbe, 0xb0, 0xb1, 0x30, 0x68, 0x69, 0x35, + 0xb5, 0xc9, 0x27, 0xcb, 0x7c, 0xcd, 0x5e, 0x3b }, + { 0xe1, 0x14, 0x98, 0x16, 0xb1, 0x0a, 0x85, 0x14, + 0xfb, 0x3e, 0x2c, 0xab, 0x2c, 0x08, 0xbe, 0xe9, + 0xf7, 0x3c, 0xe7, 0x62, 0x21, 0x70, 0x12, 0x46, + 0xa5, 0x89, 0xbb, 0xb6, 0x73, 0x02, 0xd8, 0xa9 }, + { 0x7d, 0xa3, 0xf4, 0x41, 0xde, 0x90, 0x54, 0x31, + 0x7e, 0x72, 0xb5, 0xdb, 0xf9, 0x79, 0xda, 0x01, + 0xe6, 0xbc, 0xee, 0xbb, 0x84, 0x78, 0xea, 0xe6, + 0xa2, 0x28, 0x49, 0xd9, 0x02, 0x92, 0x63, 0x5c }, + { 0x12, 0x30, 0xb1, 0xfc, 0x8a, 0x7d, 0x92, 0x15, + 0xed, 0xc2, 0xd4, 0xa2, 0xde, 0xcb, 0xdd, 0x0a, + 0x6e, 0x21, 0x6c, 0x92, 0x42, 0x78, 0xc9, 0x1f, + 0xc5, 0xd1, 0x0e, 0x7d, 0x60, 0x19, 0x2d, 0x94 }, + { 0x57, 0x50, 0xd7, 0x16, 0xb4, 0x80, 0x8f, 0x75, + 0x1f, 0xeb, 0xc3, 0x88, 0x06, 0xba, 0x17, 0x0b, + 0xf6, 0xd5, 0x19, 0x9a, 0x78, 0x16, 0xbe, 0x51, + 0x4e, 0x3f, 0x93, 0x2f, 0xbe, 0x0c, 0xb8, 0x71 }, + { 0x6f, 0xc5, 0x9b, 0x2f, 0x10, 0xfe, 0xba, 0x95, + 0x4a, 0xa6, 0x82, 0x0b, 0x3c, 0xa9, 0x87, 0xee, + 0x81, 0xd5, 0xcc, 0x1d, 0xa3, 0xc6, 0x3c, 0xe8, + 0x27, 0x30, 0x1c, 0x56, 0x9d, 0xfb, 0x39, 0xce }, + { 0xc7, 0xc3, 0xfe, 0x1e, 0xeb, 0xdc, 0x7b, 0x5a, + 0x93, 0x93, 0x26, 0xe8, 0xdd, 0xb8, 0x3e, 0x8b, + 0xf2, 0xb7, 0x80, 0xb6, 0x56, 0x78, 0xcb, 0x62, + 0xf2, 0x08, 0xb0, 0x40, 0xab, 0xdd, 0x35, 0xe2 }, + { 0x0c, 0x75, 0xc1, 0xa1, 0x5c, 0xf3, 0x4a, 0x31, + 0x4e, 0xe4, 0x78, 0xf4, 0xa5, 0xce, 0x0b, 0x8a, + 0x6b, 0x36, 0x52, 0x8e, 0xf7, 0xa8, 0x20, 0x69, + 0x6c, 0x3e, 0x42, 0x46, 0xc5, 0xa1, 0x58, 0x64 }, + { 0x21, 0x6d, 0xc1, 0x2a, 0x10, 0x85, 0x69, 0xa3, + 0xc7, 0xcd, 0xde, 0x4a, 0xed, 0x43, 0xa6, 0xc3, + 0x30, 0x13, 0x9d, 0xda, 0x3c, 0xcc, 0x4a, 0x10, + 0x89, 0x05, 0xdb, 0x38, 0x61, 0x89, 0x90, 0x50 }, + { 0xa5, 0x7b, 0xe6, 0xae, 0x67, 0x56, 0xf2, 0x8b, + 0x02, 0xf5, 0x9d, 0xad, 0xf7, 0xe0, 0xd7, 0xd8, + 0x80, 0x7f, 0x10, 0xfa, 0x15, 0xce, 0xd1, 0xad, + 0x35, 0x85, 0x52, 0x1a, 0x1d, 0x99, 0x5a, 0x89 }, + { 0x81, 0x6a, 0xef, 0x87, 0x59, 0x53, 0x71, 0x6c, + 0xd7, 0xa5, 0x81, 0xf7, 0x32, 0xf5, 0x3d, 0xd4, + 0x35, 0xda, 0xb6, 0x6d, 0x09, 0xc3, 0x61, 0xd2, + 0xd6, 0x59, 0x2d, 0xe1, 0x77, 0x55, 0xd8, 0xa8 }, + { 0x9a, 0x76, 0x89, 0x32, 0x26, 0x69, 0x3b, 0x6e, + 0xa9, 0x7e, 0x6a, 0x73, 0x8f, 0x9d, 0x10, 0xfb, + 0x3d, 0x0b, 0x43, 0xae, 0x0e, 0x8b, 0x7d, 0x81, + 0x23, 0xea, 0x76, 0xce, 0x97, 0x98, 0x9c, 0x7e }, + { 0x8d, 0xae, 0xdb, 0x9a, 0x27, 0x15, 0x29, 0xdb, + 0xb7, 0xdc, 0x3b, 0x60, 0x7f, 0xe5, 0xeb, 0x2d, + 0x32, 0x11, 0x77, 0x07, 0x58, 0xdd, 0x3b, 0x0a, + 0x35, 0x93, 0xd2, 0xd7, 0x95, 0x4e, 0x2d, 0x5b }, + { 0x16, 0xdb, 0xc0, 0xaa, 0x5d, 0xd2, 0xc7, 0x74, + 0xf5, 0x05, 0x10, 0x0f, 0x73, 0x37, 0x86, 0xd8, + 0xa1, 0x75, 0xfc, 0xbb, 0xb5, 0x9c, 0x43, 0xe1, + 0xfb, 0xff, 0x3e, 0x1e, 0xaf, 0x31, 0xcb, 0x4a }, + { 0x86, 0x06, 0xcb, 0x89, 0x9c, 0x6a, 0xea, 0xf5, + 0x1b, 0x9d, 0xb0, 0xfe, 0x49, 0x24, 0xa9, 0xfd, + 0x5d, 0xab, 0xc1, 0x9f, 0x88, 0x26, 0xf2, 0xbc, + 0x1c, 0x1d, 0x7d, 0xa1, 0x4d, 0x2c, 0x2c, 0x99 }, + { 0x84, 0x79, 0x73, 0x1a, 0xed, 0xa5, 0x7b, 0xd3, + 0x7e, 0xad, 0xb5, 0x1a, 0x50, 0x7e, 0x30, 0x7f, + 0x3b, 0xd9, 0x5e, 0x69, 0xdb, 0xca, 0x94, 0xf3, + 0xbc, 0x21, 0x72, 0x60, 0x66, 0xad, 0x6d, 0xfd }, + { 0x58, 0x47, 0x3a, 0x9e, 0xa8, 0x2e, 0xfa, 0x3f, + 0x3b, 0x3d, 0x8f, 0xc8, 0x3e, 0xd8, 0x86, 0x31, + 0x27, 0xb3, 0x3a, 0xe8, 0xde, 0xae, 0x63, 0x07, + 0x20, 0x1e, 0xdb, 0x6d, 0xde, 0x61, 0xde, 0x29 }, + { 0x9a, 0x92, 0x55, 0xd5, 0x3a, 0xf1, 0x16, 0xde, + 0x8b, 0xa2, 0x7c, 0xe3, 0x5b, 0x4c, 0x7e, 0x15, + 0x64, 0x06, 0x57, 0xa0, 0xfc, 0xb8, 0x88, 0xc7, + 0x0d, 0x95, 0x43, 0x1d, 0xac, 0xd8, 0xf8, 0x30 }, + { 0x9e, 0xb0, 0x5f, 0xfb, 0xa3, 0x9f, 0xd8, 0x59, + 0x6a, 0x45, 0x49, 0x3e, 0x18, 0xd2, 0x51, 0x0b, + 0xf3, 0xef, 0x06, 0x5c, 0x51, 0xd6, 0xe1, 0x3a, + 0xbe, 0x66, 0xaa, 0x57, 0xe0, 0x5c, 0xfd, 0xb7 }, + { 0x81, 0xdc, 0xc3, 0xa5, 0x05, 0xea, 0xce, 0x3f, + 0x87, 0x9d, 0x8f, 0x70, 0x27, 0x76, 0x77, 0x0f, + 0x9d, 0xf5, 0x0e, 0x52, 0x1d, 0x14, 0x28, 0xa8, + 0x5d, 0xaf, 0x04, 0xf9, 0xad, 0x21, 0x50, 0xe0 }, + { 0xe3, 0xe3, 0xc4, 0xaa, 0x3a, 0xcb, 0xbc, 0x85, + 0x33, 0x2a, 0xf9, 0xd5, 0x64, 0xbc, 0x24, 0x16, + 0x5e, 0x16, 0x87, 0xf6, 0xb1, 0xad, 0xcb, 0xfa, + 0xe7, 0x7a, 0x8f, 0x03, 0xc7, 0x2a, 0xc2, 0x8c }, + { 0x67, 0x46, 0xc8, 0x0b, 0x4e, 0xb5, 0x6a, 0xea, + 0x45, 0xe6, 0x4e, 0x72, 0x89, 0xbb, 0xa3, 0xed, + 0xbf, 0x45, 0xec, 0xf8, 0x20, 0x64, 0x81, 0xff, + 0x63, 0x02, 0x12, 0x29, 0x84, 0xcd, 0x52, 0x6a }, + { 0x2b, 0x62, 0x8e, 0x52, 0x76, 0x4d, 0x7d, 0x62, + 0xc0, 0x86, 0x8b, 0x21, 0x23, 0x57, 0xcd, 0xd1, + 0x2d, 0x91, 0x49, 0x82, 0x2f, 0x4e, 0x98, 0x45, + 0xd9, 0x18, 0xa0, 0x8d, 0x1a, 0xe9, 0x90, 0xc0 }, + { 0xe4, 0xbf, 0xe8, 0x0d, 0x58, 0xc9, 0x19, 0x94, + 0x61, 0x39, 0x09, 0xdc, 0x4b, 0x1a, 0x12, 0x49, + 0x68, 0x96, 0xc0, 0x04, 0xaf, 0x7b, 0x57, 0x01, + 0x48, 0x3d, 0xe4, 0x5d, 0x28, 0x23, 0xd7, 0x8e }, + { 0xeb, 0xb4, 0xba, 0x15, 0x0c, 0xef, 0x27, 0x34, + 0x34, 0x5b, 0x5d, 0x64, 0x1b, 0xbe, 0xd0, 0x3a, + 0x21, 0xea, 0xfa, 0xe9, 0x33, 0xc9, 0x9e, 0x00, + 0x92, 0x12, 0xef, 0x04, 0x57, 0x4a, 0x85, 0x30 }, + { 0x39, 0x66, 0xec, 0x73, 0xb1, 0x54, 0xac, 0xc6, + 0x97, 0xac, 0x5c, 0xf5, 0xb2, 0x4b, 0x40, 0xbd, + 0xb0, 0xdb, 0x9e, 0x39, 0x88, 0x36, 0xd7, 0x6d, + 0x4b, 0x88, 0x0e, 0x3b, 0x2a, 0xf1, 0xaa, 0x27 }, + { 0xef, 0x7e, 0x48, 0x31, 0xb3, 0xa8, 0x46, 0x36, + 0x51, 0x8d, 0x6e, 0x4b, 0xfc, 0xe6, 0x4a, 0x43, + 0xdb, 0x2a, 0x5d, 0xda, 0x9c, 0xca, 0x2b, 0x44, + 0xf3, 0x90, 0x33, 0xbd, 0xc4, 0x0d, 0x62, 0x43 }, + { 0x7a, 0xbf, 0x6a, 0xcf, 0x5c, 0x8e, 0x54, 0x9d, + 0xdb, 0xb1, 0x5a, 0xe8, 0xd8, 0xb3, 0x88, 0xc1, + 0xc1, 0x97, 0xe6, 0x98, 0x73, 0x7c, 0x97, 0x85, + 0x50, 0x1e, 0xd1, 0xf9, 0x49, 0x30, 0xb7, 0xd9 }, + { 0x88, 0x01, 0x8d, 0xed, 0x66, 0x81, 0x3f, 0x0c, + 0xa9, 0x5d, 0xef, 0x47, 0x4c, 0x63, 0x06, 0x92, + 0x01, 0x99, 0x67, 0xb9, 0xe3, 0x68, 0x88, 0xda, + 0xdd, 0x94, 0x12, 0x47, 0x19, 0xb6, 0x82, 0xf6 }, + { 0x39, 0x30, 0x87, 0x6b, 0x9f, 0xc7, 0x52, 0x90, + 0x36, 0xb0, 0x08, 0xb1, 0xb8, 0xbb, 0x99, 0x75, + 0x22, 0xa4, 0x41, 0x63, 0x5a, 0x0c, 0x25, 0xec, + 0x02, 0xfb, 0x6d, 0x90, 0x26, 0xe5, 0x5a, 0x97 }, + { 0x0a, 0x40, 0x49, 0xd5, 0x7e, 0x83, 0x3b, 0x56, + 0x95, 0xfa, 0xc9, 0x3d, 0xd1, 0xfb, 0xef, 0x31, + 0x66, 0xb4, 0x4b, 0x12, 0xad, 0x11, 0x24, 0x86, + 0x62, 0x38, 0x3a, 0xe0, 0x51, 0xe1, 0x58, 0x27 }, + { 0x81, 0xdc, 0xc0, 0x67, 0x8b, 0xb6, 0xa7, 0x65, + 0xe4, 0x8c, 0x32, 0x09, 0x65, 0x4f, 0xe9, 0x00, + 0x89, 0xce, 0x44, 0xff, 0x56, 0x18, 0x47, 0x7e, + 0x39, 0xab, 0x28, 0x64, 0x76, 0xdf, 0x05, 0x2b }, + { 0xe6, 0x9b, 0x3a, 0x36, 0xa4, 0x46, 0x19, 0x12, + 0xdc, 0x08, 0x34, 0x6b, 0x11, 0xdd, 0xcb, 0x9d, + 0xb7, 0x96, 0xf8, 0x85, 0xfd, 0x01, 0x93, 0x6e, + 0x66, 0x2f, 0xe2, 0x92, 0x97, 0xb0, 0x99, 0xa4 }, + { 0x5a, 0xc6, 0x50, 0x3b, 0x0d, 0x8d, 0xa6, 0x91, + 0x76, 0x46, 0xe6, 0xdc, 0xc8, 0x7e, 0xdc, 0x58, + 0xe9, 0x42, 0x45, 0x32, 0x4c, 0xc2, 0x04, 0xf4, + 0xdd, 0x4a, 0xf0, 0x15, 0x63, 0xac, 0xd4, 0x27 }, + { 0xdf, 0x6d, 0xda, 0x21, 0x35, 0x9a, 0x30, 0xbc, + 0x27, 0x17, 0x80, 0x97, 0x1c, 0x1a, 0xbd, 0x56, + 0xa6, 0xef, 0x16, 0x7e, 0x48, 0x08, 0x87, 0x88, + 0x8e, 0x73, 0xa8, 0x6d, 0x3b, 0xf6, 0x05, 0xe9 }, + { 0xe8, 0xe6, 0xe4, 0x70, 0x71, 0xe7, 0xb7, 0xdf, + 0x25, 0x80, 0xf2, 0x25, 0xcf, 0xbb, 0xed, 0xf8, + 0x4c, 0xe6, 0x77, 0x46, 0x62, 0x66, 0x28, 0xd3, + 0x30, 0x97, 0xe4, 0xb7, 0xdc, 0x57, 0x11, 0x07 }, + { 0x53, 0xe4, 0x0e, 0xad, 0x62, 0x05, 0x1e, 0x19, + 0xcb, 0x9b, 0xa8, 0x13, 0x3e, 0x3e, 0x5c, 0x1c, + 0xe0, 0x0d, 0xdc, 0xad, 0x8a, 0xcf, 0x34, 0x2a, + 0x22, 0x43, 0x60, 0xb0, 0xac, 0xc1, 0x47, 0x77 }, + { 0x9c, 0xcd, 0x53, 0xfe, 0x80, 0xbe, 0x78, 0x6a, + 0xa9, 0x84, 0x63, 0x84, 0x62, 0xfb, 0x28, 0xaf, + 0xdf, 0x12, 0x2b, 0x34, 0xd7, 0x8f, 0x46, 0x87, + 0xec, 0x63, 0x2b, 0xb1, 0x9d, 0xe2, 0x37, 0x1a }, + { 0xcb, 0xd4, 0x80, 0x52, 0xc4, 0x8d, 0x78, 0x84, + 0x66, 0xa3, 0xe8, 0x11, 0x8c, 0x56, 0xc9, 0x7f, + 0xe1, 0x46, 0xe5, 0x54, 0x6f, 0xaa, 0xf9, 0x3e, + 0x2b, 0xc3, 0xc4, 0x7e, 0x45, 0x93, 0x97, 0x53 }, + { 0x25, 0x68, 0x83, 0xb1, 0x4e, 0x2a, 0xf4, 0x4d, + 0xad, 0xb2, 0x8e, 0x1b, 0x34, 0xb2, 0xac, 0x0f, + 0x0f, 0x4c, 0x91, 0xc3, 0x4e, 0xc9, 0x16, 0x9e, + 0x29, 0x03, 0x61, 0x58, 0xac, 0xaa, 0x95, 0xb9 }, + { 0x44, 0x71, 0xb9, 0x1a, 0xb4, 0x2d, 0xb7, 0xc4, + 0xdd, 0x84, 0x90, 0xab, 0x95, 0xa2, 0xee, 0x8d, + 0x04, 0xe3, 0xef, 0x5c, 0x3d, 0x6f, 0xc7, 0x1a, + 0xc7, 0x4b, 0x2b, 0x26, 0x91, 0x4d, 0x16, 0x41 }, + { 0xa5, 0xeb, 0x08, 0x03, 0x8f, 0x8f, 0x11, 0x55, + 0xed, 0x86, 0xe6, 0x31, 0x90, 0x6f, 0xc1, 0x30, + 0x95, 0xf6, 0xbb, 0xa4, 0x1d, 0xe5, 0xd4, 0xe7, + 0x95, 0x75, 0x8e, 0xc8, 0xc8, 0xdf, 0x8a, 0xf1 }, + { 0xdc, 0x1d, 0xb6, 0x4e, 0xd8, 0xb4, 0x8a, 0x91, + 0x0e, 0x06, 0x0a, 0x6b, 0x86, 0x63, 0x74, 0xc5, + 0x78, 0x78, 0x4e, 0x9a, 0xc4, 0x9a, 0xb2, 0x77, + 0x40, 0x92, 0xac, 0x71, 0x50, 0x19, 0x34, 0xac }, + { 0x28, 0x54, 0x13, 0xb2, 0xf2, 0xee, 0x87, 0x3d, + 0x34, 0x31, 0x9e, 0xe0, 0xbb, 0xfb, 0xb9, 0x0f, + 0x32, 0xda, 0x43, 0x4c, 0xc8, 0x7e, 0x3d, 0xb5, + 0xed, 0x12, 0x1b, 0xb3, 0x98, 0xed, 0x96, 0x4b }, + { 0x02, 0x16, 0xe0, 0xf8, 0x1f, 0x75, 0x0f, 0x26, + 0xf1, 0x99, 0x8b, 0xc3, 0x93, 0x4e, 0x3e, 0x12, + 0x4c, 0x99, 0x45, 0xe6, 0x85, 0xa6, 0x0b, 0x25, + 0xe8, 0xfb, 0xd9, 0x62, 0x5a, 0xb6, 0xb5, 0x99 }, + { 0x38, 0xc4, 0x10, 0xf5, 0xb9, 0xd4, 0x07, 0x20, + 0x50, 0x75, 0x5b, 0x31, 0xdc, 0xa8, 0x9f, 0xd5, + 0x39, 0x5c, 0x67, 0x85, 0xee, 0xb3, 0xd7, 0x90, + 0xf3, 0x20, 0xff, 0x94, 0x1c, 0x5a, 0x93, 0xbf }, + { 0xf1, 0x84, 0x17, 0xb3, 0x9d, 0x61, 0x7a, 0xb1, + 0xc1, 0x8f, 0xdf, 0x91, 0xeb, 0xd0, 0xfc, 0x6d, + 0x55, 0x16, 0xbb, 0x34, 0xcf, 0x39, 0x36, 0x40, + 0x37, 0xbc, 0xe8, 0x1f, 0xa0, 0x4c, 0xec, 0xb1 }, + { 0x1f, 0xa8, 0x77, 0xde, 0x67, 0x25, 0x9d, 0x19, + 0x86, 0x3a, 0x2a, 0x34, 0xbc, 0xc6, 0x96, 0x2a, + 0x2b, 0x25, 0xfc, 0xbf, 0x5c, 0xbe, 0xcd, 0x7e, + 0xde, 0x8f, 0x1f, 0xa3, 0x66, 0x88, 0xa7, 0x96 }, + { 0x5b, 0xd1, 0x69, 0xe6, 0x7c, 0x82, 0xc2, 0xc2, + 0xe9, 0x8e, 0xf7, 0x00, 0x8b, 0xdf, 0x26, 0x1f, + 0x2d, 0xdf, 0x30, 0xb1, 0xc0, 0x0f, 0x9e, 0x7f, + 0x27, 0x5b, 0xb3, 0xe8, 0xa2, 0x8d, 0xc9, 0xa2 }, + { 0xc8, 0x0a, 0xbe, 0xeb, 0xb6, 0x69, 0xad, 0x5d, + 0xee, 0xb5, 0xf5, 0xec, 0x8e, 0xa6, 0xb7, 0xa0, + 0x5d, 0xdf, 0x7d, 0x31, 0xec, 0x4c, 0x0a, 0x2e, + 0xe2, 0x0b, 0x0b, 0x98, 0xca, 0xec, 0x67, 0x46 }, + { 0xe7, 0x6d, 0x3f, 0xbd, 0xa5, 0xba, 0x37, 0x4e, + 0x6b, 0xf8, 0xe5, 0x0f, 0xad, 0xc3, 0xbb, 0xb9, + 0xba, 0x5c, 0x20, 0x6e, 0xbd, 0xec, 0x89, 0xa3, + 0xa5, 0x4c, 0xf3, 0xdd, 0x84, 0xa0, 0x70, 0x16 }, + { 0x7b, 0xba, 0x9d, 0xc5, 0xb5, 0xdb, 0x20, 0x71, + 0xd1, 0x77, 0x52, 0xb1, 0x04, 0x4c, 0x1e, 0xce, + 0xd9, 0x6a, 0xaf, 0x2d, 0xd4, 0x6e, 0x9b, 0x43, + 0x37, 0x50, 0xe8, 0xea, 0x0d, 0xcc, 0x18, 0x70 }, + { 0xf2, 0x9b, 0x1b, 0x1a, 0xb9, 0xba, 0xb1, 0x63, + 0x01, 0x8e, 0xe3, 0xda, 0x15, 0x23, 0x2c, 0xca, + 0x78, 0xec, 0x52, 0xdb, 0xc3, 0x4e, 0xda, 0x5b, + 0x82, 0x2e, 0xc1, 0xd8, 0x0f, 0xc2, 0x1b, 0xd0 }, + { 0x9e, 0xe3, 0xe3, 0xe7, 0xe9, 0x00, 0xf1, 0xe1, + 0x1d, 0x30, 0x8c, 0x4b, 0x2b, 0x30, 0x76, 0xd2, + 0x72, 0xcf, 0x70, 0x12, 0x4f, 0x9f, 0x51, 0xe1, + 0xda, 0x60, 0xf3, 0x78, 0x46, 0xcd, 0xd2, 0xf4 }, + { 0x70, 0xea, 0x3b, 0x01, 0x76, 0x92, 0x7d, 0x90, + 0x96, 0xa1, 0x85, 0x08, 0xcd, 0x12, 0x3a, 0x29, + 0x03, 0x25, 0x92, 0x0a, 0x9d, 0x00, 0xa8, 0x9b, + 0x5d, 0xe0, 0x42, 0x73, 0xfb, 0xc7, 0x6b, 0x85 }, + { 0x67, 0xde, 0x25, 0xc0, 0x2a, 0x4a, 0xab, 0xa2, + 0x3b, 0xdc, 0x97, 0x3c, 0x8b, 0xb0, 0xb5, 0x79, + 0x6d, 0x47, 0xcc, 0x06, 0x59, 0xd4, 0x3d, 0xff, + 0x1f, 0x97, 0xde, 0x17, 0x49, 0x63, 0xb6, 0x8e }, + { 0xb2, 0x16, 0x8e, 0x4e, 0x0f, 0x18, 0xb0, 0xe6, + 0x41, 0x00, 0xb5, 0x17, 0xed, 0x95, 0x25, 0x7d, + 0x73, 0xf0, 0x62, 0x0d, 0xf8, 0x85, 0xc1, 0x3d, + 0x2e, 0xcf, 0x79, 0x36, 0x7b, 0x38, 0x4c, 0xee }, + { 0x2e, 0x7d, 0xec, 0x24, 0x28, 0x85, 0x3b, 0x2c, + 0x71, 0x76, 0x07, 0x45, 0x54, 0x1f, 0x7a, 0xfe, + 0x98, 0x25, 0xb5, 0xdd, 0x77, 0xdf, 0x06, 0x51, + 0x1d, 0x84, 0x41, 0xa9, 0x4b, 0xac, 0xc9, 0x27 }, + { 0xca, 0x9f, 0xfa, 0xc4, 0xc4, 0x3f, 0x0b, 0x48, + 0x46, 0x1d, 0xc5, 0xc2, 0x63, 0xbe, 0xa3, 0xf6, + 0xf0, 0x06, 0x11, 0xce, 0xac, 0xab, 0xf6, 0xf8, + 0x95, 0xba, 0x2b, 0x01, 0x01, 0xdb, 0xb6, 0x8d }, + { 0x74, 0x10, 0xd4, 0x2d, 0x8f, 0xd1, 0xd5, 0xe9, + 0xd2, 0xf5, 0x81, 0x5c, 0xb9, 0x34, 0x17, 0x99, + 0x88, 0x28, 0xef, 0x3c, 0x42, 0x30, 0xbf, 0xbd, + 0x41, 0x2d, 0xf0, 0xa4, 0xa7, 0xa2, 0x50, 0x7a }, + { 0x50, 0x10, 0xf6, 0x84, 0x51, 0x6d, 0xcc, 0xd0, + 0xb6, 0xee, 0x08, 0x52, 0xc2, 0x51, 0x2b, 0x4d, + 0xc0, 0x06, 0x6c, 0xf0, 0xd5, 0x6f, 0x35, 0x30, + 0x29, 0x78, 0xdb, 0x8a, 0xe3, 0x2c, 0x6a, 0x81 }, + { 0xac, 0xaa, 0xb5, 0x85, 0xf7, 0xb7, 0x9b, 0x71, + 0x99, 0x35, 0xce, 0xb8, 0x95, 0x23, 0xdd, 0xc5, + 0x48, 0x27, 0xf7, 0x5c, 0x56, 0x88, 0x38, 0x56, + 0x15, 0x4a, 0x56, 0xcd, 0xcd, 0x5e, 0xe9, 0x88 }, + { 0x66, 0x6d, 0xe5, 0xd1, 0x44, 0x0f, 0xee, 0x73, + 0x31, 0xaa, 0xf0, 0x12, 0x3a, 0x62, 0xef, 0x2d, + 0x8b, 0xa5, 0x74, 0x53, 0xa0, 0x76, 0x96, 0x35, + 0xac, 0x6c, 0xd0, 0x1e, 0x63, 0x3f, 0x77, 0x12 }, + { 0xa6, 0xf9, 0x86, 0x58, 0xf6, 0xea, 0xba, 0xf9, + 0x02, 0xd8, 0xb3, 0x87, 0x1a, 0x4b, 0x10, 0x1d, + 0x16, 0x19, 0x6e, 0x8a, 0x4b, 0x24, 0x1e, 0x15, + 0x58, 0xfe, 0x29, 0x96, 0x6e, 0x10, 0x3e, 0x8d }, + { 0x89, 0x15, 0x46, 0xa8, 0xb2, 0x9f, 0x30, 0x47, + 0xdd, 0xcf, 0xe5, 0xb0, 0x0e, 0x45, 0xfd, 0x55, + 0x75, 0x63, 0x73, 0x10, 0x5e, 0xa8, 0x63, 0x7d, + 0xfc, 0xff, 0x54, 0x7b, 0x6e, 0xa9, 0x53, 0x5f }, + { 0x18, 0xdf, 0xbc, 0x1a, 0xc5, 0xd2, 0x5b, 0x07, + 0x61, 0x13, 0x7d, 0xbd, 0x22, 0xc1, 0x7c, 0x82, + 0x9d, 0x0f, 0x0e, 0xf1, 0xd8, 0x23, 0x44, 0xe9, + 0xc8, 0x9c, 0x28, 0x66, 0x94, 0xda, 0x24, 0xe8 }, + { 0xb5, 0x4b, 0x9b, 0x67, 0xf8, 0xfe, 0xd5, 0x4b, + 0xbf, 0x5a, 0x26, 0x66, 0xdb, 0xdf, 0x4b, 0x23, + 0xcf, 0xf1, 0xd1, 0xb6, 0xf4, 0xaf, 0xc9, 0x85, + 0xb2, 0xe6, 0xd3, 0x30, 0x5a, 0x9f, 0xf8, 0x0f }, + { 0x7d, 0xb4, 0x42, 0xe1, 0x32, 0xba, 0x59, 0xbc, + 0x12, 0x89, 0xaa, 0x98, 0xb0, 0xd3, 0xe8, 0x06, + 0x00, 0x4f, 0x8e, 0xc1, 0x28, 0x11, 0xaf, 0x1e, + 0x2e, 0x33, 0xc6, 0x9b, 0xfd, 0xe7, 0x29, 0xe1 }, + { 0x25, 0x0f, 0x37, 0xcd, 0xc1, 0x5e, 0x81, 0x7d, + 0x2f, 0x16, 0x0d, 0x99, 0x56, 0xc7, 0x1f, 0xe3, + 0xeb, 0x5d, 0xb7, 0x45, 0x56, 0xe4, 0xad, 0xf9, + 0xa4, 0xff, 0xaf, 0xba, 0x74, 0x01, 0x03, 0x96 }, + { 0x4a, 0xb8, 0xa3, 0xdd, 0x1d, 0xdf, 0x8a, 0xd4, + 0x3d, 0xab, 0x13, 0xa2, 0x7f, 0x66, 0xa6, 0x54, + 0x4f, 0x29, 0x05, 0x97, 0xfa, 0x96, 0x04, 0x0e, + 0x0e, 0x1d, 0xb9, 0x26, 0x3a, 0xa4, 0x79, 0xf8 }, + { 0xee, 0x61, 0x72, 0x7a, 0x07, 0x66, 0xdf, 0x93, + 0x9c, 0xcd, 0xc8, 0x60, 0x33, 0x40, 0x44, 0xc7, + 0x9a, 0x3c, 0x9b, 0x15, 0x62, 0x00, 0xbc, 0x3a, + 0xa3, 0x29, 0x73, 0x48, 0x3d, 0x83, 0x41, 0xae }, + { 0x3f, 0x68, 0xc7, 0xec, 0x63, 0xac, 0x11, 0xeb, + 0xb9, 0x8f, 0x94, 0xb3, 0x39, 0xb0, 0x5c, 0x10, + 0x49, 0x84, 0xfd, 0xa5, 0x01, 0x03, 0x06, 0x01, + 0x44, 0xe5, 0xa2, 0xbf, 0xcc, 0xc9, 0xda, 0x95 }, + { 0x05, 0x6f, 0x29, 0x81, 0x6b, 0x8a, 0xf8, 0xf5, + 0x66, 0x82, 0xbc, 0x4d, 0x7c, 0xf0, 0x94, 0x11, + 0x1d, 0xa7, 0x73, 0x3e, 0x72, 0x6c, 0xd1, 0x3d, + 0x6b, 0x3e, 0x8e, 0xa0, 0x3e, 0x92, 0xa0, 0xd5 }, + { 0xf5, 0xec, 0x43, 0xa2, 0x8a, 0xcb, 0xef, 0xf1, + 0xf3, 0x31, 0x8a, 0x5b, 0xca, 0xc7, 0xc6, 0x6d, + 0xdb, 0x52, 0x30, 0xb7, 0x9d, 0xb2, 0xd1, 0x05, + 0xbc, 0xbe, 0x15, 0xf3, 0xc1, 0x14, 0x8d, 0x69 }, + { 0x2a, 0x69, 0x60, 0xad, 0x1d, 0x8d, 0xd5, 0x47, + 0x55, 0x5c, 0xfb, 0xd5, 0xe4, 0x60, 0x0f, 0x1e, + 0xaa, 0x1c, 0x8e, 0xda, 0x34, 0xde, 0x03, 0x74, + 0xec, 0x4a, 0x26, 0xea, 0xaa, 0xa3, 0x3b, 0x4e }, + { 0xdc, 0xc1, 0xea, 0x7b, 0xaa, 0xb9, 0x33, 0x84, + 0xf7, 0x6b, 0x79, 0x68, 0x66, 0x19, 0x97, 0x54, + 0x74, 0x2f, 0x7b, 0x96, 0xd6, 0xb4, 0xc1, 0x20, + 0x16, 0x5c, 0x04, 0xa6, 0xc4, 0xf5, 0xce, 0x10 }, + { 0x13, 0xd5, 0xdf, 0x17, 0x92, 0x21, 0x37, 0x9c, + 0x6a, 0x78, 0xc0, 0x7c, 0x79, 0x3f, 0xf5, 0x34, + 0x87, 0xca, 0xe6, 0xbf, 0x9f, 0xe8, 0x82, 0x54, + 0x1a, 0xb0, 0xe7, 0x35, 0xe3, 0xea, 0xda, 0x3b }, + { 0x8c, 0x59, 0xe4, 0x40, 0x76, 0x41, 0xa0, 0x1e, + 0x8f, 0xf9, 0x1f, 0x99, 0x80, 0xdc, 0x23, 0x6f, + 0x4e, 0xcd, 0x6f, 0xcf, 0x52, 0x58, 0x9a, 0x09, + 0x9a, 0x96, 0x16, 0x33, 0x96, 0x77, 0x14, 0xe1 }, + { 0x83, 0x3b, 0x1a, 0xc6, 0xa2, 0x51, 0xfd, 0x08, + 0xfd, 0x6d, 0x90, 0x8f, 0xea, 0x2a, 0x4e, 0xe1, + 0xe0, 0x40, 0xbc, 0xa9, 0x3f, 0xc1, 0xa3, 0x8e, + 0xc3, 0x82, 0x0e, 0x0c, 0x10, 0xbd, 0x82, 0xea }, + { 0xa2, 0x44, 0xf9, 0x27, 0xf3, 0xb4, 0x0b, 0x8f, + 0x6c, 0x39, 0x15, 0x70, 0xc7, 0x65, 0x41, 0x8f, + 0x2f, 0x6e, 0x70, 0x8e, 0xac, 0x90, 0x06, 0xc5, + 0x1a, 0x7f, 0xef, 0xf4, 0xaf, 0x3b, 0x2b, 0x9e }, + { 0x3d, 0x99, 0xed, 0x95, 0x50, 0xcf, 0x11, 0x96, + 0xe6, 0xc4, 0xd2, 0x0c, 0x25, 0x96, 0x20, 0xf8, + 0x58, 0xc3, 0xd7, 0x03, 0x37, 0x4c, 0x12, 0x8c, + 0xe7, 0xb5, 0x90, 0x31, 0x0c, 0x83, 0x04, 0x6d }, + { 0x2b, 0x35, 0xc4, 0x7d, 0x7b, 0x87, 0x76, 0x1f, + 0x0a, 0xe4, 0x3a, 0xc5, 0x6a, 0xc2, 0x7b, 0x9f, + 0x25, 0x83, 0x03, 0x67, 0xb5, 0x95, 0xbe, 0x8c, + 0x24, 0x0e, 0x94, 0x60, 0x0c, 0x6e, 0x33, 0x12 }, + { 0x5d, 0x11, 0xed, 0x37, 0xd2, 0x4d, 0xc7, 0x67, + 0x30, 0x5c, 0xb7, 0xe1, 0x46, 0x7d, 0x87, 0xc0, + 0x65, 0xac, 0x4b, 0xc8, 0xa4, 0x26, 0xde, 0x38, + 0x99, 0x1f, 0xf5, 0x9a, 0xa8, 0x73, 0x5d, 0x02 }, + { 0xb8, 0x36, 0x47, 0x8e, 0x1c, 0xa0, 0x64, 0x0d, + 0xce, 0x6f, 0xd9, 0x10, 0xa5, 0x09, 0x62, 0x72, + 0xc8, 0x33, 0x09, 0x90, 0xcd, 0x97, 0x86, 0x4a, + 0xc2, 0xbf, 0x14, 0xef, 0x6b, 0x23, 0x91, 0x4a }, + { 0x91, 0x00, 0xf9, 0x46, 0xd6, 0xcc, 0xde, 0x3a, + 0x59, 0x7f, 0x90, 0xd3, 0x9f, 0xc1, 0x21, 0x5b, + 0xad, 0xdc, 0x74, 0x13, 0x64, 0x3d, 0x85, 0xc2, + 0x1c, 0x3e, 0xee, 0x5d, 0x2d, 0xd3, 0x28, 0x94 }, + { 0xda, 0x70, 0xee, 0xdd, 0x23, 0xe6, 0x63, 0xaa, + 0x1a, 0x74, 0xb9, 0x76, 0x69, 0x35, 0xb4, 0x79, + 0x22, 0x2a, 0x72, 0xaf, 0xba, 0x5c, 0x79, 0x51, + 0x58, 0xda, 0xd4, 0x1a, 0x3b, 0xd7, 0x7e, 0x40 }, + { 0xf0, 0x67, 0xed, 0x6a, 0x0d, 0xbd, 0x43, 0xaa, + 0x0a, 0x92, 0x54, 0xe6, 0x9f, 0xd6, 0x6b, 0xdd, + 0x8a, 0xcb, 0x87, 0xde, 0x93, 0x6c, 0x25, 0x8c, + 0xfb, 0x02, 0x28, 0x5f, 0x2c, 0x11, 0xfa, 0x79 }, + { 0x71, 0x5c, 0x99, 0xc7, 0xd5, 0x75, 0x80, 0xcf, + 0x97, 0x53, 0xb4, 0xc1, 0xd7, 0x95, 0xe4, 0x5a, + 0x83, 0xfb, 0xb2, 0x28, 0xc0, 0xd3, 0x6f, 0xbe, + 0x20, 0xfa, 0xf3, 0x9b, 0xdd, 0x6d, 0x4e, 0x85 }, + { 0xe4, 0x57, 0xd6, 0xad, 0x1e, 0x67, 0xcb, 0x9b, + 0xbd, 0x17, 0xcb, 0xd6, 0x98, 0xfa, 0x6d, 0x7d, + 0xae, 0x0c, 0x9b, 0x7a, 0xd6, 0xcb, 0xd6, 0x53, + 0x96, 0x34, 0xe3, 0x2a, 0x71, 0x9c, 0x84, 0x92 }, + { 0xec, 0xe3, 0xea, 0x81, 0x03, 0xe0, 0x24, 0x83, + 0xc6, 0x4a, 0x70, 0xa4, 0xbd, 0xce, 0xe8, 0xce, + 0xb6, 0x27, 0x8f, 0x25, 0x33, 0xf3, 0xf4, 0x8d, + 0xbe, 0xed, 0xfb, 0xa9, 0x45, 0x31, 0xd4, 0xae }, + { 0x38, 0x8a, 0xa5, 0xd3, 0x66, 0x7a, 0x97, 0xc6, + 0x8d, 0x3d, 0x56, 0xf8, 0xf3, 0xee, 0x8d, 0x3d, + 0x36, 0x09, 0x1f, 0x17, 0xfe, 0x5d, 0x1b, 0x0d, + 0x5d, 0x84, 0xc9, 0x3b, 0x2f, 0xfe, 0x40, 0xbd }, + { 0x8b, 0x6b, 0x31, 0xb9, 0xad, 0x7c, 0x3d, 0x5c, + 0xd8, 0x4b, 0xf9, 0x89, 0x47, 0xb9, 0xcd, 0xb5, + 0x9d, 0xf8, 0xa2, 0x5f, 0xf7, 0x38, 0x10, 0x10, + 0x13, 0xbe, 0x4f, 0xd6, 0x5e, 0x1d, 0xd1, 0xa3 }, + { 0x06, 0x62, 0x91, 0xf6, 0xbb, 0xd2, 0x5f, 0x3c, + 0x85, 0x3d, 0xb7, 0xd8, 0xb9, 0x5c, 0x9a, 0x1c, + 0xfb, 0x9b, 0xf1, 0xc1, 0xc9, 0x9f, 0xb9, 0x5a, + 0x9b, 0x78, 0x69, 0xd9, 0x0f, 0x1c, 0x29, 0x03 }, + { 0xa7, 0x07, 0xef, 0xbc, 0xcd, 0xce, 0xed, 0x42, + 0x96, 0x7a, 0x66, 0xf5, 0x53, 0x9b, 0x93, 0xed, + 0x75, 0x60, 0xd4, 0x67, 0x30, 0x40, 0x16, 0xc4, + 0x78, 0x0d, 0x77, 0x55, 0xa5, 0x65, 0xd4, 0xc4 }, + { 0x38, 0xc5, 0x3d, 0xfb, 0x70, 0xbe, 0x7e, 0x79, + 0x2b, 0x07, 0xa6, 0xa3, 0x5b, 0x8a, 0x6a, 0x0a, + 0xba, 0x02, 0xc5, 0xc5, 0xf3, 0x8b, 0xaf, 0x5c, + 0x82, 0x3f, 0xdf, 0xd9, 0xe4, 0x2d, 0x65, 0x7e }, + { 0xf2, 0x91, 0x13, 0x86, 0x50, 0x1d, 0x9a, 0xb9, + 0xd7, 0x20, 0xcf, 0x8a, 0xd1, 0x05, 0x03, 0xd5, + 0x63, 0x4b, 0xf4, 0xb7, 0xd1, 0x2b, 0x56, 0xdf, + 0xb7, 0x4f, 0xec, 0xc6, 0xe4, 0x09, 0x3f, 0x68 }, + { 0xc6, 0xf2, 0xbd, 0xd5, 0x2b, 0x81, 0xe6, 0xe4, + 0xf6, 0x59, 0x5a, 0xbd, 0x4d, 0x7f, 0xb3, 0x1f, + 0x65, 0x11, 0x69, 0xd0, 0x0f, 0xf3, 0x26, 0x92, + 0x6b, 0x34, 0x94, 0x7b, 0x28, 0xa8, 0x39, 0x59 }, + { 0x29, 0x3d, 0x94, 0xb1, 0x8c, 0x98, 0xbb, 0x32, + 0x23, 0x36, 0x6b, 0x8c, 0xe7, 0x4c, 0x28, 0xfb, + 0xdf, 0x28, 0xe1, 0xf8, 0x4a, 0x33, 0x50, 0xb0, + 0xeb, 0x2d, 0x18, 0x04, 0xa5, 0x77, 0x57, 0x9b }, + { 0x2c, 0x2f, 0xa5, 0xc0, 0xb5, 0x15, 0x33, 0x16, + 0x5b, 0xc3, 0x75, 0xc2, 0x2e, 0x27, 0x81, 0x76, + 0x82, 0x70, 0xa3, 0x83, 0x98, 0x5d, 0x13, 0xbd, + 0x6b, 0x67, 0xb6, 0xfd, 0x67, 0xf8, 0x89, 0xeb }, + { 0xca, 0xa0, 0x9b, 0x82, 0xb7, 0x25, 0x62, 0xe4, + 0x3f, 0x4b, 0x22, 0x75, 0xc0, 0x91, 0x91, 0x8e, + 0x62, 0x4d, 0x91, 0x16, 0x61, 0xcc, 0x81, 0x1b, + 0xb5, 0xfa, 0xec, 0x51, 0xf6, 0x08, 0x8e, 0xf7 }, + { 0x24, 0x76, 0x1e, 0x45, 0xe6, 0x74, 0x39, 0x53, + 0x79, 0xfb, 0x17, 0x72, 0x9c, 0x78, 0xcb, 0x93, + 0x9e, 0x6f, 0x74, 0xc5, 0xdf, 0xfb, 0x9c, 0x96, + 0x1f, 0x49, 0x59, 0x82, 0xc3, 0xed, 0x1f, 0xe3 }, + { 0x55, 0xb7, 0x0a, 0x82, 0x13, 0x1e, 0xc9, 0x48, + 0x88, 0xd7, 0xab, 0x54, 0xa7, 0xc5, 0x15, 0x25, + 0x5c, 0x39, 0x38, 0xbb, 0x10, 0xbc, 0x78, 0x4d, + 0xc9, 0xb6, 0x7f, 0x07, 0x6e, 0x34, 0x1a, 0x73 }, + { 0x6a, 0xb9, 0x05, 0x7b, 0x97, 0x7e, 0xbc, 0x3c, + 0xa4, 0xd4, 0xce, 0x74, 0x50, 0x6c, 0x25, 0xcc, + 0xcd, 0xc5, 0x66, 0x49, 0x7c, 0x45, 0x0b, 0x54, + 0x15, 0xa3, 0x94, 0x86, 0xf8, 0x65, 0x7a, 0x03 }, + { 0x24, 0x06, 0x6d, 0xee, 0xe0, 0xec, 0xee, 0x15, + 0xa4, 0x5f, 0x0a, 0x32, 0x6d, 0x0f, 0x8d, 0xbc, + 0x79, 0x76, 0x1e, 0xbb, 0x93, 0xcf, 0x8c, 0x03, + 0x77, 0xaf, 0x44, 0x09, 0x78, 0xfc, 0xf9, 0x94 }, + { 0x20, 0x00, 0x0d, 0x3f, 0x66, 0xba, 0x76, 0x86, + 0x0d, 0x5a, 0x95, 0x06, 0x88, 0xb9, 0xaa, 0x0d, + 0x76, 0xcf, 0xea, 0x59, 0xb0, 0x05, 0xd8, 0x59, + 0x91, 0x4b, 0x1a, 0x46, 0x65, 0x3a, 0x93, 0x9b }, + { 0xb9, 0x2d, 0xaa, 0x79, 0x60, 0x3e, 0x3b, 0xdb, + 0xc3, 0xbf, 0xe0, 0xf4, 0x19, 0xe4, 0x09, 0xb2, + 0xea, 0x10, 0xdc, 0x43, 0x5b, 0xee, 0xfe, 0x29, + 0x59, 0xda, 0x16, 0x89, 0x5d, 0x5d, 0xca, 0x1c }, + { 0xe9, 0x47, 0x94, 0x87, 0x05, 0xb2, 0x06, 0xd5, + 0x72, 0xb0, 0xe8, 0xf6, 0x2f, 0x66, 0xa6, 0x55, + 0x1c, 0xbd, 0x6b, 0xc3, 0x05, 0xd2, 0x6c, 0xe7, + 0x53, 0x9a, 0x12, 0xf9, 0xaa, 0xdf, 0x75, 0x71 }, + { 0x3d, 0x67, 0xc1, 0xb3, 0xf9, 0xb2, 0x39, 0x10, + 0xe3, 0xd3, 0x5e, 0x6b, 0x0f, 0x2c, 0xcf, 0x44, + 0xa0, 0xb5, 0x40, 0xa4, 0x5c, 0x18, 0xba, 0x3c, + 0x36, 0x26, 0x4d, 0xd4, 0x8e, 0x96, 0xaf, 0x6a }, + { 0xc7, 0x55, 0x8b, 0xab, 0xda, 0x04, 0xbc, 0xcb, + 0x76, 0x4d, 0x0b, 0xbf, 0x33, 0x58, 0x42, 0x51, + 0x41, 0x90, 0x2d, 0x22, 0x39, 0x1d, 0x9f, 0x8c, + 0x59, 0x15, 0x9f, 0xec, 0x9e, 0x49, 0xb1, 0x51 }, + { 0x0b, 0x73, 0x2b, 0xb0, 0x35, 0x67, 0x5a, 0x50, + 0xff, 0x58, 0xf2, 0xc2, 0x42, 0xe4, 0x71, 0x0a, + 0xec, 0xe6, 0x46, 0x70, 0x07, 0x9c, 0x13, 0x04, + 0x4c, 0x79, 0xc9, 0xb7, 0x49, 0x1f, 0x70, 0x00 }, + { 0xd1, 0x20, 0xb5, 0xef, 0x6d, 0x57, 0xeb, 0xf0, + 0x6e, 0xaf, 0x96, 0xbc, 0x93, 0x3c, 0x96, 0x7b, + 0x16, 0xcb, 0xe6, 0xe2, 0xbf, 0x00, 0x74, 0x1c, + 0x30, 0xaa, 0x1c, 0x54, 0xba, 0x64, 0x80, 0x1f }, + { 0x58, 0xd2, 0x12, 0xad, 0x6f, 0x58, 0xae, 0xf0, + 0xf8, 0x01, 0x16, 0xb4, 0x41, 0xe5, 0x7f, 0x61, + 0x95, 0xbf, 0xef, 0x26, 0xb6, 0x14, 0x63, 0xed, + 0xec, 0x11, 0x83, 0xcd, 0xb0, 0x4f, 0xe7, 0x6d }, + { 0xb8, 0x83, 0x6f, 0x51, 0xd1, 0xe2, 0x9b, 0xdf, + 0xdb, 0xa3, 0x25, 0x56, 0x53, 0x60, 0x26, 0x8b, + 0x8f, 0xad, 0x62, 0x74, 0x73, 0xed, 0xec, 0xef, + 0x7e, 0xae, 0xfe, 0xe8, 0x37, 0xc7, 0x40, 0x03 }, + { 0xc5, 0x47, 0xa3, 0xc1, 0x24, 0xae, 0x56, 0x85, + 0xff, 0xa7, 0xb8, 0xed, 0xaf, 0x96, 0xec, 0x86, + 0xf8, 0xb2, 0xd0, 0xd5, 0x0c, 0xee, 0x8b, 0xe3, + 0xb1, 0xf0, 0xc7, 0x67, 0x63, 0x06, 0x9d, 0x9c }, + { 0x5d, 0x16, 0x8b, 0x76, 0x9a, 0x2f, 0x67, 0x85, + 0x3d, 0x62, 0x95, 0xf7, 0x56, 0x8b, 0xe4, 0x0b, + 0xb7, 0xa1, 0x6b, 0x8d, 0x65, 0xba, 0x87, 0x63, + 0x5d, 0x19, 0x78, 0xd2, 0xab, 0x11, 0xba, 0x2a }, + { 0xa2, 0xf6, 0x75, 0xdc, 0x73, 0x02, 0x63, 0x8c, + 0xb6, 0x02, 0x01, 0x06, 0x4c, 0xa5, 0x50, 0x77, + 0x71, 0x4d, 0x71, 0xfe, 0x09, 0x6a, 0x31, 0x5f, + 0x2f, 0xe7, 0x40, 0x12, 0x77, 0xca, 0xa5, 0xaf }, + { 0xc8, 0xaa, 0xb5, 0xcd, 0x01, 0x60, 0xae, 0x78, + 0xcd, 0x2e, 0x8a, 0xc5, 0xfb, 0x0e, 0x09, 0x3c, + 0xdb, 0x5c, 0x4b, 0x60, 0x52, 0xa0, 0xa9, 0x7b, + 0xb0, 0x42, 0x16, 0x82, 0x6f, 0xa7, 0xa4, 0x37 }, + { 0xff, 0x68, 0xca, 0x40, 0x35, 0xbf, 0xeb, 0x43, + 0xfb, 0xf1, 0x45, 0xfd, 0xdd, 0x5e, 0x43, 0xf1, + 0xce, 0xa5, 0x4f, 0x11, 0xf7, 0xbe, 0xe1, 0x30, + 0x58, 0xf0, 0x27, 0x32, 0x9a, 0x4a, 0x5f, 0xa4 }, + { 0x1d, 0x4e, 0x54, 0x87, 0xae, 0x3c, 0x74, 0x0f, + 0x2b, 0xa6, 0xe5, 0x41, 0xac, 0x91, 0xbc, 0x2b, + 0xfc, 0xd2, 0x99, 0x9c, 0x51, 0x8d, 0x80, 0x7b, + 0x42, 0x67, 0x48, 0x80, 0x3a, 0x35, 0x0f, 0xd4 }, + { 0x6d, 0x24, 0x4e, 0x1a, 0x06, 0xce, 0x4e, 0xf5, + 0x78, 0xdd, 0x0f, 0x63, 0xaf, 0xf0, 0x93, 0x67, + 0x06, 0x73, 0x51, 0x19, 0xca, 0x9c, 0x8d, 0x22, + 0xd8, 0x6c, 0x80, 0x14, 0x14, 0xab, 0x97, 0x41 }, + { 0xde, 0xcf, 0x73, 0x29, 0xdb, 0xcc, 0x82, 0x7b, + 0x8f, 0xc5, 0x24, 0xc9, 0x43, 0x1e, 0x89, 0x98, + 0x02, 0x9e, 0xce, 0x12, 0xce, 0x93, 0xb7, 0xb2, + 0xf3, 0xe7, 0x69, 0xa9, 0x41, 0xfb, 0x8c, 0xea }, + { 0x2f, 0xaf, 0xcc, 0x0f, 0x2e, 0x63, 0xcb, 0xd0, + 0x77, 0x55, 0xbe, 0x7b, 0x75, 0xec, 0xea, 0x0a, + 0xdf, 0xf9, 0xaa, 0x5e, 0xde, 0x2a, 0x52, 0xfd, + 0xab, 0x4d, 0xfd, 0x03, 0x74, 0xcd, 0x48, 0x3f }, + { 0xaa, 0x85, 0x01, 0x0d, 0xd4, 0x6a, 0x54, 0x6b, + 0x53, 0x5e, 0xf4, 0xcf, 0x5f, 0x07, 0xd6, 0x51, + 0x61, 0xe8, 0x98, 0x28, 0xf3, 0xa7, 0x7d, 0xb7, + 0xb9, 0xb5, 0x6f, 0x0d, 0xf5, 0x9a, 0xae, 0x45 }, + { 0x07, 0xe8, 0xe1, 0xee, 0x73, 0x2c, 0xb0, 0xd3, + 0x56, 0xc9, 0xc0, 0xd1, 0x06, 0x9c, 0x89, 0xd1, + 0x7a, 0xdf, 0x6a, 0x9a, 0x33, 0x4f, 0x74, 0x5e, + 0xc7, 0x86, 0x73, 0x32, 0x54, 0x8c, 0xa8, 0xe9 }, + { 0x0e, 0x01, 0xe8, 0x1c, 0xad, 0xa8, 0x16, 0x2b, + 0xfd, 0x5f, 0x8a, 0x8c, 0x81, 0x8a, 0x6c, 0x69, + 0xfe, 0xdf, 0x02, 0xce, 0xb5, 0x20, 0x85, 0x23, + 0xcb, 0xe5, 0x31, 0x3b, 0x89, 0xca, 0x10, 0x53 }, + { 0x6b, 0xb6, 0xc6, 0x47, 0x26, 0x55, 0x08, 0x43, + 0x99, 0x85, 0x2e, 0x00, 0x24, 0x9f, 0x8c, 0xb2, + 0x47, 0x89, 0x6d, 0x39, 0x2b, 0x02, 0xd7, 0x3b, + 0x7f, 0x0d, 0xd8, 0x18, 0xe1, 0xe2, 0x9b, 0x07 }, + { 0x42, 0xd4, 0x63, 0x6e, 0x20, 0x60, 0xf0, 0x8f, + 0x41, 0xc8, 0x82, 0xe7, 0x6b, 0x39, 0x6b, 0x11, + 0x2e, 0xf6, 0x27, 0xcc, 0x24, 0xc4, 0x3d, 0xd5, + 0xf8, 0x3a, 0x1d, 0x1a, 0x7e, 0xad, 0x71, 0x1a }, + { 0x48, 0x58, 0xc9, 0xa1, 0x88, 0xb0, 0x23, 0x4f, + 0xb9, 0xa8, 0xd4, 0x7d, 0x0b, 0x41, 0x33, 0x65, + 0x0a, 0x03, 0x0b, 0xd0, 0x61, 0x1b, 0x87, 0xc3, + 0x89, 0x2e, 0x94, 0x95, 0x1f, 0x8d, 0xf8, 0x52 }, + { 0x3f, 0xab, 0x3e, 0x36, 0x98, 0x8d, 0x44, 0x5a, + 0x51, 0xc8, 0x78, 0x3e, 0x53, 0x1b, 0xe3, 0xa0, + 0x2b, 0xe4, 0x0c, 0xd0, 0x47, 0x96, 0xcf, 0xb6, + 0x1d, 0x40, 0x34, 0x74, 0x42, 0xd3, 0xf7, 0x94 }, + { 0xeb, 0xab, 0xc4, 0x96, 0x36, 0xbd, 0x43, 0x3d, + 0x2e, 0xc8, 0xf0, 0xe5, 0x18, 0x73, 0x2e, 0xf8, + 0xfa, 0x21, 0xd4, 0xd0, 0x71, 0xcc, 0x3b, 0xc4, + 0x6c, 0xd7, 0x9f, 0xa3, 0x8a, 0x28, 0xb8, 0x10 }, + { 0xa1, 0xd0, 0x34, 0x35, 0x23, 0xb8, 0x93, 0xfc, + 0xa8, 0x4f, 0x47, 0xfe, 0xb4, 0xa6, 0x4d, 0x35, + 0x0a, 0x17, 0xd8, 0xee, 0xf5, 0x49, 0x7e, 0xce, + 0x69, 0x7d, 0x02, 0xd7, 0x91, 0x78, 0xb5, 0x91 }, + { 0x26, 0x2e, 0xbf, 0xd9, 0x13, 0x0b, 0x7d, 0x28, + 0x76, 0x0d, 0x08, 0xef, 0x8b, 0xfd, 0x3b, 0x86, + 0xcd, 0xd3, 0xb2, 0x11, 0x3d, 0x2c, 0xae, 0xf7, + 0xea, 0x95, 0x1a, 0x30, 0x3d, 0xfa, 0x38, 0x46 }, + { 0xf7, 0x61, 0x58, 0xed, 0xd5, 0x0a, 0x15, 0x4f, + 0xa7, 0x82, 0x03, 0xed, 0x23, 0x62, 0x93, 0x2f, + 0xcb, 0x82, 0x53, 0xaa, 0xe3, 0x78, 0x90, 0x3e, + 0xde, 0xd1, 0xe0, 0x3f, 0x70, 0x21, 0xa2, 0x57 }, + { 0x26, 0x17, 0x8e, 0x95, 0x0a, 0xc7, 0x22, 0xf6, + 0x7a, 0xe5, 0x6e, 0x57, 0x1b, 0x28, 0x4c, 0x02, + 0x07, 0x68, 0x4a, 0x63, 0x34, 0xa1, 0x77, 0x48, + 0xa9, 0x4d, 0x26, 0x0b, 0xc5, 0xf5, 0x52, 0x74 }, + { 0xc3, 0x78, 0xd1, 0xe4, 0x93, 0xb4, 0x0e, 0xf1, + 0x1f, 0xe6, 0xa1, 0x5d, 0x9c, 0x27, 0x37, 0xa3, + 0x78, 0x09, 0x63, 0x4c, 0x5a, 0xba, 0xd5, 0xb3, + 0x3d, 0x7e, 0x39, 0x3b, 0x4a, 0xe0, 0x5d, 0x03 }, + { 0x98, 0x4b, 0xd8, 0x37, 0x91, 0x01, 0xbe, 0x8f, + 0xd8, 0x06, 0x12, 0xd8, 0xea, 0x29, 0x59, 0xa7, + 0x86, 0x5e, 0xc9, 0x71, 0x85, 0x23, 0x55, 0x01, + 0x07, 0xae, 0x39, 0x38, 0xdf, 0x32, 0x01, 0x1b }, + { 0xc6, 0xf2, 0x5a, 0x81, 0x2a, 0x14, 0x48, 0x58, + 0xac, 0x5c, 0xed, 0x37, 0xa9, 0x3a, 0x9f, 0x47, + 0x59, 0xba, 0x0b, 0x1c, 0x0f, 0xdc, 0x43, 0x1d, + 0xce, 0x35, 0xf9, 0xec, 0x1f, 0x1f, 0x4a, 0x99 }, + { 0x92, 0x4c, 0x75, 0xc9, 0x44, 0x24, 0xff, 0x75, + 0xe7, 0x4b, 0x8b, 0x4e, 0x94, 0x35, 0x89, 0x58, + 0xb0, 0x27, 0xb1, 0x71, 0xdf, 0x5e, 0x57, 0x89, + 0x9a, 0xd0, 0xd4, 0xda, 0xc3, 0x73, 0x53, 0xb6 }, + { 0x0a, 0xf3, 0x58, 0x92, 0xa6, 0x3f, 0x45, 0x93, + 0x1f, 0x68, 0x46, 0xed, 0x19, 0x03, 0x61, 0xcd, + 0x07, 0x30, 0x89, 0xe0, 0x77, 0x16, 0x57, 0x14, + 0xb5, 0x0b, 0x81, 0xa2, 0xe3, 0xdd, 0x9b, 0xa1 }, + { 0xcc, 0x80, 0xce, 0xfb, 0x26, 0xc3, 0xb2, 0xb0, + 0xda, 0xef, 0x23, 0x3e, 0x60, 0x6d, 0x5f, 0xfc, + 0x80, 0xfa, 0x17, 0x42, 0x7d, 0x18, 0xe3, 0x04, + 0x89, 0x67, 0x3e, 0x06, 0xef, 0x4b, 0x87, 0xf7 }, + { 0xc2, 0xf8, 0xc8, 0x11, 0x74, 0x47, 0xf3, 0x97, + 0x8b, 0x08, 0x18, 0xdc, 0xf6, 0xf7, 0x01, 0x16, + 0xac, 0x56, 0xfd, 0x18, 0x4d, 0xd1, 0x27, 0x84, + 0x94, 0xe1, 0x03, 0xfc, 0x6d, 0x74, 0xa8, 0x87 }, + { 0xbd, 0xec, 0xf6, 0xbf, 0xc1, 0xba, 0x0d, 0xf6, + 0xe8, 0x62, 0xc8, 0x31, 0x99, 0x22, 0x07, 0x79, + 0x6a, 0xcc, 0x79, 0x79, 0x68, 0x35, 0x88, 0x28, + 0xc0, 0x6e, 0x7a, 0x51, 0xe0, 0x90, 0x09, 0x8f }, + { 0x24, 0xd1, 0xa2, 0x6e, 0x3d, 0xab, 0x02, 0xfe, + 0x45, 0x72, 0xd2, 0xaa, 0x7d, 0xbd, 0x3e, 0xc3, + 0x0f, 0x06, 0x93, 0xdb, 0x26, 0xf2, 0x73, 0xd0, + 0xab, 0x2c, 0xb0, 0xc1, 0x3b, 0x5e, 0x64, 0x51 }, + { 0xec, 0x56, 0xf5, 0x8b, 0x09, 0x29, 0x9a, 0x30, + 0x0b, 0x14, 0x05, 0x65, 0xd7, 0xd3, 0xe6, 0x87, + 0x82, 0xb6, 0xe2, 0xfb, 0xeb, 0x4b, 0x7e, 0xa9, + 0x7a, 0xc0, 0x57, 0x98, 0x90, 0x61, 0xdd, 0x3f }, + { 0x11, 0xa4, 0x37, 0xc1, 0xab, 0xa3, 0xc1, 0x19, + 0xdd, 0xfa, 0xb3, 0x1b, 0x3e, 0x8c, 0x84, 0x1d, + 0xee, 0xeb, 0x91, 0x3e, 0xf5, 0x7f, 0x7e, 0x48, + 0xf2, 0xc9, 0xcf, 0x5a, 0x28, 0xfa, 0x42, 0xbc }, + { 0x53, 0xc7, 0xe6, 0x11, 0x4b, 0x85, 0x0a, 0x2c, + 0xb4, 0x96, 0xc9, 0xb3, 0xc6, 0x9a, 0x62, 0x3e, + 0xae, 0xa2, 0xcb, 0x1d, 0x33, 0xdd, 0x81, 0x7e, + 0x47, 0x65, 0xed, 0xaa, 0x68, 0x23, 0xc2, 0x28 }, + { 0x15, 0x4c, 0x3e, 0x96, 0xfe, 0xe5, 0xdb, 0x14, + 0xf8, 0x77, 0x3e, 0x18, 0xaf, 0x14, 0x85, 0x79, + 0x13, 0x50, 0x9d, 0xa9, 0x99, 0xb4, 0x6c, 0xdd, + 0x3d, 0x4c, 0x16, 0x97, 0x60, 0xc8, 0x3a, 0xd2 }, + { 0x40, 0xb9, 0x91, 0x6f, 0x09, 0x3e, 0x02, 0x7a, + 0x87, 0x86, 0x64, 0x18, 0x18, 0x92, 0x06, 0x20, + 0x47, 0x2f, 0xbc, 0xf6, 0x8f, 0x70, 0x1d, 0x1b, + 0x68, 0x06, 0x32, 0xe6, 0x99, 0x6b, 0xde, 0xd3 }, + { 0x24, 0xc4, 0xcb, 0xba, 0x07, 0x11, 0x98, 0x31, + 0xa7, 0x26, 0xb0, 0x53, 0x05, 0xd9, 0x6d, 0xa0, + 0x2f, 0xf8, 0xb1, 0x48, 0xf0, 0xda, 0x44, 0x0f, + 0xe2, 0x33, 0xbc, 0xaa, 0x32, 0xc7, 0x2f, 0x6f }, + { 0x5d, 0x20, 0x15, 0x10, 0x25, 0x00, 0x20, 0xb7, + 0x83, 0x68, 0x96, 0x88, 0xab, 0xbf, 0x8e, 0xcf, + 0x25, 0x94, 0xa9, 0x6a, 0x08, 0xf2, 0xbf, 0xec, + 0x6c, 0xe0, 0x57, 0x44, 0x65, 0xdd, 0xed, 0x71 }, + { 0x04, 0x3b, 0x97, 0xe3, 0x36, 0xee, 0x6f, 0xdb, + 0xbe, 0x2b, 0x50, 0xf2, 0x2a, 0xf8, 0x32, 0x75, + 0xa4, 0x08, 0x48, 0x05, 0xd2, 0xd5, 0x64, 0x59, + 0x62, 0x45, 0x4b, 0x6c, 0x9b, 0x80, 0x53, 0xa0 }, + { 0x56, 0x48, 0x35, 0xcb, 0xae, 0xa7, 0x74, 0x94, + 0x85, 0x68, 0xbe, 0x36, 0xcf, 0x52, 0xfc, 0xdd, + 0x83, 0x93, 0x4e, 0xb0, 0xa2, 0x75, 0x12, 0xdb, + 0xe3, 0xe2, 0xdb, 0x47, 0xb9, 0xe6, 0x63, 0x5a }, + { 0xf2, 0x1c, 0x33, 0xf4, 0x7b, 0xde, 0x40, 0xa2, + 0xa1, 0x01, 0xc9, 0xcd, 0xe8, 0x02, 0x7a, 0xaf, + 0x61, 0xa3, 0x13, 0x7d, 0xe2, 0x42, 0x2b, 0x30, + 0x03, 0x5a, 0x04, 0xc2, 0x70, 0x89, 0x41, 0x83 }, + { 0x9d, 0xb0, 0xef, 0x74, 0xe6, 0x6c, 0xbb, 0x84, + 0x2e, 0xb0, 0xe0, 0x73, 0x43, 0xa0, 0x3c, 0x5c, + 0x56, 0x7e, 0x37, 0x2b, 0x3f, 0x23, 0xb9, 0x43, + 0xc7, 0x88, 0xa4, 0xf2, 0x50, 0xf6, 0x78, 0x91 }, + { 0xab, 0x8d, 0x08, 0x65, 0x5f, 0xf1, 0xd3, 0xfe, + 0x87, 0x58, 0xd5, 0x62, 0x23, 0x5f, 0xd2, 0x3e, + 0x7c, 0xf9, 0xdc, 0xaa, 0xd6, 0x58, 0x87, 0x2a, + 0x49, 0xe5, 0xd3, 0x18, 0x3b, 0x6c, 0xce, 0xbd }, + { 0x6f, 0x27, 0xf7, 0x7e, 0x7b, 0xcf, 0x46, 0xa1, + 0xe9, 0x63, 0xad, 0xe0, 0x30, 0x97, 0x33, 0x54, + 0x30, 0x31, 0xdc, 0xcd, 0xd4, 0x7c, 0xaa, 0xc1, + 0x74, 0xd7, 0xd2, 0x7c, 0xe8, 0x07, 0x7e, 0x8b }, + { 0xe3, 0xcd, 0x54, 0xda, 0x7e, 0x44, 0x4c, 0xaa, + 0x62, 0x07, 0x56, 0x95, 0x25, 0xa6, 0x70, 0xeb, + 0xae, 0x12, 0x78, 0xde, 0x4e, 0x3f, 0xe2, 0x68, + 0x4b, 0x3e, 0x33, 0xf5, 0xef, 0x90, 0xcc, 0x1b }, + { 0xb2, 0xc3, 0xe3, 0x3a, 0x51, 0xd2, 0x2c, 0x4c, + 0x08, 0xfc, 0x09, 0x89, 0xc8, 0x73, 0xc9, 0xcc, + 0x41, 0x50, 0x57, 0x9b, 0x1e, 0x61, 0x63, 0xfa, + 0x69, 0x4a, 0xd5, 0x1d, 0x53, 0xd7, 0x12, 0xdc }, + { 0xbe, 0x7f, 0xda, 0x98, 0x3e, 0x13, 0x18, 0x9b, + 0x4c, 0x77, 0xe0, 0xa8, 0x09, 0x20, 0xb6, 0xe0, + 0xe0, 0xea, 0x80, 0xc3, 0xb8, 0x4d, 0xbe, 0x7e, + 0x71, 0x17, 0xd2, 0x53, 0xf4, 0x81, 0x12, 0xf4 }, + { 0xb6, 0x00, 0x8c, 0x28, 0xfa, 0xe0, 0x8a, 0xa4, + 0x27, 0xe5, 0xbd, 0x3a, 0xad, 0x36, 0xf1, 0x00, + 0x21, 0xf1, 0x6c, 0x77, 0xcf, 0xea, 0xbe, 0xd0, + 0x7f, 0x97, 0xcc, 0x7d, 0xc1, 0xf1, 0x28, 0x4a }, + { 0x6e, 0x4e, 0x67, 0x60, 0xc5, 0x38, 0xf2, 0xe9, + 0x7b, 0x3a, 0xdb, 0xfb, 0xbc, 0xde, 0x57, 0xf8, + 0x96, 0x6b, 0x7e, 0xa8, 0xfc, 0xb5, 0xbf, 0x7e, + 0xfe, 0xc9, 0x13, 0xfd, 0x2a, 0x2b, 0x0c, 0x55 }, + { 0x4a, 0xe5, 0x1f, 0xd1, 0x83, 0x4a, 0xa5, 0xbd, + 0x9a, 0x6f, 0x7e, 0xc3, 0x9f, 0xc6, 0x63, 0x33, + 0x8d, 0xc5, 0xd2, 0xe2, 0x07, 0x61, 0x56, 0x6d, + 0x90, 0xcc, 0x68, 0xb1, 0xcb, 0x87, 0x5e, 0xd8 }, + { 0xb6, 0x73, 0xaa, 0xd7, 0x5a, 0xb1, 0xfd, 0xb5, + 0x40, 0x1a, 0xbf, 0xa1, 0xbf, 0x89, 0xf3, 0xad, + 0xd2, 0xeb, 0xc4, 0x68, 0xdf, 0x36, 0x24, 0xa4, + 0x78, 0xf4, 0xfe, 0x85, 0x9d, 0x8d, 0x55, 0xe2 }, + { 0x13, 0xc9, 0x47, 0x1a, 0x98, 0x55, 0x91, 0x35, + 0x39, 0x83, 0x66, 0x60, 0x39, 0x8d, 0xa0, 0xf3, + 0xf9, 0x9a, 0xda, 0x08, 0x47, 0x9c, 0x69, 0xd1, + 0xb7, 0xfc, 0xaa, 0x34, 0x61, 0xdd, 0x7e, 0x59 }, + { 0x2c, 0x11, 0xf4, 0xa7, 0xf9, 0x9a, 0x1d, 0x23, + 0xa5, 0x8b, 0xb6, 0x36, 0x35, 0x0f, 0xe8, 0x49, + 0xf2, 0x9c, 0xba, 0xc1, 0xb2, 0xa1, 0x11, 0x2d, + 0x9f, 0x1e, 0xd5, 0xbc, 0x5b, 0x31, 0x3c, 0xcd }, + { 0xc7, 0xd3, 0xc0, 0x70, 0x6b, 0x11, 0xae, 0x74, + 0x1c, 0x05, 0xa1, 0xef, 0x15, 0x0d, 0xd6, 0x5b, + 0x54, 0x94, 0xd6, 0xd5, 0x4c, 0x9a, 0x86, 0xe2, + 0x61, 0x78, 0x54, 0xe6, 0xae, 0xee, 0xbb, 0xd9 }, + { 0x19, 0x4e, 0x10, 0xc9, 0x38, 0x93, 0xaf, 0xa0, + 0x64, 0xc3, 0xac, 0x04, 0xc0, 0xdd, 0x80, 0x8d, + 0x79, 0x1c, 0x3d, 0x4b, 0x75, 0x56, 0xe8, 0x9d, + 0x8d, 0x9c, 0xb2, 0x25, 0xc4, 0xb3, 0x33, 0x39 }, + { 0x6f, 0xc4, 0x98, 0x8b, 0x8f, 0x78, 0x54, 0x6b, + 0x16, 0x88, 0x99, 0x18, 0x45, 0x90, 0x8f, 0x13, + 0x4b, 0x6a, 0x48, 0x2e, 0x69, 0x94, 0xb3, 0xd4, + 0x83, 0x17, 0xbf, 0x08, 0xdb, 0x29, 0x21, 0x85 }, + { 0x56, 0x65, 0xbe, 0xb8, 0xb0, 0x95, 0x55, 0x25, + 0x81, 0x3b, 0x59, 0x81, 0xcd, 0x14, 0x2e, 0xd4, + 0xd0, 0x3f, 0xba, 0x38, 0xa6, 0xf3, 0xe5, 0xad, + 0x26, 0x8e, 0x0c, 0xc2, 0x70, 0xd1, 0xcd, 0x11 }, + { 0xb8, 0x83, 0xd6, 0x8f, 0x5f, 0xe5, 0x19, 0x36, + 0x43, 0x1b, 0xa4, 0x25, 0x67, 0x38, 0x05, 0x3b, + 0x1d, 0x04, 0x26, 0xd4, 0xcb, 0x64, 0xb1, 0x6e, + 0x83, 0xba, 0xdc, 0x5e, 0x9f, 0xbe, 0x3b, 0x81 }, + { 0x53, 0xe7, 0xb2, 0x7e, 0xa5, 0x9c, 0x2f, 0x6d, + 0xbb, 0x50, 0x76, 0x9e, 0x43, 0x55, 0x4d, 0xf3, + 0x5a, 0xf8, 0x9f, 0x48, 0x22, 0xd0, 0x46, 0x6b, + 0x00, 0x7d, 0xd6, 0xf6, 0xde, 0xaf, 0xff, 0x02 }, + { 0x1f, 0x1a, 0x02, 0x29, 0xd4, 0x64, 0x0f, 0x01, + 0x90, 0x15, 0x88, 0xd9, 0xde, 0xc2, 0x2d, 0x13, + 0xfc, 0x3e, 0xb3, 0x4a, 0x61, 0xb3, 0x29, 0x38, + 0xef, 0xbf, 0x53, 0x34, 0xb2, 0x80, 0x0a, 0xfa }, + { 0xc2, 0xb4, 0x05, 0xaf, 0xa0, 0xfa, 0x66, 0x68, + 0x85, 0x2a, 0xee, 0x4d, 0x88, 0x04, 0x08, 0x53, + 0xfa, 0xb8, 0x00, 0xe7, 0x2b, 0x57, 0x58, 0x14, + 0x18, 0xe5, 0x50, 0x6f, 0x21, 0x4c, 0x7d, 0x1f }, + { 0xc0, 0x8a, 0xa1, 0xc2, 0x86, 0xd7, 0x09, 0xfd, + 0xc7, 0x47, 0x37, 0x44, 0x97, 0x71, 0x88, 0xc8, + 0x95, 0xba, 0x01, 0x10, 0x14, 0x24, 0x7e, 0x4e, + 0xfa, 0x8d, 0x07, 0xe7, 0x8f, 0xec, 0x69, 0x5c }, + { 0xf0, 0x3f, 0x57, 0x89, 0xd3, 0x33, 0x6b, 0x80, + 0xd0, 0x02, 0xd5, 0x9f, 0xdf, 0x91, 0x8b, 0xdb, + 0x77, 0x5b, 0x00, 0x95, 0x6e, 0xd5, 0x52, 0x8e, + 0x86, 0xaa, 0x99, 0x4a, 0xcb, 0x38, 0xfe, 0x2d } +}; + +static const u8 blake2s_keyed_testvecs[][BLAKE2S_HASH_SIZE] __initconst = { + { 0x48, 0xa8, 0x99, 0x7d, 0xa4, 0x07, 0x87, 0x6b, + 0x3d, 0x79, 0xc0, 0xd9, 0x23, 0x25, 0xad, 0x3b, + 0x89, 0xcb, 0xb7, 0x54, 0xd8, 0x6a, 0xb7, 0x1a, + 0xee, 0x04, 0x7a, 0xd3, 0x45, 0xfd, 0x2c, 0x49 }, + { 0x40, 0xd1, 0x5f, 0xee, 0x7c, 0x32, 0x88, 0x30, + 0x16, 0x6a, 0xc3, 0xf9, 0x18, 0x65, 0x0f, 0x80, + 0x7e, 0x7e, 0x01, 0xe1, 0x77, 0x25, 0x8c, 0xdc, + 0x0a, 0x39, 0xb1, 0x1f, 0x59, 0x80, 0x66, 0xf1 }, + { 0x6b, 0xb7, 0x13, 0x00, 0x64, 0x4c, 0xd3, 0x99, + 0x1b, 0x26, 0xcc, 0xd4, 0xd2, 0x74, 0xac, 0xd1, + 0xad, 0xea, 0xb8, 0xb1, 0xd7, 0x91, 0x45, 0x46, + 0xc1, 0x19, 0x8b, 0xbe, 0x9f, 0xc9, 0xd8, 0x03 }, + { 0x1d, 0x22, 0x0d, 0xbe, 0x2e, 0xe1, 0x34, 0x66, + 0x1f, 0xdf, 0x6d, 0x9e, 0x74, 0xb4, 0x17, 0x04, + 0x71, 0x05, 0x56, 0xf2, 0xf6, 0xe5, 0xa0, 0x91, + 0xb2, 0x27, 0x69, 0x74, 0x45, 0xdb, 0xea, 0x6b }, + { 0xf6, 0xc3, 0xfb, 0xad, 0xb4, 0xcc, 0x68, 0x7a, + 0x00, 0x64, 0xa5, 0xbe, 0x6e, 0x79, 0x1b, 0xec, + 0x63, 0xb8, 0x68, 0xad, 0x62, 0xfb, 0xa6, 0x1b, + 0x37, 0x57, 0xef, 0x9c, 0xa5, 0x2e, 0x05, 0xb2 }, + { 0x49, 0xc1, 0xf2, 0x11, 0x88, 0xdf, 0xd7, 0x69, + 0xae, 0xa0, 0xe9, 0x11, 0xdd, 0x6b, 0x41, 0xf1, + 0x4d, 0xab, 0x10, 0x9d, 0x2b, 0x85, 0x97, 0x7a, + 0xa3, 0x08, 0x8b, 0x5c, 0x70, 0x7e, 0x85, 0x98 }, + { 0xfd, 0xd8, 0x99, 0x3d, 0xcd, 0x43, 0xf6, 0x96, + 0xd4, 0x4f, 0x3c, 0xea, 0x0f, 0xf3, 0x53, 0x45, + 0x23, 0x4e, 0xc8, 0xee, 0x08, 0x3e, 0xb3, 0xca, + 0xda, 0x01, 0x7c, 0x7f, 0x78, 0xc1, 0x71, 0x43 }, + { 0xe6, 0xc8, 0x12, 0x56, 0x37, 0x43, 0x8d, 0x09, + 0x05, 0xb7, 0x49, 0xf4, 0x65, 0x60, 0xac, 0x89, + 0xfd, 0x47, 0x1c, 0xf8, 0x69, 0x2e, 0x28, 0xfa, + 0xb9, 0x82, 0xf7, 0x3f, 0x01, 0x9b, 0x83, 0xa9 }, + { 0x19, 0xfc, 0x8c, 0xa6, 0x97, 0x9d, 0x60, 0xe6, + 0xed, 0xd3, 0xb4, 0x54, 0x1e, 0x2f, 0x96, 0x7c, + 0xed, 0x74, 0x0d, 0xf6, 0xec, 0x1e, 0xae, 0xbb, + 0xfe, 0x81, 0x38, 0x32, 0xe9, 0x6b, 0x29, 0x74 }, + { 0xa6, 0xad, 0x77, 0x7c, 0xe8, 0x81, 0xb5, 0x2b, + 0xb5, 0xa4, 0x42, 0x1a, 0xb6, 0xcd, 0xd2, 0xdf, + 0xba, 0x13, 0xe9, 0x63, 0x65, 0x2d, 0x4d, 0x6d, + 0x12, 0x2a, 0xee, 0x46, 0x54, 0x8c, 0x14, 0xa7 }, + { 0xf5, 0xc4, 0xb2, 0xba, 0x1a, 0x00, 0x78, 0x1b, + 0x13, 0xab, 0xa0, 0x42, 0x52, 0x42, 0xc6, 0x9c, + 0xb1, 0x55, 0x2f, 0x3f, 0x71, 0xa9, 0xa3, 0xbb, + 0x22, 0xb4, 0xa6, 0xb4, 0x27, 0x7b, 0x46, 0xdd }, + { 0xe3, 0x3c, 0x4c, 0x9b, 0xd0, 0xcc, 0x7e, 0x45, + 0xc8, 0x0e, 0x65, 0xc7, 0x7f, 0xa5, 0x99, 0x7f, + 0xec, 0x70, 0x02, 0x73, 0x85, 0x41, 0x50, 0x9e, + 0x68, 0xa9, 0x42, 0x38, 0x91, 0xe8, 0x22, 0xa3 }, + { 0xfb, 0xa1, 0x61, 0x69, 0xb2, 0xc3, 0xee, 0x10, + 0x5b, 0xe6, 0xe1, 0xe6, 0x50, 0xe5, 0xcb, 0xf4, + 0x07, 0x46, 0xb6, 0x75, 0x3d, 0x03, 0x6a, 0xb5, + 0x51, 0x79, 0x01, 0x4a, 0xd7, 0xef, 0x66, 0x51 }, + { 0xf5, 0xc4, 0xbe, 0xc6, 0xd6, 0x2f, 0xc6, 0x08, + 0xbf, 0x41, 0xcc, 0x11, 0x5f, 0x16, 0xd6, 0x1c, + 0x7e, 0xfd, 0x3f, 0xf6, 0xc6, 0x56, 0x92, 0xbb, + 0xe0, 0xaf, 0xff, 0xb1, 0xfe, 0xde, 0x74, 0x75 }, + { 0xa4, 0x86, 0x2e, 0x76, 0xdb, 0x84, 0x7f, 0x05, + 0xba, 0x17, 0xed, 0xe5, 0xda, 0x4e, 0x7f, 0x91, + 0xb5, 0x92, 0x5c, 0xf1, 0xad, 0x4b, 0xa1, 0x27, + 0x32, 0xc3, 0x99, 0x57, 0x42, 0xa5, 0xcd, 0x6e }, + { 0x65, 0xf4, 0xb8, 0x60, 0xcd, 0x15, 0xb3, 0x8e, + 0xf8, 0x14, 0xa1, 0xa8, 0x04, 0x31, 0x4a, 0x55, + 0xbe, 0x95, 0x3c, 0xaa, 0x65, 0xfd, 0x75, 0x8a, + 0xd9, 0x89, 0xff, 0x34, 0xa4, 0x1c, 0x1e, 0xea }, + { 0x19, 0xba, 0x23, 0x4f, 0x0a, 0x4f, 0x38, 0x63, + 0x7d, 0x18, 0x39, 0xf9, 0xd9, 0xf7, 0x6a, 0xd9, + 0x1c, 0x85, 0x22, 0x30, 0x71, 0x43, 0xc9, 0x7d, + 0x5f, 0x93, 0xf6, 0x92, 0x74, 0xce, 0xc9, 0xa7 }, + { 0x1a, 0x67, 0x18, 0x6c, 0xa4, 0xa5, 0xcb, 0x8e, + 0x65, 0xfc, 0xa0, 0xe2, 0xec, 0xbc, 0x5d, 0xdc, + 0x14, 0xae, 0x38, 0x1b, 0xb8, 0xbf, 0xfe, 0xb9, + 0xe0, 0xa1, 0x03, 0x44, 0x9e, 0x3e, 0xf0, 0x3c }, + { 0xaf, 0xbe, 0xa3, 0x17, 0xb5, 0xa2, 0xe8, 0x9c, + 0x0b, 0xd9, 0x0c, 0xcf, 0x5d, 0x7f, 0xd0, 0xed, + 0x57, 0xfe, 0x58, 0x5e, 0x4b, 0xe3, 0x27, 0x1b, + 0x0a, 0x6b, 0xf0, 0xf5, 0x78, 0x6b, 0x0f, 0x26 }, + { 0xf1, 0xb0, 0x15, 0x58, 0xce, 0x54, 0x12, 0x62, + 0xf5, 0xec, 0x34, 0x29, 0x9d, 0x6f, 0xb4, 0x09, + 0x00, 0x09, 0xe3, 0x43, 0x4b, 0xe2, 0xf4, 0x91, + 0x05, 0xcf, 0x46, 0xaf, 0x4d, 0x2d, 0x41, 0x24 }, + { 0x13, 0xa0, 0xa0, 0xc8, 0x63, 0x35, 0x63, 0x5e, + 0xaa, 0x74, 0xca, 0x2d, 0x5d, 0x48, 0x8c, 0x79, + 0x7b, 0xbb, 0x4f, 0x47, 0xdc, 0x07, 0x10, 0x50, + 0x15, 0xed, 0x6a, 0x1f, 0x33, 0x09, 0xef, 0xce }, + { 0x15, 0x80, 0xaf, 0xee, 0xbe, 0xbb, 0x34, 0x6f, + 0x94, 0xd5, 0x9f, 0xe6, 0x2d, 0xa0, 0xb7, 0x92, + 0x37, 0xea, 0xd7, 0xb1, 0x49, 0x1f, 0x56, 0x67, + 0xa9, 0x0e, 0x45, 0xed, 0xf6, 0xca, 0x8b, 0x03 }, + { 0x20, 0xbe, 0x1a, 0x87, 0x5b, 0x38, 0xc5, 0x73, + 0xdd, 0x7f, 0xaa, 0xa0, 0xde, 0x48, 0x9d, 0x65, + 0x5c, 0x11, 0xef, 0xb6, 0xa5, 0x52, 0x69, 0x8e, + 0x07, 0xa2, 0xd3, 0x31, 0xb5, 0xf6, 0x55, 0xc3 }, + { 0xbe, 0x1f, 0xe3, 0xc4, 0xc0, 0x40, 0x18, 0xc5, + 0x4c, 0x4a, 0x0f, 0x6b, 0x9a, 0x2e, 0xd3, 0xc5, + 0x3a, 0xbe, 0x3a, 0x9f, 0x76, 0xb4, 0xd2, 0x6d, + 0xe5, 0x6f, 0xc9, 0xae, 0x95, 0x05, 0x9a, 0x99 }, + { 0xe3, 0xe3, 0xac, 0xe5, 0x37, 0xeb, 0x3e, 0xdd, + 0x84, 0x63, 0xd9, 0xad, 0x35, 0x82, 0xe1, 0x3c, + 0xf8, 0x65, 0x33, 0xff, 0xde, 0x43, 0xd6, 0x68, + 0xdd, 0x2e, 0x93, 0xbb, 0xdb, 0xd7, 0x19, 0x5a }, + { 0x11, 0x0c, 0x50, 0xc0, 0xbf, 0x2c, 0x6e, 0x7a, + 0xeb, 0x7e, 0x43, 0x5d, 0x92, 0xd1, 0x32, 0xab, + 0x66, 0x55, 0x16, 0x8e, 0x78, 0xa2, 0xde, 0xcd, + 0xec, 0x33, 0x30, 0x77, 0x76, 0x84, 0xd9, 0xc1 }, + { 0xe9, 0xba, 0x8f, 0x50, 0x5c, 0x9c, 0x80, 0xc0, + 0x86, 0x66, 0xa7, 0x01, 0xf3, 0x36, 0x7e, 0x6c, + 0xc6, 0x65, 0xf3, 0x4b, 0x22, 0xe7, 0x3c, 0x3c, + 0x04, 0x17, 0xeb, 0x1c, 0x22, 0x06, 0x08, 0x2f }, + { 0x26, 0xcd, 0x66, 0xfc, 0xa0, 0x23, 0x79, 0xc7, + 0x6d, 0xf1, 0x23, 0x17, 0x05, 0x2b, 0xca, 0xfd, + 0x6c, 0xd8, 0xc3, 0xa7, 0xb8, 0x90, 0xd8, 0x05, + 0xf3, 0x6c, 0x49, 0x98, 0x97, 0x82, 0x43, 0x3a }, + { 0x21, 0x3f, 0x35, 0x96, 0xd6, 0xe3, 0xa5, 0xd0, + 0xe9, 0x93, 0x2c, 0xd2, 0x15, 0x91, 0x46, 0x01, + 0x5e, 0x2a, 0xbc, 0x94, 0x9f, 0x47, 0x29, 0xee, + 0x26, 0x32, 0xfe, 0x1e, 0xdb, 0x78, 0xd3, 0x37 }, + { 0x10, 0x15, 0xd7, 0x01, 0x08, 0xe0, 0x3b, 0xe1, + 0xc7, 0x02, 0xfe, 0x97, 0x25, 0x36, 0x07, 0xd1, + 0x4a, 0xee, 0x59, 0x1f, 0x24, 0x13, 0xea, 0x67, + 0x87, 0x42, 0x7b, 0x64, 0x59, 0xff, 0x21, 0x9a }, + { 0x3c, 0xa9, 0x89, 0xde, 0x10, 0xcf, 0xe6, 0x09, + 0x90, 0x94, 0x72, 0xc8, 0xd3, 0x56, 0x10, 0x80, + 0x5b, 0x2f, 0x97, 0x77, 0x34, 0xcf, 0x65, 0x2c, + 0xc6, 0x4b, 0x3b, 0xfc, 0x88, 0x2d, 0x5d, 0x89 }, + { 0xb6, 0x15, 0x6f, 0x72, 0xd3, 0x80, 0xee, 0x9e, + 0xa6, 0xac, 0xd1, 0x90, 0x46, 0x4f, 0x23, 0x07, + 0xa5, 0xc1, 0x79, 0xef, 0x01, 0xfd, 0x71, 0xf9, + 0x9f, 0x2d, 0x0f, 0x7a, 0x57, 0x36, 0x0a, 0xea }, + { 0xc0, 0x3b, 0xc6, 0x42, 0xb2, 0x09, 0x59, 0xcb, + 0xe1, 0x33, 0xa0, 0x30, 0x3e, 0x0c, 0x1a, 0xbf, + 0xf3, 0xe3, 0x1e, 0xc8, 0xe1, 0xa3, 0x28, 0xec, + 0x85, 0x65, 0xc3, 0x6d, 0xec, 0xff, 0x52, 0x65 }, + { 0x2c, 0x3e, 0x08, 0x17, 0x6f, 0x76, 0x0c, 0x62, + 0x64, 0xc3, 0xa2, 0xcd, 0x66, 0xfe, 0xc6, 0xc3, + 0xd7, 0x8d, 0xe4, 0x3f, 0xc1, 0x92, 0x45, 0x7b, + 0x2a, 0x4a, 0x66, 0x0a, 0x1e, 0x0e, 0xb2, 0x2b }, + { 0xf7, 0x38, 0xc0, 0x2f, 0x3c, 0x1b, 0x19, 0x0c, + 0x51, 0x2b, 0x1a, 0x32, 0xde, 0xab, 0xf3, 0x53, + 0x72, 0x8e, 0x0e, 0x9a, 0xb0, 0x34, 0x49, 0x0e, + 0x3c, 0x34, 0x09, 0x94, 0x6a, 0x97, 0xae, 0xec }, + { 0x8b, 0x18, 0x80, 0xdf, 0x30, 0x1c, 0xc9, 0x63, + 0x41, 0x88, 0x11, 0x08, 0x89, 0x64, 0x83, 0x92, + 0x87, 0xff, 0x7f, 0xe3, 0x1c, 0x49, 0xea, 0x6e, + 0xbd, 0x9e, 0x48, 0xbd, 0xee, 0xe4, 0x97, 0xc5 }, + { 0x1e, 0x75, 0xcb, 0x21, 0xc6, 0x09, 0x89, 0x02, + 0x03, 0x75, 0xf1, 0xa7, 0xa2, 0x42, 0x83, 0x9f, + 0x0b, 0x0b, 0x68, 0x97, 0x3a, 0x4c, 0x2a, 0x05, + 0xcf, 0x75, 0x55, 0xed, 0x5a, 0xae, 0xc4, 0xc1 }, + { 0x62, 0xbf, 0x8a, 0x9c, 0x32, 0xa5, 0xbc, 0xcf, + 0x29, 0x0b, 0x6c, 0x47, 0x4d, 0x75, 0xb2, 0xa2, + 0xa4, 0x09, 0x3f, 0x1a, 0x9e, 0x27, 0x13, 0x94, + 0x33, 0xa8, 0xf2, 0xb3, 0xbc, 0xe7, 0xb8, 0xd7 }, + { 0x16, 0x6c, 0x83, 0x50, 0xd3, 0x17, 0x3b, 0x5e, + 0x70, 0x2b, 0x78, 0x3d, 0xfd, 0x33, 0xc6, 0x6e, + 0xe0, 0x43, 0x27, 0x42, 0xe9, 0xb9, 0x2b, 0x99, + 0x7f, 0xd2, 0x3c, 0x60, 0xdc, 0x67, 0x56, 0xca }, + { 0x04, 0x4a, 0x14, 0xd8, 0x22, 0xa9, 0x0c, 0xac, + 0xf2, 0xf5, 0xa1, 0x01, 0x42, 0x8a, 0xdc, 0x8f, + 0x41, 0x09, 0x38, 0x6c, 0xcb, 0x15, 0x8b, 0xf9, + 0x05, 0xc8, 0x61, 0x8b, 0x8e, 0xe2, 0x4e, 0xc3 }, + { 0x38, 0x7d, 0x39, 0x7e, 0xa4, 0x3a, 0x99, 0x4b, + 0xe8, 0x4d, 0x2d, 0x54, 0x4a, 0xfb, 0xe4, 0x81, + 0xa2, 0x00, 0x0f, 0x55, 0x25, 0x26, 0x96, 0xbb, + 0xa2, 0xc5, 0x0c, 0x8e, 0xbd, 0x10, 0x13, 0x47 }, + { 0x56, 0xf8, 0xcc, 0xf1, 0xf8, 0x64, 0x09, 0xb4, + 0x6c, 0xe3, 0x61, 0x66, 0xae, 0x91, 0x65, 0x13, + 0x84, 0x41, 0x57, 0x75, 0x89, 0xdb, 0x08, 0xcb, + 0xc5, 0xf6, 0x6c, 0xa2, 0x97, 0x43, 0xb9, 0xfd }, + { 0x97, 0x06, 0xc0, 0x92, 0xb0, 0x4d, 0x91, 0xf5, + 0x3d, 0xff, 0x91, 0xfa, 0x37, 0xb7, 0x49, 0x3d, + 0x28, 0xb5, 0x76, 0xb5, 0xd7, 0x10, 0x46, 0x9d, + 0xf7, 0x94, 0x01, 0x66, 0x22, 0x36, 0xfc, 0x03 }, + { 0x87, 0x79, 0x68, 0x68, 0x6c, 0x06, 0x8c, 0xe2, + 0xf7, 0xe2, 0xad, 0xcf, 0xf6, 0x8b, 0xf8, 0x74, + 0x8e, 0xdf, 0x3c, 0xf8, 0x62, 0xcf, 0xb4, 0xd3, + 0x94, 0x7a, 0x31, 0x06, 0x95, 0x80, 0x54, 0xe3 }, + { 0x88, 0x17, 0xe5, 0x71, 0x98, 0x79, 0xac, 0xf7, + 0x02, 0x47, 0x87, 0xec, 0xcd, 0xb2, 0x71, 0x03, + 0x55, 0x66, 0xcf, 0xa3, 0x33, 0xe0, 0x49, 0x40, + 0x7c, 0x01, 0x78, 0xcc, 0xc5, 0x7a, 0x5b, 0x9f }, + { 0x89, 0x38, 0x24, 0x9e, 0x4b, 0x50, 0xca, 0xda, + 0xcc, 0xdf, 0x5b, 0x18, 0x62, 0x13, 0x26, 0xcb, + 0xb1, 0x52, 0x53, 0xe3, 0x3a, 0x20, 0xf5, 0x63, + 0x6e, 0x99, 0x5d, 0x72, 0x47, 0x8d, 0xe4, 0x72 }, + { 0xf1, 0x64, 0xab, 0xba, 0x49, 0x63, 0xa4, 0x4d, + 0x10, 0x72, 0x57, 0xe3, 0x23, 0x2d, 0x90, 0xac, + 0xa5, 0xe6, 0x6a, 0x14, 0x08, 0x24, 0x8c, 0x51, + 0x74, 0x1e, 0x99, 0x1d, 0xb5, 0x22, 0x77, 0x56 }, + { 0xd0, 0x55, 0x63, 0xe2, 0xb1, 0xcb, 0xa0, 0xc4, + 0xa2, 0xa1, 0xe8, 0xbd, 0xe3, 0xa1, 0xa0, 0xd9, + 0xf5, 0xb4, 0x0c, 0x85, 0xa0, 0x70, 0xd6, 0xf5, + 0xfb, 0x21, 0x06, 0x6e, 0xad, 0x5d, 0x06, 0x01 }, + { 0x03, 0xfb, 0xb1, 0x63, 0x84, 0xf0, 0xa3, 0x86, + 0x6f, 0x4c, 0x31, 0x17, 0x87, 0x76, 0x66, 0xef, + 0xbf, 0x12, 0x45, 0x97, 0x56, 0x4b, 0x29, 0x3d, + 0x4a, 0xab, 0x0d, 0x26, 0x9f, 0xab, 0xdd, 0xfa }, + { 0x5f, 0xa8, 0x48, 0x6a, 0xc0, 0xe5, 0x29, 0x64, + 0xd1, 0x88, 0x1b, 0xbe, 0x33, 0x8e, 0xb5, 0x4b, + 0xe2, 0xf7, 0x19, 0x54, 0x92, 0x24, 0x89, 0x20, + 0x57, 0xb4, 0xda, 0x04, 0xba, 0x8b, 0x34, 0x75 }, + { 0xcd, 0xfa, 0xbc, 0xee, 0x46, 0x91, 0x11, 0x11, + 0x23, 0x6a, 0x31, 0x70, 0x8b, 0x25, 0x39, 0xd7, + 0x1f, 0xc2, 0x11, 0xd9, 0xb0, 0x9c, 0x0d, 0x85, + 0x30, 0xa1, 0x1e, 0x1d, 0xbf, 0x6e, 0xed, 0x01 }, + { 0x4f, 0x82, 0xde, 0x03, 0xb9, 0x50, 0x47, 0x93, + 0xb8, 0x2a, 0x07, 0xa0, 0xbd, 0xcd, 0xff, 0x31, + 0x4d, 0x75, 0x9e, 0x7b, 0x62, 0xd2, 0x6b, 0x78, + 0x49, 0x46, 0xb0, 0xd3, 0x6f, 0x91, 0x6f, 0x52 }, + { 0x25, 0x9e, 0xc7, 0xf1, 0x73, 0xbc, 0xc7, 0x6a, + 0x09, 0x94, 0xc9, 0x67, 0xb4, 0xf5, 0xf0, 0x24, + 0xc5, 0x60, 0x57, 0xfb, 0x79, 0xc9, 0x65, 0xc4, + 0xfa, 0xe4, 0x18, 0x75, 0xf0, 0x6a, 0x0e, 0x4c }, + { 0x19, 0x3c, 0xc8, 0xe7, 0xc3, 0xe0, 0x8b, 0xb3, + 0x0f, 0x54, 0x37, 0xaa, 0x27, 0xad, 0xe1, 0xf1, + 0x42, 0x36, 0x9b, 0x24, 0x6a, 0x67, 0x5b, 0x23, + 0x83, 0xe6, 0xda, 0x9b, 0x49, 0xa9, 0x80, 0x9e }, + { 0x5c, 0x10, 0x89, 0x6f, 0x0e, 0x28, 0x56, 0xb2, + 0xa2, 0xee, 0xe0, 0xfe, 0x4a, 0x2c, 0x16, 0x33, + 0x56, 0x5d, 0x18, 0xf0, 0xe9, 0x3e, 0x1f, 0xab, + 0x26, 0xc3, 0x73, 0xe8, 0xf8, 0x29, 0x65, 0x4d }, + { 0xf1, 0x60, 0x12, 0xd9, 0x3f, 0x28, 0x85, 0x1a, + 0x1e, 0xb9, 0x89, 0xf5, 0xd0, 0xb4, 0x3f, 0x3f, + 0x39, 0xca, 0x73, 0xc9, 0xa6, 0x2d, 0x51, 0x81, + 0xbf, 0xf2, 0x37, 0x53, 0x6b, 0xd3, 0x48, 0xc3 }, + { 0x29, 0x66, 0xb3, 0xcf, 0xae, 0x1e, 0x44, 0xea, + 0x99, 0x6d, 0xc5, 0xd6, 0x86, 0xcf, 0x25, 0xfa, + 0x05, 0x3f, 0xb6, 0xf6, 0x72, 0x01, 0xb9, 0xe4, + 0x6e, 0xad, 0xe8, 0x5d, 0x0a, 0xd6, 0xb8, 0x06 }, + { 0xdd, 0xb8, 0x78, 0x24, 0x85, 0xe9, 0x00, 0xbc, + 0x60, 0xbc, 0xf4, 0xc3, 0x3a, 0x6f, 0xd5, 0x85, + 0x68, 0x0c, 0xc6, 0x83, 0xd5, 0x16, 0xef, 0xa0, + 0x3e, 0xb9, 0x98, 0x5f, 0xad, 0x87, 0x15, 0xfb }, + { 0x4c, 0x4d, 0x6e, 0x71, 0xae, 0xa0, 0x57, 0x86, + 0x41, 0x31, 0x48, 0xfc, 0x7a, 0x78, 0x6b, 0x0e, + 0xca, 0xf5, 0x82, 0xcf, 0xf1, 0x20, 0x9f, 0x5a, + 0x80, 0x9f, 0xba, 0x85, 0x04, 0xce, 0x66, 0x2c }, + { 0xfb, 0x4c, 0x5e, 0x86, 0xd7, 0xb2, 0x22, 0x9b, + 0x99, 0xb8, 0xba, 0x6d, 0x94, 0xc2, 0x47, 0xef, + 0x96, 0x4a, 0xa3, 0xa2, 0xba, 0xe8, 0xed, 0xc7, + 0x75, 0x69, 0xf2, 0x8d, 0xbb, 0xff, 0x2d, 0x4e }, + { 0xe9, 0x4f, 0x52, 0x6d, 0xe9, 0x01, 0x96, 0x33, + 0xec, 0xd5, 0x4a, 0xc6, 0x12, 0x0f, 0x23, 0x95, + 0x8d, 0x77, 0x18, 0xf1, 0xe7, 0x71, 0x7b, 0xf3, + 0x29, 0x21, 0x1a, 0x4f, 0xae, 0xed, 0x4e, 0x6d }, + { 0xcb, 0xd6, 0x66, 0x0a, 0x10, 0xdb, 0x3f, 0x23, + 0xf7, 0xa0, 0x3d, 0x4b, 0x9d, 0x40, 0x44, 0xc7, + 0x93, 0x2b, 0x28, 0x01, 0xac, 0x89, 0xd6, 0x0b, + 0xc9, 0xeb, 0x92, 0xd6, 0x5a, 0x46, 0xc2, 0xa0 }, + { 0x88, 0x18, 0xbb, 0xd3, 0xdb, 0x4d, 0xc1, 0x23, + 0xb2, 0x5c, 0xbb, 0xa5, 0xf5, 0x4c, 0x2b, 0xc4, + 0xb3, 0xfc, 0xf9, 0xbf, 0x7d, 0x7a, 0x77, 0x09, + 0xf4, 0xae, 0x58, 0x8b, 0x26, 0x7c, 0x4e, 0xce }, + { 0xc6, 0x53, 0x82, 0x51, 0x3f, 0x07, 0x46, 0x0d, + 0xa3, 0x98, 0x33, 0xcb, 0x66, 0x6c, 0x5e, 0xd8, + 0x2e, 0x61, 0xb9, 0xe9, 0x98, 0xf4, 0xb0, 0xc4, + 0x28, 0x7c, 0xee, 0x56, 0xc3, 0xcc, 0x9b, 0xcd }, + { 0x89, 0x75, 0xb0, 0x57, 0x7f, 0xd3, 0x55, 0x66, + 0xd7, 0x50, 0xb3, 0x62, 0xb0, 0x89, 0x7a, 0x26, + 0xc3, 0x99, 0x13, 0x6d, 0xf0, 0x7b, 0xab, 0xab, + 0xbd, 0xe6, 0x20, 0x3f, 0xf2, 0x95, 0x4e, 0xd4 }, + { 0x21, 0xfe, 0x0c, 0xeb, 0x00, 0x52, 0xbe, 0x7f, + 0xb0, 0xf0, 0x04, 0x18, 0x7c, 0xac, 0xd7, 0xde, + 0x67, 0xfa, 0x6e, 0xb0, 0x93, 0x8d, 0x92, 0x76, + 0x77, 0xf2, 0x39, 0x8c, 0x13, 0x23, 0x17, 0xa8 }, + { 0x2e, 0xf7, 0x3f, 0x3c, 0x26, 0xf1, 0x2d, 0x93, + 0x88, 0x9f, 0x3c, 0x78, 0xb6, 0xa6, 0x6c, 0x1d, + 0x52, 0xb6, 0x49, 0xdc, 0x9e, 0x85, 0x6e, 0x2c, + 0x17, 0x2e, 0xa7, 0xc5, 0x8a, 0xc2, 0xb5, 0xe3 }, + { 0x38, 0x8a, 0x3c, 0xd5, 0x6d, 0x73, 0x86, 0x7a, + 0xbb, 0x5f, 0x84, 0x01, 0x49, 0x2b, 0x6e, 0x26, + 0x81, 0xeb, 0x69, 0x85, 0x1e, 0x76, 0x7f, 0xd8, + 0x42, 0x10, 0xa5, 0x60, 0x76, 0xfb, 0x3d, 0xd3 }, + { 0xaf, 0x53, 0x3e, 0x02, 0x2f, 0xc9, 0x43, 0x9e, + 0x4e, 0x3c, 0xb8, 0x38, 0xec, 0xd1, 0x86, 0x92, + 0x23, 0x2a, 0xdf, 0x6f, 0xe9, 0x83, 0x95, 0x26, + 0xd3, 0xc3, 0xdd, 0x1b, 0x71, 0x91, 0x0b, 0x1a }, + { 0x75, 0x1c, 0x09, 0xd4, 0x1a, 0x93, 0x43, 0x88, + 0x2a, 0x81, 0xcd, 0x13, 0xee, 0x40, 0x81, 0x8d, + 0x12, 0xeb, 0x44, 0xc6, 0xc7, 0xf4, 0x0d, 0xf1, + 0x6e, 0x4a, 0xea, 0x8f, 0xab, 0x91, 0x97, 0x2a }, + { 0x5b, 0x73, 0xdd, 0xb6, 0x8d, 0x9d, 0x2b, 0x0a, + 0xa2, 0x65, 0xa0, 0x79, 0x88, 0xd6, 0xb8, 0x8a, + 0xe9, 0xaa, 0xc5, 0x82, 0xaf, 0x83, 0x03, 0x2f, + 0x8a, 0x9b, 0x21, 0xa2, 0xe1, 0xb7, 0xbf, 0x18 }, + { 0x3d, 0xa2, 0x91, 0x26, 0xc7, 0xc5, 0xd7, 0xf4, + 0x3e, 0x64, 0x24, 0x2a, 0x79, 0xfe, 0xaa, 0x4e, + 0xf3, 0x45, 0x9c, 0xde, 0xcc, 0xc8, 0x98, 0xed, + 0x59, 0xa9, 0x7f, 0x6e, 0xc9, 0x3b, 0x9d, 0xab }, + { 0x56, 0x6d, 0xc9, 0x20, 0x29, 0x3d, 0xa5, 0xcb, + 0x4f, 0xe0, 0xaa, 0x8a, 0xbd, 0xa8, 0xbb, 0xf5, + 0x6f, 0x55, 0x23, 0x13, 0xbf, 0xf1, 0x90, 0x46, + 0x64, 0x1e, 0x36, 0x15, 0xc1, 0xe3, 0xed, 0x3f }, + { 0x41, 0x15, 0xbe, 0xa0, 0x2f, 0x73, 0xf9, 0x7f, + 0x62, 0x9e, 0x5c, 0x55, 0x90, 0x72, 0x0c, 0x01, + 0xe7, 0xe4, 0x49, 0xae, 0x2a, 0x66, 0x97, 0xd4, + 0xd2, 0x78, 0x33, 0x21, 0x30, 0x36, 0x92, 0xf9 }, + { 0x4c, 0xe0, 0x8f, 0x47, 0x62, 0x46, 0x8a, 0x76, + 0x70, 0x01, 0x21, 0x64, 0x87, 0x8d, 0x68, 0x34, + 0x0c, 0x52, 0xa3, 0x5e, 0x66, 0xc1, 0x88, 0x4d, + 0x5c, 0x86, 0x48, 0x89, 0xab, 0xc9, 0x66, 0x77 }, + { 0x81, 0xea, 0x0b, 0x78, 0x04, 0x12, 0x4e, 0x0c, + 0x22, 0xea, 0x5f, 0xc7, 0x11, 0x04, 0xa2, 0xaf, + 0xcb, 0x52, 0xa1, 0xfa, 0x81, 0x6f, 0x3e, 0xcb, + 0x7d, 0xcb, 0x5d, 0x9d, 0xea, 0x17, 0x86, 0xd0 }, + { 0xfe, 0x36, 0x27, 0x33, 0xb0, 0x5f, 0x6b, 0xed, + 0xaf, 0x93, 0x79, 0xd7, 0xf7, 0x93, 0x6e, 0xde, + 0x20, 0x9b, 0x1f, 0x83, 0x23, 0xc3, 0x92, 0x25, + 0x49, 0xd9, 0xe7, 0x36, 0x81, 0xb5, 0xdb, 0x7b }, + { 0xef, 0xf3, 0x7d, 0x30, 0xdf, 0xd2, 0x03, 0x59, + 0xbe, 0x4e, 0x73, 0xfd, 0xf4, 0x0d, 0x27, 0x73, + 0x4b, 0x3d, 0xf9, 0x0a, 0x97, 0xa5, 0x5e, 0xd7, + 0x45, 0x29, 0x72, 0x94, 0xca, 0x85, 0xd0, 0x9f }, + { 0x17, 0x2f, 0xfc, 0x67, 0x15, 0x3d, 0x12, 0xe0, + 0xca, 0x76, 0xa8, 0xb6, 0xcd, 0x5d, 0x47, 0x31, + 0x88, 0x5b, 0x39, 0xce, 0x0c, 0xac, 0x93, 0xa8, + 0x97, 0x2a, 0x18, 0x00, 0x6c, 0x8b, 0x8b, 0xaf }, + { 0xc4, 0x79, 0x57, 0xf1, 0xcc, 0x88, 0xe8, 0x3e, + 0xf9, 0x44, 0x58, 0x39, 0x70, 0x9a, 0x48, 0x0a, + 0x03, 0x6b, 0xed, 0x5f, 0x88, 0xac, 0x0f, 0xcc, + 0x8e, 0x1e, 0x70, 0x3f, 0xfa, 0xac, 0x13, 0x2c }, + { 0x30, 0xf3, 0x54, 0x83, 0x70, 0xcf, 0xdc, 0xed, + 0xa5, 0xc3, 0x7b, 0x56, 0x9b, 0x61, 0x75, 0xe7, + 0x99, 0xee, 0xf1, 0xa6, 0x2a, 0xaa, 0x94, 0x32, + 0x45, 0xae, 0x76, 0x69, 0xc2, 0x27, 0xa7, 0xb5 }, + { 0xc9, 0x5d, 0xcb, 0x3c, 0xf1, 0xf2, 0x7d, 0x0e, + 0xef, 0x2f, 0x25, 0xd2, 0x41, 0x38, 0x70, 0x90, + 0x4a, 0x87, 0x7c, 0x4a, 0x56, 0xc2, 0xde, 0x1e, + 0x83, 0xe2, 0xbc, 0x2a, 0xe2, 0xe4, 0x68, 0x21 }, + { 0xd5, 0xd0, 0xb5, 0xd7, 0x05, 0x43, 0x4c, 0xd4, + 0x6b, 0x18, 0x57, 0x49, 0xf6, 0x6b, 0xfb, 0x58, + 0x36, 0xdc, 0xdf, 0x6e, 0xe5, 0x49, 0xa2, 0xb7, + 0xa4, 0xae, 0xe7, 0xf5, 0x80, 0x07, 0xca, 0xaf }, + { 0xbb, 0xc1, 0x24, 0xa7, 0x12, 0xf1, 0x5d, 0x07, + 0xc3, 0x00, 0xe0, 0x5b, 0x66, 0x83, 0x89, 0xa4, + 0x39, 0xc9, 0x17, 0x77, 0xf7, 0x21, 0xf8, 0x32, + 0x0c, 0x1c, 0x90, 0x78, 0x06, 0x6d, 0x2c, 0x7e }, + { 0xa4, 0x51, 0xb4, 0x8c, 0x35, 0xa6, 0xc7, 0x85, + 0x4c, 0xfa, 0xae, 0x60, 0x26, 0x2e, 0x76, 0x99, + 0x08, 0x16, 0x38, 0x2a, 0xc0, 0x66, 0x7e, 0x5a, + 0x5c, 0x9e, 0x1b, 0x46, 0xc4, 0x34, 0x2d, 0xdf }, + { 0xb0, 0xd1, 0x50, 0xfb, 0x55, 0xe7, 0x78, 0xd0, + 0x11, 0x47, 0xf0, 0xb5, 0xd8, 0x9d, 0x99, 0xec, + 0xb2, 0x0f, 0xf0, 0x7e, 0x5e, 0x67, 0x60, 0xd6, + 0xb6, 0x45, 0xeb, 0x5b, 0x65, 0x4c, 0x62, 0x2b }, + { 0x34, 0xf7, 0x37, 0xc0, 0xab, 0x21, 0x99, 0x51, + 0xee, 0xe8, 0x9a, 0x9f, 0x8d, 0xac, 0x29, 0x9c, + 0x9d, 0x4c, 0x38, 0xf3, 0x3f, 0xa4, 0x94, 0xc5, + 0xc6, 0xee, 0xfc, 0x92, 0xb6, 0xdb, 0x08, 0xbc }, + { 0x1a, 0x62, 0xcc, 0x3a, 0x00, 0x80, 0x0d, 0xcb, + 0xd9, 0x98, 0x91, 0x08, 0x0c, 0x1e, 0x09, 0x84, + 0x58, 0x19, 0x3a, 0x8c, 0xc9, 0xf9, 0x70, 0xea, + 0x99, 0xfb, 0xef, 0xf0, 0x03, 0x18, 0xc2, 0x89 }, + { 0xcf, 0xce, 0x55, 0xeb, 0xaf, 0xc8, 0x40, 0xd7, + 0xae, 0x48, 0x28, 0x1c, 0x7f, 0xd5, 0x7e, 0xc8, + 0xb4, 0x82, 0xd4, 0xb7, 0x04, 0x43, 0x74, 0x95, + 0x49, 0x5a, 0xc4, 0x14, 0xcf, 0x4a, 0x37, 0x4b }, + { 0x67, 0x46, 0xfa, 0xcf, 0x71, 0x14, 0x6d, 0x99, + 0x9d, 0xab, 0xd0, 0x5d, 0x09, 0x3a, 0xe5, 0x86, + 0x64, 0x8d, 0x1e, 0xe2, 0x8e, 0x72, 0x61, 0x7b, + 0x99, 0xd0, 0xf0, 0x08, 0x6e, 0x1e, 0x45, 0xbf }, + { 0x57, 0x1c, 0xed, 0x28, 0x3b, 0x3f, 0x23, 0xb4, + 0xe7, 0x50, 0xbf, 0x12, 0xa2, 0xca, 0xf1, 0x78, + 0x18, 0x47, 0xbd, 0x89, 0x0e, 0x43, 0x60, 0x3c, + 0xdc, 0x59, 0x76, 0x10, 0x2b, 0x7b, 0xb1, 0x1b }, + { 0xcf, 0xcb, 0x76, 0x5b, 0x04, 0x8e, 0x35, 0x02, + 0x2c, 0x5d, 0x08, 0x9d, 0x26, 0xe8, 0x5a, 0x36, + 0xb0, 0x05, 0xa2, 0xb8, 0x04, 0x93, 0xd0, 0x3a, + 0x14, 0x4e, 0x09, 0xf4, 0x09, 0xb6, 0xaf, 0xd1 }, + { 0x40, 0x50, 0xc7, 0xa2, 0x77, 0x05, 0xbb, 0x27, + 0xf4, 0x20, 0x89, 0xb2, 0x99, 0xf3, 0xcb, 0xe5, + 0x05, 0x4e, 0xad, 0x68, 0x72, 0x7e, 0x8e, 0xf9, + 0x31, 0x8c, 0xe6, 0xf2, 0x5c, 0xd6, 0xf3, 0x1d }, + { 0x18, 0x40, 0x70, 0xbd, 0x5d, 0x26, 0x5f, 0xbd, + 0xc1, 0x42, 0xcd, 0x1c, 0x5c, 0xd0, 0xd7, 0xe4, + 0x14, 0xe7, 0x03, 0x69, 0xa2, 0x66, 0xd6, 0x27, + 0xc8, 0xfb, 0xa8, 0x4f, 0xa5, 0xe8, 0x4c, 0x34 }, + { 0x9e, 0xdd, 0xa9, 0xa4, 0x44, 0x39, 0x02, 0xa9, + 0x58, 0x8c, 0x0d, 0x0c, 0xcc, 0x62, 0xb9, 0x30, + 0x21, 0x84, 0x79, 0xa6, 0x84, 0x1e, 0x6f, 0xe7, + 0xd4, 0x30, 0x03, 0xf0, 0x4b, 0x1f, 0xd6, 0x43 }, + { 0xe4, 0x12, 0xfe, 0xef, 0x79, 0x08, 0x32, 0x4a, + 0x6d, 0xa1, 0x84, 0x16, 0x29, 0xf3, 0x5d, 0x3d, + 0x35, 0x86, 0x42, 0x01, 0x93, 0x10, 0xec, 0x57, + 0xc6, 0x14, 0x83, 0x6b, 0x63, 0xd3, 0x07, 0x63 }, + { 0x1a, 0x2b, 0x8e, 0xdf, 0xf3, 0xf9, 0xac, 0xc1, + 0x55, 0x4f, 0xcb, 0xae, 0x3c, 0xf1, 0xd6, 0x29, + 0x8c, 0x64, 0x62, 0xe2, 0x2e, 0x5e, 0xb0, 0x25, + 0x96, 0x84, 0xf8, 0x35, 0x01, 0x2b, 0xd1, 0x3f }, + { 0x28, 0x8c, 0x4a, 0xd9, 0xb9, 0x40, 0x97, 0x62, + 0xea, 0x07, 0xc2, 0x4a, 0x41, 0xf0, 0x4f, 0x69, + 0xa7, 0xd7, 0x4b, 0xee, 0x2d, 0x95, 0x43, 0x53, + 0x74, 0xbd, 0xe9, 0x46, 0xd7, 0x24, 0x1c, 0x7b }, + { 0x80, 0x56, 0x91, 0xbb, 0x28, 0x67, 0x48, 0xcf, + 0xb5, 0x91, 0xd3, 0xae, 0xbe, 0x7e, 0x6f, 0x4e, + 0x4d, 0xc6, 0xe2, 0x80, 0x8c, 0x65, 0x14, 0x3c, + 0xc0, 0x04, 0xe4, 0xeb, 0x6f, 0xd0, 0x9d, 0x43 }, + { 0xd4, 0xac, 0x8d, 0x3a, 0x0a, 0xfc, 0x6c, 0xfa, + 0x7b, 0x46, 0x0a, 0xe3, 0x00, 0x1b, 0xae, 0xb3, + 0x6d, 0xad, 0xb3, 0x7d, 0xa0, 0x7d, 0x2e, 0x8a, + 0xc9, 0x18, 0x22, 0xdf, 0x34, 0x8a, 0xed, 0x3d }, + { 0xc3, 0x76, 0x61, 0x70, 0x14, 0xd2, 0x01, 0x58, + 0xbc, 0xed, 0x3d, 0x3b, 0xa5, 0x52, 0xb6, 0xec, + 0xcf, 0x84, 0xe6, 0x2a, 0xa3, 0xeb, 0x65, 0x0e, + 0x90, 0x02, 0x9c, 0x84, 0xd1, 0x3e, 0xea, 0x69 }, + { 0xc4, 0x1f, 0x09, 0xf4, 0x3c, 0xec, 0xae, 0x72, + 0x93, 0xd6, 0x00, 0x7c, 0xa0, 0xa3, 0x57, 0x08, + 0x7d, 0x5a, 0xe5, 0x9b, 0xe5, 0x00, 0xc1, 0xcd, + 0x5b, 0x28, 0x9e, 0xe8, 0x10, 0xc7, 0xb0, 0x82 }, + { 0x03, 0xd1, 0xce, 0xd1, 0xfb, 0xa5, 0xc3, 0x91, + 0x55, 0xc4, 0x4b, 0x77, 0x65, 0xcb, 0x76, 0x0c, + 0x78, 0x70, 0x8d, 0xcf, 0xc8, 0x0b, 0x0b, 0xd8, + 0xad, 0xe3, 0xa5, 0x6d, 0xa8, 0x83, 0x0b, 0x29 }, + { 0x09, 0xbd, 0xe6, 0xf1, 0x52, 0x21, 0x8d, 0xc9, + 0x2c, 0x41, 0xd7, 0xf4, 0x53, 0x87, 0xe6, 0x3e, + 0x58, 0x69, 0xd8, 0x07, 0xec, 0x70, 0xb8, 0x21, + 0x40, 0x5d, 0xbd, 0x88, 0x4b, 0x7f, 0xcf, 0x4b }, + { 0x71, 0xc9, 0x03, 0x6e, 0x18, 0x17, 0x9b, 0x90, + 0xb3, 0x7d, 0x39, 0xe9, 0xf0, 0x5e, 0xb8, 0x9c, + 0xc5, 0xfc, 0x34, 0x1f, 0xd7, 0xc4, 0x77, 0xd0, + 0xd7, 0x49, 0x32, 0x85, 0xfa, 0xca, 0x08, 0xa4 }, + { 0x59, 0x16, 0x83, 0x3e, 0xbb, 0x05, 0xcd, 0x91, + 0x9c, 0xa7, 0xfe, 0x83, 0xb6, 0x92, 0xd3, 0x20, + 0x5b, 0xef, 0x72, 0x39, 0x2b, 0x2c, 0xf6, 0xbb, + 0x0a, 0x6d, 0x43, 0xf9, 0x94, 0xf9, 0x5f, 0x11 }, + { 0xf6, 0x3a, 0xab, 0x3e, 0xc6, 0x41, 0xb3, 0xb0, + 0x24, 0x96, 0x4c, 0x2b, 0x43, 0x7c, 0x04, 0xf6, + 0x04, 0x3c, 0x4c, 0x7e, 0x02, 0x79, 0x23, 0x99, + 0x95, 0x40, 0x19, 0x58, 0xf8, 0x6b, 0xbe, 0x54 }, + { 0xf1, 0x72, 0xb1, 0x80, 0xbf, 0xb0, 0x97, 0x40, + 0x49, 0x31, 0x20, 0xb6, 0x32, 0x6c, 0xbd, 0xc5, + 0x61, 0xe4, 0x77, 0xde, 0xf9, 0xbb, 0xcf, 0xd2, + 0x8c, 0xc8, 0xc1, 0xc5, 0xe3, 0x37, 0x9a, 0x31 }, + { 0xcb, 0x9b, 0x89, 0xcc, 0x18, 0x38, 0x1d, 0xd9, + 0x14, 0x1a, 0xde, 0x58, 0x86, 0x54, 0xd4, 0xe6, + 0xa2, 0x31, 0xd5, 0xbf, 0x49, 0xd4, 0xd5, 0x9a, + 0xc2, 0x7d, 0x86, 0x9c, 0xbe, 0x10, 0x0c, 0xf3 }, + { 0x7b, 0xd8, 0x81, 0x50, 0x46, 0xfd, 0xd8, 0x10, + 0xa9, 0x23, 0xe1, 0x98, 0x4a, 0xae, 0xbd, 0xcd, + 0xf8, 0x4d, 0x87, 0xc8, 0x99, 0x2d, 0x68, 0xb5, + 0xee, 0xb4, 0x60, 0xf9, 0x3e, 0xb3, 0xc8, 0xd7 }, + { 0x60, 0x7b, 0xe6, 0x68, 0x62, 0xfd, 0x08, 0xee, + 0x5b, 0x19, 0xfa, 0xca, 0xc0, 0x9d, 0xfd, 0xbc, + 0xd4, 0x0c, 0x31, 0x21, 0x01, 0xd6, 0x6e, 0x6e, + 0xbd, 0x2b, 0x84, 0x1f, 0x1b, 0x9a, 0x93, 0x25 }, + { 0x9f, 0xe0, 0x3b, 0xbe, 0x69, 0xab, 0x18, 0x34, + 0xf5, 0x21, 0x9b, 0x0d, 0xa8, 0x8a, 0x08, 0xb3, + 0x0a, 0x66, 0xc5, 0x91, 0x3f, 0x01, 0x51, 0x96, + 0x3c, 0x36, 0x05, 0x60, 0xdb, 0x03, 0x87, 0xb3 }, + { 0x90, 0xa8, 0x35, 0x85, 0x71, 0x7b, 0x75, 0xf0, + 0xe9, 0xb7, 0x25, 0xe0, 0x55, 0xee, 0xee, 0xb9, + 0xe7, 0xa0, 0x28, 0xea, 0x7e, 0x6c, 0xbc, 0x07, + 0xb2, 0x09, 0x17, 0xec, 0x03, 0x63, 0xe3, 0x8c }, + { 0x33, 0x6e, 0xa0, 0x53, 0x0f, 0x4a, 0x74, 0x69, + 0x12, 0x6e, 0x02, 0x18, 0x58, 0x7e, 0xbb, 0xde, + 0x33, 0x58, 0xa0, 0xb3, 0x1c, 0x29, 0xd2, 0x00, + 0xf7, 0xdc, 0x7e, 0xb1, 0x5c, 0x6a, 0xad, 0xd8 }, + { 0xa7, 0x9e, 0x76, 0xdc, 0x0a, 0xbc, 0xa4, 0x39, + 0x6f, 0x07, 0x47, 0xcd, 0x7b, 0x74, 0x8d, 0xf9, + 0x13, 0x00, 0x76, 0x26, 0xb1, 0xd6, 0x59, 0xda, + 0x0c, 0x1f, 0x78, 0xb9, 0x30, 0x3d, 0x01, 0xa3 }, + { 0x44, 0xe7, 0x8a, 0x77, 0x37, 0x56, 0xe0, 0x95, + 0x15, 0x19, 0x50, 0x4d, 0x70, 0x38, 0xd2, 0x8d, + 0x02, 0x13, 0xa3, 0x7e, 0x0c, 0xe3, 0x75, 0x37, + 0x17, 0x57, 0xbc, 0x99, 0x63, 0x11, 0xe3, 0xb8 }, + { 0x77, 0xac, 0x01, 0x2a, 0x3f, 0x75, 0x4d, 0xcf, + 0xea, 0xb5, 0xeb, 0x99, 0x6b, 0xe9, 0xcd, 0x2d, + 0x1f, 0x96, 0x11, 0x1b, 0x6e, 0x49, 0xf3, 0x99, + 0x4d, 0xf1, 0x81, 0xf2, 0x85, 0x69, 0xd8, 0x25 }, + { 0xce, 0x5a, 0x10, 0xdb, 0x6f, 0xcc, 0xda, 0xf1, + 0x40, 0xaa, 0xa4, 0xde, 0xd6, 0x25, 0x0a, 0x9c, + 0x06, 0xe9, 0x22, 0x2b, 0xc9, 0xf9, 0xf3, 0x65, + 0x8a, 0x4a, 0xff, 0x93, 0x5f, 0x2b, 0x9f, 0x3a }, + { 0xec, 0xc2, 0x03, 0xa7, 0xfe, 0x2b, 0xe4, 0xab, + 0xd5, 0x5b, 0xb5, 0x3e, 0x6e, 0x67, 0x35, 0x72, + 0xe0, 0x07, 0x8d, 0xa8, 0xcd, 0x37, 0x5e, 0xf4, + 0x30, 0xcc, 0x97, 0xf9, 0xf8, 0x00, 0x83, 0xaf }, + { 0x14, 0xa5, 0x18, 0x6d, 0xe9, 0xd7, 0xa1, 0x8b, + 0x04, 0x12, 0xb8, 0x56, 0x3e, 0x51, 0xcc, 0x54, + 0x33, 0x84, 0x0b, 0x4a, 0x12, 0x9a, 0x8f, 0xf9, + 0x63, 0xb3, 0x3a, 0x3c, 0x4a, 0xfe, 0x8e, 0xbb }, + { 0x13, 0xf8, 0xef, 0x95, 0xcb, 0x86, 0xe6, 0xa6, + 0x38, 0x93, 0x1c, 0x8e, 0x10, 0x76, 0x73, 0xeb, + 0x76, 0xba, 0x10, 0xd7, 0xc2, 0xcd, 0x70, 0xb9, + 0xd9, 0x92, 0x0b, 0xbe, 0xed, 0x92, 0x94, 0x09 }, + { 0x0b, 0x33, 0x8f, 0x4e, 0xe1, 0x2f, 0x2d, 0xfc, + 0xb7, 0x87, 0x13, 0x37, 0x79, 0x41, 0xe0, 0xb0, + 0x63, 0x21, 0x52, 0x58, 0x1d, 0x13, 0x32, 0x51, + 0x6e, 0x4a, 0x2c, 0xab, 0x19, 0x42, 0xcc, 0xa4 }, + { 0xea, 0xab, 0x0e, 0xc3, 0x7b, 0x3b, 0x8a, 0xb7, + 0x96, 0xe9, 0xf5, 0x72, 0x38, 0xde, 0x14, 0xa2, + 0x64, 0xa0, 0x76, 0xf3, 0x88, 0x7d, 0x86, 0xe2, + 0x9b, 0xb5, 0x90, 0x6d, 0xb5, 0xa0, 0x0e, 0x02 }, + { 0x23, 0xcb, 0x68, 0xb8, 0xc0, 0xe6, 0xdc, 0x26, + 0xdc, 0x27, 0x76, 0x6d, 0xdc, 0x0a, 0x13, 0xa9, + 0x94, 0x38, 0xfd, 0x55, 0x61, 0x7a, 0xa4, 0x09, + 0x5d, 0x8f, 0x96, 0x97, 0x20, 0xc8, 0x72, 0xdf }, + { 0x09, 0x1d, 0x8e, 0xe3, 0x0d, 0x6f, 0x29, 0x68, + 0xd4, 0x6b, 0x68, 0x7d, 0xd6, 0x52, 0x92, 0x66, + 0x57, 0x42, 0xde, 0x0b, 0xb8, 0x3d, 0xcc, 0x00, + 0x04, 0xc7, 0x2c, 0xe1, 0x00, 0x07, 0xa5, 0x49 }, + { 0x7f, 0x50, 0x7a, 0xbc, 0x6d, 0x19, 0xba, 0x00, + 0xc0, 0x65, 0xa8, 0x76, 0xec, 0x56, 0x57, 0x86, + 0x88, 0x82, 0xd1, 0x8a, 0x22, 0x1b, 0xc4, 0x6c, + 0x7a, 0x69, 0x12, 0x54, 0x1f, 0x5b, 0xc7, 0xba }, + { 0xa0, 0x60, 0x7c, 0x24, 0xe1, 0x4e, 0x8c, 0x22, + 0x3d, 0xb0, 0xd7, 0x0b, 0x4d, 0x30, 0xee, 0x88, + 0x01, 0x4d, 0x60, 0x3f, 0x43, 0x7e, 0x9e, 0x02, + 0xaa, 0x7d, 0xaf, 0xa3, 0xcd, 0xfb, 0xad, 0x94 }, + { 0xdd, 0xbf, 0xea, 0x75, 0xcc, 0x46, 0x78, 0x82, + 0xeb, 0x34, 0x83, 0xce, 0x5e, 0x2e, 0x75, 0x6a, + 0x4f, 0x47, 0x01, 0xb7, 0x6b, 0x44, 0x55, 0x19, + 0xe8, 0x9f, 0x22, 0xd6, 0x0f, 0xa8, 0x6e, 0x06 }, + { 0x0c, 0x31, 0x1f, 0x38, 0xc3, 0x5a, 0x4f, 0xb9, + 0x0d, 0x65, 0x1c, 0x28, 0x9d, 0x48, 0x68, 0x56, + 0xcd, 0x14, 0x13, 0xdf, 0x9b, 0x06, 0x77, 0xf5, + 0x3e, 0xce, 0x2c, 0xd9, 0xe4, 0x77, 0xc6, 0x0a }, + { 0x46, 0xa7, 0x3a, 0x8d, 0xd3, 0xe7, 0x0f, 0x59, + 0xd3, 0x94, 0x2c, 0x01, 0xdf, 0x59, 0x9d, 0xef, + 0x78, 0x3c, 0x9d, 0xa8, 0x2f, 0xd8, 0x32, 0x22, + 0xcd, 0x66, 0x2b, 0x53, 0xdc, 0xe7, 0xdb, 0xdf }, + { 0xad, 0x03, 0x8f, 0xf9, 0xb1, 0x4d, 0xe8, 0x4a, + 0x80, 0x1e, 0x4e, 0x62, 0x1c, 0xe5, 0xdf, 0x02, + 0x9d, 0xd9, 0x35, 0x20, 0xd0, 0xc2, 0xfa, 0x38, + 0xbf, 0xf1, 0x76, 0xa8, 0xb1, 0xd1, 0x69, 0x8c }, + { 0xab, 0x70, 0xc5, 0xdf, 0xbd, 0x1e, 0xa8, 0x17, + 0xfe, 0xd0, 0xcd, 0x06, 0x72, 0x93, 0xab, 0xf3, + 0x19, 0xe5, 0xd7, 0x90, 0x1c, 0x21, 0x41, 0xd5, + 0xd9, 0x9b, 0x23, 0xf0, 0x3a, 0x38, 0xe7, 0x48 }, + { 0x1f, 0xff, 0xda, 0x67, 0x93, 0x2b, 0x73, 0xc8, + 0xec, 0xaf, 0x00, 0x9a, 0x34, 0x91, 0xa0, 0x26, + 0x95, 0x3b, 0xab, 0xfe, 0x1f, 0x66, 0x3b, 0x06, + 0x97, 0xc3, 0xc4, 0xae, 0x8b, 0x2e, 0x7d, 0xcb }, + { 0xb0, 0xd2, 0xcc, 0x19, 0x47, 0x2d, 0xd5, 0x7f, + 0x2b, 0x17, 0xef, 0xc0, 0x3c, 0x8d, 0x58, 0xc2, + 0x28, 0x3d, 0xbb, 0x19, 0xda, 0x57, 0x2f, 0x77, + 0x55, 0x85, 0x5a, 0xa9, 0x79, 0x43, 0x17, 0xa0 }, + { 0xa0, 0xd1, 0x9a, 0x6e, 0xe3, 0x39, 0x79, 0xc3, + 0x25, 0x51, 0x0e, 0x27, 0x66, 0x22, 0xdf, 0x41, + 0xf7, 0x15, 0x83, 0xd0, 0x75, 0x01, 0xb8, 0x70, + 0x71, 0x12, 0x9a, 0x0a, 0xd9, 0x47, 0x32, 0xa5 }, + { 0x72, 0x46, 0x42, 0xa7, 0x03, 0x2d, 0x10, 0x62, + 0xb8, 0x9e, 0x52, 0xbe, 0xa3, 0x4b, 0x75, 0xdf, + 0x7d, 0x8f, 0xe7, 0x72, 0xd9, 0xfe, 0x3c, 0x93, + 0xdd, 0xf3, 0xc4, 0x54, 0x5a, 0xb5, 0xa9, 0x9b }, + { 0xad, 0xe5, 0xea, 0xa7, 0xe6, 0x1f, 0x67, 0x2d, + 0x58, 0x7e, 0xa0, 0x3d, 0xae, 0x7d, 0x7b, 0x55, + 0x22, 0x9c, 0x01, 0xd0, 0x6b, 0xc0, 0xa5, 0x70, + 0x14, 0x36, 0xcb, 0xd1, 0x83, 0x66, 0xa6, 0x26 }, + { 0x01, 0x3b, 0x31, 0xeb, 0xd2, 0x28, 0xfc, 0xdd, + 0xa5, 0x1f, 0xab, 0xb0, 0x3b, 0xb0, 0x2d, 0x60, + 0xac, 0x20, 0xca, 0x21, 0x5a, 0xaf, 0xa8, 0x3b, + 0xdd, 0x85, 0x5e, 0x37, 0x55, 0xa3, 0x5f, 0x0b }, + { 0x33, 0x2e, 0xd4, 0x0b, 0xb1, 0x0d, 0xde, 0x3c, + 0x95, 0x4a, 0x75, 0xd7, 0xb8, 0x99, 0x9d, 0x4b, + 0x26, 0xa1, 0xc0, 0x63, 0xc1, 0xdc, 0x6e, 0x32, + 0xc1, 0xd9, 0x1b, 0xab, 0x7b, 0xbb, 0x7d, 0x16 }, + { 0xc7, 0xa1, 0x97, 0xb3, 0xa0, 0x5b, 0x56, 0x6b, + 0xcc, 0x9f, 0xac, 0xd2, 0x0e, 0x44, 0x1d, 0x6f, + 0x6c, 0x28, 0x60, 0xac, 0x96, 0x51, 0xcd, 0x51, + 0xd6, 0xb9, 0xd2, 0xcd, 0xee, 0xea, 0x03, 0x90 }, + { 0xbd, 0x9c, 0xf6, 0x4e, 0xa8, 0x95, 0x3c, 0x03, + 0x71, 0x08, 0xe6, 0xf6, 0x54, 0x91, 0x4f, 0x39, + 0x58, 0xb6, 0x8e, 0x29, 0xc1, 0x67, 0x00, 0xdc, + 0x18, 0x4d, 0x94, 0xa2, 0x17, 0x08, 0xff, 0x60 }, + { 0x88, 0x35, 0xb0, 0xac, 0x02, 0x11, 0x51, 0xdf, + 0x71, 0x64, 0x74, 0xce, 0x27, 0xce, 0x4d, 0x3c, + 0x15, 0xf0, 0xb2, 0xda, 0xb4, 0x80, 0x03, 0xcf, + 0x3f, 0x3e, 0xfd, 0x09, 0x45, 0x10, 0x6b, 0x9a }, + { 0x3b, 0xfe, 0xfa, 0x33, 0x01, 0xaa, 0x55, 0xc0, + 0x80, 0x19, 0x0c, 0xff, 0xda, 0x8e, 0xae, 0x51, + 0xd9, 0xaf, 0x48, 0x8b, 0x4c, 0x1f, 0x24, 0xc3, + 0xd9, 0xa7, 0x52, 0x42, 0xfd, 0x8e, 0xa0, 0x1d }, + { 0x08, 0x28, 0x4d, 0x14, 0x99, 0x3c, 0xd4, 0x7d, + 0x53, 0xeb, 0xae, 0xcf, 0x0d, 0xf0, 0x47, 0x8c, + 0xc1, 0x82, 0xc8, 0x9c, 0x00, 0xe1, 0x85, 0x9c, + 0x84, 0x85, 0x16, 0x86, 0xdd, 0xf2, 0xc1, 0xb7 }, + { 0x1e, 0xd7, 0xef, 0x9f, 0x04, 0xc2, 0xac, 0x8d, + 0xb6, 0xa8, 0x64, 0xdb, 0x13, 0x10, 0x87, 0xf2, + 0x70, 0x65, 0x09, 0x8e, 0x69, 0xc3, 0xfe, 0x78, + 0x71, 0x8d, 0x9b, 0x94, 0x7f, 0x4a, 0x39, 0xd0 }, + { 0xc1, 0x61, 0xf2, 0xdc, 0xd5, 0x7e, 0x9c, 0x14, + 0x39, 0xb3, 0x1a, 0x9d, 0xd4, 0x3d, 0x8f, 0x3d, + 0x7d, 0xd8, 0xf0, 0xeb, 0x7c, 0xfa, 0xc6, 0xfb, + 0x25, 0xa0, 0xf2, 0x8e, 0x30, 0x6f, 0x06, 0x61 }, + { 0xc0, 0x19, 0x69, 0xad, 0x34, 0xc5, 0x2c, 0xaf, + 0x3d, 0xc4, 0xd8, 0x0d, 0x19, 0x73, 0x5c, 0x29, + 0x73, 0x1a, 0xc6, 0xe7, 0xa9, 0x20, 0x85, 0xab, + 0x92, 0x50, 0xc4, 0x8d, 0xea, 0x48, 0xa3, 0xfc }, + { 0x17, 0x20, 0xb3, 0x65, 0x56, 0x19, 0xd2, 0xa5, + 0x2b, 0x35, 0x21, 0xae, 0x0e, 0x49, 0xe3, 0x45, + 0xcb, 0x33, 0x89, 0xeb, 0xd6, 0x20, 0x8a, 0xca, + 0xf9, 0xf1, 0x3f, 0xda, 0xcc, 0xa8, 0xbe, 0x49 }, + { 0x75, 0x62, 0x88, 0x36, 0x1c, 0x83, 0xe2, 0x4c, + 0x61, 0x7c, 0xf9, 0x5c, 0x90, 0x5b, 0x22, 0xd0, + 0x17, 0xcd, 0xc8, 0x6f, 0x0b, 0xf1, 0xd6, 0x58, + 0xf4, 0x75, 0x6c, 0x73, 0x79, 0x87, 0x3b, 0x7f }, + { 0xe7, 0xd0, 0xed, 0xa3, 0x45, 0x26, 0x93, 0xb7, + 0x52, 0xab, 0xcd, 0xa1, 0xb5, 0x5e, 0x27, 0x6f, + 0x82, 0x69, 0x8f, 0x5f, 0x16, 0x05, 0x40, 0x3e, + 0xff, 0x83, 0x0b, 0xea, 0x00, 0x71, 0xa3, 0x94 }, + { 0x2c, 0x82, 0xec, 0xaa, 0x6b, 0x84, 0x80, 0x3e, + 0x04, 0x4a, 0xf6, 0x31, 0x18, 0xaf, 0xe5, 0x44, + 0x68, 0x7c, 0xb6, 0xe6, 0xc7, 0xdf, 0x49, 0xed, + 0x76, 0x2d, 0xfd, 0x7c, 0x86, 0x93, 0xa1, 0xbc }, + { 0x61, 0x36, 0xcb, 0xf4, 0xb4, 0x41, 0x05, 0x6f, + 0xa1, 0xe2, 0x72, 0x24, 0x98, 0x12, 0x5d, 0x6d, + 0xed, 0x45, 0xe1, 0x7b, 0x52, 0x14, 0x39, 0x59, + 0xc7, 0xf4, 0xd4, 0xe3, 0x95, 0x21, 0x8a, 0xc2 }, + { 0x72, 0x1d, 0x32, 0x45, 0xaa, 0xfe, 0xf2, 0x7f, + 0x6a, 0x62, 0x4f, 0x47, 0x95, 0x4b, 0x6c, 0x25, + 0x50, 0x79, 0x52, 0x6f, 0xfa, 0x25, 0xe9, 0xff, + 0x77, 0xe5, 0xdc, 0xff, 0x47, 0x3b, 0x15, 0x97 }, + { 0x9d, 0xd2, 0xfb, 0xd8, 0xce, 0xf1, 0x6c, 0x35, + 0x3c, 0x0a, 0xc2, 0x11, 0x91, 0xd5, 0x09, 0xeb, + 0x28, 0xdd, 0x9e, 0x3e, 0x0d, 0x8c, 0xea, 0x5d, + 0x26, 0xca, 0x83, 0x93, 0x93, 0x85, 0x1c, 0x3a }, + { 0xb2, 0x39, 0x4c, 0xea, 0xcd, 0xeb, 0xf2, 0x1b, + 0xf9, 0xdf, 0x2c, 0xed, 0x98, 0xe5, 0x8f, 0x1c, + 0x3a, 0x4b, 0xbb, 0xff, 0x66, 0x0d, 0xd9, 0x00, + 0xf6, 0x22, 0x02, 0xd6, 0x78, 0x5c, 0xc4, 0x6e }, + { 0x57, 0x08, 0x9f, 0x22, 0x27, 0x49, 0xad, 0x78, + 0x71, 0x76, 0x5f, 0x06, 0x2b, 0x11, 0x4f, 0x43, + 0xba, 0x20, 0xec, 0x56, 0x42, 0x2a, 0x8b, 0x1e, + 0x3f, 0x87, 0x19, 0x2c, 0x0e, 0xa7, 0x18, 0xc6 }, + { 0xe4, 0x9a, 0x94, 0x59, 0x96, 0x1c, 0xd3, 0x3c, + 0xdf, 0x4a, 0xae, 0x1b, 0x10, 0x78, 0xa5, 0xde, + 0xa7, 0xc0, 0x40, 0xe0, 0xfe, 0xa3, 0x40, 0xc9, + 0x3a, 0x72, 0x48, 0x72, 0xfc, 0x4a, 0xf8, 0x06 }, + { 0xed, 0xe6, 0x7f, 0x72, 0x0e, 0xff, 0xd2, 0xca, + 0x9c, 0x88, 0x99, 0x41, 0x52, 0xd0, 0x20, 0x1d, + 0xee, 0x6b, 0x0a, 0x2d, 0x2c, 0x07, 0x7a, 0xca, + 0x6d, 0xae, 0x29, 0xf7, 0x3f, 0x8b, 0x63, 0x09 }, + { 0xe0, 0xf4, 0x34, 0xbf, 0x22, 0xe3, 0x08, 0x80, + 0x39, 0xc2, 0x1f, 0x71, 0x9f, 0xfc, 0x67, 0xf0, + 0xf2, 0xcb, 0x5e, 0x98, 0xa7, 0xa0, 0x19, 0x4c, + 0x76, 0xe9, 0x6b, 0xf4, 0xe8, 0xe1, 0x7e, 0x61 }, + { 0x27, 0x7c, 0x04, 0xe2, 0x85, 0x34, 0x84, 0xa4, + 0xeb, 0xa9, 0x10, 0xad, 0x33, 0x6d, 0x01, 0xb4, + 0x77, 0xb6, 0x7c, 0xc2, 0x00, 0xc5, 0x9f, 0x3c, + 0x8d, 0x77, 0xee, 0xf8, 0x49, 0x4f, 0x29, 0xcd }, + { 0x15, 0x6d, 0x57, 0x47, 0xd0, 0xc9, 0x9c, 0x7f, + 0x27, 0x09, 0x7d, 0x7b, 0x7e, 0x00, 0x2b, 0x2e, + 0x18, 0x5c, 0xb7, 0x2d, 0x8d, 0xd7, 0xeb, 0x42, + 0x4a, 0x03, 0x21, 0x52, 0x81, 0x61, 0x21, 0x9f }, + { 0x20, 0xdd, 0xd1, 0xed, 0x9b, 0x1c, 0xa8, 0x03, + 0x94, 0x6d, 0x64, 0xa8, 0x3a, 0xe4, 0x65, 0x9d, + 0xa6, 0x7f, 0xba, 0x7a, 0x1a, 0x3e, 0xdd, 0xb1, + 0xe1, 0x03, 0xc0, 0xf5, 0xe0, 0x3e, 0x3a, 0x2c }, + { 0xf0, 0xaf, 0x60, 0x4d, 0x3d, 0xab, 0xbf, 0x9a, + 0x0f, 0x2a, 0x7d, 0x3d, 0xda, 0x6b, 0xd3, 0x8b, + 0xba, 0x72, 0xc6, 0xd0, 0x9b, 0xe4, 0x94, 0xfc, + 0xef, 0x71, 0x3f, 0xf1, 0x01, 0x89, 0xb6, 0xe6 }, + { 0x98, 0x02, 0xbb, 0x87, 0xde, 0xf4, 0xcc, 0x10, + 0xc4, 0xa5, 0xfd, 0x49, 0xaa, 0x58, 0xdf, 0xe2, + 0xf3, 0xfd, 0xdb, 0x46, 0xb4, 0x70, 0x88, 0x14, + 0xea, 0xd8, 0x1d, 0x23, 0xba, 0x95, 0x13, 0x9b }, + { 0x4f, 0x8c, 0xe1, 0xe5, 0x1d, 0x2f, 0xe7, 0xf2, + 0x40, 0x43, 0xa9, 0x04, 0xd8, 0x98, 0xeb, 0xfc, + 0x91, 0x97, 0x54, 0x18, 0x75, 0x34, 0x13, 0xaa, + 0x09, 0x9b, 0x79, 0x5e, 0xcb, 0x35, 0xce, 0xdb }, + { 0xbd, 0xdc, 0x65, 0x14, 0xd7, 0xee, 0x6a, 0xce, + 0x0a, 0x4a, 0xc1, 0xd0, 0xe0, 0x68, 0x11, 0x22, + 0x88, 0xcb, 0xcf, 0x56, 0x04, 0x54, 0x64, 0x27, + 0x05, 0x63, 0x01, 0x77, 0xcb, 0xa6, 0x08, 0xbd }, + { 0xd6, 0x35, 0x99, 0x4f, 0x62, 0x91, 0x51, 0x7b, + 0x02, 0x81, 0xff, 0xdd, 0x49, 0x6a, 0xfa, 0x86, + 0x27, 0x12, 0xe5, 0xb3, 0xc4, 0xe5, 0x2e, 0x4c, + 0xd5, 0xfd, 0xae, 0x8c, 0x0e, 0x72, 0xfb, 0x08 }, + { 0x87, 0x8d, 0x9c, 0xa6, 0x00, 0xcf, 0x87, 0xe7, + 0x69, 0xcc, 0x30, 0x5c, 0x1b, 0x35, 0x25, 0x51, + 0x86, 0x61, 0x5a, 0x73, 0xa0, 0xda, 0x61, 0x3b, + 0x5f, 0x1c, 0x98, 0xdb, 0xf8, 0x12, 0x83, 0xea }, + { 0xa6, 0x4e, 0xbe, 0x5d, 0xc1, 0x85, 0xde, 0x9f, + 0xdd, 0xe7, 0x60, 0x7b, 0x69, 0x98, 0x70, 0x2e, + 0xb2, 0x34, 0x56, 0x18, 0x49, 0x57, 0x30, 0x7d, + 0x2f, 0xa7, 0x2e, 0x87, 0xa4, 0x77, 0x02, 0xd6 }, + { 0xce, 0x50, 0xea, 0xb7, 0xb5, 0xeb, 0x52, 0xbd, + 0xc9, 0xad, 0x8e, 0x5a, 0x48, 0x0a, 0xb7, 0x80, + 0xca, 0x93, 0x20, 0xe4, 0x43, 0x60, 0xb1, 0xfe, + 0x37, 0xe0, 0x3f, 0x2f, 0x7a, 0xd7, 0xde, 0x01 }, + { 0xee, 0xdd, 0xb7, 0xc0, 0xdb, 0x6e, 0x30, 0xab, + 0xe6, 0x6d, 0x79, 0xe3, 0x27, 0x51, 0x1e, 0x61, + 0xfc, 0xeb, 0xbc, 0x29, 0xf1, 0x59, 0xb4, 0x0a, + 0x86, 0xb0, 0x46, 0xec, 0xf0, 0x51, 0x38, 0x23 }, + { 0x78, 0x7f, 0xc9, 0x34, 0x40, 0xc1, 0xec, 0x96, + 0xb5, 0xad, 0x01, 0xc1, 0x6c, 0xf7, 0x79, 0x16, + 0xa1, 0x40, 0x5f, 0x94, 0x26, 0x35, 0x6e, 0xc9, + 0x21, 0xd8, 0xdf, 0xf3, 0xea, 0x63, 0xb7, 0xe0 }, + { 0x7f, 0x0d, 0x5e, 0xab, 0x47, 0xee, 0xfd, 0xa6, + 0x96, 0xc0, 0xbf, 0x0f, 0xbf, 0x86, 0xab, 0x21, + 0x6f, 0xce, 0x46, 0x1e, 0x93, 0x03, 0xab, 0xa6, + 0xac, 0x37, 0x41, 0x20, 0xe8, 0x90, 0xe8, 0xdf }, + { 0xb6, 0x80, 0x04, 0xb4, 0x2f, 0x14, 0xad, 0x02, + 0x9f, 0x4c, 0x2e, 0x03, 0xb1, 0xd5, 0xeb, 0x76, + 0xd5, 0x71, 0x60, 0xe2, 0x64, 0x76, 0xd2, 0x11, + 0x31, 0xbe, 0xf2, 0x0a, 0xda, 0x7d, 0x27, 0xf4 }, + { 0xb0, 0xc4, 0xeb, 0x18, 0xae, 0x25, 0x0b, 0x51, + 0xa4, 0x13, 0x82, 0xea, 0xd9, 0x2d, 0x0d, 0xc7, + 0x45, 0x5f, 0x93, 0x79, 0xfc, 0x98, 0x84, 0x42, + 0x8e, 0x47, 0x70, 0x60, 0x8d, 0xb0, 0xfa, 0xec }, + { 0xf9, 0x2b, 0x7a, 0x87, 0x0c, 0x05, 0x9f, 0x4d, + 0x46, 0x46, 0x4c, 0x82, 0x4e, 0xc9, 0x63, 0x55, + 0x14, 0x0b, 0xdc, 0xe6, 0x81, 0x32, 0x2c, 0xc3, + 0xa9, 0x92, 0xff, 0x10, 0x3e, 0x3f, 0xea, 0x52 }, + { 0x53, 0x64, 0x31, 0x26, 0x14, 0x81, 0x33, 0x98, + 0xcc, 0x52, 0x5d, 0x4c, 0x4e, 0x14, 0x6e, 0xde, + 0xb3, 0x71, 0x26, 0x5f, 0xba, 0x19, 0x13, 0x3a, + 0x2c, 0x3d, 0x21, 0x59, 0x29, 0x8a, 0x17, 0x42 }, + { 0xf6, 0x62, 0x0e, 0x68, 0xd3, 0x7f, 0xb2, 0xaf, + 0x50, 0x00, 0xfc, 0x28, 0xe2, 0x3b, 0x83, 0x22, + 0x97, 0xec, 0xd8, 0xbc, 0xe9, 0x9e, 0x8b, 0xe4, + 0xd0, 0x4e, 0x85, 0x30, 0x9e, 0x3d, 0x33, 0x74 }, + { 0x53, 0x16, 0xa2, 0x79, 0x69, 0xd7, 0xfe, 0x04, + 0xff, 0x27, 0xb2, 0x83, 0x96, 0x1b, 0xff, 0xc3, + 0xbf, 0x5d, 0xfb, 0x32, 0xfb, 0x6a, 0x89, 0xd1, + 0x01, 0xc6, 0xc3, 0xb1, 0x93, 0x7c, 0x28, 0x71 }, + { 0x81, 0xd1, 0x66, 0x4f, 0xdf, 0x3c, 0xb3, 0x3c, + 0x24, 0xee, 0xba, 0xc0, 0xbd, 0x64, 0x24, 0x4b, + 0x77, 0xc4, 0xab, 0xea, 0x90, 0xbb, 0xe8, 0xb5, + 0xee, 0x0b, 0x2a, 0xaf, 0xcf, 0x2d, 0x6a, 0x53 }, + { 0x34, 0x57, 0x82, 0xf2, 0x95, 0xb0, 0x88, 0x03, + 0x52, 0xe9, 0x24, 0xa0, 0x46, 0x7b, 0x5f, 0xbc, + 0x3e, 0x8f, 0x3b, 0xfb, 0xc3, 0xc7, 0xe4, 0x8b, + 0x67, 0x09, 0x1f, 0xb5, 0xe8, 0x0a, 0x94, 0x42 }, + { 0x79, 0x41, 0x11, 0xea, 0x6c, 0xd6, 0x5e, 0x31, + 0x1f, 0x74, 0xee, 0x41, 0xd4, 0x76, 0xcb, 0x63, + 0x2c, 0xe1, 0xe4, 0xb0, 0x51, 0xdc, 0x1d, 0x9e, + 0x9d, 0x06, 0x1a, 0x19, 0xe1, 0xd0, 0xbb, 0x49 }, + { 0x2a, 0x85, 0xda, 0xf6, 0x13, 0x88, 0x16, 0xb9, + 0x9b, 0xf8, 0xd0, 0x8b, 0xa2, 0x11, 0x4b, 0x7a, + 0xb0, 0x79, 0x75, 0xa7, 0x84, 0x20, 0xc1, 0xa3, + 0xb0, 0x6a, 0x77, 0x7c, 0x22, 0xdd, 0x8b, 0xcb }, + { 0x89, 0xb0, 0xd5, 0xf2, 0x89, 0xec, 0x16, 0x40, + 0x1a, 0x06, 0x9a, 0x96, 0x0d, 0x0b, 0x09, 0x3e, + 0x62, 0x5d, 0xa3, 0xcf, 0x41, 0xee, 0x29, 0xb5, + 0x9b, 0x93, 0x0c, 0x58, 0x20, 0x14, 0x54, 0x55 }, + { 0xd0, 0xfd, 0xcb, 0x54, 0x39, 0x43, 0xfc, 0x27, + 0xd2, 0x08, 0x64, 0xf5, 0x21, 0x81, 0x47, 0x1b, + 0x94, 0x2c, 0xc7, 0x7c, 0xa6, 0x75, 0xbc, 0xb3, + 0x0d, 0xf3, 0x1d, 0x35, 0x8e, 0xf7, 0xb1, 0xeb }, + { 0xb1, 0x7e, 0xa8, 0xd7, 0x70, 0x63, 0xc7, 0x09, + 0xd4, 0xdc, 0x6b, 0x87, 0x94, 0x13, 0xc3, 0x43, + 0xe3, 0x79, 0x0e, 0x9e, 0x62, 0xca, 0x85, 0xb7, + 0x90, 0x0b, 0x08, 0x6f, 0x6b, 0x75, 0xc6, 0x72 }, + { 0xe7, 0x1a, 0x3e, 0x2c, 0x27, 0x4d, 0xb8, 0x42, + 0xd9, 0x21, 0x14, 0xf2, 0x17, 0xe2, 0xc0, 0xea, + 0xc8, 0xb4, 0x50, 0x93, 0xfd, 0xfd, 0x9d, 0xf4, + 0xca, 0x71, 0x62, 0x39, 0x48, 0x62, 0xd5, 0x01 }, + { 0xc0, 0x47, 0x67, 0x59, 0xab, 0x7a, 0xa3, 0x33, + 0x23, 0x4f, 0x6b, 0x44, 0xf5, 0xfd, 0x85, 0x83, + 0x90, 0xec, 0x23, 0x69, 0x4c, 0x62, 0x2c, 0xb9, + 0x86, 0xe7, 0x69, 0xc7, 0x8e, 0xdd, 0x73, 0x3e }, + { 0x9a, 0xb8, 0xea, 0xbb, 0x14, 0x16, 0x43, 0x4d, + 0x85, 0x39, 0x13, 0x41, 0xd5, 0x69, 0x93, 0xc5, + 0x54, 0x58, 0x16, 0x7d, 0x44, 0x18, 0xb1, 0x9a, + 0x0f, 0x2a, 0xd8, 0xb7, 0x9a, 0x83, 0xa7, 0x5b }, + { 0x79, 0x92, 0xd0, 0xbb, 0xb1, 0x5e, 0x23, 0x82, + 0x6f, 0x44, 0x3e, 0x00, 0x50, 0x5d, 0x68, 0xd3, + 0xed, 0x73, 0x72, 0x99, 0x5a, 0x5c, 0x3e, 0x49, + 0x86, 0x54, 0x10, 0x2f, 0xbc, 0xd0, 0x96, 0x4e }, + { 0xc0, 0x21, 0xb3, 0x00, 0x85, 0x15, 0x14, 0x35, + 0xdf, 0x33, 0xb0, 0x07, 0xcc, 0xec, 0xc6, 0x9d, + 0xf1, 0x26, 0x9f, 0x39, 0xba, 0x25, 0x09, 0x2b, + 0xed, 0x59, 0xd9, 0x32, 0xac, 0x0f, 0xdc, 0x28 }, + { 0x91, 0xa2, 0x5e, 0xc0, 0xec, 0x0d, 0x9a, 0x56, + 0x7f, 0x89, 0xc4, 0xbf, 0xe1, 0xa6, 0x5a, 0x0e, + 0x43, 0x2d, 0x07, 0x06, 0x4b, 0x41, 0x90, 0xe2, + 0x7d, 0xfb, 0x81, 0x90, 0x1f, 0xd3, 0x13, 0x9b }, + { 0x59, 0x50, 0xd3, 0x9a, 0x23, 0xe1, 0x54, 0x5f, + 0x30, 0x12, 0x70, 0xaa, 0x1a, 0x12, 0xf2, 0xe6, + 0xc4, 0x53, 0x77, 0x6e, 0x4d, 0x63, 0x55, 0xde, + 0x42, 0x5c, 0xc1, 0x53, 0xf9, 0x81, 0x88, 0x67 }, + { 0xd7, 0x9f, 0x14, 0x72, 0x0c, 0x61, 0x0a, 0xf1, + 0x79, 0xa3, 0x76, 0x5d, 0x4b, 0x7c, 0x09, 0x68, + 0xf9, 0x77, 0x96, 0x2d, 0xbf, 0x65, 0x5b, 0x52, + 0x12, 0x72, 0xb6, 0xf1, 0xe1, 0x94, 0x48, 0x8e }, + { 0xe9, 0x53, 0x1b, 0xfc, 0x8b, 0x02, 0x99, 0x5a, + 0xea, 0xa7, 0x5b, 0xa2, 0x70, 0x31, 0xfa, 0xdb, + 0xcb, 0xf4, 0xa0, 0xda, 0xb8, 0x96, 0x1d, 0x92, + 0x96, 0xcd, 0x7e, 0x84, 0xd2, 0x5d, 0x60, 0x06 }, + { 0x34, 0xe9, 0xc2, 0x6a, 0x01, 0xd7, 0xf1, 0x61, + 0x81, 0xb4, 0x54, 0xa9, 0xd1, 0x62, 0x3c, 0x23, + 0x3c, 0xb9, 0x9d, 0x31, 0xc6, 0x94, 0x65, 0x6e, + 0x94, 0x13, 0xac, 0xa3, 0xe9, 0x18, 0x69, 0x2f }, + { 0xd9, 0xd7, 0x42, 0x2f, 0x43, 0x7b, 0xd4, 0x39, + 0xdd, 0xd4, 0xd8, 0x83, 0xda, 0xe2, 0xa0, 0x83, + 0x50, 0x17, 0x34, 0x14, 0xbe, 0x78, 0x15, 0x51, + 0x33, 0xff, 0xf1, 0x96, 0x4c, 0x3d, 0x79, 0x72 }, + { 0x4a, 0xee, 0x0c, 0x7a, 0xaf, 0x07, 0x54, 0x14, + 0xff, 0x17, 0x93, 0xea, 0xd7, 0xea, 0xca, 0x60, + 0x17, 0x75, 0xc6, 0x15, 0xdb, 0xd6, 0x0b, 0x64, + 0x0b, 0x0a, 0x9f, 0x0c, 0xe5, 0x05, 0xd4, 0x35 }, + { 0x6b, 0xfd, 0xd1, 0x54, 0x59, 0xc8, 0x3b, 0x99, + 0xf0, 0x96, 0xbf, 0xb4, 0x9e, 0xe8, 0x7b, 0x06, + 0x3d, 0x69, 0xc1, 0x97, 0x4c, 0x69, 0x28, 0xac, + 0xfc, 0xfb, 0x40, 0x99, 0xf8, 0xc4, 0xef, 0x67 }, + { 0x9f, 0xd1, 0xc4, 0x08, 0xfd, 0x75, 0xc3, 0x36, + 0x19, 0x3a, 0x2a, 0x14, 0xd9, 0x4f, 0x6a, 0xf5, + 0xad, 0xf0, 0x50, 0xb8, 0x03, 0x87, 0xb4, 0xb0, + 0x10, 0xfb, 0x29, 0xf4, 0xcc, 0x72, 0x70, 0x7c }, + { 0x13, 0xc8, 0x84, 0x80, 0xa5, 0xd0, 0x0d, 0x6c, + 0x8c, 0x7a, 0xd2, 0x11, 0x0d, 0x76, 0xa8, 0x2d, + 0x9b, 0x70, 0xf4, 0xfa, 0x66, 0x96, 0xd4, 0xe5, + 0xdd, 0x42, 0xa0, 0x66, 0xdc, 0xaf, 0x99, 0x20 }, + { 0x82, 0x0e, 0x72, 0x5e, 0xe2, 0x5f, 0xe8, 0xfd, + 0x3a, 0x8d, 0x5a, 0xbe, 0x4c, 0x46, 0xc3, 0xba, + 0x88, 0x9d, 0xe6, 0xfa, 0x91, 0x91, 0xaa, 0x22, + 0xba, 0x67, 0xd5, 0x70, 0x54, 0x21, 0x54, 0x2b }, + { 0x32, 0xd9, 0x3a, 0x0e, 0xb0, 0x2f, 0x42, 0xfb, + 0xbc, 0xaf, 0x2b, 0xad, 0x00, 0x85, 0xb2, 0x82, + 0xe4, 0x60, 0x46, 0xa4, 0xdf, 0x7a, 0xd1, 0x06, + 0x57, 0xc9, 0xd6, 0x47, 0x63, 0x75, 0xb9, 0x3e }, + { 0xad, 0xc5, 0x18, 0x79, 0x05, 0xb1, 0x66, 0x9c, + 0xd8, 0xec, 0x9c, 0x72, 0x1e, 0x19, 0x53, 0x78, + 0x6b, 0x9d, 0x89, 0xa9, 0xba, 0xe3, 0x07, 0x80, + 0xf1, 0xe1, 0xea, 0xb2, 0x4a, 0x00, 0x52, 0x3c }, + { 0xe9, 0x07, 0x56, 0xff, 0x7f, 0x9a, 0xd8, 0x10, + 0xb2, 0x39, 0xa1, 0x0c, 0xed, 0x2c, 0xf9, 0xb2, + 0x28, 0x43, 0x54, 0xc1, 0xf8, 0xc7, 0xe0, 0xac, + 0xcc, 0x24, 0x61, 0xdc, 0x79, 0x6d, 0x6e, 0x89 }, + { 0x12, 0x51, 0xf7, 0x6e, 0x56, 0x97, 0x84, 0x81, + 0x87, 0x53, 0x59, 0x80, 0x1d, 0xb5, 0x89, 0xa0, + 0xb2, 0x2f, 0x86, 0xd8, 0xd6, 0x34, 0xdc, 0x04, + 0x50, 0x6f, 0x32, 0x2e, 0xd7, 0x8f, 0x17, 0xe8 }, + { 0x3a, 0xfa, 0x89, 0x9f, 0xd9, 0x80, 0xe7, 0x3e, + 0xcb, 0x7f, 0x4d, 0x8b, 0x8f, 0x29, 0x1d, 0xc9, + 0xaf, 0x79, 0x6b, 0xc6, 0x5d, 0x27, 0xf9, 0x74, + 0xc6, 0xf1, 0x93, 0xc9, 0x19, 0x1a, 0x09, 0xfd }, + { 0xaa, 0x30, 0x5b, 0xe2, 0x6e, 0x5d, 0xed, 0xdc, + 0x3c, 0x10, 0x10, 0xcb, 0xc2, 0x13, 0xf9, 0x5f, + 0x05, 0x1c, 0x78, 0x5c, 0x5b, 0x43, 0x1e, 0x6a, + 0x7c, 0xd0, 0x48, 0xf1, 0x61, 0x78, 0x75, 0x28 }, + { 0x8e, 0xa1, 0x88, 0x4f, 0xf3, 0x2e, 0x9d, 0x10, + 0xf0, 0x39, 0xb4, 0x07, 0xd0, 0xd4, 0x4e, 0x7e, + 0x67, 0x0a, 0xbd, 0x88, 0x4a, 0xee, 0xe0, 0xfb, + 0x75, 0x7a, 0xe9, 0x4e, 0xaa, 0x97, 0x37, 0x3d }, + { 0xd4, 0x82, 0xb2, 0x15, 0x5d, 0x4d, 0xec, 0x6b, + 0x47, 0x36, 0xa1, 0xf1, 0x61, 0x7b, 0x53, 0xaa, + 0xa3, 0x73, 0x10, 0x27, 0x7d, 0x3f, 0xef, 0x0c, + 0x37, 0xad, 0x41, 0x76, 0x8f, 0xc2, 0x35, 0xb4 }, + { 0x4d, 0x41, 0x39, 0x71, 0x38, 0x7e, 0x7a, 0x88, + 0x98, 0xa8, 0xdc, 0x2a, 0x27, 0x50, 0x07, 0x78, + 0x53, 0x9e, 0xa2, 0x14, 0xa2, 0xdf, 0xe9, 0xb3, + 0xd7, 0xe8, 0xeb, 0xdc, 0xe5, 0xcf, 0x3d, 0xb3 }, + { 0x69, 0x6e, 0x5d, 0x46, 0xe6, 0xc5, 0x7e, 0x87, + 0x96, 0xe4, 0x73, 0x5d, 0x08, 0x91, 0x6e, 0x0b, + 0x79, 0x29, 0xb3, 0xcf, 0x29, 0x8c, 0x29, 0x6d, + 0x22, 0xe9, 0xd3, 0x01, 0x96, 0x53, 0x37, 0x1c }, + { 0x1f, 0x56, 0x47, 0xc1, 0xd3, 0xb0, 0x88, 0x22, + 0x88, 0x85, 0x86, 0x5c, 0x89, 0x40, 0x90, 0x8b, + 0xf4, 0x0d, 0x1a, 0x82, 0x72, 0x82, 0x19, 0x73, + 0xb1, 0x60, 0x00, 0x8e, 0x7a, 0x3c, 0xe2, 0xeb }, + { 0xb6, 0xe7, 0x6c, 0x33, 0x0f, 0x02, 0x1a, 0x5b, + 0xda, 0x65, 0x87, 0x50, 0x10, 0xb0, 0xed, 0xf0, + 0x91, 0x26, 0xc0, 0xf5, 0x10, 0xea, 0x84, 0x90, + 0x48, 0x19, 0x20, 0x03, 0xae, 0xf4, 0xc6, 0x1c }, + { 0x3c, 0xd9, 0x52, 0xa0, 0xbe, 0xad, 0xa4, 0x1a, + 0xbb, 0x42, 0x4c, 0xe4, 0x7f, 0x94, 0xb4, 0x2b, + 0xe6, 0x4e, 0x1f, 0xfb, 0x0f, 0xd0, 0x78, 0x22, + 0x76, 0x80, 0x79, 0x46, 0xd0, 0xd0, 0xbc, 0x55 }, + { 0x98, 0xd9, 0x26, 0x77, 0x43, 0x9b, 0x41, 0xb7, + 0xbb, 0x51, 0x33, 0x12, 0xaf, 0xb9, 0x2b, 0xcc, + 0x8e, 0xe9, 0x68, 0xb2, 0xe3, 0xb2, 0x38, 0xce, + 0xcb, 0x9b, 0x0f, 0x34, 0xc9, 0xbb, 0x63, 0xd0 }, + { 0xec, 0xbc, 0xa2, 0xcf, 0x08, 0xae, 0x57, 0xd5, + 0x17, 0xad, 0x16, 0x15, 0x8a, 0x32, 0xbf, 0xa7, + 0xdc, 0x03, 0x82, 0xea, 0xed, 0xa1, 0x28, 0xe9, + 0x18, 0x86, 0x73, 0x4c, 0x24, 0xa0, 0xb2, 0x9d }, + { 0x94, 0x2c, 0xc7, 0xc0, 0xb5, 0x2e, 0x2b, 0x16, + 0xa4, 0xb8, 0x9f, 0xa4, 0xfc, 0x7e, 0x0b, 0xf6, + 0x09, 0xe2, 0x9a, 0x08, 0xc1, 0xa8, 0x54, 0x34, + 0x52, 0xb7, 0x7c, 0x7b, 0xfd, 0x11, 0xbb, 0x28 }, + { 0x8a, 0x06, 0x5d, 0x8b, 0x61, 0xa0, 0xdf, 0xfb, + 0x17, 0x0d, 0x56, 0x27, 0x73, 0x5a, 0x76, 0xb0, + 0xe9, 0x50, 0x60, 0x37, 0x80, 0x8c, 0xba, 0x16, + 0xc3, 0x45, 0x00, 0x7c, 0x9f, 0x79, 0xcf, 0x8f }, + { 0x1b, 0x9f, 0xa1, 0x97, 0x14, 0x65, 0x9c, 0x78, + 0xff, 0x41, 0x38, 0x71, 0x84, 0x92, 0x15, 0x36, + 0x10, 0x29, 0xac, 0x80, 0x2b, 0x1c, 0xbc, 0xd5, + 0x4e, 0x40, 0x8b, 0xd8, 0x72, 0x87, 0xf8, 0x1f }, + { 0x8d, 0xab, 0x07, 0x1b, 0xcd, 0x6c, 0x72, 0x92, + 0xa9, 0xef, 0x72, 0x7b, 0x4a, 0xe0, 0xd8, 0x67, + 0x13, 0x30, 0x1d, 0xa8, 0x61, 0x8d, 0x9a, 0x48, + 0xad, 0xce, 0x55, 0xf3, 0x03, 0xa8, 0x69, 0xa1 }, + { 0x82, 0x53, 0xe3, 0xe7, 0xc7, 0xb6, 0x84, 0xb9, + 0xcb, 0x2b, 0xeb, 0x01, 0x4c, 0xe3, 0x30, 0xff, + 0x3d, 0x99, 0xd1, 0x7a, 0xbb, 0xdb, 0xab, 0xe4, + 0xf4, 0xd6, 0x74, 0xde, 0xd5, 0x3f, 0xfc, 0x6b }, + { 0xf1, 0x95, 0xf3, 0x21, 0xe9, 0xe3, 0xd6, 0xbd, + 0x7d, 0x07, 0x45, 0x04, 0xdd, 0x2a, 0xb0, 0xe6, + 0x24, 0x1f, 0x92, 0xe7, 0x84, 0xb1, 0xaa, 0x27, + 0x1f, 0xf6, 0x48, 0xb1, 0xca, 0xb6, 0xd7, 0xf6 }, + { 0x27, 0xe4, 0xcc, 0x72, 0x09, 0x0f, 0x24, 0x12, + 0x66, 0x47, 0x6a, 0x7c, 0x09, 0x49, 0x5f, 0x2d, + 0xb1, 0x53, 0xd5, 0xbc, 0xbd, 0x76, 0x19, 0x03, + 0xef, 0x79, 0x27, 0x5e, 0xc5, 0x6b, 0x2e, 0xd8 }, + { 0x89, 0x9c, 0x24, 0x05, 0x78, 0x8e, 0x25, 0xb9, + 0x9a, 0x18, 0x46, 0x35, 0x5e, 0x64, 0x6d, 0x77, + 0xcf, 0x40, 0x00, 0x83, 0x41, 0x5f, 0x7d, 0xc5, + 0xaf, 0xe6, 0x9d, 0x6e, 0x17, 0xc0, 0x00, 0x23 }, + { 0xa5, 0x9b, 0x78, 0xc4, 0x90, 0x57, 0x44, 0x07, + 0x6b, 0xfe, 0xe8, 0x94, 0xde, 0x70, 0x7d, 0x4f, + 0x12, 0x0b, 0x5c, 0x68, 0x93, 0xea, 0x04, 0x00, + 0x29, 0x7d, 0x0b, 0xb8, 0x34, 0x72, 0x76, 0x32 }, + { 0x59, 0xdc, 0x78, 0xb1, 0x05, 0x64, 0x97, 0x07, + 0xa2, 0xbb, 0x44, 0x19, 0xc4, 0x8f, 0x00, 0x54, + 0x00, 0xd3, 0x97, 0x3d, 0xe3, 0x73, 0x66, 0x10, + 0x23, 0x04, 0x35, 0xb1, 0x04, 0x24, 0xb2, 0x4f }, + { 0xc0, 0x14, 0x9d, 0x1d, 0x7e, 0x7a, 0x63, 0x53, + 0xa6, 0xd9, 0x06, 0xef, 0xe7, 0x28, 0xf2, 0xf3, + 0x29, 0xfe, 0x14, 0xa4, 0x14, 0x9a, 0x3e, 0xa7, + 0x76, 0x09, 0xbc, 0x42, 0xb9, 0x75, 0xdd, 0xfa }, + { 0xa3, 0x2f, 0x24, 0x14, 0x74, 0xa6, 0xc1, 0x69, + 0x32, 0xe9, 0x24, 0x3b, 0xe0, 0xcf, 0x09, 0xbc, + 0xdc, 0x7e, 0x0c, 0xa0, 0xe7, 0xa6, 0xa1, 0xb9, + 0xb1, 0xa0, 0xf0, 0x1e, 0x41, 0x50, 0x23, 0x77 }, + { 0xb2, 0x39, 0xb2, 0xe4, 0xf8, 0x18, 0x41, 0x36, + 0x1c, 0x13, 0x39, 0xf6, 0x8e, 0x2c, 0x35, 0x9f, + 0x92, 0x9a, 0xf9, 0xad, 0x9f, 0x34, 0xe0, 0x1a, + 0xab, 0x46, 0x31, 0xad, 0x6d, 0x55, 0x00, 0xb0 }, + { 0x85, 0xfb, 0x41, 0x9c, 0x70, 0x02, 0xa3, 0xe0, + 0xb4, 0xb6, 0xea, 0x09, 0x3b, 0x4c, 0x1a, 0xc6, + 0x93, 0x66, 0x45, 0xb6, 0x5d, 0xac, 0x5a, 0xc1, + 0x5a, 0x85, 0x28, 0xb7, 0xb9, 0x4c, 0x17, 0x54 }, + { 0x96, 0x19, 0x72, 0x06, 0x25, 0xf1, 0x90, 0xb9, + 0x3a, 0x3f, 0xad, 0x18, 0x6a, 0xb3, 0x14, 0x18, + 0x96, 0x33, 0xc0, 0xd3, 0xa0, 0x1e, 0x6f, 0x9b, + 0xc8, 0xc4, 0xa8, 0xf8, 0x2f, 0x38, 0x3d, 0xbf }, + { 0x7d, 0x62, 0x0d, 0x90, 0xfe, 0x69, 0xfa, 0x46, + 0x9a, 0x65, 0x38, 0x38, 0x89, 0x70, 0xa1, 0xaa, + 0x09, 0xbb, 0x48, 0xa2, 0xd5, 0x9b, 0x34, 0x7b, + 0x97, 0xe8, 0xce, 0x71, 0xf4, 0x8c, 0x7f, 0x46 }, + { 0x29, 0x43, 0x83, 0x56, 0x85, 0x96, 0xfb, 0x37, + 0xc7, 0x5b, 0xba, 0xcd, 0x97, 0x9c, 0x5f, 0xf6, + 0xf2, 0x0a, 0x55, 0x6b, 0xf8, 0x87, 0x9c, 0xc7, + 0x29, 0x24, 0x85, 0x5d, 0xf9, 0xb8, 0x24, 0x0e }, + { 0x16, 0xb1, 0x8a, 0xb3, 0x14, 0x35, 0x9c, 0x2b, + 0x83, 0x3c, 0x1c, 0x69, 0x86, 0xd4, 0x8c, 0x55, + 0xa9, 0xfc, 0x97, 0xcd, 0xe9, 0xa3, 0xc1, 0xf1, + 0x0a, 0x31, 0x77, 0x14, 0x0f, 0x73, 0xf7, 0x38 }, + { 0x8c, 0xbb, 0xdd, 0x14, 0xbc, 0x33, 0xf0, 0x4c, + 0xf4, 0x58, 0x13, 0xe4, 0xa1, 0x53, 0xa2, 0x73, + 0xd3, 0x6a, 0xda, 0xd5, 0xce, 0x71, 0xf4, 0x99, + 0xee, 0xb8, 0x7f, 0xb8, 0xac, 0x63, 0xb7, 0x29 }, + { 0x69, 0xc9, 0xa4, 0x98, 0xdb, 0x17, 0x4e, 0xca, + 0xef, 0xcc, 0x5a, 0x3a, 0xc9, 0xfd, 0xed, 0xf0, + 0xf8, 0x13, 0xa5, 0xbe, 0xc7, 0x27, 0xf1, 0xe7, + 0x75, 0xba, 0xbd, 0xec, 0x77, 0x18, 0x81, 0x6e }, + { 0xb4, 0x62, 0xc3, 0xbe, 0x40, 0x44, 0x8f, 0x1d, + 0x4f, 0x80, 0x62, 0x62, 0x54, 0xe5, 0x35, 0xb0, + 0x8b, 0xc9, 0xcd, 0xcf, 0xf5, 0x99, 0xa7, 0x68, + 0x57, 0x8d, 0x4b, 0x28, 0x81, 0xa8, 0xe3, 0xf0 }, + { 0x55, 0x3e, 0x9d, 0x9c, 0x5f, 0x36, 0x0a, 0xc0, + 0xb7, 0x4a, 0x7d, 0x44, 0xe5, 0xa3, 0x91, 0xda, + 0xd4, 0xce, 0xd0, 0x3e, 0x0c, 0x24, 0x18, 0x3b, + 0x7e, 0x8e, 0xca, 0xbd, 0xf1, 0x71, 0x5a, 0x64 }, + { 0x7a, 0x7c, 0x55, 0xa5, 0x6f, 0xa9, 0xae, 0x51, + 0xe6, 0x55, 0xe0, 0x19, 0x75, 0xd8, 0xa6, 0xff, + 0x4a, 0xe9, 0xe4, 0xb4, 0x86, 0xfc, 0xbe, 0x4e, + 0xac, 0x04, 0x45, 0x88, 0xf2, 0x45, 0xeb, 0xea }, + { 0x2a, 0xfd, 0xf3, 0xc8, 0x2a, 0xbc, 0x48, 0x67, + 0xf5, 0xde, 0x11, 0x12, 0x86, 0xc2, 0xb3, 0xbe, + 0x7d, 0x6e, 0x48, 0x65, 0x7b, 0xa9, 0x23, 0xcf, + 0xbf, 0x10, 0x1a, 0x6d, 0xfc, 0xf9, 0xdb, 0x9a }, + { 0x41, 0x03, 0x7d, 0x2e, 0xdc, 0xdc, 0xe0, 0xc4, + 0x9b, 0x7f, 0xb4, 0xa6, 0xaa, 0x09, 0x99, 0xca, + 0x66, 0x97, 0x6c, 0x74, 0x83, 0xaf, 0xe6, 0x31, + 0xd4, 0xed, 0xa2, 0x83, 0x14, 0x4f, 0x6d, 0xfc }, + { 0xc4, 0x46, 0x6f, 0x84, 0x97, 0xca, 0x2e, 0xeb, + 0x45, 0x83, 0xa0, 0xb0, 0x8e, 0x9d, 0x9a, 0xc7, + 0x43, 0x95, 0x70, 0x9f, 0xda, 0x10, 0x9d, 0x24, + 0xf2, 0xe4, 0x46, 0x21, 0x96, 0x77, 0x9c, 0x5d }, + { 0x75, 0xf6, 0x09, 0x33, 0x8a, 0xa6, 0x7d, 0x96, + 0x9a, 0x2a, 0xe2, 0xa2, 0x36, 0x2b, 0x2d, 0xa9, + 0xd7, 0x7c, 0x69, 0x5d, 0xfd, 0x1d, 0xf7, 0x22, + 0x4a, 0x69, 0x01, 0xdb, 0x93, 0x2c, 0x33, 0x64 }, + { 0x68, 0x60, 0x6c, 0xeb, 0x98, 0x9d, 0x54, 0x88, + 0xfc, 0x7c, 0xf6, 0x49, 0xf3, 0xd7, 0xc2, 0x72, + 0xef, 0x05, 0x5d, 0xa1, 0xa9, 0x3f, 0xae, 0xcd, + 0x55, 0xfe, 0x06, 0xf6, 0x96, 0x70, 0x98, 0xca }, + { 0x44, 0x34, 0x6b, 0xde, 0xb7, 0xe0, 0x52, 0xf6, + 0x25, 0x50, 0x48, 0xf0, 0xd9, 0xb4, 0x2c, 0x42, + 0x5b, 0xab, 0x9c, 0x3d, 0xd2, 0x41, 0x68, 0x21, + 0x2c, 0x3e, 0xcf, 0x1e, 0xbf, 0x34, 0xe6, 0xae }, + { 0x8e, 0x9c, 0xf6, 0xe1, 0xf3, 0x66, 0x47, 0x1f, + 0x2a, 0xc7, 0xd2, 0xee, 0x9b, 0x5e, 0x62, 0x66, + 0xfd, 0xa7, 0x1f, 0x8f, 0x2e, 0x41, 0x09, 0xf2, + 0x23, 0x7e, 0xd5, 0xf8, 0x81, 0x3f, 0xc7, 0x18 }, + { 0x84, 0xbb, 0xeb, 0x84, 0x06, 0xd2, 0x50, 0x95, + 0x1f, 0x8c, 0x1b, 0x3e, 0x86, 0xa7, 0xc0, 0x10, + 0x08, 0x29, 0x21, 0x83, 0x3d, 0xfd, 0x95, 0x55, + 0xa2, 0xf9, 0x09, 0xb1, 0x08, 0x6e, 0xb4, 0xb8 }, + { 0xee, 0x66, 0x6f, 0x3e, 0xef, 0x0f, 0x7e, 0x2a, + 0x9c, 0x22, 0x29, 0x58, 0xc9, 0x7e, 0xaf, 0x35, + 0xf5, 0x1c, 0xed, 0x39, 0x3d, 0x71, 0x44, 0x85, + 0xab, 0x09, 0xa0, 0x69, 0x34, 0x0f, 0xdf, 0x88 }, + { 0xc1, 0x53, 0xd3, 0x4a, 0x65, 0xc4, 0x7b, 0x4a, + 0x62, 0xc5, 0xca, 0xcf, 0x24, 0x01, 0x09, 0x75, + 0xd0, 0x35, 0x6b, 0x2f, 0x32, 0xc8, 0xf5, 0xda, + 0x53, 0x0d, 0x33, 0x88, 0x16, 0xad, 0x5d, 0xe6 }, + { 0x9f, 0xc5, 0x45, 0x01, 0x09, 0xe1, 0xb7, 0x79, + 0xf6, 0xc7, 0xae, 0x79, 0xd5, 0x6c, 0x27, 0x63, + 0x5c, 0x8d, 0xd4, 0x26, 0xc5, 0xa9, 0xd5, 0x4e, + 0x25, 0x78, 0xdb, 0x98, 0x9b, 0x8c, 0x3b, 0x4e }, + { 0xd1, 0x2b, 0xf3, 0x73, 0x2e, 0xf4, 0xaf, 0x5c, + 0x22, 0xfa, 0x90, 0x35, 0x6a, 0xf8, 0xfc, 0x50, + 0xfc, 0xb4, 0x0f, 0x8f, 0x2e, 0xa5, 0xc8, 0x59, + 0x47, 0x37, 0xa3, 0xb3, 0xd5, 0xab, 0xdb, 0xd7 }, + { 0x11, 0x03, 0x0b, 0x92, 0x89, 0xbb, 0xa5, 0xaf, + 0x65, 0x26, 0x06, 0x72, 0xab, 0x6f, 0xee, 0x88, + 0xb8, 0x74, 0x20, 0xac, 0xef, 0x4a, 0x17, 0x89, + 0xa2, 0x07, 0x3b, 0x7e, 0xc2, 0xf2, 0xa0, 0x9e }, + { 0x69, 0xcb, 0x19, 0x2b, 0x84, 0x44, 0x00, 0x5c, + 0x8c, 0x0c, 0xeb, 0x12, 0xc8, 0x46, 0x86, 0x07, + 0x68, 0x18, 0x8c, 0xda, 0x0a, 0xec, 0x27, 0xa9, + 0xc8, 0xa5, 0x5c, 0xde, 0xe2, 0x12, 0x36, 0x32 }, + { 0xdb, 0x44, 0x4c, 0x15, 0x59, 0x7b, 0x5f, 0x1a, + 0x03, 0xd1, 0xf9, 0xed, 0xd1, 0x6e, 0x4a, 0x9f, + 0x43, 0xa6, 0x67, 0xcc, 0x27, 0x51, 0x75, 0xdf, + 0xa2, 0xb7, 0x04, 0xe3, 0xbb, 0x1a, 0x9b, 0x83 }, + { 0x3f, 0xb7, 0x35, 0x06, 0x1a, 0xbc, 0x51, 0x9d, + 0xfe, 0x97, 0x9e, 0x54, 0xc1, 0xee, 0x5b, 0xfa, + 0xd0, 0xa9, 0xd8, 0x58, 0xb3, 0x31, 0x5b, 0xad, + 0x34, 0xbd, 0xe9, 0x99, 0xef, 0xd7, 0x24, 0xdd } +}; + +static bool __init blake2s_selftest(void) +{ + u8 key[BLAKE2S_KEY_SIZE]; + u8 buf[ARRAY_SIZE(blake2s_testvecs)]; + u8 hash[BLAKE2S_HASH_SIZE]; + size_t i; + bool success = true; + + for (i = 0; i < BLAKE2S_KEY_SIZE; ++i) + key[i] = (u8)i; + + for (i = 0; i < ARRAY_SIZE(blake2s_testvecs); ++i) + buf[i] = (u8)i; + + for (i = 0; i < ARRAY_SIZE(blake2s_keyed_testvecs); ++i) { + blake2s(hash, buf, key, BLAKE2S_HASH_SIZE, i, BLAKE2S_KEY_SIZE); + if (memcmp(hash, blake2s_keyed_testvecs[i], BLAKE2S_HASH_SIZE)) { + pr_err("blake2s keyed self-test %zu: FAIL\n", i + 1); + success = false; + } + } + + for (i = 0; i < ARRAY_SIZE(blake2s_testvecs); ++i) { + blake2s(hash, buf, NULL, BLAKE2S_HASH_SIZE, i, 0); + if (memcmp(hash, blake2s_testvecs[i], BLAKE2S_HASH_SIZE)) { + pr_err("blake2s unkeyed self-test %zu: FAIL\n", i + i); + success = false; + } + } + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/chacha20.c b/net/wireguard/crypto/zinc/selftest/chacha20.c new file mode 100644 index 0000000000000..1a2390aaf6c23 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/chacha20.c @@ -0,0 +1,2698 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct chacha20_testvec { + const u8 *input, *output, *key; + u64 nonce; + size_t ilen; +}; + +struct hchacha20_testvec { + u8 key[HCHACHA20_KEY_SIZE]; + u8 nonce[HCHACHA20_NONCE_SIZE]; + u8 output[CHACHA20_KEY_SIZE]; +}; + +/* These test vectors are generated by reference implementations and are + * designed to check chacha20 implementation block handling, as well as from + * the draft-arciszewski-xchacha-01 document. + */ + +static const u8 input01[] __initconst = { }; +static const u8 output01[] __initconst = { }; +static const u8 key01[] __initconst = { + 0x09, 0xf4, 0xe8, 0x57, 0x10, 0xf2, 0x12, 0xc3, + 0xc6, 0x91, 0xc4, 0x09, 0x97, 0x46, 0xef, 0xfe, + 0x02, 0x00, 0xe4, 0x5c, 0x82, 0xed, 0x16, 0xf3, + 0x32, 0xbe, 0xec, 0x7a, 0xe6, 0x68, 0x12, 0x26 +}; +enum { nonce01 = 0x3834e2afca3c66d3ULL }; + +static const u8 input02[] __initconst = { + 0x9d +}; +static const u8 output02[] __initconst = { + 0x94 +}; +static const u8 key02[] __initconst = { + 0x8c, 0x01, 0xac, 0xaf, 0x62, 0x63, 0x56, 0x7a, + 0xad, 0x23, 0x4c, 0x58, 0x29, 0x29, 0xbe, 0xab, + 0xe9, 0xf8, 0xdf, 0x6c, 0x8c, 0x74, 0x4d, 0x7d, + 0x13, 0x94, 0x10, 0x02, 0x3d, 0x8e, 0x9f, 0x94 +}; +enum { nonce02 = 0x5d1b3bfdedd9f73aULL }; + +static const u8 input03[] __initconst = { + 0x04, 0x16 +}; +static const u8 output03[] __initconst = { + 0x92, 0x07 +}; +static const u8 key03[] __initconst = { + 0x22, 0x0c, 0x79, 0x2c, 0x38, 0x51, 0xbe, 0x99, + 0xa9, 0x59, 0x24, 0x50, 0xef, 0x87, 0x38, 0xa6, + 0xa0, 0x97, 0x20, 0xcb, 0xb4, 0x0c, 0x94, 0x67, + 0x1f, 0x98, 0xdc, 0xc4, 0x83, 0xbc, 0x35, 0x4d +}; +enum { nonce03 = 0x7a3353ad720a3e2eULL }; + +static const u8 input04[] __initconst = { + 0xc7, 0xcc, 0xd0 +}; +static const u8 output04[] __initconst = { + 0xd8, 0x41, 0x80 +}; +static const u8 key04[] __initconst = { + 0x81, 0x5e, 0x12, 0x01, 0xc4, 0x36, 0x15, 0x03, + 0x11, 0xa0, 0xe9, 0x86, 0xbb, 0x5a, 0xdc, 0x45, + 0x7d, 0x5e, 0x98, 0xf8, 0x06, 0x76, 0x1c, 0xec, + 0xc0, 0xf7, 0xca, 0x4e, 0x99, 0xd9, 0x42, 0x38 +}; +enum { nonce04 = 0x6816e2fc66176da2ULL }; + +static const u8 input05[] __initconst = { + 0x48, 0xf1, 0x31, 0x5f +}; +static const u8 output05[] __initconst = { + 0x48, 0xf7, 0x13, 0x67 +}; +static const u8 key05[] __initconst = { + 0x3f, 0xd6, 0xb6, 0x5e, 0x2f, 0xda, 0x82, 0x39, + 0x97, 0x06, 0xd3, 0x62, 0x4f, 0xbd, 0xcb, 0x9b, + 0x1d, 0xe6, 0x4a, 0x76, 0xab, 0xdd, 0x14, 0x50, + 0x59, 0x21, 0xe3, 0xb2, 0xc7, 0x95, 0xbc, 0x45 +}; +enum { nonce05 = 0xc41a7490e228cc42ULL }; + +static const u8 input06[] __initconst = { + 0xae, 0xa2, 0x85, 0x1d, 0xc8 +}; +static const u8 output06[] __initconst = { + 0xfa, 0xff, 0x45, 0x6b, 0x6f +}; +static const u8 key06[] __initconst = { + 0x04, 0x8d, 0xea, 0x67, 0x20, 0x78, 0xfb, 0x8f, + 0x49, 0x80, 0x35, 0xb5, 0x7b, 0xe4, 0x31, 0x74, + 0x57, 0x43, 0x3a, 0x64, 0x64, 0xb9, 0xe6, 0x23, + 0x4d, 0xfe, 0xb8, 0x7b, 0x71, 0x4d, 0x9d, 0x21 +}; +enum { nonce06 = 0x251366db50b10903ULL }; + +static const u8 input07[] __initconst = { + 0x1a, 0x32, 0x85, 0xb6, 0xe8, 0x52 +}; +static const u8 output07[] __initconst = { + 0xd3, 0x5f, 0xf0, 0x07, 0x69, 0xec +}; +static const u8 key07[] __initconst = { + 0xbf, 0x2d, 0x42, 0x99, 0x97, 0x76, 0x04, 0xad, + 0xd3, 0x8f, 0x6e, 0x6a, 0x34, 0x85, 0xaf, 0x81, + 0xef, 0x36, 0x33, 0xd5, 0x43, 0xa2, 0xaa, 0x08, + 0x0f, 0x77, 0x42, 0x83, 0x58, 0xc5, 0x42, 0x2a +}; +enum { nonce07 = 0xe0796da17dba9b58ULL }; + +static const u8 input08[] __initconst = { + 0x40, 0xae, 0xcd, 0xe4, 0x3d, 0x22, 0xe0 +}; +static const u8 output08[] __initconst = { + 0xfd, 0x8a, 0x9f, 0x3d, 0x05, 0xc9, 0xd3 +}; +static const u8 key08[] __initconst = { + 0xdc, 0x3f, 0x41, 0xe3, 0x23, 0x2a, 0x8d, 0xf6, + 0x41, 0x2a, 0xa7, 0x66, 0x05, 0x68, 0xe4, 0x7b, + 0xc4, 0x58, 0xd6, 0xcc, 0xdf, 0x0d, 0xc6, 0x25, + 0x1b, 0x61, 0x32, 0x12, 0x4e, 0xf1, 0xe6, 0x29 +}; +enum { nonce08 = 0xb1d2536d9e159832ULL }; + +static const u8 input09[] __initconst = { + 0xba, 0x1d, 0x14, 0x16, 0x9f, 0x83, 0x67, 0x24 +}; +static const u8 output09[] __initconst = { + 0x7c, 0xe3, 0x78, 0x1d, 0xa2, 0xe7, 0xe9, 0x39 +}; +static const u8 key09[] __initconst = { + 0x17, 0x55, 0x90, 0x52, 0xa4, 0xce, 0x12, 0xae, + 0xd4, 0xfd, 0xd4, 0xfb, 0xd5, 0x18, 0x59, 0x50, + 0x4e, 0x51, 0x99, 0x32, 0x09, 0x31, 0xfc, 0xf7, + 0x27, 0x10, 0x8e, 0xa2, 0x4b, 0xa5, 0xf5, 0x62 +}; +enum { nonce09 = 0x495fc269536d003ULL }; + +static const u8 input10[] __initconst = { + 0x09, 0xfd, 0x3c, 0x0b, 0x3d, 0x0e, 0xf3, 0x9d, + 0x27 +}; +static const u8 output10[] __initconst = { + 0xdc, 0xe4, 0x33, 0x60, 0x0c, 0x07, 0xcb, 0x51, + 0x6b +}; +static const u8 key10[] __initconst = { + 0x4e, 0x00, 0x72, 0x37, 0x0f, 0x52, 0x4d, 0x6f, + 0x37, 0x50, 0x3c, 0xb3, 0x51, 0x81, 0x49, 0x16, + 0x7e, 0xfd, 0xb1, 0x51, 0x72, 0x2e, 0xe4, 0x16, + 0x68, 0x5c, 0x5b, 0x8a, 0xc3, 0x90, 0x70, 0x04 +}; +enum { nonce10 = 0x1ad9d1114d88cbbdULL }; + +static const u8 input11[] __initconst = { + 0x70, 0x18, 0x52, 0x85, 0xba, 0x66, 0xff, 0x2c, + 0x9a, 0x46 +}; +static const u8 output11[] __initconst = { + 0xf5, 0x2a, 0x7a, 0xfd, 0x31, 0x7c, 0x91, 0x41, + 0xb1, 0xcf +}; +static const u8 key11[] __initconst = { + 0x48, 0xb4, 0xd0, 0x7c, 0x88, 0xd1, 0x96, 0x0d, + 0x80, 0x33, 0xb4, 0xd5, 0x31, 0x9a, 0x88, 0xca, + 0x14, 0xdc, 0xf0, 0xa8, 0xf3, 0xac, 0xb8, 0x47, + 0x75, 0x86, 0x7c, 0x88, 0x50, 0x11, 0x43, 0x40 +}; +enum { nonce11 = 0x47c35dd1f4f8aa4fULL }; + +static const u8 input12[] __initconst = { + 0x9e, 0x8e, 0x3d, 0x2a, 0x05, 0xfd, 0xe4, 0x90, + 0x24, 0x1c, 0xd3 +}; +static const u8 output12[] __initconst = { + 0x97, 0x72, 0x40, 0x9f, 0xc0, 0x6b, 0x05, 0x33, + 0x42, 0x7e, 0x28 +}; +static const u8 key12[] __initconst = { + 0xee, 0xff, 0x33, 0x33, 0xe0, 0x28, 0xdf, 0xa2, + 0xb6, 0x5e, 0x25, 0x09, 0x52, 0xde, 0xa5, 0x9c, + 0x8f, 0x95, 0xa9, 0x03, 0x77, 0x0f, 0xbe, 0xa1, + 0xd0, 0x7d, 0x73, 0x2f, 0xf8, 0x7e, 0x51, 0x44 +}; +enum { nonce12 = 0xc22d044dc6ea4af3ULL }; + +static const u8 input13[] __initconst = { + 0x9c, 0x16, 0xa2, 0x22, 0x4d, 0xbe, 0x04, 0x9a, + 0xb3, 0xb5, 0xc6, 0x58 +}; +static const u8 output13[] __initconst = { + 0xf0, 0x81, 0xdb, 0x6d, 0xa3, 0xe9, 0xb2, 0xc6, + 0x32, 0x50, 0x16, 0x9f +}; +static const u8 key13[] __initconst = { + 0x96, 0xb3, 0x01, 0xd2, 0x7a, 0x8c, 0x94, 0x09, + 0x4f, 0x58, 0xbe, 0x80, 0xcc, 0xa9, 0x7e, 0x2d, + 0xad, 0x58, 0x3b, 0x63, 0xb8, 0x5c, 0x17, 0xce, + 0xbf, 0x43, 0x33, 0x7a, 0x7b, 0x82, 0x28, 0x2f +}; +enum { nonce13 = 0x2a5d05d88cd7b0daULL }; + +static const u8 input14[] __initconst = { + 0x57, 0x4f, 0xaa, 0x30, 0xe6, 0x23, 0x50, 0x86, + 0x91, 0xa5, 0x60, 0x96, 0x2b +}; +static const u8 output14[] __initconst = { + 0x6c, 0x1f, 0x3b, 0x42, 0xb6, 0x2f, 0xf0, 0xbd, + 0x76, 0x60, 0xc7, 0x7e, 0x8d +}; +static const u8 key14[] __initconst = { + 0x22, 0x85, 0xaf, 0x8f, 0xa3, 0x53, 0xa0, 0xc4, + 0xb5, 0x75, 0xc0, 0xba, 0x30, 0x92, 0xc3, 0x32, + 0x20, 0x5a, 0x8f, 0x7e, 0x93, 0xda, 0x65, 0x18, + 0xd1, 0xf6, 0x9a, 0x9b, 0x8f, 0x85, 0x30, 0xe6 +}; +enum { nonce14 = 0xf9946c166aa4475fULL }; + +static const u8 input15[] __initconst = { + 0x89, 0x81, 0xc7, 0xe2, 0x00, 0xac, 0x52, 0x70, + 0xa4, 0x79, 0xab, 0xeb, 0x74, 0xf7 +}; +static const u8 output15[] __initconst = { + 0xb4, 0xd0, 0xa9, 0x9d, 0x15, 0x5f, 0x48, 0xd6, + 0x00, 0x7e, 0x4c, 0x77, 0x5a, 0x46 +}; +static const u8 key15[] __initconst = { + 0x0a, 0x66, 0x36, 0xca, 0x5d, 0x82, 0x23, 0xb6, + 0xe4, 0x9b, 0xad, 0x5e, 0xd0, 0x7f, 0xf6, 0x7a, + 0x7b, 0x03, 0xa7, 0x4c, 0xfd, 0xec, 0xd5, 0xa1, + 0xfc, 0x25, 0x54, 0xda, 0x5a, 0x5c, 0xf0, 0x2c +}; +enum { nonce15 = 0x9ab2b87a35e772c8ULL }; + +static const u8 input16[] __initconst = { + 0x5f, 0x09, 0xc0, 0x8b, 0x1e, 0xde, 0xca, 0xd9, + 0xb7, 0x5c, 0x23, 0xc9, 0x55, 0x1e, 0xcf +}; +static const u8 output16[] __initconst = { + 0x76, 0x9b, 0x53, 0xf3, 0x66, 0x88, 0x28, 0x60, + 0x98, 0x80, 0x2c, 0xa8, 0x80, 0xa6, 0x48 +}; +static const u8 key16[] __initconst = { + 0x80, 0xb5, 0x51, 0xdf, 0x17, 0x5b, 0xb0, 0xef, + 0x8b, 0x5b, 0x2e, 0x3e, 0xc5, 0xe3, 0xa5, 0x86, + 0xac, 0x0d, 0x8e, 0x32, 0x90, 0x9d, 0x82, 0x27, + 0xf1, 0x23, 0x26, 0xc3, 0xea, 0x55, 0xb6, 0x63 +}; +enum { nonce16 = 0xa82e9d39e4d02ef5ULL }; + +static const u8 input17[] __initconst = { + 0x87, 0x0b, 0x36, 0x71, 0x7c, 0xb9, 0x0b, 0x80, + 0x4d, 0x77, 0x5c, 0x4f, 0xf5, 0x51, 0x0e, 0x1a +}; +static const u8 output17[] __initconst = { + 0xf1, 0x12, 0x4a, 0x8a, 0xd9, 0xd0, 0x08, 0x67, + 0x66, 0xd7, 0x34, 0xea, 0x32, 0x3b, 0x54, 0x0e +}; +static const u8 key17[] __initconst = { + 0xfb, 0x71, 0x5f, 0x3f, 0x7a, 0xc0, 0x9a, 0xc8, + 0xc8, 0xcf, 0xe8, 0xbc, 0xfb, 0x09, 0xbf, 0x89, + 0x6a, 0xef, 0xd5, 0xe5, 0x36, 0x87, 0x14, 0x76, + 0x00, 0xb9, 0x32, 0x28, 0xb2, 0x00, 0x42, 0x53 +}; +enum { nonce17 = 0x229b87e73d557b96ULL }; + +static const u8 input18[] __initconst = { + 0x38, 0x42, 0xb5, 0x37, 0xb4, 0x3d, 0xfe, 0x59, + 0x38, 0x68, 0x88, 0xfa, 0x89, 0x8a, 0x5f, 0x90, + 0x3c +}; +static const u8 output18[] __initconst = { + 0xac, 0xad, 0x14, 0xe8, 0x7e, 0xd7, 0xce, 0x96, + 0x3d, 0xb3, 0x78, 0x85, 0x22, 0x5a, 0xcb, 0x39, + 0xd4 +}; +static const u8 key18[] __initconst = { + 0xe1, 0xc1, 0xa8, 0xe0, 0x91, 0xe7, 0x38, 0x66, + 0x80, 0x17, 0x12, 0x3c, 0x5e, 0x2d, 0xbb, 0xea, + 0xeb, 0x6c, 0x8b, 0xc8, 0x1b, 0x6f, 0x7c, 0xea, + 0x50, 0x57, 0x23, 0x1e, 0x65, 0x6f, 0x6d, 0x81 +}; +enum { nonce18 = 0xfaf5fcf8f30e57a9ULL }; + +static const u8 input19[] __initconst = { + 0x1c, 0x4a, 0x30, 0x26, 0xef, 0x9a, 0x32, 0xa7, + 0x8f, 0xe5, 0xc0, 0x0f, 0x30, 0x3a, 0xbf, 0x38, + 0x54, 0xba +}; +static const u8 output19[] __initconst = { + 0x57, 0x67, 0x54, 0x4f, 0x31, 0xd6, 0xef, 0x35, + 0x0b, 0xd9, 0x52, 0xa7, 0x46, 0x7d, 0x12, 0x17, + 0x1e, 0xe3 +}; +static const u8 key19[] __initconst = { + 0x5a, 0x79, 0xc1, 0xea, 0x33, 0xb3, 0xc7, 0x21, + 0xec, 0xf8, 0xcb, 0xd2, 0x58, 0x96, 0x23, 0xd6, + 0x4d, 0xed, 0x2f, 0xdf, 0x8a, 0x79, 0xe6, 0x8b, + 0x38, 0xa3, 0xc3, 0x7a, 0x33, 0xda, 0x02, 0xc7 +}; +enum { nonce19 = 0x2b23b61840429604ULL }; + +static const u8 input20[] __initconst = { + 0xab, 0xe9, 0x32, 0xbb, 0x35, 0x17, 0xe0, 0x60, + 0x80, 0xb1, 0x27, 0xdc, 0xe6, 0x62, 0x9e, 0x0c, + 0x77, 0xf4, 0x50 +}; +static const u8 output20[] __initconst = { + 0x54, 0x6d, 0xaa, 0xfc, 0x08, 0xfb, 0x71, 0xa8, + 0xd6, 0x1d, 0x7d, 0xf3, 0x45, 0x10, 0xb5, 0x4c, + 0xcc, 0x4b, 0x45 +}; +static const u8 key20[] __initconst = { + 0xa3, 0xfd, 0x3d, 0xa9, 0xeb, 0xea, 0x2c, 0x69, + 0xcf, 0x59, 0x38, 0x13, 0x5b, 0xa7, 0x53, 0x8f, + 0x5e, 0xa2, 0x33, 0x86, 0x4c, 0x75, 0x26, 0xaf, + 0x35, 0x12, 0x09, 0x71, 0x81, 0xea, 0x88, 0x66 +}; +enum { nonce20 = 0x7459667a8fadff58ULL }; + +static const u8 input21[] __initconst = { + 0xa6, 0x82, 0x21, 0x23, 0xad, 0x27, 0x3f, 0xc6, + 0xd7, 0x16, 0x0d, 0x6d, 0x24, 0x15, 0x54, 0xc5, + 0x96, 0x72, 0x59, 0x8a +}; +static const u8 output21[] __initconst = { + 0x5f, 0x34, 0x32, 0xea, 0x06, 0xd4, 0x9e, 0x01, + 0xdc, 0x32, 0x32, 0x40, 0x66, 0x73, 0x6d, 0x4a, + 0x6b, 0x12, 0x20, 0xe8 +}; +static const u8 key21[] __initconst = { + 0x96, 0xfd, 0x13, 0x23, 0xa9, 0x89, 0x04, 0xe6, + 0x31, 0xa5, 0x2c, 0xc1, 0x40, 0xd5, 0x69, 0x5c, + 0x32, 0x79, 0x56, 0xe0, 0x29, 0x93, 0x8f, 0xe8, + 0x5f, 0x65, 0x53, 0x7f, 0xc1, 0xe9, 0xaf, 0xaf +}; +enum { nonce21 = 0xba8defee9d8e13b5ULL }; + +static const u8 input22[] __initconst = { + 0xb8, 0x32, 0x1a, 0x81, 0xd8, 0x38, 0x89, 0x5a, + 0xb0, 0x05, 0xbe, 0xf4, 0xd2, 0x08, 0xc6, 0xee, + 0x79, 0x7b, 0x3a, 0x76, 0x59 +}; +static const u8 output22[] __initconst = { + 0xb7, 0xba, 0xae, 0x80, 0xe4, 0x9f, 0x79, 0x84, + 0x5a, 0x48, 0x50, 0x6d, 0xcb, 0xd0, 0x06, 0x0c, + 0x15, 0x63, 0xa7, 0x5e, 0xbd +}; +static const u8 key22[] __initconst = { + 0x0f, 0x35, 0x3d, 0xeb, 0x5f, 0x0a, 0x82, 0x0d, + 0x24, 0x59, 0x71, 0xd8, 0xe6, 0x2d, 0x5f, 0xe1, + 0x7e, 0x0c, 0xae, 0xf6, 0xdc, 0x2c, 0xc5, 0x4a, + 0x38, 0x88, 0xf2, 0xde, 0xd9, 0x5f, 0x76, 0x7c +}; +enum { nonce22 = 0xe77f1760e9f5e192ULL }; + +static const u8 input23[] __initconst = { + 0x4b, 0x1e, 0x79, 0x99, 0xcf, 0xef, 0x64, 0x4b, + 0xb0, 0x66, 0xae, 0x99, 0x2e, 0x68, 0x97, 0xf5, + 0x5d, 0x9b, 0x3f, 0x7a, 0xa9, 0xd9 +}; +static const u8 output23[] __initconst = { + 0x5f, 0xa4, 0x08, 0x39, 0xca, 0xfa, 0x2b, 0x83, + 0x5d, 0x95, 0x70, 0x7c, 0x2e, 0xd4, 0xae, 0xfa, + 0x45, 0x4a, 0x77, 0x7f, 0xa7, 0x65 +}; +static const u8 key23[] __initconst = { + 0x4a, 0x06, 0x83, 0x64, 0xaa, 0xe3, 0x38, 0x32, + 0x28, 0x5d, 0xa4, 0xb2, 0x5a, 0xee, 0xcf, 0x8e, + 0x19, 0x67, 0xf1, 0x09, 0xe8, 0xc9, 0xf6, 0x40, + 0x02, 0x6d, 0x0b, 0xde, 0xfa, 0x81, 0x03, 0xb1 +}; +enum { nonce23 = 0x9b3f349158709849ULL }; + +static const u8 input24[] __initconst = { + 0xc6, 0xfc, 0x47, 0x5e, 0xd8, 0xed, 0xa9, 0xe5, + 0x4f, 0x82, 0x79, 0x35, 0xee, 0x3e, 0x7e, 0x3e, + 0x35, 0x70, 0x6e, 0xfa, 0x6d, 0x08, 0xe8 +}; +static const u8 output24[] __initconst = { + 0x3b, 0xc5, 0xf8, 0xc2, 0xbf, 0x2b, 0x90, 0x33, + 0xa6, 0xae, 0xf5, 0x5a, 0x65, 0xb3, 0x3d, 0xe1, + 0xcd, 0x5f, 0x55, 0xfa, 0xe7, 0xa5, 0x4a +}; +static const u8 key24[] __initconst = { + 0x00, 0x24, 0xc3, 0x65, 0x5f, 0xe6, 0x31, 0xbb, + 0x6d, 0xfc, 0x20, 0x7b, 0x1b, 0xa8, 0x96, 0x26, + 0x55, 0x21, 0x62, 0x25, 0x7e, 0xba, 0x23, 0x97, + 0xc9, 0xb8, 0x53, 0xa8, 0xef, 0xab, 0xad, 0x61 +}; +enum { nonce24 = 0x13ee0b8f526177c3ULL }; + +static const u8 input25[] __initconst = { + 0x33, 0x07, 0x16, 0xb1, 0x34, 0x33, 0x67, 0x04, + 0x9b, 0x0a, 0xce, 0x1b, 0xe9, 0xde, 0x1a, 0xec, + 0xd0, 0x55, 0xfb, 0xc6, 0x33, 0xaf, 0x2d, 0xe3 +}; +static const u8 output25[] __initconst = { + 0x05, 0x93, 0x10, 0xd1, 0x58, 0x6f, 0x68, 0x62, + 0x45, 0xdb, 0x91, 0xae, 0x70, 0xcf, 0xd4, 0x5f, + 0xee, 0xdf, 0xd5, 0xba, 0x9e, 0xde, 0x68, 0xe6 +}; +static const u8 key25[] __initconst = { + 0x83, 0xa9, 0x4f, 0x5d, 0x74, 0xd5, 0x91, 0xb3, + 0xc9, 0x97, 0x19, 0x15, 0xdb, 0x0d, 0x0b, 0x4a, + 0x3d, 0x55, 0xcf, 0xab, 0xb2, 0x05, 0x21, 0x35, + 0x45, 0x50, 0xeb, 0xf8, 0xf5, 0xbf, 0x36, 0x35 +}; +enum { nonce25 = 0x7c6f459e49ebfebcULL }; + +static const u8 input26[] __initconst = { + 0xc2, 0xd4, 0x7a, 0xa3, 0x92, 0xe1, 0xac, 0x46, + 0x1a, 0x15, 0x38, 0xc9, 0xb5, 0xfd, 0xdf, 0x84, + 0x38, 0xbc, 0x6b, 0x1d, 0xb0, 0x83, 0x43, 0x04, + 0x39 +}; +static const u8 output26[] __initconst = { + 0x7f, 0xde, 0xd6, 0x87, 0xcc, 0x34, 0xf4, 0x12, + 0xae, 0x55, 0xa5, 0x89, 0x95, 0x29, 0xfc, 0x18, + 0xd8, 0xc7, 0x7c, 0xd3, 0xcb, 0x85, 0x95, 0x21, + 0xd2 +}; +static const u8 key26[] __initconst = { + 0xe4, 0xd0, 0x54, 0x1d, 0x7d, 0x47, 0xa8, 0xc1, + 0x08, 0xca, 0xe2, 0x42, 0x52, 0x95, 0x16, 0x43, + 0xa3, 0x01, 0x23, 0x03, 0xcc, 0x3b, 0x81, 0x78, + 0x23, 0xcc, 0xa7, 0x36, 0xd7, 0xa0, 0x97, 0x8d +}; +enum { nonce26 = 0x524401012231683ULL }; + +static const u8 input27[] __initconst = { + 0x0d, 0xb0, 0xcf, 0xec, 0xfc, 0x38, 0x9d, 0x9d, + 0x89, 0x00, 0x96, 0xf2, 0x79, 0x8a, 0xa1, 0x8d, + 0x32, 0x5e, 0xc6, 0x12, 0x22, 0xec, 0xf6, 0x52, + 0xc1, 0x0b +}; +static const u8 output27[] __initconst = { + 0xef, 0xe1, 0xf2, 0x67, 0x8e, 0x2c, 0x00, 0x9f, + 0x1d, 0x4c, 0x66, 0x1f, 0x94, 0x58, 0xdc, 0xbb, + 0xb9, 0x11, 0x8f, 0x74, 0xfd, 0x0e, 0x14, 0x01, + 0xa8, 0x21 +}; +static const u8 key27[] __initconst = { + 0x78, 0x71, 0xa4, 0xe6, 0xb2, 0x95, 0x44, 0x12, + 0x81, 0xaa, 0x7e, 0x94, 0xa7, 0x8d, 0x44, 0xea, + 0xc4, 0xbc, 0x01, 0xb7, 0x9e, 0xf7, 0x82, 0x9e, + 0x3b, 0x23, 0x9f, 0x31, 0xdd, 0xb8, 0x0d, 0x18 +}; +enum { nonce27 = 0xd58fe0e58fb254d6ULL }; + +static const u8 input28[] __initconst = { + 0xaa, 0xb7, 0xaa, 0xd9, 0xa8, 0x91, 0xd7, 0x8a, + 0x97, 0x9b, 0xdb, 0x7c, 0x47, 0x2b, 0xdb, 0xd2, + 0xda, 0x77, 0xb1, 0xfa, 0x2d, 0x12, 0xe3, 0xe9, + 0xc4, 0x7f, 0x54 +}; +static const u8 output28[] __initconst = { + 0x87, 0x84, 0xa9, 0xa6, 0xad, 0x8f, 0xe6, 0x0f, + 0x69, 0xf8, 0x21, 0xc3, 0x54, 0x95, 0x0f, 0xb0, + 0x4e, 0xc7, 0x02, 0xe4, 0x04, 0xb0, 0x6c, 0x42, + 0x8c, 0x63, 0xe3 +}; +static const u8 key28[] __initconst = { + 0x12, 0x23, 0x37, 0x95, 0x04, 0xb4, 0x21, 0xe8, + 0xbc, 0x65, 0x46, 0x7a, 0xf4, 0x01, 0x05, 0x3f, + 0xb1, 0x34, 0x73, 0xd2, 0x49, 0xbf, 0x6f, 0x20, + 0xbd, 0x23, 0x58, 0x5f, 0xd1, 0x73, 0x57, 0xa6 +}; +enum { nonce28 = 0x3a04d51491eb4e07ULL }; + +static const u8 input29[] __initconst = { + 0x55, 0xd0, 0xd4, 0x4b, 0x17, 0xc8, 0xc4, 0x2b, + 0xc0, 0x28, 0xbd, 0x9d, 0x65, 0x4d, 0xaf, 0x77, + 0x72, 0x7c, 0x36, 0x68, 0xa7, 0xb6, 0x87, 0x4d, + 0xb9, 0x27, 0x25, 0x6c +}; +static const u8 output29[] __initconst = { + 0x0e, 0xac, 0x4c, 0xf5, 0x12, 0xb5, 0x56, 0xa5, + 0x00, 0x9a, 0xd6, 0xe5, 0x1a, 0x59, 0x2c, 0xf6, + 0x42, 0x22, 0xcf, 0x23, 0x98, 0x34, 0x29, 0xac, + 0x6e, 0xe3, 0x37, 0x6d +}; +static const u8 key29[] __initconst = { + 0xda, 0x9d, 0x05, 0x0c, 0x0c, 0xba, 0x75, 0xb9, + 0x9e, 0xb1, 0x8d, 0xd9, 0x73, 0x26, 0x2c, 0xa9, + 0x3a, 0xb5, 0xcb, 0x19, 0x49, 0xa7, 0x4f, 0xf7, + 0x64, 0x35, 0x23, 0x20, 0x2a, 0x45, 0x78, 0xc7 +}; +enum { nonce29 = 0xc25ac9982431cbfULL }; + +static const u8 input30[] __initconst = { + 0x4e, 0xd6, 0x85, 0xbb, 0xe7, 0x99, 0xfa, 0x04, + 0x33, 0x24, 0xfd, 0x75, 0x18, 0xe3, 0xd3, 0x25, + 0xcd, 0xca, 0xae, 0x00, 0xbe, 0x52, 0x56, 0x4a, + 0x31, 0xe9, 0x4f, 0xae, 0x8a +}; +static const u8 output30[] __initconst = { + 0x30, 0x36, 0x32, 0xa2, 0x3c, 0xb6, 0xf9, 0xf9, + 0x76, 0x70, 0xad, 0xa6, 0x10, 0x41, 0x00, 0x4a, + 0xfa, 0xce, 0x1b, 0x86, 0x05, 0xdb, 0x77, 0x96, + 0xb3, 0xb7, 0x8f, 0x61, 0x24 +}; +static const u8 key30[] __initconst = { + 0x49, 0x35, 0x4c, 0x15, 0x98, 0xfb, 0xc6, 0x57, + 0x62, 0x6d, 0x06, 0xc3, 0xd4, 0x79, 0x20, 0x96, + 0x05, 0x2a, 0x31, 0x63, 0xc0, 0x44, 0x42, 0x09, + 0x13, 0x13, 0xff, 0x1b, 0xc8, 0x63, 0x1f, 0x0b +}; +enum { nonce30 = 0x4967f9c08e41568bULL }; + +static const u8 input31[] __initconst = { + 0x91, 0x04, 0x20, 0x47, 0x59, 0xee, 0xa6, 0x0f, + 0x04, 0x75, 0xc8, 0x18, 0x95, 0x44, 0x01, 0x28, + 0x20, 0x6f, 0x73, 0x68, 0x66, 0xb5, 0x03, 0xb3, + 0x58, 0x27, 0x6e, 0x7a, 0x76, 0xb8 +}; +static const u8 output31[] __initconst = { + 0xe8, 0x03, 0x78, 0x9d, 0x13, 0x15, 0x98, 0xef, + 0x64, 0x68, 0x12, 0x41, 0xb0, 0x29, 0x94, 0x0c, + 0x83, 0x35, 0x46, 0xa9, 0x74, 0xe1, 0x75, 0xf0, + 0xb6, 0x96, 0xc3, 0x6f, 0xd7, 0x70 +}; +static const u8 key31[] __initconst = { + 0xef, 0xcd, 0x5a, 0x4a, 0xf4, 0x7e, 0x6a, 0x3a, + 0x11, 0x88, 0x72, 0x94, 0xb8, 0xae, 0x84, 0xc3, + 0x66, 0xe0, 0xde, 0x4b, 0x00, 0xa5, 0xd6, 0x2d, + 0x50, 0xb7, 0x28, 0xff, 0x76, 0x57, 0x18, 0x1f +}; +enum { nonce31 = 0xcb6f428fa4192e19ULL }; + +static const u8 input32[] __initconst = { + 0x90, 0x06, 0x50, 0x4b, 0x98, 0x14, 0x30, 0xf1, + 0xb8, 0xd7, 0xf0, 0xa4, 0x3e, 0x4e, 0xd8, 0x00, + 0xea, 0xdb, 0x4f, 0x93, 0x05, 0xef, 0x02, 0x71, + 0x1a, 0xcd, 0xa3, 0xb1, 0xae, 0xd3, 0x18 +}; +static const u8 output32[] __initconst = { + 0xcb, 0x4a, 0x37, 0x3f, 0xea, 0x40, 0xab, 0x86, + 0xfe, 0xcc, 0x07, 0xd5, 0xdc, 0xb2, 0x25, 0xb6, + 0xfd, 0x2a, 0x72, 0xbc, 0x5e, 0xd4, 0x75, 0xff, + 0x71, 0xfc, 0xce, 0x1e, 0x6f, 0x22, 0xc1 +}; +static const u8 key32[] __initconst = { + 0xfc, 0x6d, 0xc3, 0x80, 0xce, 0xa4, 0x31, 0xa1, + 0xcc, 0xfa, 0x9d, 0x10, 0x0b, 0xc9, 0x11, 0x77, + 0x34, 0xdb, 0xad, 0x1b, 0xc4, 0xfc, 0xeb, 0x79, + 0x91, 0xda, 0x59, 0x3b, 0x0d, 0xb1, 0x19, 0x3b +}; +enum { nonce32 = 0x88551bf050059467ULL }; + +static const u8 input33[] __initconst = { + 0x88, 0x94, 0x71, 0x92, 0xe8, 0xd7, 0xf9, 0xbd, + 0x55, 0xe3, 0x22, 0xdb, 0x99, 0x51, 0xfb, 0x50, + 0xbf, 0x82, 0xb5, 0x70, 0x8b, 0x2b, 0x6a, 0x03, + 0x37, 0xa0, 0xc6, 0x19, 0x5d, 0xc9, 0xbc, 0xcc +}; +static const u8 output33[] __initconst = { + 0xb6, 0x17, 0x51, 0xc8, 0xea, 0x8a, 0x14, 0xdc, + 0x23, 0x1b, 0xd4, 0xed, 0xbf, 0x50, 0xb9, 0x38, + 0x00, 0xc2, 0x3f, 0x78, 0x3d, 0xbf, 0xa0, 0x84, + 0xef, 0x45, 0xb2, 0x7d, 0x48, 0x7b, 0x62, 0xa7 +}; +static const u8 key33[] __initconst = { + 0xb9, 0x8f, 0x6a, 0xad, 0xb4, 0x6f, 0xb5, 0xdc, + 0x48, 0xfa, 0x43, 0x57, 0x62, 0x97, 0xef, 0x89, + 0x4c, 0x5a, 0x7b, 0x67, 0xb8, 0x9d, 0xf0, 0x42, + 0x2b, 0x8f, 0xf3, 0x18, 0x05, 0x2e, 0x48, 0xd0 +}; +enum { nonce33 = 0x31f16488fe8447f5ULL }; + +static const u8 input34[] __initconst = { + 0xda, 0x2b, 0x3d, 0x63, 0x9e, 0x4f, 0xc2, 0xb8, + 0x7f, 0xc2, 0x1a, 0x8b, 0x0d, 0x95, 0x65, 0x55, + 0x52, 0xba, 0x51, 0x51, 0xc0, 0x61, 0x9f, 0x0a, + 0x5d, 0xb0, 0x59, 0x8c, 0x64, 0x6a, 0xab, 0xf5, + 0x57 +}; +static const u8 output34[] __initconst = { + 0x5c, 0xf6, 0x62, 0x24, 0x8c, 0x45, 0xa3, 0x26, + 0xd0, 0xe4, 0x88, 0x1c, 0xed, 0xc4, 0x26, 0x58, + 0xb5, 0x5d, 0x92, 0xc4, 0x17, 0x44, 0x1c, 0xb8, + 0x2c, 0xf3, 0x55, 0x7e, 0xd6, 0xe5, 0xb3, 0x65, + 0xa8 +}; +static const u8 key34[] __initconst = { + 0xde, 0xd1, 0x27, 0xb7, 0x7c, 0xfa, 0xa6, 0x78, + 0x39, 0x80, 0xdf, 0xb7, 0x46, 0xac, 0x71, 0x26, + 0xd0, 0x2a, 0x56, 0x79, 0x12, 0xeb, 0x26, 0x37, + 0x01, 0x0d, 0x30, 0xe0, 0xe3, 0x66, 0xb2, 0xf4 +}; +enum { nonce34 = 0x92d0d9b252c24149ULL }; + +static const u8 input35[] __initconst = { + 0x3a, 0x15, 0x5b, 0x75, 0x6e, 0xd0, 0x52, 0x20, + 0x6c, 0x82, 0xfa, 0xce, 0x5b, 0xea, 0xf5, 0x43, + 0xc1, 0x81, 0x7c, 0xb2, 0xac, 0x16, 0x3f, 0xd3, + 0x5a, 0xaf, 0x55, 0x98, 0xf4, 0xc6, 0xba, 0x71, + 0x25, 0x8b +}; +static const u8 output35[] __initconst = { + 0xb3, 0xaf, 0xac, 0x6d, 0x4d, 0xc7, 0x68, 0x56, + 0x50, 0x5b, 0x69, 0x2a, 0xe5, 0x90, 0xf9, 0x5f, + 0x99, 0x88, 0xff, 0x0c, 0xa6, 0xb1, 0x83, 0xd6, + 0x80, 0xa6, 0x1b, 0xde, 0x94, 0xa4, 0x2c, 0xc3, + 0x74, 0xfa +}; +static const u8 key35[] __initconst = { + 0xd8, 0x24, 0xe2, 0x06, 0xd7, 0x7a, 0xce, 0x81, + 0x52, 0x72, 0x02, 0x69, 0x89, 0xc4, 0xe9, 0x53, + 0x3b, 0x08, 0x5f, 0x98, 0x1e, 0x1b, 0x99, 0x6e, + 0x28, 0x17, 0x6d, 0xba, 0xc0, 0x96, 0xf9, 0x3c +}; +enum { nonce35 = 0x7baf968c4c8e3a37ULL }; + +static const u8 input36[] __initconst = { + 0x31, 0x5d, 0x4f, 0xe3, 0xac, 0xad, 0x17, 0xa6, + 0xb5, 0x01, 0xe2, 0xc6, 0xd4, 0x7e, 0xc4, 0x80, + 0xc0, 0x59, 0x72, 0xbb, 0x4b, 0x74, 0x6a, 0x41, + 0x0f, 0x9c, 0xf6, 0xca, 0x20, 0xb3, 0x73, 0x07, + 0x6b, 0x02, 0x2a +}; +static const u8 output36[] __initconst = { + 0xf9, 0x09, 0x92, 0x94, 0x7e, 0x31, 0xf7, 0x53, + 0xe8, 0x8a, 0x5b, 0x20, 0xef, 0x9b, 0x45, 0x81, + 0xba, 0x5e, 0x45, 0x63, 0xc1, 0xc7, 0x9e, 0x06, + 0x0e, 0xd9, 0x62, 0x8e, 0x96, 0xf9, 0xfa, 0x43, + 0x4d, 0xd4, 0x28 +}; +static const u8 key36[] __initconst = { + 0x13, 0x30, 0x4c, 0x06, 0xae, 0x18, 0xde, 0x03, + 0x1d, 0x02, 0x40, 0xf5, 0xbb, 0x19, 0xe3, 0x88, + 0x41, 0xb1, 0x29, 0x15, 0x97, 0xc2, 0x69, 0x3f, + 0x32, 0x2a, 0x0c, 0x8b, 0xcf, 0x83, 0x8b, 0x6c +}; +enum { nonce36 = 0x226d251d475075a0ULL }; + +static const u8 input37[] __initconst = { + 0x10, 0x18, 0xbe, 0xfd, 0x66, 0xc9, 0x77, 0xcc, + 0x43, 0xe5, 0x46, 0x0b, 0x08, 0x8b, 0xae, 0x11, + 0x86, 0x15, 0xc2, 0xf6, 0x45, 0xd4, 0x5f, 0xd6, + 0xb6, 0x5f, 0x9f, 0x3e, 0x97, 0xb7, 0xd4, 0xad, + 0x0b, 0xe8, 0x31, 0x94 +}; +static const u8 output37[] __initconst = { + 0x03, 0x2c, 0x1c, 0xee, 0xc6, 0xdd, 0xed, 0x38, + 0x80, 0x6d, 0x84, 0x16, 0xc3, 0xc2, 0x04, 0x63, + 0xcd, 0xa7, 0x6e, 0x36, 0x8b, 0xed, 0x78, 0x63, + 0x95, 0xfc, 0x69, 0x7a, 0x3f, 0x8d, 0x75, 0x6b, + 0x6c, 0x26, 0x56, 0x4d +}; +static const u8 key37[] __initconst = { + 0xac, 0x84, 0x4d, 0xa9, 0x29, 0x49, 0x3c, 0x39, + 0x7f, 0xd9, 0xa6, 0x01, 0xf3, 0x7e, 0xfa, 0x4a, + 0x14, 0x80, 0x22, 0x74, 0xf0, 0x29, 0x30, 0x2d, + 0x07, 0x21, 0xda, 0xc0, 0x4d, 0x70, 0x56, 0xa2 +}; +enum { nonce37 = 0x167823ce3b64925aULL }; + +static const u8 input38[] __initconst = { + 0x30, 0x8f, 0xfa, 0x24, 0x29, 0xb1, 0xfb, 0xce, + 0x31, 0x62, 0xdc, 0xd0, 0x46, 0xab, 0xe1, 0x31, + 0xd9, 0xae, 0x60, 0x0d, 0xca, 0x0a, 0x49, 0x12, + 0x3d, 0x92, 0xe9, 0x91, 0x67, 0x12, 0x62, 0x18, + 0x89, 0xe2, 0xf9, 0x1c, 0xcc +}; +static const u8 output38[] __initconst = { + 0x56, 0x9c, 0xc8, 0x7a, 0xc5, 0x98, 0xa3, 0x0f, + 0xba, 0xd5, 0x3e, 0xe1, 0xc9, 0x33, 0x64, 0x33, + 0xf0, 0xd5, 0xf7, 0x43, 0x66, 0x0e, 0x08, 0x9a, + 0x6e, 0x09, 0xe4, 0x01, 0x0d, 0x1e, 0x2f, 0x4b, + 0xed, 0x9c, 0x08, 0x8c, 0x03 +}; +static const u8 key38[] __initconst = { + 0x77, 0x52, 0x2a, 0x23, 0xf1, 0xc5, 0x96, 0x2b, + 0x89, 0x4f, 0x3e, 0xf3, 0xff, 0x0e, 0x94, 0xce, + 0xf1, 0xbd, 0x53, 0xf5, 0x77, 0xd6, 0x9e, 0x47, + 0x49, 0x3d, 0x16, 0x64, 0xff, 0x95, 0x42, 0x42 +}; +enum { nonce38 = 0xff629d7b82cef357ULL }; + +static const u8 input39[] __initconst = { + 0x38, 0x26, 0x27, 0xd0, 0xc2, 0xf5, 0x34, 0xba, + 0xda, 0x0f, 0x1c, 0x1c, 0x9a, 0x70, 0xe5, 0x8a, + 0x78, 0x2d, 0x8f, 0x9a, 0xbf, 0x89, 0x6a, 0xfd, + 0xd4, 0x9c, 0x33, 0xf1, 0xb6, 0x89, 0x16, 0xe3, + 0x6a, 0x00, 0xfa, 0x3a, 0x0f, 0x26 +}; +static const u8 output39[] __initconst = { + 0x0f, 0xaf, 0x91, 0x6d, 0x9c, 0x99, 0xa4, 0xf7, + 0x3b, 0x9d, 0x9a, 0x98, 0xca, 0xbb, 0x50, 0x48, + 0xee, 0xcb, 0x5d, 0xa1, 0x37, 0x2d, 0x36, 0x09, + 0x2a, 0xe2, 0x1c, 0x3d, 0x98, 0x40, 0x1c, 0x16, + 0x56, 0xa7, 0x98, 0xe9, 0x7d, 0x2b +}; +static const u8 key39[] __initconst = { + 0x6e, 0x83, 0x15, 0x4d, 0xf8, 0x78, 0xa8, 0x0e, + 0x71, 0x37, 0xd4, 0x6e, 0x28, 0x5c, 0x06, 0xa1, + 0x2d, 0x6c, 0x72, 0x7a, 0xfd, 0xf8, 0x65, 0x1a, + 0xb8, 0xe6, 0x29, 0x7b, 0xe5, 0xb3, 0x23, 0x79 +}; +enum { nonce39 = 0xa4d8c491cf093e9dULL }; + +static const u8 input40[] __initconst = { + 0x8f, 0x32, 0x7c, 0x40, 0x37, 0x95, 0x08, 0x00, + 0x00, 0xfe, 0x2f, 0x95, 0x20, 0x12, 0x40, 0x18, + 0x5e, 0x7e, 0x5e, 0x99, 0xee, 0x8d, 0x91, 0x7d, + 0x50, 0x7d, 0x21, 0x45, 0x27, 0xe1, 0x7f, 0xd4, + 0x73, 0x10, 0xe1, 0x33, 0xbc, 0xf8, 0xdd +}; +static const u8 output40[] __initconst = { + 0x78, 0x7c, 0xdc, 0x55, 0x2b, 0xd9, 0x2b, 0x3a, + 0xdd, 0x56, 0x11, 0x52, 0xd3, 0x2e, 0xe0, 0x0d, + 0x23, 0x20, 0x8a, 0xf1, 0x4f, 0xee, 0xf1, 0x68, + 0xf6, 0xdc, 0x53, 0xcf, 0x17, 0xd4, 0xf0, 0x6c, + 0xdc, 0x80, 0x5f, 0x1c, 0xa4, 0x91, 0x05 +}; +static const u8 key40[] __initconst = { + 0x0d, 0x86, 0xbf, 0x8a, 0xba, 0x9e, 0x39, 0x91, + 0xa8, 0xe7, 0x22, 0xf0, 0x0c, 0x43, 0x18, 0xe4, + 0x1f, 0xb0, 0xaf, 0x8a, 0x34, 0x31, 0xf4, 0x41, + 0xf0, 0x89, 0x85, 0xca, 0x5d, 0x05, 0x3b, 0x94 +}; +enum { nonce40 = 0xae7acc4f5986439eULL }; + +static const u8 input41[] __initconst = { + 0x20, 0x5f, 0xc1, 0x83, 0x36, 0x02, 0x76, 0x96, + 0xf0, 0xbf, 0x8e, 0x0e, 0x1a, 0xd1, 0xc7, 0x88, + 0x18, 0xc7, 0x09, 0xc4, 0x15, 0xd9, 0x4f, 0x5e, + 0x1f, 0xb3, 0xb4, 0x6d, 0xcb, 0xa0, 0xd6, 0x8a, + 0x3b, 0x40, 0x8e, 0x80, 0xf1, 0xe8, 0x8f, 0x5f +}; +static const u8 output41[] __initconst = { + 0x0b, 0xd1, 0x49, 0x9a, 0x9d, 0xe8, 0x97, 0xb8, + 0xd1, 0xeb, 0x90, 0x62, 0x37, 0xd2, 0x99, 0x15, + 0x67, 0x6d, 0x27, 0x93, 0xce, 0x37, 0x65, 0xa2, + 0x94, 0x88, 0xd6, 0x17, 0xbc, 0x1c, 0x6e, 0xa2, + 0xcc, 0xfb, 0x81, 0x0e, 0x30, 0x60, 0x5a, 0x6f +}; +static const u8 key41[] __initconst = { + 0x36, 0x27, 0x57, 0x01, 0x21, 0x68, 0x97, 0xc7, + 0x00, 0x67, 0x7b, 0xe9, 0x0f, 0x55, 0x49, 0xbb, + 0x92, 0x18, 0x98, 0xf5, 0x5e, 0xbc, 0xe7, 0x5a, + 0x9d, 0x3d, 0xc7, 0xbd, 0x59, 0xec, 0x82, 0x8e +}; +enum { nonce41 = 0x5da05e4c8dfab464ULL }; + +static const u8 input42[] __initconst = { + 0xca, 0x30, 0xcd, 0x63, 0xf0, 0x2d, 0xf1, 0x03, + 0x4d, 0x0d, 0xf2, 0xf7, 0x6f, 0xae, 0xd6, 0x34, + 0xea, 0xf6, 0x13, 0xcf, 0x1c, 0xa0, 0xd0, 0xe8, + 0xa4, 0x78, 0x80, 0x3b, 0x1e, 0xa5, 0x32, 0x4c, + 0x73, 0x12, 0xd4, 0x6a, 0x94, 0xbc, 0xba, 0x80, + 0x5e +}; +static const u8 output42[] __initconst = { + 0xec, 0x3f, 0x18, 0x31, 0xc0, 0x7b, 0xb5, 0xe2, + 0xad, 0xf3, 0xec, 0xa0, 0x16, 0x9d, 0xef, 0xce, + 0x05, 0x65, 0x59, 0x9d, 0x5a, 0xca, 0x3e, 0x13, + 0xb9, 0x5d, 0x5d, 0xb5, 0xeb, 0xae, 0xc0, 0x87, + 0xbb, 0xfd, 0xe7, 0xe4, 0x89, 0x5b, 0xd2, 0x6c, + 0x56 +}; +static const u8 key42[] __initconst = { + 0x7c, 0x6b, 0x7e, 0x77, 0xcc, 0x8c, 0x1b, 0x03, + 0x8b, 0x2a, 0xb3, 0x7c, 0x5a, 0x73, 0xcc, 0xac, + 0xdd, 0x53, 0x54, 0x0c, 0x85, 0xed, 0xcd, 0x47, + 0x24, 0xc1, 0xb8, 0x9b, 0x2e, 0x41, 0x92, 0x36 +}; +enum { nonce42 = 0xe4d7348b09682c9cULL }; + +static const u8 input43[] __initconst = { + 0x52, 0xf2, 0x4b, 0x7c, 0xe5, 0x58, 0xe8, 0xd2, + 0xb7, 0xf3, 0xa1, 0x29, 0x68, 0xa2, 0x50, 0x50, + 0xae, 0x9c, 0x1b, 0xe2, 0x67, 0x77, 0xe2, 0xdb, + 0x85, 0x55, 0x7e, 0x84, 0x8a, 0x12, 0x3c, 0xb6, + 0x2e, 0xed, 0xd3, 0xec, 0x47, 0x68, 0xfa, 0x52, + 0x46, 0x9d +}; +static const u8 output43[] __initconst = { + 0x1b, 0xf0, 0x05, 0xe4, 0x1c, 0xd8, 0x74, 0x9a, + 0xf0, 0xee, 0x00, 0x54, 0xce, 0x02, 0x83, 0x15, + 0xfb, 0x23, 0x35, 0x78, 0xc3, 0xda, 0x98, 0xd8, + 0x9d, 0x1b, 0xb2, 0x51, 0x82, 0xb0, 0xff, 0xbe, + 0x05, 0xa9, 0xa4, 0x04, 0xba, 0xea, 0x4b, 0x73, + 0x47, 0x6e +}; +static const u8 key43[] __initconst = { + 0xeb, 0xec, 0x0e, 0xa1, 0x65, 0xe2, 0x99, 0x46, + 0xd8, 0x54, 0x8c, 0x4a, 0x93, 0xdf, 0x6d, 0xbf, + 0x93, 0x34, 0x94, 0x57, 0xc9, 0x12, 0x9d, 0x68, + 0x05, 0xc5, 0x05, 0xad, 0x5a, 0xc9, 0x2a, 0x3b +}; +enum { nonce43 = 0xe14f6a902b7827fULL }; + +static const u8 input44[] __initconst = { + 0x3e, 0x22, 0x3e, 0x8e, 0xcd, 0x18, 0xe2, 0xa3, + 0x8d, 0x8b, 0x38, 0xc3, 0x02, 0xa3, 0x31, 0x48, + 0xc6, 0x0e, 0xec, 0x99, 0x51, 0x11, 0x6d, 0x8b, + 0x32, 0x35, 0x3b, 0x08, 0x58, 0x76, 0x25, 0x30, + 0xe2, 0xfc, 0xa2, 0x46, 0x7d, 0x6e, 0x34, 0x87, + 0xac, 0x42, 0xbf +}; +static const u8 output44[] __initconst = { + 0x08, 0x92, 0x58, 0x02, 0x1a, 0xf4, 0x1f, 0x3d, + 0x38, 0x7b, 0x6b, 0xf6, 0x84, 0x07, 0xa3, 0x19, + 0x17, 0x2a, 0xed, 0x57, 0x1c, 0xf9, 0x55, 0x37, + 0x4e, 0xf4, 0x68, 0x68, 0x82, 0x02, 0x4f, 0xca, + 0x21, 0x00, 0xc6, 0x66, 0x79, 0x53, 0x19, 0xef, + 0x7f, 0xdd, 0x74 +}; +static const u8 key44[] __initconst = { + 0x73, 0xb6, 0x3e, 0xf4, 0x57, 0x52, 0xa6, 0x43, + 0x51, 0xd8, 0x25, 0x00, 0xdb, 0xb4, 0x52, 0x69, + 0xd6, 0x27, 0x49, 0xeb, 0x9b, 0xf1, 0x7b, 0xa0, + 0xd6, 0x7c, 0x9c, 0xd8, 0x95, 0x03, 0x69, 0x26 +}; +enum { nonce44 = 0xf5e6dc4f35ce24e5ULL }; + +static const u8 input45[] __initconst = { + 0x55, 0x76, 0xc0, 0xf1, 0x74, 0x03, 0x7a, 0x6d, + 0x14, 0xd8, 0x36, 0x2c, 0x9f, 0x9a, 0x59, 0x7a, + 0x2a, 0xf5, 0x77, 0x84, 0x70, 0x7c, 0x1d, 0x04, + 0x90, 0x45, 0xa4, 0xc1, 0x5e, 0xdd, 0x2e, 0x07, + 0x18, 0x34, 0xa6, 0x85, 0x56, 0x4f, 0x09, 0xaf, + 0x2f, 0x83, 0xe1, 0xc6 +}; +static const u8 output45[] __initconst = { + 0x22, 0x46, 0xe4, 0x0b, 0x3a, 0x55, 0xcc, 0x9b, + 0xf0, 0xc0, 0x53, 0xcd, 0x95, 0xc7, 0x57, 0x6c, + 0x77, 0x46, 0x41, 0x72, 0x07, 0xbf, 0xa8, 0xe5, + 0x68, 0x69, 0xd8, 0x1e, 0x45, 0xc1, 0xa2, 0x50, + 0xa5, 0xd1, 0x62, 0xc9, 0x5a, 0x7d, 0x08, 0x14, + 0xae, 0x44, 0x16, 0xb9 +}; +static const u8 key45[] __initconst = { + 0x41, 0xf3, 0x88, 0xb2, 0x51, 0x25, 0x47, 0x02, + 0x39, 0xe8, 0x15, 0x3a, 0x22, 0x78, 0x86, 0x0b, + 0xf9, 0x1e, 0x8d, 0x98, 0xb2, 0x22, 0x82, 0xac, + 0x42, 0x94, 0xde, 0x64, 0xf0, 0xfd, 0xb3, 0x6c +}; +enum { nonce45 = 0xf51a582daf4aa01aULL }; + +static const u8 input46[] __initconst = { + 0xf6, 0xff, 0x20, 0xf9, 0x26, 0x7e, 0x0f, 0xa8, + 0x6a, 0x45, 0x5a, 0x91, 0x73, 0xc4, 0x4c, 0x63, + 0xe5, 0x61, 0x59, 0xca, 0xec, 0xc0, 0x20, 0x35, + 0xbc, 0x9f, 0x58, 0x9c, 0x5e, 0xa1, 0x17, 0x46, + 0xcc, 0xab, 0x6e, 0xd0, 0x4f, 0x24, 0xeb, 0x05, + 0x4d, 0x40, 0x41, 0xe0, 0x9d +}; +static const u8 output46[] __initconst = { + 0x31, 0x6e, 0x63, 0x3f, 0x9c, 0xe6, 0xb1, 0xb7, + 0xef, 0x47, 0x46, 0xd7, 0xb1, 0x53, 0x42, 0x2f, + 0x2c, 0xc8, 0x01, 0xae, 0x8b, 0xec, 0x42, 0x2c, + 0x6b, 0x2c, 0x9c, 0xb2, 0xf0, 0x29, 0x06, 0xa5, + 0xcd, 0x7e, 0xc7, 0x3a, 0x38, 0x98, 0x8a, 0xde, + 0x03, 0x29, 0x14, 0x8f, 0xf9 +}; +static const u8 key46[] __initconst = { + 0xac, 0xa6, 0x44, 0x4a, 0x0d, 0x42, 0x10, 0xbc, + 0xd3, 0xc9, 0x8e, 0x9e, 0x71, 0xa3, 0x1c, 0x14, + 0x9d, 0x65, 0x0d, 0x49, 0x4d, 0x8c, 0xec, 0x46, + 0xe1, 0x41, 0xcd, 0xf5, 0xfc, 0x82, 0x75, 0x34 +}; +enum { nonce46 = 0x25f85182df84dec5ULL }; + +static const u8 input47[] __initconst = { + 0xa1, 0xd2, 0xf2, 0x52, 0x2f, 0x79, 0x50, 0xb2, + 0x42, 0x29, 0x5b, 0x44, 0x20, 0xf9, 0xbd, 0x85, + 0xb7, 0x65, 0x77, 0x86, 0xce, 0x3e, 0x1c, 0xe4, + 0x70, 0x80, 0xdd, 0x72, 0x07, 0x48, 0x0f, 0x84, + 0x0d, 0xfd, 0x97, 0xc0, 0xb7, 0x48, 0x9b, 0xb4, + 0xec, 0xff, 0x73, 0x14, 0x99, 0xe4 +}; +static const u8 output47[] __initconst = { + 0xe5, 0x3c, 0x78, 0x66, 0x31, 0x1e, 0xd6, 0xc4, + 0x9e, 0x71, 0xb3, 0xd7, 0xd5, 0xad, 0x84, 0xf2, + 0x78, 0x61, 0x77, 0xf8, 0x31, 0xf0, 0x13, 0xad, + 0x66, 0xf5, 0x31, 0x7d, 0xeb, 0xdf, 0xaf, 0xcb, + 0xac, 0x28, 0x6c, 0xc2, 0x9e, 0xe7, 0x78, 0xa2, + 0xa2, 0x58, 0xce, 0x84, 0x76, 0x70 +}; +static const u8 key47[] __initconst = { + 0x05, 0x7f, 0xc0, 0x7f, 0x37, 0x20, 0x71, 0x02, + 0x3a, 0xe7, 0x20, 0x5a, 0x0a, 0x8f, 0x79, 0x5a, + 0xfe, 0xbb, 0x43, 0x4d, 0x2f, 0xcb, 0xf6, 0x9e, + 0xa2, 0x97, 0x00, 0xad, 0x0d, 0x51, 0x7e, 0x17 +}; +enum { nonce47 = 0xae707c60f54de32bULL }; + +static const u8 input48[] __initconst = { + 0x80, 0x93, 0x77, 0x2e, 0x8d, 0xe8, 0xe6, 0xc1, + 0x27, 0xe6, 0xf2, 0x89, 0x5b, 0x33, 0x62, 0x18, + 0x80, 0x6e, 0x17, 0x22, 0x8e, 0x83, 0x31, 0x40, + 0x8f, 0xc9, 0x5c, 0x52, 0x6c, 0x0e, 0xa5, 0xe9, + 0x6c, 0x7f, 0xd4, 0x6a, 0x27, 0x56, 0x99, 0xce, + 0x8d, 0x37, 0x59, 0xaf, 0xc0, 0x0e, 0xe1 +}; +static const u8 output48[] __initconst = { + 0x02, 0xa4, 0x2e, 0x33, 0xb7, 0x7c, 0x2b, 0x9a, + 0x18, 0x5a, 0xba, 0x53, 0x38, 0xaf, 0x00, 0xeb, + 0xd8, 0x3d, 0x02, 0x77, 0x43, 0x45, 0x03, 0x91, + 0xe2, 0x5e, 0x4e, 0xeb, 0x50, 0xd5, 0x5b, 0xe0, + 0xf3, 0x33, 0xa7, 0xa2, 0xac, 0x07, 0x6f, 0xeb, + 0x3f, 0x6c, 0xcd, 0xf2, 0x6c, 0x61, 0x64 +}; +static const u8 key48[] __initconst = { + 0xf3, 0x79, 0xe7, 0xf8, 0x0e, 0x02, 0x05, 0x6b, + 0x83, 0x1a, 0xe7, 0x86, 0x6b, 0xe6, 0x8f, 0x3f, + 0xd3, 0xa3, 0xe4, 0x6e, 0x29, 0x06, 0xad, 0xbc, + 0xe8, 0x33, 0x56, 0x39, 0xdf, 0xb0, 0xe2, 0xfe +}; +enum { nonce48 = 0xd849b938c6569da0ULL }; + +static const u8 input49[] __initconst = { + 0x89, 0x3b, 0x88, 0x9e, 0x7b, 0x38, 0x16, 0x9f, + 0xa1, 0x28, 0xf6, 0xf5, 0x23, 0x74, 0x28, 0xb0, + 0xdf, 0x6c, 0x9e, 0x8a, 0x71, 0xaf, 0xed, 0x7a, + 0x39, 0x21, 0x57, 0x7d, 0x31, 0x6c, 0xee, 0x0d, + 0x11, 0x8d, 0x41, 0x9a, 0x5f, 0xb7, 0x27, 0x40, + 0x08, 0xad, 0xc6, 0xe0, 0x00, 0x43, 0x9e, 0xae +}; +static const u8 output49[] __initconst = { + 0x4d, 0xfd, 0xdb, 0x4c, 0x77, 0xc1, 0x05, 0x07, + 0x4d, 0x6d, 0x32, 0xcb, 0x2e, 0x0e, 0xff, 0x65, + 0xc9, 0x27, 0xeb, 0xa9, 0x46, 0x5b, 0xab, 0x06, + 0xe6, 0xb6, 0x5a, 0x1e, 0x00, 0xfb, 0xcf, 0xe4, + 0xb9, 0x71, 0x40, 0x10, 0xef, 0x12, 0x39, 0xf0, + 0xea, 0x40, 0xb8, 0x9a, 0xa2, 0x85, 0x38, 0x48 +}; +static const u8 key49[] __initconst = { + 0xe7, 0x10, 0x40, 0xd9, 0x66, 0xc0, 0xa8, 0x6d, + 0xa3, 0xcc, 0x8b, 0xdd, 0x93, 0xf2, 0x6e, 0xe0, + 0x90, 0x7f, 0xd0, 0xf4, 0x37, 0x0c, 0x8b, 0x9b, + 0x4c, 0x4d, 0xe6, 0xf2, 0x1f, 0xe9, 0x95, 0x24 +}; +enum { nonce49 = 0xf269817bdae01bc0ULL }; + +static const u8 input50[] __initconst = { + 0xda, 0x5b, 0x60, 0xcd, 0xed, 0x58, 0x8e, 0x7f, + 0xae, 0xdd, 0xc8, 0x2e, 0x16, 0x90, 0xea, 0x4b, + 0x0c, 0x74, 0x14, 0x35, 0xeb, 0xee, 0x2c, 0xff, + 0x46, 0x99, 0x97, 0x6e, 0xae, 0xa7, 0x8e, 0x6e, + 0x38, 0xfe, 0x63, 0xe7, 0x51, 0xd9, 0xaa, 0xce, + 0x7b, 0x1e, 0x7e, 0x5d, 0xc0, 0xe8, 0x10, 0x06, + 0x14 +}; +static const u8 output50[] __initconst = { + 0xe4, 0xe5, 0x86, 0x1b, 0x66, 0x19, 0xac, 0x49, + 0x1c, 0xbd, 0xee, 0x03, 0xaf, 0x11, 0xfc, 0x1f, + 0x6a, 0xd2, 0x50, 0x5c, 0xea, 0x2c, 0xa5, 0x75, + 0xfd, 0xb7, 0x0e, 0x80, 0x8f, 0xed, 0x3f, 0x31, + 0x47, 0xac, 0x67, 0x43, 0xb8, 0x2e, 0xb4, 0x81, + 0x6d, 0xe4, 0x1e, 0xb7, 0x8b, 0x0c, 0x53, 0xa9, + 0x26 +}; +static const u8 key50[] __initconst = { + 0xd7, 0xb2, 0x04, 0x76, 0x30, 0xcc, 0x38, 0x45, + 0xef, 0xdb, 0xc5, 0x86, 0x08, 0x61, 0xf0, 0xee, + 0x6d, 0xd8, 0x22, 0x04, 0x8c, 0xfb, 0xcb, 0x37, + 0xa6, 0xfb, 0x95, 0x22, 0xe1, 0x87, 0xb7, 0x6f +}; +enum { nonce50 = 0x3b44d09c45607d38ULL }; + +static const u8 input51[] __initconst = { + 0xa9, 0x41, 0x02, 0x4b, 0xd7, 0xd5, 0xd1, 0xf1, + 0x21, 0x55, 0xb2, 0x75, 0x6d, 0x77, 0x1b, 0x86, + 0xa9, 0xc8, 0x90, 0xfd, 0xed, 0x4a, 0x7b, 0x6c, + 0xb2, 0x5f, 0x9b, 0x5f, 0x16, 0xa1, 0x54, 0xdb, + 0xd6, 0x3f, 0x6a, 0x7f, 0x2e, 0x51, 0x9d, 0x49, + 0x5b, 0xa5, 0x0e, 0xf9, 0xfb, 0x2a, 0x38, 0xff, + 0x20, 0x8c +}; +static const u8 output51[] __initconst = { + 0x18, 0xf7, 0x88, 0xc1, 0x72, 0xfd, 0x90, 0x4b, + 0xa9, 0x2d, 0xdb, 0x47, 0xb0, 0xa5, 0xc4, 0x37, + 0x01, 0x95, 0xc4, 0xb1, 0xab, 0xc5, 0x5b, 0xcd, + 0xe1, 0x97, 0x78, 0x13, 0xde, 0x6a, 0xff, 0x36, + 0xce, 0xa4, 0x67, 0xc5, 0x4a, 0x45, 0x2b, 0xd9, + 0xff, 0x8f, 0x06, 0x7c, 0x63, 0xbb, 0x83, 0x17, + 0xb4, 0x6b +}; +static const u8 key51[] __initconst = { + 0x82, 0x1a, 0x79, 0xab, 0x9a, 0xb5, 0x49, 0x6a, + 0x30, 0x6b, 0x99, 0x19, 0x11, 0xc7, 0xa2, 0xf4, + 0xca, 0x55, 0xb9, 0xdd, 0xe7, 0x2f, 0xe7, 0xc1, + 0xdd, 0x27, 0xad, 0x80, 0xf2, 0x56, 0xad, 0xf3 +}; +enum { nonce51 = 0xe93aff94ca71a4a6ULL }; + +static const u8 input52[] __initconst = { + 0x89, 0xdd, 0xf3, 0xfa, 0xb6, 0xc1, 0xaa, 0x9a, + 0xc8, 0xad, 0x6b, 0x00, 0xa1, 0x65, 0xea, 0x14, + 0x55, 0x54, 0x31, 0x8f, 0xf0, 0x03, 0x84, 0x51, + 0x17, 0x1e, 0x0a, 0x93, 0x6e, 0x79, 0x96, 0xa3, + 0x2a, 0x85, 0x9c, 0x89, 0xf8, 0xd1, 0xe2, 0x15, + 0x95, 0x05, 0xf4, 0x43, 0x4d, 0x6b, 0xf0, 0x71, + 0x3b, 0x3e, 0xba +}; +static const u8 output52[] __initconst = { + 0x0c, 0x42, 0x6a, 0xb3, 0x66, 0x63, 0x5d, 0x2c, + 0x9f, 0x3d, 0xa6, 0x6e, 0xc7, 0x5f, 0x79, 0x2f, + 0x50, 0xe3, 0xd6, 0x07, 0x56, 0xa4, 0x2b, 0x2d, + 0x8d, 0x10, 0xc0, 0x6c, 0xa2, 0xfc, 0x97, 0xec, + 0x3f, 0x5c, 0x8d, 0x59, 0xbe, 0x84, 0xf1, 0x3e, + 0x38, 0x47, 0x4f, 0x75, 0x25, 0x66, 0x88, 0x14, + 0x03, 0xdd, 0xde +}; +static const u8 key52[] __initconst = { + 0x4f, 0xb0, 0x27, 0xb6, 0xdd, 0x24, 0x0c, 0xdb, + 0x6b, 0x71, 0x2e, 0xac, 0xfc, 0x3f, 0xa6, 0x48, + 0x5d, 0xd5, 0xff, 0x53, 0xb5, 0x62, 0xf1, 0xe0, + 0x93, 0xfe, 0x39, 0x4c, 0x9f, 0x03, 0x11, 0xa7 +}; +enum { nonce52 = 0xed8becec3bdf6f25ULL }; + +static const u8 input53[] __initconst = { + 0x68, 0xd1, 0xc7, 0x74, 0x44, 0x1c, 0x84, 0xde, + 0x27, 0x27, 0x35, 0xf0, 0x18, 0x0b, 0x57, 0xaa, + 0xd0, 0x1a, 0xd3, 0x3b, 0x5e, 0x5c, 0x62, 0x93, + 0xd7, 0x6b, 0x84, 0x3b, 0x71, 0x83, 0x77, 0x01, + 0x3e, 0x59, 0x45, 0xf4, 0x77, 0x6c, 0x6b, 0xcb, + 0x88, 0x45, 0x09, 0x1d, 0xc6, 0x45, 0x6e, 0xdc, + 0x6e, 0x51, 0xb8, 0x28 +}; +static const u8 output53[] __initconst = { + 0xc5, 0x90, 0x96, 0x78, 0x02, 0xf5, 0xc4, 0x3c, + 0xde, 0xd4, 0xd4, 0xc6, 0xa7, 0xad, 0x12, 0x47, + 0x45, 0xce, 0xcd, 0x8c, 0x35, 0xcc, 0xa6, 0x9e, + 0x5a, 0xc6, 0x60, 0xbb, 0xe3, 0xed, 0xec, 0x68, + 0x3f, 0x64, 0xf7, 0x06, 0x63, 0x9c, 0x8c, 0xc8, + 0x05, 0x3a, 0xad, 0x32, 0x79, 0x8b, 0x45, 0x96, + 0x93, 0x73, 0x4c, 0xe0 +}; +static const u8 key53[] __initconst = { + 0x42, 0x4b, 0x20, 0x81, 0x49, 0x50, 0xe9, 0xc2, + 0x43, 0x69, 0x36, 0xe7, 0x68, 0xae, 0xd5, 0x7e, + 0x42, 0x1a, 0x1b, 0xb4, 0x06, 0x4d, 0xa7, 0x17, + 0xb5, 0x31, 0xd6, 0x0c, 0xb0, 0x5c, 0x41, 0x0b +}; +enum { nonce53 = 0xf44ce1931fbda3d7ULL }; + +static const u8 input54[] __initconst = { + 0x7b, 0xf6, 0x8b, 0xae, 0xc0, 0xcb, 0x10, 0x8e, + 0xe8, 0xd8, 0x2e, 0x3b, 0x14, 0xba, 0xb4, 0xd2, + 0x58, 0x6b, 0x2c, 0xec, 0xc1, 0x81, 0x71, 0xb4, + 0xc6, 0xea, 0x08, 0xc5, 0xc9, 0x78, 0xdb, 0xa2, + 0xfa, 0x44, 0x50, 0x9b, 0xc8, 0x53, 0x8d, 0x45, + 0x42, 0xe7, 0x09, 0xc4, 0x29, 0xd8, 0x75, 0x02, + 0xbb, 0xb2, 0x78, 0xcf, 0xe7 +}; +static const u8 output54[] __initconst = { + 0xaf, 0x2c, 0x83, 0x26, 0x6e, 0x7f, 0xa6, 0xe9, + 0x03, 0x75, 0xfe, 0xfe, 0x87, 0x58, 0xcf, 0xb5, + 0xbc, 0x3c, 0x9d, 0xa1, 0x6e, 0x13, 0xf1, 0x0f, + 0x9e, 0xbc, 0xe0, 0x54, 0x24, 0x32, 0xce, 0x95, + 0xe6, 0xa5, 0x59, 0x3d, 0x24, 0x1d, 0x8f, 0xb1, + 0x74, 0x6c, 0x56, 0xe7, 0x96, 0xc1, 0x91, 0xc8, + 0x2d, 0x0e, 0xb7, 0x51, 0x10 +}; +static const u8 key54[] __initconst = { + 0x00, 0x68, 0x74, 0xdc, 0x30, 0x9e, 0xe3, 0x52, + 0xa9, 0xae, 0xb6, 0x7c, 0xa1, 0xdc, 0x12, 0x2d, + 0x98, 0x32, 0x7a, 0x77, 0xe1, 0xdd, 0xa3, 0x76, + 0x72, 0x34, 0x83, 0xd8, 0xb7, 0x69, 0xba, 0x77 +}; +enum { nonce54 = 0xbea57d79b798b63aULL }; + +static const u8 input55[] __initconst = { + 0xb5, 0xf4, 0x2f, 0xc1, 0x5e, 0x10, 0xa7, 0x4e, + 0x74, 0x3d, 0xa3, 0x96, 0xc0, 0x4d, 0x7b, 0x92, + 0x8f, 0xdb, 0x2d, 0x15, 0x52, 0x6a, 0x95, 0x5e, + 0x40, 0x81, 0x4f, 0x70, 0x73, 0xea, 0x84, 0x65, + 0x3d, 0x9a, 0x4e, 0x03, 0x95, 0xf8, 0x5d, 0x2f, + 0x07, 0x02, 0x13, 0x13, 0xdd, 0x82, 0xe6, 0x3b, + 0xe1, 0x5f, 0xb3, 0x37, 0x9b, 0x88 +}; +static const u8 output55[] __initconst = { + 0xc1, 0x88, 0xbd, 0x92, 0x77, 0xad, 0x7c, 0x5f, + 0xaf, 0xa8, 0x57, 0x0e, 0x40, 0x0a, 0xdc, 0x70, + 0xfb, 0xc6, 0x71, 0xfd, 0xc4, 0x74, 0x60, 0xcc, + 0xa0, 0x89, 0x8e, 0x99, 0xf0, 0x06, 0xa6, 0x7c, + 0x97, 0x42, 0x21, 0x81, 0x6a, 0x07, 0xe7, 0xb3, + 0xf7, 0xa5, 0x03, 0x71, 0x50, 0x05, 0x63, 0x17, + 0xa9, 0x46, 0x0b, 0xff, 0x30, 0x78 +}; +static const u8 key55[] __initconst = { + 0x19, 0x8f, 0xe7, 0xd7, 0x6b, 0x7f, 0x6f, 0x69, + 0x86, 0x91, 0x0f, 0xa7, 0x4a, 0x69, 0x8e, 0x34, + 0xf3, 0xdb, 0xde, 0xaf, 0xf2, 0x66, 0x1d, 0x64, + 0x97, 0x0c, 0xcf, 0xfa, 0x33, 0x84, 0xfd, 0x0c +}; +enum { nonce55 = 0x80aa3d3e2c51ef06ULL }; + +static const u8 input56[] __initconst = { + 0x6b, 0xe9, 0x73, 0x42, 0x27, 0x5e, 0x12, 0xcd, + 0xaa, 0x45, 0x12, 0x8b, 0xb3, 0xe6, 0x54, 0x33, + 0x31, 0x7d, 0xe2, 0x25, 0xc6, 0x86, 0x47, 0x67, + 0x86, 0x83, 0xe4, 0x46, 0xb5, 0x8f, 0x2c, 0xbb, + 0xe4, 0xb8, 0x9f, 0xa2, 0xa4, 0xe8, 0x75, 0x96, + 0x92, 0x51, 0x51, 0xac, 0x8e, 0x2e, 0x6f, 0xfc, + 0xbd, 0x0d, 0xa3, 0x9f, 0x16, 0x55, 0x3e +}; +static const u8 output56[] __initconst = { + 0x42, 0x99, 0x73, 0x6c, 0xd9, 0x4b, 0x16, 0xe5, + 0x18, 0x63, 0x1a, 0xd9, 0x0e, 0xf1, 0x15, 0x2e, + 0x0f, 0x4b, 0xe4, 0x5f, 0xa0, 0x4d, 0xde, 0x9f, + 0xa7, 0x18, 0xc1, 0x0c, 0x0b, 0xae, 0x55, 0xe4, + 0x89, 0x18, 0xa4, 0x78, 0x9d, 0x25, 0x0d, 0xd5, + 0x94, 0x0f, 0xf9, 0x78, 0xa3, 0xa6, 0xe9, 0x9e, + 0x2c, 0x73, 0xf0, 0xf7, 0x35, 0xf3, 0x2b +}; +static const u8 key56[] __initconst = { + 0x7d, 0x12, 0xad, 0x51, 0xd5, 0x6f, 0x8f, 0x96, + 0xc0, 0x5d, 0x9a, 0xd1, 0x7e, 0x20, 0x98, 0x0e, + 0x3c, 0x0a, 0x67, 0x6b, 0x1b, 0x88, 0x69, 0xd4, + 0x07, 0x8c, 0xaf, 0x0f, 0x3a, 0x28, 0xe4, 0x5d +}; +enum { nonce56 = 0x70f4c372fb8b5984ULL }; + +static const u8 input57[] __initconst = { + 0x28, 0xa3, 0x06, 0xe8, 0xe7, 0x08, 0xb9, 0xef, + 0x0d, 0x63, 0x15, 0x99, 0xb2, 0x78, 0x7e, 0xaf, + 0x30, 0x50, 0xcf, 0xea, 0xc9, 0x91, 0x41, 0x2f, + 0x3b, 0x38, 0x70, 0xc4, 0x87, 0xb0, 0x3a, 0xee, + 0x4a, 0xea, 0xe3, 0x83, 0x68, 0x8b, 0xcf, 0xda, + 0x04, 0xa5, 0xbd, 0xb2, 0xde, 0x3c, 0x55, 0x13, + 0xfe, 0x96, 0xad, 0xc1, 0x61, 0x1b, 0x98, 0xde +}; +static const u8 output57[] __initconst = { + 0xf4, 0x44, 0xe9, 0xd2, 0x6d, 0xc2, 0x5a, 0xe9, + 0xfd, 0x7e, 0x41, 0x54, 0x3f, 0xf4, 0x12, 0xd8, + 0x55, 0x0d, 0x12, 0x9b, 0xd5, 0x2e, 0x95, 0xe5, + 0x77, 0x42, 0x3f, 0x2c, 0xfb, 0x28, 0x9d, 0x72, + 0x6d, 0x89, 0x82, 0x27, 0x64, 0x6f, 0x0d, 0x57, + 0xa1, 0x25, 0xa3, 0x6b, 0x88, 0x9a, 0xac, 0x0c, + 0x76, 0x19, 0x90, 0xe2, 0x50, 0x5a, 0xf8, 0x12 +}; +static const u8 key57[] __initconst = { + 0x08, 0x26, 0xb8, 0xac, 0xf3, 0xa5, 0xc6, 0xa3, + 0x7f, 0x09, 0x87, 0xf5, 0x6c, 0x5a, 0x85, 0x6c, + 0x3d, 0xbd, 0xde, 0xd5, 0x87, 0xa3, 0x98, 0x7a, + 0xaa, 0x40, 0x3e, 0xf7, 0xff, 0x44, 0x5d, 0xee +}; +enum { nonce57 = 0xc03a6130bf06b089ULL }; + +static const u8 input58[] __initconst = { + 0x82, 0xa5, 0x38, 0x6f, 0xaa, 0xb4, 0xaf, 0xb2, + 0x42, 0x01, 0xa8, 0x39, 0x3f, 0x15, 0x51, 0xa8, + 0x11, 0x1b, 0x93, 0xca, 0x9c, 0xa0, 0x57, 0x68, + 0x8f, 0xdb, 0x68, 0x53, 0x51, 0x6d, 0x13, 0x22, + 0x12, 0x9b, 0xbd, 0x33, 0xa8, 0x52, 0x40, 0x57, + 0x80, 0x9b, 0x98, 0xef, 0x56, 0x70, 0x11, 0xfa, + 0x36, 0x69, 0x7d, 0x15, 0x48, 0xf9, 0x3b, 0xeb, + 0x42 +}; +static const u8 output58[] __initconst = { + 0xff, 0x3a, 0x74, 0xc3, 0x3e, 0x44, 0x64, 0x4d, + 0x0e, 0x5f, 0x9d, 0xa8, 0xdb, 0xbe, 0x12, 0xef, + 0xba, 0x56, 0x65, 0x50, 0x76, 0xaf, 0xa4, 0x4e, + 0x01, 0xc1, 0xd3, 0x31, 0x14, 0xe2, 0xbe, 0x7b, + 0xa5, 0x67, 0xb4, 0xe3, 0x68, 0x40, 0x9c, 0xb0, + 0xb1, 0x78, 0xef, 0x49, 0x03, 0x0f, 0x2d, 0x56, + 0xb4, 0x37, 0xdb, 0xbc, 0x2d, 0x68, 0x1c, 0x3c, + 0xf1 +}; +static const u8 key58[] __initconst = { + 0x7e, 0xf1, 0x7c, 0x20, 0x65, 0xed, 0xcd, 0xd7, + 0x57, 0xe8, 0xdb, 0x90, 0x87, 0xdb, 0x5f, 0x63, + 0x3d, 0xdd, 0xb8, 0x2b, 0x75, 0x8e, 0x04, 0xb5, + 0xf4, 0x12, 0x79, 0xa9, 0x4d, 0x42, 0x16, 0x7f +}; +enum { nonce58 = 0x92838183f80d2f7fULL }; + +static const u8 input59[] __initconst = { + 0x37, 0xf1, 0x9d, 0xdd, 0xd7, 0x08, 0x9f, 0x13, + 0xc5, 0x21, 0x82, 0x75, 0x08, 0x9e, 0x25, 0x16, + 0xb1, 0xd1, 0x71, 0x42, 0x28, 0x63, 0xac, 0x47, + 0x71, 0x54, 0xb1, 0xfc, 0x39, 0xf0, 0x61, 0x4f, + 0x7c, 0x6d, 0x4f, 0xc8, 0x33, 0xef, 0x7e, 0xc8, + 0xc0, 0x97, 0xfc, 0x1a, 0x61, 0xb4, 0x87, 0x6f, + 0xdd, 0x5a, 0x15, 0x7b, 0x1b, 0x95, 0x50, 0x94, + 0x1d, 0xba +}; +static const u8 output59[] __initconst = { + 0x73, 0x67, 0xc5, 0x07, 0xbb, 0x57, 0x79, 0xd5, + 0xc9, 0x04, 0xdd, 0x88, 0xf3, 0x86, 0xe5, 0x70, + 0x49, 0x31, 0xe0, 0xcc, 0x3b, 0x1d, 0xdf, 0xb0, + 0xaf, 0xf4, 0x2d, 0xe0, 0x06, 0x10, 0x91, 0x8d, + 0x1c, 0xcf, 0x31, 0x0b, 0xf6, 0x73, 0xda, 0x1c, + 0xf0, 0x17, 0x52, 0x9e, 0x20, 0x2e, 0x9f, 0x8c, + 0xb3, 0x59, 0xce, 0xd4, 0xd3, 0xc1, 0x81, 0xe9, + 0x11, 0x36 +}; +static const u8 key59[] __initconst = { + 0xbd, 0x07, 0xd0, 0x53, 0x2c, 0xb3, 0xcc, 0x3f, + 0xc4, 0x95, 0xfd, 0xe7, 0x81, 0xb3, 0x29, 0x99, + 0x05, 0x45, 0xd6, 0x95, 0x25, 0x0b, 0x72, 0xd3, + 0xcd, 0xbb, 0x73, 0xf8, 0xfa, 0xc0, 0x9b, 0x7a +}; +enum { nonce59 = 0x4a0db819b0d519e2ULL }; + +static const u8 input60[] __initconst = { + 0x58, 0x4e, 0xdf, 0x94, 0x3c, 0x76, 0x0a, 0x79, + 0x47, 0xf1, 0xbe, 0x88, 0xd3, 0xba, 0x94, 0xd8, + 0xe2, 0x8f, 0xe3, 0x2f, 0x2f, 0x74, 0x82, 0x55, + 0xc3, 0xda, 0xe2, 0x4e, 0x2c, 0x8c, 0x45, 0x1d, + 0x72, 0x8f, 0x54, 0x41, 0xb5, 0xb7, 0x69, 0xe4, + 0xdc, 0xd2, 0x36, 0x21, 0x5c, 0x28, 0x52, 0xf7, + 0x98, 0x8e, 0x72, 0xa7, 0x6d, 0x57, 0xed, 0xdc, + 0x3c, 0xe6, 0x6a +}; +static const u8 output60[] __initconst = { + 0xda, 0xaf, 0xb5, 0xe3, 0x30, 0x65, 0x5c, 0xb1, + 0x48, 0x08, 0x43, 0x7b, 0x9e, 0xd2, 0x6a, 0x62, + 0x56, 0x7c, 0xad, 0xd9, 0xe5, 0xf6, 0x09, 0x71, + 0xcd, 0xe6, 0x05, 0x6b, 0x3f, 0x44, 0x3a, 0x5c, + 0xf6, 0xf8, 0xd7, 0xce, 0x7d, 0xd1, 0xe0, 0x4f, + 0x88, 0x15, 0x04, 0xd8, 0x20, 0xf0, 0x3e, 0xef, + 0xae, 0xa6, 0x27, 0xa3, 0x0e, 0xfc, 0x18, 0x90, + 0x33, 0xcd, 0xd3 +}; +static const u8 key60[] __initconst = { + 0xbf, 0xfd, 0x25, 0xb5, 0xb2, 0xfc, 0x78, 0x0c, + 0x8e, 0xb9, 0x57, 0x2f, 0x26, 0x4a, 0x7e, 0x71, + 0xcc, 0xf2, 0xe0, 0xfd, 0x24, 0x11, 0x20, 0x23, + 0x57, 0x00, 0xff, 0x80, 0x11, 0x0c, 0x1e, 0xff +}; +enum { nonce60 = 0xf18df56fdb7954adULL }; + +static const u8 input61[] __initconst = { + 0xb0, 0xf3, 0x06, 0xbc, 0x22, 0xae, 0x49, 0x40, + 0xae, 0xff, 0x1b, 0x31, 0xa7, 0x98, 0xab, 0x1d, + 0xe7, 0x40, 0x23, 0x18, 0x4f, 0xab, 0x8e, 0x93, + 0x82, 0xf4, 0x56, 0x61, 0xfd, 0x2b, 0xcf, 0xa7, + 0xc4, 0xb4, 0x0a, 0xf4, 0xcb, 0xc7, 0x8c, 0x40, + 0x57, 0xac, 0x0b, 0x3e, 0x2a, 0x0a, 0x67, 0x83, + 0x50, 0xbf, 0xec, 0xb0, 0xc7, 0xf1, 0x32, 0x26, + 0x98, 0x80, 0x33, 0xb4 +}; +static const u8 output61[] __initconst = { + 0x9d, 0x23, 0x0e, 0xff, 0xcc, 0x7c, 0xd5, 0xcf, + 0x1a, 0xb8, 0x59, 0x1e, 0x92, 0xfd, 0x7f, 0xca, + 0xca, 0x3c, 0x18, 0x81, 0xde, 0xfa, 0x59, 0xc8, + 0x6f, 0x9c, 0x24, 0x3f, 0x3a, 0xe6, 0x0b, 0xb4, + 0x34, 0x48, 0x69, 0xfc, 0xb6, 0xea, 0xb2, 0xde, + 0x9f, 0xfd, 0x92, 0x36, 0x18, 0x98, 0x99, 0xaa, + 0x65, 0xe2, 0xea, 0xf4, 0xb1, 0x47, 0x8e, 0xb0, + 0xe7, 0xd4, 0x7a, 0x2c +}; +static const u8 key61[] __initconst = { + 0xd7, 0xfd, 0x9b, 0xbd, 0x8f, 0x65, 0x0d, 0x00, + 0xca, 0xa1, 0x6c, 0x85, 0x85, 0xa4, 0x6d, 0xf1, + 0xb1, 0x68, 0x0c, 0x8b, 0x5d, 0x37, 0x72, 0xd0, + 0xd8, 0xd2, 0x25, 0xab, 0x9f, 0x7b, 0x7d, 0x95 +}; +enum { nonce61 = 0xd82caf72a9c4864fULL }; + +static const u8 input62[] __initconst = { + 0x10, 0x77, 0xf3, 0x2f, 0xc2, 0x50, 0xd6, 0x0c, + 0xba, 0xa8, 0x8d, 0xce, 0x0d, 0x58, 0x9e, 0x87, + 0xb1, 0x59, 0x66, 0x0a, 0x4a, 0xb3, 0xd8, 0xca, + 0x0a, 0x6b, 0xf8, 0xc6, 0x2b, 0x3f, 0x8e, 0x09, + 0xe0, 0x0a, 0x15, 0x85, 0xfe, 0xaa, 0xc6, 0xbd, + 0x30, 0xef, 0xe4, 0x10, 0x78, 0x03, 0xc1, 0xc7, + 0x8a, 0xd9, 0xde, 0x0b, 0x51, 0x07, 0xc4, 0x7b, + 0xe2, 0x2e, 0x36, 0x3a, 0xc2 +}; +static const u8 output62[] __initconst = { + 0xa0, 0x0c, 0xfc, 0xc1, 0xf6, 0xaf, 0xc2, 0xb8, + 0x5c, 0xef, 0x6e, 0xf3, 0xce, 0x15, 0x48, 0x05, + 0xb5, 0x78, 0x49, 0x51, 0x1f, 0x9d, 0xf4, 0xbf, + 0x2f, 0x53, 0xa2, 0xd1, 0x15, 0x20, 0x82, 0x6b, + 0xd2, 0x22, 0x6c, 0x4e, 0x14, 0x87, 0xe3, 0xd7, + 0x49, 0x45, 0x84, 0xdb, 0x5f, 0x68, 0x60, 0xc4, + 0xb3, 0xe6, 0x3f, 0xd1, 0xfc, 0xa5, 0x73, 0xf3, + 0xfc, 0xbb, 0xbe, 0xc8, 0x9d +}; +static const u8 key62[] __initconst = { + 0x6e, 0xc9, 0xaf, 0xce, 0x35, 0xb9, 0x86, 0xd1, + 0xce, 0x5f, 0xd9, 0xbb, 0xd5, 0x1f, 0x7c, 0xcd, + 0xfe, 0x19, 0xaa, 0x3d, 0xea, 0x64, 0xc1, 0x28, + 0x40, 0xba, 0xa1, 0x28, 0xcd, 0x40, 0xb6, 0xf2 +}; +enum { nonce62 = 0xa1c0c265f900cde8ULL }; + +static const u8 input63[] __initconst = { + 0x7a, 0x70, 0x21, 0x2c, 0xef, 0xa6, 0x36, 0xd4, + 0xe0, 0xab, 0x8c, 0x25, 0x73, 0x34, 0xc8, 0x94, + 0x6c, 0x81, 0xcb, 0x19, 0x8d, 0x5a, 0x49, 0xaa, + 0x6f, 0xba, 0x83, 0x72, 0x02, 0x5e, 0xf5, 0x89, + 0xce, 0x79, 0x7e, 0x13, 0x3d, 0x5b, 0x98, 0x60, + 0x5d, 0xd9, 0xfb, 0x15, 0x93, 0x4c, 0xf3, 0x51, + 0x49, 0x55, 0xd1, 0x58, 0xdd, 0x7e, 0x6d, 0xfe, + 0xdd, 0x84, 0x23, 0x05, 0xba, 0xe9 +}; +static const u8 output63[] __initconst = { + 0x20, 0xb3, 0x5c, 0x03, 0x03, 0x78, 0x17, 0xfc, + 0x3b, 0x35, 0x30, 0x9a, 0x00, 0x18, 0xf5, 0xc5, + 0x06, 0x53, 0xf5, 0x04, 0x24, 0x9d, 0xd1, 0xb2, + 0xac, 0x5a, 0xb6, 0x2a, 0xa5, 0xda, 0x50, 0x00, + 0xec, 0xff, 0xa0, 0x7a, 0x14, 0x7b, 0xe4, 0x6b, + 0x63, 0xe8, 0x66, 0x86, 0x34, 0xfd, 0x74, 0x44, + 0xa2, 0x50, 0x97, 0x0d, 0xdc, 0xc3, 0x84, 0xf8, + 0x71, 0x02, 0x31, 0x95, 0xed, 0x54 +}; +static const u8 key63[] __initconst = { + 0x7d, 0x64, 0xb4, 0x12, 0x81, 0xe4, 0xe6, 0x8f, + 0xcc, 0xe7, 0xd1, 0x1f, 0x70, 0x20, 0xfd, 0xb8, + 0x3a, 0x7d, 0xa6, 0x53, 0x65, 0x30, 0x5d, 0xe3, + 0x1a, 0x44, 0xbe, 0x62, 0xed, 0x90, 0xc4, 0xd1 +}; +enum { nonce63 = 0xe8e849596c942276ULL }; + +static const u8 input64[] __initconst = { + 0x84, 0xf8, 0xda, 0x87, 0x23, 0x39, 0x60, 0xcf, + 0xc5, 0x50, 0x7e, 0xc5, 0x47, 0x29, 0x7c, 0x05, + 0xc2, 0xb4, 0xf4, 0xb2, 0xec, 0x5d, 0x48, 0x36, + 0xbf, 0xfc, 0x06, 0x8c, 0xf2, 0x0e, 0x88, 0xe7, + 0xc9, 0xc5, 0xa4, 0xa2, 0x83, 0x20, 0xa1, 0x6f, + 0x37, 0xe5, 0x2d, 0xa1, 0x72, 0xa1, 0x19, 0xef, + 0x05, 0x42, 0x08, 0xf2, 0x57, 0x47, 0x31, 0x1e, + 0x17, 0x76, 0x13, 0xd3, 0xcc, 0x75, 0x2c +}; +static const u8 output64[] __initconst = { + 0xcb, 0xec, 0x90, 0x88, 0xeb, 0x31, 0x69, 0x20, + 0xa6, 0xdc, 0xff, 0x76, 0x98, 0xb0, 0x24, 0x49, + 0x7b, 0x20, 0xd9, 0xd1, 0x1b, 0xe3, 0x61, 0xdc, + 0xcf, 0x51, 0xf6, 0x70, 0x72, 0x33, 0x28, 0x94, + 0xac, 0x73, 0x18, 0xcf, 0x93, 0xfd, 0xca, 0x08, + 0x0d, 0xa2, 0xb9, 0x57, 0x1e, 0x51, 0xb6, 0x07, + 0x5c, 0xc1, 0x13, 0x64, 0x1d, 0x18, 0x6f, 0xe6, + 0x0b, 0xb7, 0x14, 0x03, 0x43, 0xb6, 0xaf +}; +static const u8 key64[] __initconst = { + 0xbf, 0x82, 0x65, 0xe4, 0x50, 0xf9, 0x5e, 0xea, + 0x28, 0x91, 0xd1, 0xd2, 0x17, 0x7c, 0x13, 0x7e, + 0xf5, 0xd5, 0x6b, 0x06, 0x1c, 0x20, 0xc2, 0x82, + 0xa1, 0x7a, 0xa2, 0x14, 0xa1, 0xb0, 0x54, 0x58 +}; +enum { nonce64 = 0xe57c5095aa5723c9ULL }; + +static const u8 input65[] __initconst = { + 0x1c, 0xfb, 0xd3, 0x3f, 0x85, 0xd7, 0xba, 0x7b, + 0xae, 0xb1, 0xa5, 0xd2, 0xe5, 0x40, 0xce, 0x4d, + 0x3e, 0xab, 0x17, 0x9d, 0x7d, 0x9f, 0x03, 0x98, + 0x3f, 0x9f, 0xc8, 0xdd, 0x36, 0x17, 0x43, 0x5c, + 0x34, 0xd1, 0x23, 0xe0, 0x77, 0xbf, 0x35, 0x5d, + 0x8f, 0xb1, 0xcb, 0x82, 0xbb, 0x39, 0x69, 0xd8, + 0x90, 0x45, 0x37, 0xfd, 0x98, 0x25, 0xf7, 0x5b, + 0xce, 0x06, 0x43, 0xba, 0x61, 0xa8, 0x47, 0xb9 +}; +static const u8 output65[] __initconst = { + 0x73, 0xa5, 0x68, 0xab, 0x8b, 0xa5, 0xc3, 0x7e, + 0x74, 0xf8, 0x9d, 0xf5, 0x93, 0x6e, 0xf2, 0x71, + 0x6d, 0xde, 0x82, 0xc5, 0x40, 0xa0, 0x46, 0xb3, + 0x9a, 0x78, 0xa8, 0xf7, 0xdf, 0xb1, 0xc3, 0xdd, + 0x8d, 0x90, 0x00, 0x68, 0x21, 0x48, 0xe8, 0xba, + 0x56, 0x9f, 0x8f, 0xe7, 0xa4, 0x4d, 0x36, 0x55, + 0xd0, 0x34, 0x99, 0xa6, 0x1c, 0x4c, 0xc1, 0xe2, + 0x65, 0x98, 0x14, 0x8e, 0x6a, 0x05, 0xb1, 0x2b +}; +static const u8 key65[] __initconst = { + 0xbd, 0x5c, 0x8a, 0xb0, 0x11, 0x29, 0xf3, 0x00, + 0x7a, 0x78, 0x32, 0x63, 0x34, 0x00, 0xe6, 0x7d, + 0x30, 0x54, 0xde, 0x37, 0xda, 0xc2, 0xc4, 0x3d, + 0x92, 0x6b, 0x4c, 0xc2, 0x92, 0xe9, 0x9e, 0x2a +}; +enum { nonce65 = 0xf654a3031de746f2ULL }; + +static const u8 input66[] __initconst = { + 0x4b, 0x27, 0x30, 0x8f, 0x28, 0xd8, 0x60, 0x46, + 0x39, 0x06, 0x49, 0xea, 0x1b, 0x71, 0x26, 0xe0, + 0x99, 0x2b, 0xd4, 0x8f, 0x64, 0x64, 0xcd, 0xac, + 0x1d, 0x78, 0x88, 0x90, 0xe1, 0x5c, 0x24, 0x4b, + 0xdc, 0x2d, 0xb7, 0xee, 0x3a, 0xe6, 0x86, 0x2c, + 0x21, 0xe4, 0x2b, 0xfc, 0xe8, 0x19, 0xca, 0x65, + 0xe7, 0xdd, 0x6f, 0x52, 0xb3, 0x11, 0xe1, 0xe2, + 0xbf, 0xe8, 0x70, 0xe3, 0x0d, 0x45, 0xb8, 0xa5, + 0x20, 0xb7, 0xb5, 0xaf, 0xff, 0x08, 0xcf, 0x23, + 0x65, 0xdf, 0x8d, 0xc3, 0x31, 0xf3, 0x1e, 0x6a, + 0x58, 0x8d, 0xcc, 0x45, 0x16, 0x86, 0x1f, 0x31, + 0x5c, 0x27, 0xcd, 0xc8, 0x6b, 0x19, 0x1e, 0xec, + 0x44, 0x75, 0x63, 0x97, 0xfd, 0x79, 0xf6, 0x62, + 0xc5, 0xba, 0x17, 0xc7, 0xab, 0x8f, 0xbb, 0xed, + 0x85, 0x2a, 0x98, 0x79, 0x21, 0xec, 0x6e, 0x4d, + 0xdc, 0xfa, 0x72, 0x52, 0xba, 0xc8, 0x4c +}; +static const u8 output66[] __initconst = { + 0x76, 0x5b, 0x2c, 0xa7, 0x62, 0xb9, 0x08, 0x4a, + 0xc6, 0x4a, 0x92, 0xc3, 0xbb, 0x10, 0xb3, 0xee, + 0xff, 0xb9, 0x07, 0xc7, 0x27, 0xcb, 0x1e, 0xcf, + 0x58, 0x6f, 0xa1, 0x64, 0xe8, 0xf1, 0x4e, 0xe1, + 0xef, 0x18, 0x96, 0xab, 0x97, 0x28, 0xd1, 0x7c, + 0x71, 0x6c, 0xd1, 0xe2, 0xfa, 0xd9, 0x75, 0xcb, + 0xeb, 0xea, 0x0c, 0x86, 0x82, 0xd8, 0xf4, 0xcc, + 0xea, 0xa3, 0x00, 0xfa, 0x82, 0xd2, 0xcd, 0xcb, + 0xdb, 0x63, 0x28, 0xe2, 0x82, 0xe9, 0x01, 0xed, + 0x31, 0xe6, 0x71, 0x45, 0x08, 0x89, 0x8a, 0x23, + 0xa8, 0xb5, 0xc2, 0xe2, 0x9f, 0xe9, 0xb8, 0x9a, + 0xc4, 0x79, 0x6d, 0x71, 0x52, 0x61, 0x74, 0x6c, + 0x1b, 0xd7, 0x65, 0x6d, 0x03, 0xc4, 0x1a, 0xc0, + 0x50, 0xba, 0xd6, 0xc9, 0x43, 0x50, 0xbe, 0x09, + 0x09, 0x8a, 0xdb, 0xaa, 0x76, 0x4e, 0x3b, 0x61, + 0x3c, 0x7c, 0x44, 0xe7, 0xdb, 0x10, 0xa7 +}; +static const u8 key66[] __initconst = { + 0x88, 0xdf, 0xca, 0x68, 0xaf, 0x4f, 0xb3, 0xfd, + 0x6e, 0xa7, 0x95, 0x35, 0x8a, 0xe8, 0x37, 0xe8, + 0xc8, 0x55, 0xa2, 0x2a, 0x6d, 0x77, 0xf8, 0x93, + 0x7a, 0x41, 0xf3, 0x7b, 0x95, 0xdf, 0x89, 0xf5 +}; +enum { nonce66 = 0x1024b4fdd415cf82ULL }; + +static const u8 input67[] __initconst = { + 0xd4, 0x2e, 0xfa, 0x92, 0xe9, 0x29, 0x68, 0xb7, + 0x54, 0x2c, 0xf7, 0xa4, 0x2d, 0xb7, 0x50, 0xb5, + 0xc5, 0xb2, 0x9d, 0x17, 0x5e, 0x0a, 0xca, 0x37, + 0xbf, 0x60, 0xae, 0xd2, 0x98, 0xe9, 0xfa, 0x59, + 0x67, 0x62, 0xe6, 0x43, 0x0c, 0x77, 0x80, 0x82, + 0x33, 0x61, 0xa3, 0xff, 0xc1, 0xa0, 0x8f, 0x56, + 0xbc, 0xec, 0x65, 0x43, 0x88, 0xa5, 0xff, 0x51, + 0x64, 0x30, 0xee, 0x34, 0xb7, 0x5c, 0x28, 0x68, + 0xc3, 0x52, 0xd2, 0xac, 0x78, 0x2a, 0xa6, 0x10, + 0xb8, 0xb2, 0x4c, 0x80, 0x4f, 0x99, 0xb2, 0x36, + 0x94, 0x8f, 0x66, 0xcb, 0xa1, 0x91, 0xed, 0x06, + 0x42, 0x6d, 0xc1, 0xae, 0x55, 0x93, 0xdd, 0x93, + 0x9e, 0x88, 0x34, 0x7f, 0x98, 0xeb, 0xbe, 0x61, + 0xf9, 0xa9, 0x0f, 0xd9, 0xc4, 0x87, 0xd5, 0xef, + 0xcc, 0x71, 0x8c, 0x0e, 0xce, 0xad, 0x02, 0xcf, + 0xa2, 0x61, 0xdf, 0xb1, 0xfe, 0x3b, 0xdc, 0xc0, + 0x58, 0xb5, 0x71, 0xa1, 0x83, 0xc9, 0xb4, 0xaf, + 0x9d, 0x54, 0x12, 0xcd, 0xea, 0x06, 0xd6, 0x4e, + 0xe5, 0x27, 0x0c, 0xc3, 0xbb, 0xa8, 0x0a, 0x81, + 0x75, 0xc3, 0xc9, 0xd4, 0x35, 0x3e, 0x53, 0x9f, + 0xaa, 0x20, 0xc0, 0x68, 0x39, 0x2c, 0x96, 0x39, + 0x53, 0x81, 0xda, 0x07, 0x0f, 0x44, 0xa5, 0x47, + 0x0e, 0xb3, 0x87, 0x0d, 0x1b, 0xc1, 0xe5, 0x41, + 0x35, 0x12, 0x58, 0x96, 0x69, 0x8a, 0x1a, 0xa3, + 0x9d, 0x3d, 0xd4, 0xb1, 0x8e, 0x1f, 0x96, 0x87, + 0xda, 0xd3, 0x19, 0xe2, 0xb1, 0x3a, 0x19, 0x74, + 0xa0, 0x00, 0x9f, 0x4d, 0xbc, 0xcb, 0x0c, 0xe9, + 0xec, 0x10, 0xdf, 0x2a, 0x88, 0xdc, 0x30, 0x51, + 0x46, 0x56, 0x53, 0x98, 0x6a, 0x26, 0x14, 0x05, + 0x54, 0x81, 0x55, 0x0b, 0x3c, 0x85, 0xdd, 0x33, + 0x81, 0x11, 0x29, 0x82, 0x46, 0x35, 0xe1, 0xdb, + 0x59, 0x7b +}; +static const u8 output67[] __initconst = { + 0x64, 0x6c, 0xda, 0x7f, 0xd4, 0xa9, 0x2a, 0x5e, + 0x22, 0xae, 0x8d, 0x67, 0xdb, 0xee, 0xfd, 0xd0, + 0x44, 0x80, 0x17, 0xb2, 0xe3, 0x87, 0xad, 0x57, + 0x15, 0xcb, 0x88, 0x64, 0xc0, 0xf1, 0x49, 0x3d, + 0xfa, 0xbe, 0xa8, 0x9f, 0x12, 0xc3, 0x57, 0x56, + 0x70, 0xa5, 0xc5, 0x6b, 0xf1, 0xab, 0xd5, 0xde, + 0x77, 0x92, 0x6a, 0x56, 0x03, 0xf5, 0x21, 0x0d, + 0xb6, 0xc4, 0xcc, 0x62, 0x44, 0x3f, 0xb1, 0xc1, + 0x61, 0x41, 0x90, 0xb2, 0xd5, 0xb8, 0xf3, 0x57, + 0xfb, 0xc2, 0x6b, 0x25, 0x58, 0xc8, 0x45, 0x20, + 0x72, 0x29, 0x6f, 0x9d, 0xb5, 0x81, 0x4d, 0x2b, + 0xb2, 0x89, 0x9e, 0x91, 0x53, 0x97, 0x1c, 0xd9, + 0x3d, 0x79, 0xdc, 0x14, 0xae, 0x01, 0x73, 0x75, + 0xf0, 0xca, 0xd5, 0xab, 0x62, 0x5c, 0x7a, 0x7d, + 0x3f, 0xfe, 0x22, 0x7d, 0xee, 0xe2, 0xcb, 0x76, + 0x55, 0xec, 0x06, 0xdd, 0x41, 0x47, 0x18, 0x62, + 0x1d, 0x57, 0xd0, 0xd6, 0xb6, 0x0f, 0x4b, 0xfc, + 0x79, 0x19, 0xf4, 0xd6, 0x37, 0x86, 0x18, 0x1f, + 0x98, 0x0d, 0x9e, 0x15, 0x2d, 0xb6, 0x9a, 0x8a, + 0x8c, 0x80, 0x22, 0x2f, 0x82, 0xc4, 0xc7, 0x36, + 0xfa, 0xfa, 0x07, 0xbd, 0xc2, 0x2a, 0xe2, 0xea, + 0x93, 0xc8, 0xb2, 0x90, 0x33, 0xf2, 0xee, 0x4b, + 0x1b, 0xf4, 0x37, 0x92, 0x13, 0xbb, 0xe2, 0xce, + 0xe3, 0x03, 0xcf, 0x07, 0x94, 0xab, 0x9a, 0xc9, + 0xff, 0x83, 0x69, 0x3a, 0xda, 0x2c, 0xd0, 0x47, + 0x3d, 0x6c, 0x1a, 0x60, 0x68, 0x47, 0xb9, 0x36, + 0x52, 0xdd, 0x16, 0xef, 0x6c, 0xbf, 0x54, 0x11, + 0x72, 0x62, 0xce, 0x8c, 0x9d, 0x90, 0xa0, 0x25, + 0x06, 0x92, 0x3e, 0x12, 0x7e, 0x1a, 0x1d, 0xe5, + 0xa2, 0x71, 0xce, 0x1c, 0x4c, 0x6a, 0x7c, 0xdc, + 0x3d, 0xe3, 0x6e, 0x48, 0x9d, 0xb3, 0x64, 0x7d, + 0x78, 0x40 +}; +static const u8 key67[] __initconst = { + 0xa9, 0x20, 0x75, 0x89, 0x7e, 0x37, 0x85, 0x48, + 0xa3, 0xfb, 0x7b, 0xe8, 0x30, 0xa7, 0xe3, 0x6e, + 0xa6, 0xc1, 0x71, 0x17, 0xc1, 0x6c, 0x9b, 0xc2, + 0xde, 0xf0, 0xa7, 0x19, 0xec, 0xce, 0xc6, 0x53 +}; +enum { nonce67 = 0x4adc4d1f968c8a10ULL }; + +static const u8 input68[] __initconst = { + 0x99, 0xae, 0x72, 0xfb, 0x16, 0xe1, 0xf1, 0x59, + 0x43, 0x15, 0x4e, 0x33, 0xa0, 0x95, 0xe7, 0x6c, + 0x74, 0x24, 0x31, 0xca, 0x3b, 0x2e, 0xeb, 0xd7, + 0x11, 0xd8, 0xe0, 0x56, 0x92, 0x91, 0x61, 0x57, + 0xe2, 0x82, 0x9f, 0x8f, 0x37, 0xf5, 0x3d, 0x24, + 0x92, 0x9d, 0x87, 0x00, 0x8d, 0x89, 0xe0, 0x25, + 0x8b, 0xe4, 0x20, 0x5b, 0x8a, 0x26, 0x2c, 0x61, + 0x78, 0xb0, 0xa6, 0x3e, 0x82, 0x18, 0xcf, 0xdc, + 0x2d, 0x24, 0xdd, 0x81, 0x42, 0xc4, 0x95, 0xf0, + 0x48, 0x60, 0x71, 0xe3, 0xe3, 0xac, 0xec, 0xbe, + 0x98, 0x6b, 0x0c, 0xb5, 0x6a, 0xa9, 0xc8, 0x79, + 0x23, 0x2e, 0x38, 0x0b, 0x72, 0x88, 0x8c, 0xe7, + 0x71, 0x8b, 0x36, 0xe3, 0x58, 0x3d, 0x9c, 0xa0, + 0xa2, 0xea, 0xcf, 0x0c, 0x6a, 0x6c, 0x64, 0xdf, + 0x97, 0x21, 0x8f, 0x93, 0xfb, 0xba, 0xf3, 0x5a, + 0xd7, 0x8f, 0xa6, 0x37, 0x15, 0x50, 0x43, 0x02, + 0x46, 0x7f, 0x93, 0x46, 0x86, 0x31, 0xe2, 0xaa, + 0x24, 0xa8, 0x26, 0xae, 0xe6, 0xc0, 0x05, 0x73, + 0x0b, 0x4f, 0x7e, 0xed, 0x65, 0xeb, 0x56, 0x1e, + 0xb6, 0xb3, 0x0b, 0xc3, 0x0e, 0x31, 0x95, 0xa9, + 0x18, 0x4d, 0xaf, 0x38, 0xd7, 0xec, 0xc6, 0x44, + 0x72, 0x77, 0x4e, 0x25, 0x4b, 0x25, 0xdd, 0x1e, + 0x8c, 0xa2, 0xdf, 0xf6, 0x2a, 0x97, 0x1a, 0x88, + 0x2c, 0x8a, 0x5d, 0xfe, 0xe8, 0xfb, 0x35, 0xe8, + 0x0f, 0x2b, 0x7a, 0x18, 0x69, 0x43, 0x31, 0x1d, + 0x38, 0x6a, 0x62, 0x95, 0x0f, 0x20, 0x4b, 0xbb, + 0x97, 0x3c, 0xe0, 0x64, 0x2f, 0x52, 0xc9, 0x2d, + 0x4d, 0x9d, 0x54, 0x04, 0x3d, 0xc9, 0xea, 0xeb, + 0xd0, 0x86, 0x52, 0xff, 0x42, 0xe1, 0x0d, 0x7a, + 0xad, 0x88, 0xf9, 0x9b, 0x1e, 0x5e, 0x12, 0x27, + 0x95, 0x3e, 0x0c, 0x2c, 0x13, 0x00, 0x6f, 0x8e, + 0x93, 0x69, 0x0e, 0x01, 0x8c, 0xc1, 0xfd, 0xb3 +}; +static const u8 output68[] __initconst = { + 0x26, 0x3e, 0xf2, 0xb1, 0xf5, 0xef, 0x81, 0xa4, + 0xb7, 0x42, 0xd4, 0x26, 0x18, 0x4b, 0xdd, 0x6a, + 0x47, 0x15, 0xcb, 0x0e, 0x57, 0xdb, 0xa7, 0x29, + 0x7e, 0x7b, 0x3f, 0x47, 0x89, 0x57, 0xab, 0xea, + 0x14, 0x7b, 0xcf, 0x37, 0xdb, 0x1c, 0xe1, 0x11, + 0x77, 0xae, 0x2e, 0x4c, 0xd2, 0x08, 0x3f, 0xa6, + 0x62, 0x86, 0xa6, 0xb2, 0x07, 0xd5, 0x3f, 0x9b, + 0xdc, 0xc8, 0x50, 0x4b, 0x7b, 0xb9, 0x06, 0xe6, + 0xeb, 0xac, 0x98, 0x8c, 0x36, 0x0c, 0x1e, 0xb2, + 0xc8, 0xfb, 0x24, 0x60, 0x2c, 0x08, 0x17, 0x26, + 0x5b, 0xc8, 0xc2, 0xdf, 0x9c, 0x73, 0x67, 0x4a, + 0xdb, 0xcf, 0xd5, 0x2c, 0x2b, 0xca, 0x24, 0xcc, + 0xdb, 0xc9, 0xa8, 0xf2, 0x5d, 0x67, 0xdf, 0x5c, + 0x62, 0x0b, 0x58, 0xc0, 0x83, 0xde, 0x8b, 0xf6, + 0x15, 0x0a, 0xd6, 0x32, 0xd8, 0xf5, 0xf2, 0x5f, + 0x33, 0xce, 0x7e, 0xab, 0x76, 0xcd, 0x14, 0x91, + 0xd8, 0x41, 0x90, 0x93, 0xa1, 0xaf, 0xf3, 0x45, + 0x6c, 0x1b, 0x25, 0xbd, 0x48, 0x51, 0x6d, 0x15, + 0x47, 0xe6, 0x23, 0x50, 0x32, 0x69, 0x1e, 0xb5, + 0x94, 0xd3, 0x97, 0xba, 0xd7, 0x37, 0x4a, 0xba, + 0xb9, 0xcd, 0xfb, 0x96, 0x9a, 0x90, 0xe0, 0x37, + 0xf8, 0xdf, 0x91, 0x6c, 0x62, 0x13, 0x19, 0x21, + 0x4b, 0xa9, 0xf1, 0x12, 0x66, 0xe2, 0x74, 0xd7, + 0x81, 0xa0, 0x74, 0x8d, 0x7e, 0x7e, 0xc9, 0xb1, + 0x69, 0x8f, 0xed, 0xb3, 0xf6, 0x97, 0xcd, 0x72, + 0x78, 0x93, 0xd3, 0x54, 0x6b, 0x43, 0xac, 0x29, + 0xb4, 0xbc, 0x7d, 0xa4, 0x26, 0x4b, 0x7b, 0xab, + 0xd6, 0x67, 0x22, 0xff, 0x03, 0x92, 0xb6, 0xd4, + 0x96, 0x94, 0x5a, 0xe5, 0x02, 0x35, 0x77, 0xfa, + 0x3f, 0x54, 0x1d, 0xdd, 0x35, 0x39, 0xfe, 0x03, + 0xdd, 0x8e, 0x3c, 0x8c, 0xc2, 0x69, 0x2a, 0xb1, + 0xb7, 0xb3, 0xa1, 0x89, 0x84, 0xea, 0x16, 0xe2 +}; +static const u8 key68[] __initconst = { + 0xd2, 0x49, 0x7f, 0xd7, 0x49, 0x66, 0x0d, 0xb3, + 0x5a, 0x7e, 0x3c, 0xfc, 0x37, 0x83, 0x0e, 0xf7, + 0x96, 0xd8, 0xd6, 0x33, 0x79, 0x2b, 0x84, 0x53, + 0x06, 0xbc, 0x6c, 0x0a, 0x55, 0x84, 0xfe, 0xab +}; +enum { nonce68 = 0x6a6df7ff0a20de06ULL }; + +static const u8 input69[] __initconst = { + 0xf9, 0x18, 0x4c, 0xd2, 0x3f, 0xf7, 0x22, 0xd9, + 0x58, 0xb6, 0x3b, 0x38, 0x69, 0x79, 0xf4, 0x71, + 0x5f, 0x38, 0x52, 0x1f, 0x17, 0x6f, 0x6f, 0xd9, + 0x09, 0x2b, 0xfb, 0x67, 0xdc, 0xc9, 0xe8, 0x4a, + 0x70, 0x9f, 0x2e, 0x3c, 0x06, 0xe5, 0x12, 0x20, + 0x25, 0x29, 0xd0, 0xdc, 0x81, 0xc5, 0xc6, 0x0f, + 0xd2, 0xa8, 0x81, 0x15, 0x98, 0xb2, 0x71, 0x5a, + 0x9a, 0xe9, 0xfb, 0xaf, 0x0e, 0x5f, 0x8a, 0xf3, + 0x16, 0x4a, 0x47, 0xf2, 0x5c, 0xbf, 0xda, 0x52, + 0x9a, 0xa6, 0x36, 0xfd, 0xc6, 0xf7, 0x66, 0x00, + 0xcc, 0x6c, 0xd4, 0xb3, 0x07, 0x6d, 0xeb, 0xfe, + 0x92, 0x71, 0x25, 0xd0, 0xcf, 0x9c, 0xe8, 0x65, + 0x45, 0x10, 0xcf, 0x62, 0x74, 0x7d, 0xf2, 0x1b, + 0x57, 0xa0, 0xf1, 0x6b, 0xa4, 0xd5, 0xfa, 0x12, + 0x27, 0x5a, 0xf7, 0x99, 0xfc, 0xca, 0xf3, 0xb8, + 0x2c, 0x8b, 0xba, 0x28, 0x74, 0xde, 0x8f, 0x78, + 0xa2, 0x8c, 0xaf, 0x89, 0x4b, 0x05, 0xe2, 0xf3, + 0xf8, 0xd2, 0xef, 0xac, 0xa4, 0xc4, 0xe2, 0xe2, + 0x36, 0xbb, 0x5e, 0xae, 0xe6, 0x87, 0x3d, 0x88, + 0x9f, 0xb8, 0x11, 0xbb, 0xcf, 0x57, 0xce, 0xd0, + 0xba, 0x62, 0xf4, 0xf8, 0x9b, 0x95, 0x04, 0xc9, + 0xcf, 0x01, 0xe9, 0xf1, 0xc8, 0xc6, 0x22, 0xa4, + 0xf2, 0x8b, 0x2f, 0x24, 0x0a, 0xf5, 0x6e, 0xb7, + 0xd4, 0x2c, 0xb6, 0xf7, 0x5c, 0x97, 0x61, 0x0b, + 0xd9, 0xb5, 0x06, 0xcd, 0xed, 0x3e, 0x1f, 0xc5, + 0xb2, 0x6c, 0xa3, 0xea, 0xb8, 0xad, 0xa6, 0x42, + 0x88, 0x7a, 0x52, 0xd5, 0x64, 0xba, 0xb5, 0x20, + 0x10, 0xa0, 0x0f, 0x0d, 0xea, 0xef, 0x5a, 0x9b, + 0x27, 0xb8, 0xca, 0x20, 0x19, 0x6d, 0xa8, 0xc4, + 0x46, 0x04, 0xb3, 0xe8, 0xf8, 0x66, 0x1b, 0x0a, + 0xce, 0x76, 0x5d, 0x59, 0x58, 0x05, 0xee, 0x3e, + 0x3c, 0x86, 0x5b, 0x49, 0x1c, 0x72, 0x18, 0x01, + 0x62, 0x92, 0x0f, 0x3e, 0xd1, 0x57, 0x5e, 0x20, + 0x7b, 0xfb, 0x4d, 0x3c, 0xc5, 0x35, 0x43, 0x2f, + 0xb0, 0xc5, 0x7c, 0xe4, 0xa2, 0x84, 0x13, 0x77 +}; +static const u8 output69[] __initconst = { + 0xbb, 0x4a, 0x7f, 0x7c, 0xd5, 0x2f, 0x89, 0x06, + 0xec, 0x20, 0xf1, 0x9a, 0x11, 0x09, 0x14, 0x2e, + 0x17, 0x50, 0xf9, 0xd5, 0xf5, 0x48, 0x7c, 0x7a, + 0x55, 0xc0, 0x57, 0x03, 0xe3, 0xc4, 0xb2, 0xb7, + 0x18, 0x47, 0x95, 0xde, 0xaf, 0x80, 0x06, 0x3c, + 0x5a, 0xf2, 0xc3, 0x53, 0xe3, 0x29, 0x92, 0xf8, + 0xff, 0x64, 0x85, 0xb9, 0xf7, 0xd3, 0x80, 0xd2, + 0x0c, 0x5d, 0x7b, 0x57, 0x0c, 0x51, 0x79, 0x86, + 0xf3, 0x20, 0xd2, 0xb8, 0x6e, 0x0c, 0x5a, 0xce, + 0xeb, 0x88, 0x02, 0x8b, 0x82, 0x1b, 0x7f, 0xf5, + 0xde, 0x7f, 0x48, 0x48, 0xdf, 0xa0, 0x55, 0xc6, + 0x0c, 0x22, 0xa1, 0x80, 0x8d, 0x3b, 0xcb, 0x40, + 0x2d, 0x3d, 0x0b, 0xf2, 0xe0, 0x22, 0x13, 0x99, + 0xe1, 0xa7, 0x27, 0x68, 0x31, 0xe1, 0x24, 0x5d, + 0xd2, 0xee, 0x16, 0xc1, 0xd7, 0xa8, 0x14, 0x19, + 0x23, 0x72, 0x67, 0x27, 0xdc, 0x5e, 0xb9, 0xc7, + 0xd8, 0xe3, 0x55, 0x50, 0x40, 0x98, 0x7b, 0xe7, + 0x34, 0x1c, 0x3b, 0x18, 0x14, 0xd8, 0x62, 0xc1, + 0x93, 0x84, 0xf3, 0x5b, 0xdd, 0x9e, 0x1f, 0x3b, + 0x0b, 0xbc, 0x4e, 0x5b, 0x79, 0xa3, 0xca, 0x74, + 0x2a, 0x98, 0xe8, 0x04, 0x39, 0xef, 0xc6, 0x76, + 0x6d, 0xee, 0x9f, 0x67, 0x5b, 0x59, 0x3a, 0xe5, + 0xf2, 0x3b, 0xca, 0x89, 0xe8, 0x9b, 0x03, 0x3d, + 0x11, 0xd2, 0x4a, 0x70, 0xaf, 0x88, 0xb0, 0x94, + 0x96, 0x26, 0xab, 0x3c, 0xc1, 0xb8, 0xe4, 0xe7, + 0x14, 0x61, 0x64, 0x3a, 0x61, 0x08, 0x0f, 0xa9, + 0xce, 0x64, 0xb2, 0x40, 0xf8, 0x20, 0x3a, 0xa9, + 0x31, 0xbd, 0x7e, 0x16, 0xca, 0xf5, 0x62, 0x0f, + 0x91, 0x9f, 0x8e, 0x1d, 0xa4, 0x77, 0xf3, 0x87, + 0x61, 0xe8, 0x14, 0xde, 0x18, 0x68, 0x4e, 0x9d, + 0x73, 0xcd, 0x8a, 0xe4, 0x80, 0x84, 0x23, 0xaa, + 0x9d, 0x64, 0x1c, 0x80, 0x41, 0xca, 0x82, 0x40, + 0x94, 0x55, 0xe3, 0x28, 0xa1, 0x97, 0x71, 0xba, + 0xf2, 0x2c, 0x39, 0x62, 0x29, 0x56, 0xd0, 0xff, + 0xb2, 0x82, 0x20, 0x59, 0x1f, 0xc3, 0x64, 0x57 +}; +static const u8 key69[] __initconst = { + 0x19, 0x09, 0xe9, 0x7c, 0xd9, 0x02, 0x4a, 0x0c, + 0x52, 0x25, 0xad, 0x5c, 0x2e, 0x8d, 0x86, 0x10, + 0x85, 0x2b, 0xba, 0xa4, 0x44, 0x5b, 0x39, 0x3e, + 0x18, 0xaa, 0xce, 0x0e, 0xe2, 0x69, 0x3c, 0xcf +}; +enum { nonce69 = 0xdb925a1948f0f060ULL }; + +static const u8 input70[] __initconst = { + 0x10, 0xe7, 0x83, 0xcf, 0x42, 0x9f, 0xf2, 0x41, + 0xc7, 0xe4, 0xdb, 0xf9, 0xa3, 0x02, 0x1d, 0x8d, + 0x50, 0x81, 0x2c, 0x6b, 0x92, 0xe0, 0x4e, 0xea, + 0x26, 0x83, 0x2a, 0xd0, 0x31, 0xf1, 0x23, 0xf3, + 0x0e, 0x88, 0x14, 0x31, 0xf9, 0x01, 0x63, 0x59, + 0x21, 0xd1, 0x8b, 0xdd, 0x06, 0xd0, 0xc6, 0xab, + 0x91, 0x71, 0x82, 0x4d, 0xd4, 0x62, 0x37, 0x17, + 0xf9, 0x50, 0xf9, 0xb5, 0x74, 0xce, 0x39, 0x80, + 0x80, 0x78, 0xf8, 0xdc, 0x1c, 0xdb, 0x7c, 0x3d, + 0xd4, 0x86, 0x31, 0x00, 0x75, 0x7b, 0xd1, 0x42, + 0x9f, 0x1b, 0x97, 0x88, 0x0e, 0x14, 0x0e, 0x1e, + 0x7d, 0x7b, 0xc4, 0xd2, 0xf3, 0xc1, 0x6d, 0x17, + 0x5d, 0xc4, 0x75, 0x54, 0x0f, 0x38, 0x65, 0x89, + 0xd8, 0x7d, 0xab, 0xc9, 0xa7, 0x0a, 0x21, 0x0b, + 0x37, 0x12, 0x05, 0x07, 0xb5, 0x68, 0x32, 0x32, + 0xb9, 0xf8, 0x97, 0x17, 0x03, 0xed, 0x51, 0x8f, + 0x3d, 0x5a, 0xd0, 0x12, 0x01, 0x6e, 0x2e, 0x91, + 0x1c, 0xbe, 0x6b, 0xa3, 0xcc, 0x75, 0x62, 0x06, + 0x8e, 0x65, 0xbb, 0xe2, 0x29, 0x71, 0x4b, 0x89, + 0x6a, 0x9d, 0x85, 0x8c, 0x8c, 0xdf, 0x94, 0x95, + 0x23, 0x66, 0xf8, 0x92, 0xee, 0x56, 0xeb, 0xb3, + 0xeb, 0xd2, 0x4a, 0x3b, 0x77, 0x8a, 0x6e, 0xf6, + 0xca, 0xd2, 0x34, 0x00, 0xde, 0xbe, 0x1d, 0x7a, + 0x73, 0xef, 0x2b, 0x80, 0x56, 0x16, 0x29, 0xbf, + 0x6e, 0x33, 0xed, 0x0d, 0xe2, 0x02, 0x60, 0x74, + 0xe9, 0x0a, 0xbc, 0xd1, 0xc5, 0xe8, 0x53, 0x02, + 0x79, 0x0f, 0x25, 0x0c, 0xef, 0xab, 0xd3, 0xbc, + 0xb7, 0xfc, 0xf3, 0xb0, 0x34, 0xd1, 0x07, 0xd2, + 0x5a, 0x31, 0x1f, 0xec, 0x1f, 0x87, 0xed, 0xdd, + 0x6a, 0xc1, 0xe8, 0xb3, 0x25, 0x4c, 0xc6, 0x9b, + 0x91, 0x73, 0xec, 0x06, 0x73, 0x9e, 0x57, 0x65, + 0x32, 0x75, 0x11, 0x74, 0x6e, 0xa4, 0x7d, 0x0d, + 0x74, 0x9f, 0x51, 0x10, 0x10, 0x47, 0xc9, 0x71, + 0x6e, 0x97, 0xae, 0x44, 0x41, 0xef, 0x98, 0x78, + 0xf4, 0xc5, 0xbd, 0x5e, 0x00, 0xe5, 0xfd, 0xe2, + 0xbe, 0x8c, 0xc2, 0xae, 0xc2, 0xee, 0x59, 0xf6, + 0xcb, 0x20, 0x54, 0x84, 0xc3, 0x31, 0x7e, 0x67, + 0x71, 0xb6, 0x76, 0xbe, 0x81, 0x8f, 0x82, 0xad, + 0x01, 0x8f, 0xc4, 0x00, 0x04, 0x3d, 0x8d, 0x34, + 0xaa, 0xea, 0xc0, 0xea, 0x91, 0x42, 0xb6, 0xb8, + 0x43, 0xf3, 0x17, 0xb2, 0x73, 0x64, 0x82, 0x97, + 0xd5, 0xc9, 0x07, 0x77, 0xb1, 0x26, 0xe2, 0x00, + 0x6a, 0xae, 0x70, 0x0b, 0xbe, 0xe6, 0xb8, 0x42, + 0x81, 0x55, 0xf7, 0xb8, 0x96, 0x41, 0x9d, 0xd4, + 0x2c, 0x27, 0x00, 0xcc, 0x91, 0x28, 0x22, 0xa4, + 0x7b, 0x42, 0x51, 0x9e, 0xd6, 0xec, 0xf3, 0x6b, + 0x00, 0xff, 0x5c, 0xa2, 0xac, 0x47, 0x33, 0x2d, + 0xf8, 0x11, 0x65, 0x5f, 0x4d, 0x79, 0x8b, 0x4f, + 0xad, 0xf0, 0x9d, 0xcd, 0xb9, 0x7b, 0x08, 0xf7, + 0x32, 0x51, 0xfa, 0x39, 0xaa, 0x78, 0x05, 0xb1, + 0xf3, 0x5d, 0xe8, 0x7c, 0x8e, 0x4f, 0xa2, 0xe0, + 0x98, 0x0c, 0xb2, 0xa7, 0xf0, 0x35, 0x8e, 0x70, + 0x7c, 0x82, 0xf3, 0x1b, 0x26, 0x28, 0x12, 0xe5, + 0x23, 0x57, 0xe4, 0xb4, 0x9b, 0x00, 0x39, 0x97, + 0xef, 0x7c, 0x46, 0x9b, 0x34, 0x6b, 0xe7, 0x0e, + 0xa3, 0x2a, 0x18, 0x11, 0x64, 0xc6, 0x7c, 0x8b, + 0x06, 0x02, 0xf5, 0x69, 0x76, 0xf9, 0xaa, 0x09, + 0x5f, 0x68, 0xf8, 0x4a, 0x79, 0x58, 0xec, 0x37, + 0xcf, 0x3a, 0xcc, 0x97, 0x70, 0x1d, 0x3e, 0x52, + 0x18, 0x0a, 0xad, 0x28, 0x5b, 0x3b, 0xe9, 0x03, + 0x84, 0xe9, 0x68, 0x50, 0xce, 0xc4, 0xbc, 0x3e, + 0x21, 0xad, 0x63, 0xfe, 0xc6, 0xfd, 0x6e, 0x69, + 0x84, 0xa9, 0x30, 0xb1, 0x7a, 0xc4, 0x31, 0x10, + 0xc1, 0x1f, 0x6e, 0xeb, 0xa5, 0xa6, 0x01 +}; +static const u8 output70[] __initconst = { + 0x0f, 0x93, 0x2a, 0x20, 0xb3, 0x87, 0x2d, 0xce, + 0xd1, 0x3b, 0x30, 0xfd, 0x06, 0x6d, 0x0a, 0xaa, + 0x3e, 0xc4, 0x29, 0x02, 0x8a, 0xde, 0xa6, 0x4b, + 0x45, 0x1b, 0x4f, 0x25, 0x59, 0xd5, 0x56, 0x6a, + 0x3b, 0x37, 0xbd, 0x3e, 0x47, 0x12, 0x2c, 0x4e, + 0x60, 0x5f, 0x05, 0x75, 0x61, 0x23, 0x05, 0x74, + 0xcb, 0xfc, 0x5a, 0xb3, 0xac, 0x5c, 0x3d, 0xab, + 0x52, 0x5f, 0x05, 0xbc, 0x57, 0xc0, 0x7e, 0xcf, + 0x34, 0x5d, 0x7f, 0x41, 0xa3, 0x17, 0x78, 0xd5, + 0x9f, 0xec, 0x0f, 0x1e, 0xf9, 0xfe, 0xa3, 0xbd, + 0x28, 0xb0, 0xba, 0x4d, 0x84, 0xdb, 0xae, 0x8f, + 0x1d, 0x98, 0xb7, 0xdc, 0xf9, 0xad, 0x55, 0x9c, + 0x89, 0xfe, 0x9b, 0x9c, 0xa9, 0x89, 0xf6, 0x97, + 0x9c, 0x3f, 0x09, 0x3e, 0xc6, 0x02, 0xc2, 0x55, + 0x58, 0x09, 0x54, 0x66, 0xe4, 0x36, 0x81, 0x35, + 0xca, 0x88, 0x17, 0x89, 0x80, 0x24, 0x2b, 0x21, + 0x89, 0xee, 0x45, 0x5a, 0xe7, 0x1f, 0xd5, 0xa5, + 0x16, 0xa4, 0xda, 0x70, 0x7e, 0xe9, 0x4f, 0x24, + 0x61, 0x97, 0xab, 0xa0, 0xe0, 0xe7, 0xb8, 0x5c, + 0x0f, 0x25, 0x17, 0x37, 0x75, 0x12, 0xb5, 0x40, + 0xde, 0x1c, 0x0d, 0x8a, 0x77, 0x62, 0x3c, 0x86, + 0xd9, 0x70, 0x2e, 0x96, 0x30, 0xd2, 0x55, 0xb3, + 0x6b, 0xc3, 0xf2, 0x9c, 0x47, 0xf3, 0x3a, 0x24, + 0x52, 0xc6, 0x38, 0xd8, 0x22, 0xb3, 0x0c, 0xfd, + 0x2f, 0xa3, 0x3c, 0xb5, 0xe8, 0x26, 0xe1, 0xa3, + 0xad, 0xb0, 0x82, 0x17, 0xc1, 0x53, 0xb8, 0x34, + 0x48, 0xee, 0x39, 0xae, 0x51, 0x43, 0xec, 0x82, + 0xce, 0x87, 0xc6, 0x76, 0xb9, 0x76, 0xd3, 0x53, + 0xfe, 0x49, 0x24, 0x7d, 0x02, 0x42, 0x2b, 0x72, + 0xfb, 0xcb, 0xd8, 0x96, 0x02, 0xc6, 0x9a, 0x20, + 0xf3, 0x5a, 0x67, 0xe8, 0x13, 0xf8, 0xb2, 0xcb, + 0xa2, 0xec, 0x18, 0x20, 0x4a, 0xb0, 0x73, 0x53, + 0x21, 0xb0, 0x77, 0x53, 0xd8, 0x76, 0xa1, 0x30, + 0x17, 0x72, 0x2e, 0x33, 0x5f, 0x33, 0x6b, 0x28, + 0xfb, 0xb0, 0xf4, 0xec, 0x8e, 0xed, 0x20, 0x7d, + 0x57, 0x8c, 0x74, 0x28, 0x64, 0x8b, 0xeb, 0x59, + 0x38, 0x3f, 0xe7, 0x83, 0x2e, 0xe5, 0x64, 0x4d, + 0x5c, 0x1f, 0xe1, 0x3b, 0xd9, 0x84, 0xdb, 0xc9, + 0xec, 0xd8, 0xc1, 0x7c, 0x1f, 0x1b, 0x68, 0x35, + 0xc6, 0x34, 0x10, 0xef, 0x19, 0xc9, 0x0a, 0xd6, + 0x43, 0x7f, 0xa6, 0xcb, 0x9d, 0xf4, 0xf0, 0x16, + 0xb1, 0xb1, 0x96, 0x64, 0xec, 0x8d, 0x22, 0x4c, + 0x4b, 0xe8, 0x1a, 0xba, 0x6f, 0xb7, 0xfc, 0xa5, + 0x69, 0x3e, 0xad, 0x78, 0x79, 0x19, 0xb5, 0x04, + 0x69, 0xe5, 0x3f, 0xff, 0x60, 0x8c, 0xda, 0x0b, + 0x7b, 0xf7, 0xe7, 0xe6, 0x29, 0x3a, 0x85, 0xba, + 0xb5, 0xb0, 0x35, 0xbd, 0x38, 0xce, 0x34, 0x5e, + 0xf2, 0xdc, 0xd1, 0x8f, 0xc3, 0x03, 0x24, 0xa2, + 0x03, 0xf7, 0x4e, 0x49, 0x5b, 0xcf, 0x6d, 0xb0, + 0xeb, 0xe3, 0x30, 0x28, 0xd5, 0x5b, 0x82, 0x5f, + 0xe4, 0x7c, 0x1e, 0xec, 0xd2, 0x39, 0xf9, 0x6f, + 0x2e, 0xb3, 0xcd, 0x01, 0xb1, 0x67, 0xaa, 0xea, + 0xaa, 0xb3, 0x63, 0xaf, 0xd9, 0xb2, 0x1f, 0xba, + 0x05, 0x20, 0xeb, 0x19, 0x32, 0xf0, 0x6c, 0x3f, + 0x40, 0xcc, 0x93, 0xb3, 0xd8, 0x25, 0xa6, 0xe4, + 0xce, 0xd7, 0x7e, 0x48, 0x99, 0x65, 0x7f, 0x86, + 0xc5, 0xd4, 0x79, 0x6b, 0xab, 0x43, 0xb8, 0x6b, + 0xf1, 0x2f, 0xea, 0x4c, 0x5e, 0xf0, 0x3b, 0xb4, + 0xb8, 0xb0, 0x94, 0x0c, 0x6b, 0xe7, 0x22, 0x93, + 0xaa, 0x01, 0xcb, 0xf1, 0x11, 0x60, 0xf6, 0x69, + 0xcf, 0x14, 0xde, 0xfb, 0x90, 0x05, 0x27, 0x0c, + 0x1a, 0x9e, 0xf0, 0xb4, 0xc6, 0xa1, 0xe8, 0xdd, + 0xd0, 0x4c, 0x25, 0x4f, 0x9c, 0xb7, 0xb1, 0xb0, + 0x21, 0xdb, 0x87, 0x09, 0x03, 0xf2, 0xb3 +}; +static const u8 key70[] __initconst = { + 0x3b, 0x5b, 0x59, 0x36, 0x44, 0xd1, 0xba, 0x71, + 0x55, 0x87, 0x4d, 0x62, 0x3d, 0xc2, 0xfc, 0xaa, + 0x3f, 0x4e, 0x1a, 0xe4, 0xca, 0x09, 0xfc, 0x6a, + 0xb2, 0xd6, 0x5d, 0x79, 0xf9, 0x1a, 0x91, 0xa7 +}; +enum { nonce70 = 0x3fd6786dd147a85ULL }; + +static const u8 input71[] __initconst = { + 0x18, 0x78, 0xd6, 0x79, 0xe4, 0x9a, 0x6c, 0x73, + 0x17, 0xd4, 0x05, 0x0f, 0x1e, 0x9f, 0xd9, 0x2b, + 0x86, 0x48, 0x7d, 0xf4, 0xd9, 0x1c, 0x76, 0xfc, + 0x8e, 0x22, 0x34, 0xe1, 0x48, 0x4a, 0x8d, 0x79, + 0xb7, 0xbb, 0x88, 0xab, 0x90, 0xde, 0xc5, 0xb4, + 0xb4, 0xe7, 0x85, 0x49, 0xda, 0x57, 0xeb, 0xc9, + 0xcd, 0x21, 0xfc, 0x45, 0x6e, 0x32, 0x67, 0xf2, + 0x4f, 0xa6, 0x54, 0xe5, 0x20, 0xed, 0xcf, 0xc6, + 0x62, 0x25, 0x8e, 0x00, 0xf8, 0x6b, 0xa2, 0x80, + 0xac, 0x88, 0xa6, 0x59, 0x27, 0x83, 0x95, 0x11, + 0x3f, 0x70, 0x5e, 0x3f, 0x11, 0xfb, 0x26, 0xbf, + 0xe1, 0x48, 0x75, 0xf9, 0x86, 0xbf, 0xa6, 0x5d, + 0x15, 0x61, 0x66, 0xbf, 0x78, 0x8f, 0x6b, 0x9b, + 0xda, 0x98, 0xb7, 0x19, 0xe2, 0xf2, 0xa3, 0x9c, + 0x7c, 0x6a, 0x9a, 0xd8, 0x3d, 0x4c, 0x2c, 0xe1, + 0x09, 0xb4, 0x28, 0x82, 0x4e, 0xab, 0x0c, 0x75, + 0x63, 0xeb, 0xbc, 0xd0, 0x71, 0xa2, 0x73, 0x85, + 0xed, 0x53, 0x7a, 0x3f, 0x68, 0x9f, 0xd0, 0xa9, + 0x00, 0x5a, 0x9e, 0x80, 0x55, 0x00, 0xe6, 0xae, + 0x0c, 0x03, 0x40, 0xed, 0xfc, 0x68, 0x4a, 0xb7, + 0x1e, 0x09, 0x65, 0x30, 0x5a, 0x3d, 0x97, 0x4d, + 0x5e, 0x51, 0x8e, 0xda, 0xc3, 0x55, 0x8c, 0xfb, + 0xcf, 0x83, 0x05, 0x35, 0x0d, 0x08, 0x1b, 0xf3, + 0x3a, 0x57, 0x96, 0xac, 0x58, 0x8b, 0xfa, 0x00, + 0x49, 0x15, 0x78, 0xd2, 0x4b, 0xed, 0xb8, 0x59, + 0x78, 0x9b, 0x7f, 0xaa, 0xfc, 0xe7, 0x46, 0xdc, + 0x7b, 0x34, 0xd0, 0x34, 0xe5, 0x10, 0xff, 0x4d, + 0x5a, 0x4d, 0x60, 0xa7, 0x16, 0x54, 0xc4, 0xfd, + 0xca, 0x5d, 0x68, 0xc7, 0x4a, 0x01, 0x8d, 0x7f, + 0x74, 0x5d, 0xff, 0xb8, 0x37, 0x15, 0x62, 0xfa, + 0x44, 0x45, 0xcf, 0x77, 0x3b, 0x1d, 0xb2, 0xd2, + 0x0d, 0x42, 0x00, 0x39, 0x68, 0x1f, 0xcc, 0x89, + 0x73, 0x5d, 0xa9, 0x2e, 0xfd, 0x58, 0x62, 0xca, + 0x35, 0x8e, 0x70, 0x70, 0xaa, 0x6e, 0x14, 0xe9, + 0xa4, 0xe2, 0x10, 0x66, 0x71, 0xdc, 0x4c, 0xfc, + 0xa9, 0xdc, 0x8f, 0x57, 0x4d, 0xc5, 0xac, 0xd7, + 0xa9, 0xf3, 0xf3, 0xa1, 0xff, 0x62, 0xa0, 0x8f, + 0xe4, 0x96, 0x3e, 0xcb, 0x9f, 0x76, 0x42, 0x39, + 0x1f, 0x24, 0xfd, 0xfd, 0x79, 0xe8, 0x27, 0xdf, + 0xa8, 0xf6, 0x33, 0x8b, 0x31, 0x59, 0x69, 0xcf, + 0x6a, 0xef, 0x89, 0x4d, 0xa7, 0xf6, 0x7e, 0x97, + 0x14, 0xbd, 0xda, 0xdd, 0xb4, 0x84, 0x04, 0x24, + 0xe0, 0x17, 0xe1, 0x0f, 0x1f, 0x8a, 0x6a, 0x71, + 0x74, 0x41, 0xdc, 0x59, 0x5c, 0x8f, 0x01, 0x25, + 0x92, 0xf0, 0x2e, 0x15, 0x62, 0x71, 0x9a, 0x9f, + 0x87, 0xdf, 0x62, 0x49, 0x7f, 0x86, 0x62, 0xfc, + 0x20, 0x84, 0xd7, 0xe3, 0x3a, 0xd9, 0x37, 0x85, + 0xb7, 0x84, 0x5a, 0xf9, 0xed, 0x21, 0x32, 0x94, + 0x3e, 0x04, 0xe7, 0x8c, 0x46, 0x76, 0x21, 0x67, + 0xf6, 0x95, 0x64, 0x92, 0xb7, 0x15, 0xf6, 0xe3, + 0x41, 0x27, 0x9d, 0xd7, 0xe3, 0x79, 0x75, 0x92, + 0xd0, 0xc1, 0xf3, 0x40, 0x92, 0x08, 0xde, 0x90, + 0x22, 0x82, 0xb2, 0x69, 0xae, 0x1a, 0x35, 0x11, + 0x89, 0xc8, 0x06, 0x82, 0x95, 0x23, 0x44, 0x08, + 0x22, 0xf2, 0x71, 0x73, 0x1b, 0x88, 0x11, 0xcf, + 0x1c, 0x7e, 0x8a, 0x2e, 0xdc, 0x79, 0x57, 0xce, + 0x1f, 0xe7, 0x6c, 0x07, 0xd8, 0x06, 0xbe, 0xec, + 0xa3, 0xcf, 0xf9, 0x68, 0xa5, 0xb8, 0xf0, 0xe3, + 0x3f, 0x01, 0x92, 0xda, 0xf1, 0xa0, 0x2d, 0x7b, + 0xab, 0x57, 0x58, 0x2a, 0xaf, 0xab, 0xbd, 0xf2, + 0xe5, 0xaf, 0x7e, 0x1f, 0x46, 0x24, 0x9e, 0x20, + 0x22, 0x0f, 0x84, 0x4c, 0xb7, 0xd8, 0x03, 0xe8, + 0x09, 0x73, 0x6c, 0xc6, 0x9b, 0x90, 0xe0, 0xdb, + 0xf2, 0x71, 0xba, 0xad, 0xb3, 0xec, 0xda, 0x7a +}; +static const u8 output71[] __initconst = { + 0x28, 0xc5, 0x9b, 0x92, 0xf9, 0x21, 0x4f, 0xbb, + 0xef, 0x3b, 0xf0, 0xf5, 0x3a, 0x6d, 0x7f, 0xd6, + 0x6a, 0x8d, 0xa1, 0x01, 0x5c, 0x62, 0x20, 0x8b, + 0x5b, 0x39, 0xd5, 0xd3, 0xc2, 0xf6, 0x9d, 0x5e, + 0xcc, 0xe1, 0xa2, 0x61, 0x16, 0xe2, 0xce, 0xe9, + 0x86, 0xd0, 0xfc, 0xce, 0x9a, 0x28, 0x27, 0xc4, + 0x0c, 0xb9, 0xaa, 0x8d, 0x48, 0xdb, 0xbf, 0x82, + 0x7d, 0xd0, 0x35, 0xc4, 0x06, 0x34, 0xb4, 0x19, + 0x51, 0x73, 0xf4, 0x7a, 0xf4, 0xfd, 0xe9, 0x1d, + 0xdc, 0x0f, 0x7e, 0xf7, 0x96, 0x03, 0xe3, 0xb1, + 0x2e, 0x22, 0x59, 0xb7, 0x6d, 0x1c, 0x97, 0x8c, + 0xd7, 0x31, 0x08, 0x26, 0x4c, 0x6d, 0xc6, 0x14, + 0xa5, 0xeb, 0x45, 0x6a, 0x88, 0xa3, 0xa2, 0x36, + 0xc4, 0x35, 0xb1, 0x5a, 0xa0, 0xad, 0xf7, 0x06, + 0x9b, 0x5d, 0xc1, 0x15, 0xc1, 0xce, 0x0a, 0xb0, + 0x57, 0x2e, 0x3f, 0x6f, 0x0d, 0x10, 0xd9, 0x11, + 0x2c, 0x9c, 0xad, 0x2d, 0xa5, 0x81, 0xfb, 0x4e, + 0x8f, 0xd5, 0x32, 0x4e, 0xaf, 0x5c, 0xc1, 0x86, + 0xde, 0x56, 0x5a, 0x33, 0x29, 0xf7, 0x67, 0xc6, + 0x37, 0x6f, 0xb2, 0x37, 0x4e, 0xd4, 0x69, 0x79, + 0xaf, 0xd5, 0x17, 0x79, 0xe0, 0xba, 0x62, 0xa3, + 0x68, 0xa4, 0x87, 0x93, 0x8d, 0x7e, 0x8f, 0xa3, + 0x9c, 0xef, 0xda, 0xe3, 0xa5, 0x1f, 0xcd, 0x30, + 0xa6, 0x55, 0xac, 0x4c, 0x69, 0x74, 0x02, 0xc7, + 0x5d, 0x95, 0x81, 0x4a, 0x68, 0x11, 0xd3, 0xa9, + 0x98, 0xb1, 0x0b, 0x0d, 0xae, 0x40, 0x86, 0x65, + 0xbf, 0xcc, 0x2d, 0xef, 0x57, 0xca, 0x1f, 0xe4, + 0x34, 0x4e, 0xa6, 0x5e, 0x82, 0x6e, 0x61, 0xad, + 0x0b, 0x3c, 0xf8, 0xeb, 0x01, 0x43, 0x7f, 0x87, + 0xa2, 0xa7, 0x6a, 0xe9, 0x62, 0x23, 0x24, 0x61, + 0xf1, 0xf7, 0x36, 0xdb, 0x10, 0xe5, 0x57, 0x72, + 0x3a, 0xc2, 0xae, 0xcc, 0x75, 0xc7, 0x80, 0x05, + 0x0a, 0x5c, 0x4c, 0x95, 0xda, 0x02, 0x01, 0x14, + 0x06, 0x6b, 0x5c, 0x65, 0xc2, 0xb8, 0x4a, 0xd6, + 0xd3, 0xb4, 0xd8, 0x12, 0x52, 0xb5, 0x60, 0xd3, + 0x8e, 0x5f, 0x5c, 0x76, 0x33, 0x7a, 0x05, 0xe5, + 0xcb, 0xef, 0x4f, 0x89, 0xf1, 0xba, 0x32, 0x6f, + 0x33, 0xcd, 0x15, 0x8d, 0xa3, 0x0c, 0x3f, 0x63, + 0x11, 0xe7, 0x0e, 0xe0, 0x00, 0x01, 0xe9, 0xe8, + 0x8e, 0x36, 0x34, 0x8d, 0x96, 0xb5, 0x03, 0xcf, + 0x55, 0x62, 0x49, 0x7a, 0x34, 0x44, 0xa5, 0xee, + 0x8c, 0x46, 0x06, 0x22, 0xab, 0x1d, 0x53, 0x9c, + 0xa1, 0xf9, 0x67, 0x18, 0x57, 0x89, 0xf9, 0xc2, + 0xd1, 0x7e, 0xbe, 0x36, 0x40, 0xcb, 0xe9, 0x04, + 0xde, 0xb1, 0x3b, 0x29, 0x52, 0xc5, 0x9a, 0xb5, + 0xa2, 0x7c, 0x7b, 0xfe, 0xe5, 0x92, 0x73, 0xea, + 0xea, 0x7b, 0xba, 0x0a, 0x8c, 0x88, 0x15, 0xe6, + 0x53, 0xbf, 0x1c, 0x33, 0xf4, 0x9b, 0x9a, 0x5e, + 0x8d, 0xae, 0x60, 0xdc, 0xcb, 0x5d, 0xfa, 0xbe, + 0x06, 0xc3, 0x3f, 0x06, 0xe7, 0x00, 0x40, 0x7b, + 0xaa, 0x94, 0xfa, 0x6d, 0x1f, 0xe4, 0xc5, 0xa9, + 0x1b, 0x5f, 0x36, 0xea, 0x5a, 0xdd, 0xa5, 0x48, + 0x6a, 0x55, 0xd2, 0x47, 0x28, 0xbf, 0x96, 0xf1, + 0x9f, 0xb6, 0x11, 0x4b, 0xd3, 0x44, 0x7d, 0x48, + 0x41, 0x61, 0xdb, 0x12, 0xd4, 0xc2, 0x59, 0x82, + 0x4c, 0x47, 0x5c, 0x04, 0xf6, 0x7b, 0xd3, 0x92, + 0x2e, 0xe8, 0x40, 0xef, 0x15, 0x32, 0x97, 0xdc, + 0x35, 0x4c, 0x6e, 0xa4, 0x97, 0xe9, 0x24, 0xde, + 0x63, 0x8b, 0xb1, 0x6b, 0x48, 0xbb, 0x46, 0x1f, + 0x84, 0xd6, 0x17, 0xb0, 0x5a, 0x4a, 0x4e, 0xd5, + 0x31, 0xd7, 0xcf, 0xa0, 0x39, 0xc6, 0x2e, 0xfc, + 0xa6, 0xa3, 0xd3, 0x0f, 0xa4, 0x28, 0xac, 0xb2, + 0xf4, 0x48, 0x8d, 0x50, 0xa5, 0x1c, 0x44, 0x5d, + 0x6e, 0x38, 0xb7, 0x2b, 0x8a, 0x45, 0xa7, 0x3d +}; +static const u8 key71[] __initconst = { + 0x8b, 0x68, 0xc4, 0xb7, 0x0d, 0x81, 0xef, 0x52, + 0x1e, 0x05, 0x96, 0x72, 0x62, 0x89, 0x27, 0x83, + 0xd0, 0xc7, 0x33, 0x6d, 0xf2, 0xcc, 0x69, 0xf9, + 0x23, 0xae, 0x99, 0xb1, 0xd1, 0x05, 0x4e, 0x54 +}; +enum { nonce71 = 0x983f03656d64b5f6ULL }; + +static const u8 input72[] __initconst = { + 0x6b, 0x09, 0xc9, 0x57, 0x3d, 0x79, 0x04, 0x8c, + 0x65, 0xad, 0x4a, 0x0f, 0xa1, 0x31, 0x3a, 0xdd, + 0x14, 0x8e, 0xe8, 0xfe, 0xbf, 0x42, 0x87, 0x98, + 0x2e, 0x8d, 0x83, 0xa3, 0xf8, 0x55, 0x3d, 0x84, + 0x1e, 0x0e, 0x05, 0x4a, 0x38, 0x9e, 0xe7, 0xfe, + 0xd0, 0x4d, 0x79, 0x74, 0x3a, 0x0b, 0x9b, 0xe1, + 0xfd, 0x51, 0x84, 0x4e, 0xb2, 0x25, 0xe4, 0x64, + 0x4c, 0xda, 0xcf, 0x46, 0xec, 0xba, 0x12, 0xeb, + 0x5a, 0x33, 0x09, 0x6e, 0x78, 0x77, 0x8f, 0x30, + 0xb1, 0x7d, 0x3f, 0x60, 0x8c, 0xf2, 0x1d, 0x8e, + 0xb4, 0x70, 0xa2, 0x90, 0x7c, 0x79, 0x1a, 0x2c, + 0xf6, 0x28, 0x79, 0x7c, 0x53, 0xc5, 0xfa, 0xcc, + 0x65, 0x9b, 0xe1, 0x51, 0xd1, 0x7f, 0x1d, 0xc4, + 0xdb, 0xd4, 0xd9, 0x04, 0x61, 0x7d, 0xbe, 0x12, + 0xfc, 0xcd, 0xaf, 0xe4, 0x0f, 0x9c, 0x20, 0xb5, + 0x22, 0x40, 0x18, 0xda, 0xe4, 0xda, 0x8c, 0x2d, + 0x84, 0xe3, 0x5f, 0x53, 0x17, 0xed, 0x78, 0xdc, + 0x2f, 0xe8, 0x31, 0xc7, 0xe6, 0x39, 0x71, 0x40, + 0xb4, 0x0f, 0xc9, 0xa9, 0x7e, 0x78, 0x87, 0xc1, + 0x05, 0x78, 0xbb, 0x01, 0xf2, 0x8f, 0x33, 0xb0, + 0x6e, 0x84, 0xcd, 0x36, 0x33, 0x5c, 0x5b, 0x8e, + 0xf1, 0xac, 0x30, 0xfe, 0x33, 0xec, 0x08, 0xf3, + 0x7e, 0xf2, 0xf0, 0x4c, 0xf2, 0xad, 0xd8, 0xc1, + 0xd4, 0x4e, 0x87, 0x06, 0xd4, 0x75, 0xe7, 0xe3, + 0x09, 0xd3, 0x4d, 0xe3, 0x21, 0x32, 0xba, 0xb4, + 0x68, 0x68, 0xcb, 0x4c, 0xa3, 0x1e, 0xb3, 0x87, + 0x7b, 0xd3, 0x0c, 0x63, 0x37, 0x71, 0x79, 0xfb, + 0x58, 0x36, 0x57, 0x0f, 0x34, 0x1d, 0xc1, 0x42, + 0x02, 0x17, 0xe7, 0xed, 0xe8, 0xe7, 0x76, 0xcb, + 0x42, 0xc4, 0x4b, 0xe2, 0xb2, 0x5e, 0x42, 0xd5, + 0xec, 0x9d, 0xc1, 0x32, 0x71, 0xe4, 0xeb, 0x10, + 0x68, 0x1a, 0x6e, 0x99, 0x8e, 0x73, 0x12, 0x1f, + 0x97, 0x0c, 0x9e, 0xcd, 0x02, 0x3e, 0x4c, 0xa0, + 0xf2, 0x8d, 0xe5, 0x44, 0xca, 0x6d, 0xfe, 0x07, + 0xe3, 0xe8, 0x9b, 0x76, 0xc1, 0x6d, 0xb7, 0x6e, + 0x0d, 0x14, 0x00, 0x6f, 0x8a, 0xfd, 0x43, 0xc6, + 0x43, 0xa5, 0x9c, 0x02, 0x47, 0x10, 0xd4, 0xb4, + 0x9b, 0x55, 0x67, 0xc8, 0x7f, 0xc1, 0x8a, 0x1f, + 0x1e, 0xd1, 0xbc, 0x99, 0x5d, 0x50, 0x4f, 0x89, + 0xf1, 0xe6, 0x5d, 0x91, 0x40, 0xdc, 0x20, 0x67, + 0x56, 0xc2, 0xef, 0xbd, 0x2c, 0xa2, 0x99, 0x38, + 0xe0, 0x45, 0xec, 0x44, 0x05, 0x52, 0x65, 0x11, + 0xfc, 0x3b, 0x19, 0xcb, 0x71, 0xc2, 0x8e, 0x0e, + 0x03, 0x2a, 0x03, 0x3b, 0x63, 0x06, 0x31, 0x9a, + 0xac, 0x53, 0x04, 0x14, 0xd4, 0x80, 0x9d, 0x6b, + 0x42, 0x7e, 0x7e, 0x4e, 0xdc, 0xc7, 0x01, 0x49, + 0x9f, 0xf5, 0x19, 0x86, 0x13, 0x28, 0x2b, 0xa6, + 0xa6, 0xbe, 0xa1, 0x7e, 0x71, 0x05, 0x00, 0xff, + 0x59, 0x2d, 0xb6, 0x63, 0xf0, 0x1e, 0x2e, 0x69, + 0x9b, 0x85, 0xf1, 0x1e, 0x8a, 0x64, 0x39, 0xab, + 0x00, 0x12, 0xe4, 0x33, 0x4b, 0xb5, 0xd8, 0xb3, + 0x6b, 0x5b, 0x8b, 0x5c, 0xd7, 0x6f, 0x23, 0xcf, + 0x3f, 0x2e, 0x5e, 0x47, 0xb9, 0xb8, 0x1f, 0xf0, + 0x1d, 0xda, 0xe7, 0x4f, 0x6e, 0xab, 0xc3, 0x36, + 0xb4, 0x74, 0x6b, 0xeb, 0xc7, 0x5d, 0x91, 0xe5, + 0xda, 0xf2, 0xc2, 0x11, 0x17, 0x48, 0xf8, 0x9c, + 0xc9, 0x8b, 0xc1, 0xa2, 0xf4, 0xcd, 0x16, 0xf8, + 0x27, 0xd9, 0x6c, 0x6f, 0xb5, 0x8f, 0x77, 0xca, + 0x1b, 0xd8, 0xef, 0x84, 0x68, 0x71, 0x53, 0xc1, + 0x43, 0x0f, 0x9f, 0x98, 0xae, 0x7e, 0x31, 0xd2, + 0x98, 0xfb, 0x20, 0xa2, 0xad, 0x00, 0x10, 0x83, + 0x00, 0x8b, 0xeb, 0x56, 0xd2, 0xc4, 0xcc, 0x7f, + 0x2f, 0x4e, 0xfa, 0x88, 0x13, 0xa4, 0x2c, 0xde, + 0x6b, 0x77, 0x86, 0x10, 0x6a, 0xab, 0x43, 0x0a, + 0x02 +}; +static const u8 output72[] __initconst = { + 0x42, 0x89, 0xa4, 0x80, 0xd2, 0xcb, 0x5f, 0x7f, + 0x2a, 0x1a, 0x23, 0x00, 0xa5, 0x6a, 0x95, 0xa3, + 0x9a, 0x41, 0xa1, 0xd0, 0x2d, 0x1e, 0xd6, 0x13, + 0x34, 0x40, 0x4e, 0x7f, 0x1a, 0xbe, 0xa0, 0x3d, + 0x33, 0x9c, 0x56, 0x2e, 0x89, 0x25, 0x45, 0xf9, + 0xf0, 0xba, 0x9c, 0x6d, 0xd1, 0xd1, 0xde, 0x51, + 0x47, 0x63, 0xc9, 0xbd, 0xfa, 0xa2, 0x9e, 0xad, + 0x6a, 0x7b, 0x21, 0x1a, 0x6c, 0x3e, 0xff, 0x46, + 0xbe, 0xf3, 0x35, 0x7a, 0x6e, 0xb3, 0xb9, 0xf7, + 0xda, 0x5e, 0xf0, 0x14, 0xb5, 0x70, 0xa4, 0x2b, + 0xdb, 0xbb, 0xc7, 0x31, 0x4b, 0x69, 0x5a, 0x83, + 0x70, 0xd9, 0x58, 0xd4, 0x33, 0x84, 0x23, 0xf0, + 0xae, 0xbb, 0x6d, 0x26, 0x7c, 0xc8, 0x30, 0xf7, + 0x24, 0xad, 0xbd, 0xe4, 0x2c, 0x38, 0x38, 0xac, + 0xe1, 0x4a, 0x9b, 0xac, 0x33, 0x0e, 0x4a, 0xf4, + 0x93, 0xed, 0x07, 0x82, 0x81, 0x4f, 0x8f, 0xb1, + 0xdd, 0x73, 0xd5, 0x50, 0x6d, 0x44, 0x1e, 0xbe, + 0xa7, 0xcd, 0x17, 0x57, 0xd5, 0x3b, 0x62, 0x36, + 0xcf, 0x7d, 0xc8, 0xd8, 0xd1, 0x78, 0xd7, 0x85, + 0x46, 0x76, 0x5d, 0xcc, 0xfe, 0xe8, 0x94, 0xc5, + 0xad, 0xbc, 0x5e, 0xbc, 0x8d, 0x1d, 0xdf, 0x03, + 0xc9, 0x6b, 0x1b, 0x81, 0xd1, 0xb6, 0x5a, 0x24, + 0xe3, 0xdc, 0x3f, 0x20, 0xc9, 0x07, 0x73, 0x4c, + 0x43, 0x13, 0x87, 0x58, 0x34, 0x0d, 0x14, 0x63, + 0x0f, 0x6f, 0xad, 0x8d, 0xac, 0x7c, 0x67, 0x68, + 0xa3, 0x9d, 0x7f, 0x00, 0xdf, 0x28, 0xee, 0x67, + 0xf4, 0x5c, 0x26, 0xcb, 0xef, 0x56, 0x71, 0xc8, + 0xc6, 0x67, 0x5f, 0x38, 0xbb, 0xa0, 0xb1, 0x5c, + 0x1f, 0xb3, 0x08, 0xd9, 0x38, 0xcf, 0x74, 0x54, + 0xc6, 0xa4, 0xc4, 0xc0, 0x9f, 0xb3, 0xd0, 0xda, + 0x62, 0x67, 0x8b, 0x81, 0x33, 0xf0, 0xa9, 0x73, + 0xa4, 0xd1, 0x46, 0x88, 0x8d, 0x85, 0x12, 0x40, + 0xba, 0x1a, 0xcd, 0x82, 0xd8, 0x8d, 0xc4, 0x52, + 0xe7, 0x01, 0x94, 0x2e, 0x0e, 0xd0, 0xaf, 0xe7, + 0x2d, 0x3f, 0x3c, 0xaa, 0xf4, 0xf5, 0xa7, 0x01, + 0x4c, 0x14, 0xe2, 0xc2, 0x96, 0x76, 0xbe, 0x05, + 0xaa, 0x19, 0xb1, 0xbd, 0x95, 0xbb, 0x5a, 0xf9, + 0xa5, 0xa7, 0xe6, 0x16, 0x38, 0x34, 0xf7, 0x9d, + 0x19, 0x66, 0x16, 0x8e, 0x7f, 0x2b, 0x5a, 0xfb, + 0xb5, 0x29, 0x79, 0xbf, 0x52, 0xae, 0x30, 0x95, + 0x3f, 0x31, 0x33, 0x28, 0xde, 0xc5, 0x0d, 0x55, + 0x89, 0xec, 0x21, 0x11, 0x0f, 0x8b, 0xfe, 0x63, + 0x3a, 0xf1, 0x95, 0x5c, 0xcd, 0x50, 0xe4, 0x5d, + 0x8f, 0xa7, 0xc8, 0xca, 0x93, 0xa0, 0x67, 0x82, + 0x63, 0x5c, 0xd0, 0xed, 0xe7, 0x08, 0xc5, 0x60, + 0xf8, 0xb4, 0x47, 0xf0, 0x1a, 0x65, 0x4e, 0xa3, + 0x51, 0x68, 0xc7, 0x14, 0xa1, 0xd9, 0x39, 0x72, + 0xa8, 0x6f, 0x7c, 0x7e, 0xf6, 0x03, 0x0b, 0x25, + 0x9b, 0xf2, 0xca, 0x49, 0xae, 0x5b, 0xf8, 0x0f, + 0x71, 0x51, 0x01, 0xa6, 0x23, 0xa9, 0xdf, 0xd0, + 0x7a, 0x39, 0x19, 0xf5, 0xc5, 0x26, 0x44, 0x7b, + 0x0a, 0x4a, 0x41, 0xbf, 0xf2, 0x8e, 0x83, 0x50, + 0x91, 0x96, 0x72, 0x02, 0xf6, 0x80, 0xbf, 0x95, + 0x41, 0xac, 0xda, 0xb0, 0xba, 0xe3, 0x76, 0xb1, + 0x9d, 0xff, 0x1f, 0x33, 0x02, 0x85, 0xfc, 0x2a, + 0x29, 0xe6, 0xe3, 0x9d, 0xd0, 0xef, 0xc2, 0xd6, + 0x9c, 0x4a, 0x62, 0xac, 0xcb, 0xea, 0x8b, 0xc3, + 0x08, 0x6e, 0x49, 0x09, 0x26, 0x19, 0xc1, 0x30, + 0xcc, 0x27, 0xaa, 0xc6, 0x45, 0x88, 0xbd, 0xae, + 0xd6, 0x79, 0xff, 0x4e, 0xfc, 0x66, 0x4d, 0x02, + 0xa5, 0xee, 0x8e, 0xa5, 0xb6, 0x15, 0x72, 0x24, + 0xb1, 0xbf, 0xbf, 0x64, 0xcf, 0xcc, 0x93, 0xe9, + 0xb6, 0xfd, 0xb4, 0xb6, 0x21, 0xb5, 0x48, 0x08, + 0x0f, 0x11, 0x65, 0xe1, 0x47, 0xee, 0x93, 0x29, + 0xad +}; +static const u8 key72[] __initconst = { + 0xb9, 0xa2, 0xfc, 0x59, 0x06, 0x3f, 0x77, 0xa5, + 0x66, 0xd0, 0x2b, 0x22, 0x74, 0x22, 0x4c, 0x1e, + 0x6a, 0x39, 0xdf, 0xe1, 0x0d, 0x4c, 0x64, 0x99, + 0x54, 0x8a, 0xba, 0x1d, 0x2c, 0x21, 0x5f, 0xc3 +}; +enum { nonce72 = 0x3d069308fa3db04bULL }; + +static const u8 input73[] __initconst = { + 0xe4, 0xdd, 0x36, 0xd4, 0xf5, 0x70, 0x51, 0x73, + 0x97, 0x1d, 0x45, 0x05, 0x92, 0xe7, 0xeb, 0xb7, + 0x09, 0x82, 0x6e, 0x25, 0x6c, 0x50, 0xf5, 0x40, + 0x19, 0xba, 0xbc, 0xf4, 0x39, 0x14, 0xc5, 0x15, + 0x83, 0x40, 0xbd, 0x26, 0xe0, 0xff, 0x3b, 0x22, + 0x7c, 0x7c, 0xd7, 0x0b, 0xe9, 0x25, 0x0c, 0x3d, + 0x92, 0x38, 0xbe, 0xe4, 0x22, 0x75, 0x65, 0xf1, + 0x03, 0x85, 0x34, 0x09, 0xb8, 0x77, 0xfb, 0x48, + 0xb1, 0x2e, 0x21, 0x67, 0x9b, 0x9d, 0xad, 0x18, + 0x82, 0x0d, 0x6b, 0xc3, 0xcf, 0x00, 0x61, 0x6e, + 0xda, 0xdc, 0xa7, 0x0b, 0x5c, 0x02, 0x1d, 0xa6, + 0x4e, 0x0d, 0x7f, 0x37, 0x01, 0x5a, 0x37, 0xf3, + 0x2b, 0xbf, 0xba, 0xe2, 0x1c, 0xb3, 0xa3, 0xbc, + 0x1c, 0x93, 0x1a, 0xb1, 0x71, 0xaf, 0xe2, 0xdd, + 0x17, 0xee, 0x53, 0xfa, 0xfb, 0x02, 0x40, 0x3e, + 0x03, 0xca, 0xe7, 0xc3, 0x51, 0x81, 0xcc, 0x8c, + 0xca, 0xcf, 0x4e, 0xc5, 0x78, 0x99, 0xfd, 0xbf, + 0xea, 0xab, 0x38, 0x81, 0xfc, 0xd1, 0x9e, 0x41, + 0x0b, 0x84, 0x25, 0xf1, 0x6b, 0x3c, 0xf5, 0x40, + 0x0d, 0xc4, 0x3e, 0xb3, 0x6a, 0xec, 0x6e, 0x75, + 0xdc, 0x9b, 0xdf, 0x08, 0x21, 0x16, 0xfb, 0x7a, + 0x8e, 0x19, 0x13, 0x02, 0xa7, 0xfc, 0x58, 0x21, + 0xc3, 0xb3, 0x59, 0x5a, 0x9c, 0xef, 0x38, 0xbd, + 0x87, 0x55, 0xd7, 0x0d, 0x1f, 0x84, 0xdc, 0x98, + 0x22, 0xca, 0x87, 0x96, 0x71, 0x6d, 0x68, 0x00, + 0xcb, 0x4f, 0x2f, 0xc4, 0x64, 0x0c, 0xc1, 0x53, + 0x0c, 0x90, 0xe7, 0x3c, 0x88, 0xca, 0xc5, 0x85, + 0xa3, 0x2a, 0x96, 0x7c, 0x82, 0x6d, 0x45, 0xf5, + 0xb7, 0x8d, 0x17, 0x69, 0xd6, 0xcd, 0x3c, 0xd3, + 0xe7, 0x1c, 0xce, 0x93, 0x50, 0xd4, 0x59, 0xa2, + 0xd8, 0x8b, 0x72, 0x60, 0x5b, 0x25, 0x14, 0xcd, + 0x5a, 0xe8, 0x8c, 0xdb, 0x23, 0x8d, 0x2b, 0x59, + 0x12, 0x13, 0x10, 0x47, 0xa4, 0xc8, 0x3c, 0xc1, + 0x81, 0x89, 0x6c, 0x98, 0xec, 0x8f, 0x7b, 0x32, + 0xf2, 0x87, 0xd9, 0xa2, 0x0d, 0xc2, 0x08, 0xf9, + 0xd5, 0xf3, 0x91, 0xe7, 0xb3, 0x87, 0xa7, 0x0b, + 0x64, 0x8f, 0xb9, 0x55, 0x1c, 0x81, 0x96, 0x6c, + 0xa1, 0xc9, 0x6e, 0x3b, 0xcd, 0x17, 0x1b, 0xfc, + 0xa6, 0x05, 0xba, 0x4a, 0x7d, 0x03, 0x3c, 0x59, + 0xc8, 0xee, 0x50, 0xb2, 0x5b, 0xe1, 0x4d, 0x6a, + 0x1f, 0x09, 0xdc, 0xa2, 0x51, 0xd1, 0x93, 0x3a, + 0x5f, 0x72, 0x1d, 0x26, 0x14, 0x62, 0xa2, 0x41, + 0x3d, 0x08, 0x70, 0x7b, 0x27, 0x3d, 0xbc, 0xdf, + 0x15, 0xfa, 0xb9, 0x5f, 0xb5, 0x38, 0x84, 0x0b, + 0x58, 0x3d, 0xee, 0x3f, 0x32, 0x65, 0x6d, 0xd7, + 0xce, 0x97, 0x3c, 0x8d, 0xfb, 0x63, 0xb9, 0xb0, + 0xa8, 0x4a, 0x72, 0x99, 0x97, 0x58, 0xc8, 0xa7, + 0xf9, 0x4c, 0xae, 0xc1, 0x63, 0xb9, 0x57, 0x18, + 0x8a, 0xfa, 0xab, 0xe9, 0xf3, 0x67, 0xe6, 0xfd, + 0xd2, 0x9d, 0x5c, 0xa9, 0x8e, 0x11, 0x0a, 0xf4, + 0x4b, 0xf1, 0xec, 0x1a, 0xaf, 0x50, 0x5d, 0x16, + 0x13, 0x69, 0x2e, 0xbd, 0x0d, 0xe6, 0xf0, 0xb2, + 0xed, 0xb4, 0x4c, 0x59, 0x77, 0x37, 0x00, 0x0b, + 0xc7, 0xa7, 0x9e, 0x37, 0xf3, 0x60, 0x70, 0xef, + 0xf3, 0xc1, 0x74, 0x52, 0x87, 0xc6, 0xa1, 0x81, + 0xbd, 0x0a, 0x2c, 0x5d, 0x2c, 0x0c, 0x6a, 0x81, + 0xa1, 0xfe, 0x26, 0x78, 0x6c, 0x03, 0x06, 0x07, + 0x34, 0xaa, 0xd1, 0x1b, 0x40, 0x03, 0x39, 0x56, + 0xcf, 0x2a, 0x92, 0xc1, 0x4e, 0xdf, 0x29, 0x24, + 0x83, 0x22, 0x7a, 0xea, 0x67, 0x1e, 0xe7, 0x54, + 0x64, 0xd3, 0xbd, 0x3a, 0x5d, 0xae, 0xca, 0xf0, + 0x9c, 0xd6, 0x5a, 0x9a, 0x62, 0xc8, 0xc7, 0x83, + 0xf9, 0x89, 0xde, 0x2d, 0x53, 0x64, 0x61, 0xf7, + 0xa3, 0xa7, 0x31, 0x38, 0xc6, 0x22, 0x9c, 0xb4, + 0x87, 0xe0 +}; +static const u8 output73[] __initconst = { + 0x34, 0xed, 0x05, 0xb0, 0x14, 0xbc, 0x8c, 0xcc, + 0x95, 0xbd, 0x99, 0x0f, 0xb1, 0x98, 0x17, 0x10, + 0xae, 0xe0, 0x08, 0x53, 0xa3, 0x69, 0xd2, 0xed, + 0x66, 0xdb, 0x2a, 0x34, 0x8d, 0x0c, 0x6e, 0xce, + 0x63, 0x69, 0xc9, 0xe4, 0x57, 0xc3, 0x0c, 0x8b, + 0xa6, 0x2c, 0xa7, 0xd2, 0x08, 0xff, 0x4f, 0xec, + 0x61, 0x8c, 0xee, 0x0d, 0xfa, 0x6b, 0xe0, 0xe8, + 0x71, 0xbc, 0x41, 0x46, 0xd7, 0x33, 0x1d, 0xc0, + 0xfd, 0xad, 0xca, 0x8b, 0x34, 0x56, 0xa4, 0x86, + 0x71, 0x62, 0xae, 0x5e, 0x3d, 0x2b, 0x66, 0x3e, + 0xae, 0xd8, 0xc0, 0xe1, 0x21, 0x3b, 0xca, 0xd2, + 0x6b, 0xa2, 0xb8, 0xc7, 0x98, 0x4a, 0xf3, 0xcf, + 0xb8, 0x62, 0xd8, 0x33, 0xe6, 0x80, 0xdb, 0x2f, + 0x0a, 0xaf, 0x90, 0x3c, 0xe1, 0xec, 0xe9, 0x21, + 0x29, 0x42, 0x9e, 0xa5, 0x50, 0xe9, 0x93, 0xd3, + 0x53, 0x1f, 0xac, 0x2a, 0x24, 0x07, 0xb8, 0xed, + 0xed, 0x38, 0x2c, 0xc4, 0xa1, 0x2b, 0x31, 0x5d, + 0x9c, 0x24, 0x7b, 0xbf, 0xd9, 0xbb, 0x4e, 0x87, + 0x8f, 0x32, 0x30, 0xf1, 0x11, 0x29, 0x54, 0x94, + 0x00, 0x95, 0x1d, 0x1d, 0x24, 0xc0, 0xd4, 0x34, + 0x49, 0x1d, 0xd5, 0xe3, 0xa6, 0xde, 0x8b, 0xbf, + 0x5a, 0x9f, 0x58, 0x5a, 0x9b, 0x70, 0xe5, 0x9b, + 0xb3, 0xdb, 0xe8, 0xb8, 0xca, 0x1b, 0x43, 0xe3, + 0xc6, 0x6f, 0x0a, 0xd6, 0x32, 0x11, 0xd4, 0x04, + 0xef, 0xa3, 0xe4, 0x3f, 0x12, 0xd8, 0xc1, 0x73, + 0x51, 0x87, 0x03, 0xbd, 0xba, 0x60, 0x79, 0xee, + 0x08, 0xcc, 0xf7, 0xc0, 0xaa, 0x4c, 0x33, 0xc4, + 0xc7, 0x09, 0xf5, 0x91, 0xcb, 0x74, 0x57, 0x08, + 0x1b, 0x90, 0xa9, 0x1b, 0x60, 0x02, 0xd2, 0x3f, + 0x7a, 0xbb, 0xfd, 0x78, 0xf0, 0x15, 0xf9, 0x29, + 0x82, 0x8f, 0xc4, 0xb2, 0x88, 0x1f, 0xbc, 0xcc, + 0x53, 0x27, 0x8b, 0x07, 0x5f, 0xfc, 0x91, 0x29, + 0x82, 0x80, 0x59, 0x0a, 0x3c, 0xea, 0xc4, 0x7e, + 0xad, 0xd2, 0x70, 0x46, 0xbd, 0x9e, 0x3b, 0x1c, + 0x8a, 0x62, 0xea, 0x69, 0xbd, 0xf6, 0x96, 0x15, + 0xb5, 0x57, 0xe8, 0x63, 0x5f, 0x65, 0x46, 0x84, + 0x58, 0x50, 0x87, 0x4b, 0x0e, 0x5b, 0x52, 0x90, + 0xb0, 0xae, 0x37, 0x0f, 0xdd, 0x7e, 0xa2, 0xa0, + 0x8b, 0x78, 0xc8, 0x5a, 0x1f, 0x53, 0xdb, 0xc5, + 0xbf, 0x73, 0x20, 0xa9, 0x44, 0xfb, 0x1e, 0xc7, + 0x97, 0xb2, 0x3a, 0x5a, 0x17, 0xe6, 0x8b, 0x9b, + 0xe8, 0xf8, 0x2a, 0x01, 0x27, 0xa3, 0x71, 0x28, + 0xe3, 0x19, 0xc6, 0xaf, 0xf5, 0x3a, 0x26, 0xc0, + 0x5c, 0x69, 0x30, 0x78, 0x75, 0x27, 0xf2, 0x0c, + 0x22, 0x71, 0x65, 0xc6, 0x8e, 0x7b, 0x47, 0xe3, + 0x31, 0xaf, 0x7b, 0xc6, 0xc2, 0x55, 0x68, 0x81, + 0xaa, 0x1b, 0x21, 0x65, 0xfb, 0x18, 0x35, 0x45, + 0x36, 0x9a, 0x44, 0xba, 0x5c, 0xff, 0x06, 0xde, + 0x3a, 0xc8, 0x44, 0x0b, 0xaa, 0x8e, 0x34, 0xe2, + 0x84, 0xac, 0x18, 0xfe, 0x9b, 0xe1, 0x4f, 0xaa, + 0xb6, 0x90, 0x0b, 0x1c, 0x2c, 0xd9, 0x9a, 0x10, + 0x18, 0xf9, 0x49, 0x41, 0x42, 0x1b, 0xb5, 0xe1, + 0x26, 0xac, 0x2d, 0x38, 0x00, 0x00, 0xe4, 0xb4, + 0x50, 0x6f, 0x14, 0x18, 0xd6, 0x3d, 0x00, 0x59, + 0x3c, 0x45, 0xf3, 0x42, 0x13, 0x44, 0xb8, 0x57, + 0xd4, 0x43, 0x5c, 0x8a, 0x2a, 0xb4, 0xfc, 0x0a, + 0x25, 0x5a, 0xdc, 0x8f, 0x11, 0x0b, 0x11, 0x44, + 0xc7, 0x0e, 0x54, 0x8b, 0x22, 0x01, 0x7e, 0x67, + 0x2e, 0x15, 0x3a, 0xb9, 0xee, 0x84, 0x10, 0xd4, + 0x80, 0x57, 0xd7, 0x75, 0xcf, 0x8b, 0xcb, 0x03, + 0xc9, 0x92, 0x2b, 0x69, 0xd8, 0x5a, 0x9b, 0x06, + 0x85, 0x47, 0xaa, 0x4c, 0x28, 0xde, 0x49, 0x58, + 0xe6, 0x11, 0x1e, 0x5e, 0x64, 0x8e, 0x3b, 0xe0, + 0x40, 0x2e, 0xac, 0x96, 0x97, 0x15, 0x37, 0x1e, + 0x30, 0xdd +}; +static const u8 key73[] __initconst = { + 0x96, 0x06, 0x1e, 0xc1, 0x6d, 0xba, 0x49, 0x5b, + 0x65, 0x80, 0x79, 0xdd, 0xf3, 0x67, 0xa8, 0x6e, + 0x2d, 0x9c, 0x54, 0x46, 0xd8, 0x4a, 0xeb, 0x7e, + 0x23, 0x86, 0x51, 0xd8, 0x49, 0x49, 0x56, 0xe0 +}; +enum { nonce73 = 0xbefb83cb67e11ffdULL }; + +static const u8 input74[] __initconst = { + 0x47, 0x22, 0x70, 0xe5, 0x2f, 0x41, 0x18, 0x45, + 0x07, 0xd3, 0x6d, 0x32, 0x0d, 0x43, 0x92, 0x2b, + 0x9b, 0x65, 0x73, 0x13, 0x1a, 0x4f, 0x49, 0x8f, + 0xff, 0xf8, 0xcc, 0xae, 0x15, 0xab, 0x9d, 0x7d, + 0xee, 0x22, 0x5d, 0x8b, 0xde, 0x81, 0x5b, 0x81, + 0x83, 0x49, 0x35, 0x9b, 0xb4, 0xbc, 0x4e, 0x01, + 0xc2, 0x29, 0xa7, 0xf1, 0xca, 0x3a, 0xce, 0x3f, + 0xf5, 0x31, 0x93, 0xa8, 0xe2, 0xc9, 0x7d, 0x03, + 0x26, 0xa4, 0xbc, 0xa8, 0x9c, 0xb9, 0x68, 0xf3, + 0xb3, 0x91, 0xe8, 0xe6, 0xc7, 0x2b, 0x1a, 0xce, + 0xd2, 0x41, 0x53, 0xbd, 0xa3, 0x2c, 0x54, 0x94, + 0x21, 0xa1, 0x40, 0xae, 0xc9, 0x0c, 0x11, 0x92, + 0xfd, 0x91, 0xa9, 0x40, 0xca, 0xde, 0x21, 0x4e, + 0x1e, 0x3d, 0xcc, 0x2c, 0x87, 0x11, 0xef, 0x46, + 0xed, 0x52, 0x03, 0x11, 0x19, 0x43, 0x25, 0xc7, + 0x0d, 0xc3, 0x37, 0x5f, 0xd3, 0x6f, 0x0c, 0x6a, + 0x45, 0x30, 0x88, 0xec, 0xf0, 0x21, 0xef, 0x1d, + 0x7b, 0x38, 0x63, 0x4b, 0x49, 0x0c, 0x72, 0xf6, + 0x4c, 0x40, 0xc3, 0xcc, 0x03, 0xa7, 0xae, 0xa8, + 0x8c, 0x37, 0x03, 0x1c, 0x11, 0xae, 0x0d, 0x1b, + 0x62, 0x97, 0x27, 0xfc, 0x56, 0x4b, 0xb7, 0xfd, + 0xbc, 0xfb, 0x0e, 0xfc, 0x61, 0xad, 0xc6, 0xb5, + 0x9c, 0x8c, 0xc6, 0x38, 0x27, 0x91, 0x29, 0x3d, + 0x29, 0xc8, 0x37, 0xc9, 0x96, 0x69, 0xe3, 0xdc, + 0x3e, 0x61, 0x35, 0x9b, 0x99, 0x4f, 0xb9, 0x4e, + 0x5a, 0x29, 0x1c, 0x2e, 0xcf, 0x16, 0xcb, 0x69, + 0x87, 0xe4, 0x1a, 0xc4, 0x6e, 0x78, 0x43, 0x00, + 0x03, 0xb2, 0x8b, 0x03, 0xd0, 0xb4, 0xf1, 0xd2, + 0x7d, 0x2d, 0x7e, 0xfc, 0x19, 0x66, 0x5b, 0xa3, + 0x60, 0x3f, 0x9d, 0xbd, 0xfa, 0x3e, 0xca, 0x7b, + 0x26, 0x08, 0x19, 0x16, 0x93, 0x5d, 0x83, 0xfd, + 0xf9, 0x21, 0xc6, 0x31, 0x34, 0x6f, 0x0c, 0xaa, + 0x28, 0xf9, 0x18, 0xa2, 0xc4, 0x78, 0x3b, 0x56, + 0xc0, 0x88, 0x16, 0xba, 0x22, 0x2c, 0x07, 0x2f, + 0x70, 0xd0, 0xb0, 0x46, 0x35, 0xc7, 0x14, 0xdc, + 0xbb, 0x56, 0x23, 0x1e, 0x36, 0x36, 0x2d, 0x73, + 0x78, 0xc7, 0xce, 0xf3, 0x58, 0xf7, 0x58, 0xb5, + 0x51, 0xff, 0x33, 0x86, 0x0e, 0x3b, 0x39, 0xfb, + 0x1a, 0xfd, 0xf8, 0x8b, 0x09, 0x33, 0x1b, 0x83, + 0xf2, 0xe6, 0x38, 0x37, 0xef, 0x47, 0x84, 0xd9, + 0x82, 0x77, 0x2b, 0x82, 0xcc, 0xf9, 0xee, 0x94, + 0x71, 0x78, 0x81, 0xc8, 0x4d, 0x91, 0xd7, 0x35, + 0x29, 0x31, 0x30, 0x5c, 0x4a, 0x23, 0x23, 0xb1, + 0x38, 0x6b, 0xac, 0x22, 0x3f, 0x80, 0xc7, 0xe0, + 0x7d, 0xfa, 0x76, 0x47, 0xd4, 0x6f, 0x93, 0xa0, + 0xa0, 0x93, 0x5d, 0x68, 0xf7, 0x43, 0x25, 0x8f, + 0x1b, 0xc7, 0x87, 0xea, 0x59, 0x0c, 0xa2, 0xfa, + 0xdb, 0x2f, 0x72, 0x43, 0xcf, 0x90, 0xf1, 0xd6, + 0x58, 0xf3, 0x17, 0x6a, 0xdf, 0xb3, 0x4e, 0x0e, + 0x38, 0x24, 0x48, 0x1f, 0xb7, 0x01, 0xec, 0x81, + 0xb1, 0x87, 0x5b, 0xec, 0x9c, 0x11, 0x1a, 0xff, + 0xa5, 0xca, 0x5a, 0x63, 0x31, 0xb2, 0xe4, 0xc6, + 0x3c, 0x1d, 0xaf, 0x27, 0xb2, 0xd4, 0x19, 0xa2, + 0xcc, 0x04, 0x92, 0x42, 0xd2, 0xc1, 0x8c, 0x3b, + 0xce, 0xf5, 0x74, 0xc1, 0x81, 0xf8, 0x20, 0x23, + 0x6f, 0x20, 0x6d, 0x78, 0x36, 0x72, 0x2c, 0x52, + 0xdf, 0x5e, 0xe8, 0x75, 0xce, 0x1c, 0x49, 0x9d, + 0x93, 0x6f, 0x65, 0xeb, 0xb1, 0xbd, 0x8e, 0x5e, + 0xe5, 0x89, 0xc4, 0x8a, 0x81, 0x3d, 0x9a, 0xa7, + 0x11, 0x82, 0x8e, 0x38, 0x5b, 0x5b, 0xca, 0x7d, + 0x4b, 0x72, 0xc2, 0x9c, 0x30, 0x5e, 0x7f, 0xc0, + 0x6f, 0x91, 0xd5, 0x67, 0x8c, 0x3e, 0xae, 0xda, + 0x2b, 0x3c, 0x53, 0xcc, 0x50, 0x97, 0x36, 0x0b, + 0x79, 0xd6, 0x73, 0x6e, 0x7d, 0x42, 0x56, 0xe1, + 0xaa, 0xfc, 0xb3, 0xa7, 0xc8, 0x01, 0xaa, 0xc1, + 0xfc, 0x5c, 0x72, 0x8e, 0x63, 0xa8, 0x46, 0x18, + 0xee, 0x11, 0xe7, 0x30, 0x09, 0x83, 0x6c, 0xd9, + 0xf4, 0x7a, 0x7b, 0xb5, 0x1f, 0x6d, 0xc7, 0xbc, + 0xcb, 0x55, 0xea, 0x40, 0x58, 0x7a, 0x00, 0x00, + 0x90, 0x60, 0xc5, 0x64, 0x69, 0x05, 0x99, 0xd2, + 0x49, 0x62, 0x4f, 0xcb, 0x97, 0xdf, 0xdd, 0x6b, + 0x60, 0x75, 0xe2, 0xe0, 0x6f, 0x76, 0xd0, 0x37, + 0x67, 0x0a, 0xcf, 0xff, 0xc8, 0x61, 0x84, 0x14, + 0x80, 0x7c, 0x1d, 0x31, 0x8d, 0x90, 0xde, 0x0b, + 0x1c, 0x74, 0x9f, 0x82, 0x96, 0x80, 0xda, 0xaf, + 0x8d, 0x99, 0x86, 0x9f, 0x24, 0x99, 0x28, 0x3e, + 0xe0, 0xa3, 0xc3, 0x90, 0x2d, 0x14, 0x65, 0x1e, + 0x3b, 0xb9, 0xba, 0x13, 0xa5, 0x77, 0x73, 0x63, + 0x9a, 0x06, 0x3d, 0xa9, 0x28, 0x9b, 0xba, 0x25, + 0x61, 0xc9, 0xcd, 0xcf, 0x7a, 0x4d, 0x96, 0x09, + 0xcb, 0xca, 0x03, 0x9c, 0x54, 0x34, 0x31, 0x85, + 0xa0, 0x3d, 0xe5, 0xbc, 0xa5, 0x5f, 0x1b, 0xd3, + 0x10, 0x63, 0x74, 0x9d, 0x01, 0x92, 0x88, 0xf0, + 0x27, 0x9c, 0x28, 0xd9, 0xfd, 0xe2, 0x4e, 0x01, + 0x8d, 0x61, 0x79, 0x60, 0x61, 0x5b, 0x76, 0xab, + 0x06, 0xd3, 0x44, 0x87, 0x43, 0x52, 0xcd, 0x06, + 0x68, 0x1e, 0x2d, 0xc5, 0xb0, 0x07, 0x25, 0xdf, + 0x0a, 0x50, 0xd7, 0xd9, 0x08, 0x53, 0x65, 0xf1, + 0x0c, 0x2c, 0xde, 0x3f, 0x9d, 0x03, 0x1f, 0xe1, + 0x49, 0x43, 0x3c, 0x83, 0x81, 0x37, 0xf8, 0xa2, + 0x0b, 0xf9, 0x61, 0x1c, 0xc1, 0xdb, 0x79, 0xbc, + 0x64, 0xce, 0x06, 0x4e, 0x87, 0x89, 0x62, 0x73, + 0x51, 0xbc, 0xa4, 0x32, 0xd4, 0x18, 0x62, 0xab, + 0x65, 0x7e, 0xad, 0x1e, 0x91, 0xa3, 0xfa, 0x2d, + 0x58, 0x9e, 0x2a, 0xe9, 0x74, 0x44, 0x64, 0x11, + 0xe6, 0xb6, 0xb3, 0x00, 0x7e, 0xa3, 0x16, 0xef, + 0x72 +}; +static const u8 output74[] __initconst = { + 0xf5, 0xca, 0x45, 0x65, 0x50, 0x35, 0x47, 0x67, + 0x6f, 0x4f, 0x67, 0xff, 0x34, 0xd9, 0xc3, 0x37, + 0x2a, 0x26, 0xb0, 0x4f, 0x08, 0x1e, 0x45, 0x13, + 0xc7, 0x2c, 0x14, 0x75, 0x33, 0xd8, 0x8e, 0x1e, + 0x1b, 0x11, 0x0d, 0x97, 0x04, 0x33, 0x8a, 0xe4, + 0xd8, 0x8d, 0x0e, 0x12, 0x8d, 0xdb, 0x6e, 0x02, + 0xfa, 0xe5, 0xbd, 0x3a, 0xb5, 0x28, 0x07, 0x7d, + 0x20, 0xf0, 0x12, 0x64, 0x83, 0x2f, 0x59, 0x79, + 0x17, 0x88, 0x3c, 0x2d, 0x08, 0x2f, 0x55, 0xda, + 0xcc, 0x02, 0x3a, 0x82, 0xcd, 0x03, 0x94, 0xdf, + 0xdf, 0xab, 0x8a, 0x13, 0xf5, 0xe6, 0x74, 0xdf, + 0x7b, 0xe2, 0xab, 0x34, 0xbc, 0x00, 0x85, 0xbf, + 0x5a, 0x48, 0xc8, 0xff, 0x8d, 0x6c, 0x27, 0x48, + 0x19, 0x2d, 0x08, 0xfa, 0x82, 0x62, 0x39, 0x55, + 0x32, 0x11, 0xa8, 0xd7, 0xb9, 0x08, 0x2c, 0xd6, + 0x7a, 0xd9, 0x83, 0x9f, 0x9b, 0xfb, 0xec, 0x3a, + 0xd1, 0x08, 0xc7, 0xad, 0xdc, 0x98, 0x4c, 0xbc, + 0x98, 0xeb, 0x36, 0xb0, 0x39, 0xf4, 0x3a, 0xd6, + 0x53, 0x02, 0xa0, 0xa9, 0x73, 0xa1, 0xca, 0xef, + 0xd8, 0xd2, 0xec, 0x0e, 0xf8, 0xf5, 0xac, 0x8d, + 0x34, 0x41, 0x06, 0xa8, 0xc6, 0xc3, 0x31, 0xbc, + 0xe5, 0xcc, 0x7e, 0x72, 0x63, 0x59, 0x3e, 0x63, + 0xc2, 0x8d, 0x2b, 0xd5, 0xb9, 0xfd, 0x1e, 0x31, + 0x69, 0x32, 0x05, 0xd6, 0xde, 0xc9, 0xe6, 0x4c, + 0xac, 0x68, 0xf7, 0x1f, 0x9d, 0xcd, 0x0e, 0xa2, + 0x15, 0x3d, 0xd6, 0x47, 0x99, 0xab, 0x08, 0x5f, + 0x28, 0xc3, 0x4c, 0xc2, 0xd5, 0xdd, 0x10, 0xb7, + 0xbd, 0xdb, 0x9b, 0xcf, 0x85, 0x27, 0x29, 0x76, + 0x98, 0xeb, 0xad, 0x31, 0x64, 0xe7, 0xfb, 0x61, + 0xe0, 0xd8, 0x1a, 0xa6, 0xe2, 0xe7, 0x43, 0x42, + 0x77, 0xc9, 0x82, 0x00, 0xac, 0x85, 0xe0, 0xa2, + 0xd4, 0x62, 0xe3, 0xb7, 0x17, 0x6e, 0xb2, 0x9e, + 0x21, 0x58, 0x73, 0xa9, 0x53, 0x2d, 0x3c, 0xe1, + 0xdd, 0xd6, 0x6e, 0x92, 0xf2, 0x1d, 0xc2, 0x22, + 0x5f, 0x9a, 0x7e, 0xd0, 0x52, 0xbf, 0x54, 0x19, + 0xd7, 0x80, 0x63, 0x3e, 0xd0, 0x08, 0x2d, 0x37, + 0x0c, 0x15, 0xf7, 0xde, 0xab, 0x2b, 0xe3, 0x16, + 0x21, 0x3a, 0xee, 0xa5, 0xdc, 0xdf, 0xde, 0xa3, + 0x69, 0xcb, 0xfd, 0x92, 0x89, 0x75, 0xcf, 0xc9, + 0x8a, 0xa4, 0xc8, 0xdd, 0xcc, 0x21, 0xe6, 0xfe, + 0x9e, 0x43, 0x76, 0xb2, 0x45, 0x22, 0xb9, 0xb5, + 0xac, 0x7e, 0x3d, 0x26, 0xb0, 0x53, 0xc8, 0xab, + 0xfd, 0xea, 0x2c, 0xd1, 0x44, 0xc5, 0x60, 0x1b, + 0x8a, 0x99, 0x0d, 0xa5, 0x0e, 0x67, 0x6e, 0x3a, + 0x96, 0x55, 0xec, 0xe8, 0xcc, 0xbe, 0x49, 0xd9, + 0xf2, 0x72, 0x9f, 0x30, 0x21, 0x97, 0x57, 0x19, + 0xbe, 0x5e, 0x33, 0x0c, 0xee, 0xc0, 0x72, 0x0d, + 0x2e, 0xd1, 0xe1, 0x52, 0xc2, 0xea, 0x41, 0xbb, + 0xe1, 0x6d, 0xd4, 0x17, 0xa9, 0x8d, 0x89, 0xa9, + 0xd6, 0x4b, 0xc6, 0x4c, 0xf2, 0x88, 0x97, 0x54, + 0x3f, 0x4f, 0x57, 0xb7, 0x37, 0xf0, 0x2c, 0x11, + 0x15, 0x56, 0xdb, 0x28, 0xb5, 0x16, 0x84, 0x66, + 0xce, 0x45, 0x3f, 0x61, 0x75, 0xb6, 0xbe, 0x00, + 0xd1, 0xe4, 0xf5, 0x27, 0x54, 0x7f, 0xc2, 0xf1, + 0xb3, 0x32, 0x9a, 0xe8, 0x07, 0x02, 0xf3, 0xdb, + 0xa9, 0xd1, 0xc2, 0xdf, 0xee, 0xad, 0xe5, 0x8a, + 0x3c, 0xfa, 0x67, 0xec, 0x6b, 0xa4, 0x08, 0xfe, + 0xba, 0x5a, 0x58, 0x0b, 0x78, 0x11, 0x91, 0x76, + 0xe3, 0x1a, 0x28, 0x54, 0x5e, 0xbd, 0x71, 0x1b, + 0x8b, 0xdc, 0x6c, 0xf4, 0x6f, 0xd7, 0xf4, 0xf3, + 0xe1, 0x03, 0xa4, 0x3c, 0x8d, 0x91, 0x2e, 0xba, + 0x5f, 0x7f, 0x8c, 0xaf, 0x69, 0x89, 0x29, 0x0a, + 0x5b, 0x25, 0x13, 0xc4, 0x2e, 0x16, 0xc2, 0x15, + 0x07, 0x5d, 0x58, 0x33, 0x7c, 0xe0, 0xf0, 0x55, + 0x5f, 0xbf, 0x5e, 0xf0, 0x71, 0x48, 0x8f, 0xf7, + 0x48, 0xb3, 0xf7, 0x0d, 0xa1, 0xd0, 0x63, 0xb1, + 0xad, 0xae, 0xb5, 0xb0, 0x5f, 0x71, 0xaf, 0x24, + 0x8b, 0xb9, 0x1c, 0x44, 0xd2, 0x1a, 0x53, 0xd1, + 0xd5, 0xb4, 0xa9, 0xff, 0x88, 0x73, 0xb5, 0xaa, + 0x15, 0x32, 0x5f, 0x59, 0x9d, 0x2e, 0xb5, 0xcb, + 0xde, 0x21, 0x2e, 0xe9, 0x35, 0xed, 0xfd, 0x0f, + 0xb6, 0xbb, 0xe6, 0x4b, 0x16, 0xf1, 0x45, 0x1e, + 0xb4, 0x84, 0xe9, 0x58, 0x1c, 0x0c, 0x95, 0xc0, + 0xcf, 0x49, 0x8b, 0x59, 0xa1, 0x78, 0xe6, 0x80, + 0x12, 0x49, 0x7a, 0xd4, 0x66, 0x62, 0xdf, 0x9c, + 0x18, 0xc8, 0x8c, 0xda, 0xc1, 0xa6, 0xbc, 0x65, + 0x28, 0xd2, 0xa4, 0xe8, 0xf1, 0x35, 0xdb, 0x5a, + 0x75, 0x1f, 0x73, 0x60, 0xec, 0xa8, 0xda, 0x5a, + 0x43, 0x15, 0x83, 0x9b, 0xe7, 0xb1, 0xa6, 0x81, + 0xbb, 0xef, 0xf3, 0x8f, 0x0f, 0xd3, 0x79, 0xa2, + 0xe5, 0xaa, 0x42, 0xef, 0xa0, 0x13, 0x4e, 0x91, + 0x2d, 0xcb, 0x61, 0x7a, 0x9a, 0x33, 0x14, 0x50, + 0x77, 0x4a, 0xd0, 0x91, 0x48, 0xe0, 0x0c, 0xe0, + 0x11, 0xcb, 0xdf, 0xb0, 0xce, 0x06, 0xd2, 0x79, + 0x4d, 0x69, 0xb9, 0xc9, 0x36, 0x74, 0x8f, 0x81, + 0x72, 0x73, 0xf3, 0x17, 0xb7, 0x13, 0xcb, 0x5b, + 0xd2, 0x5c, 0x33, 0x61, 0xb7, 0x61, 0x79, 0xb0, + 0xc0, 0x4d, 0xa1, 0xc7, 0x5d, 0x98, 0xc9, 0xe1, + 0x98, 0xbd, 0x78, 0x5a, 0x2c, 0x64, 0x53, 0xaf, + 0xaf, 0x66, 0x51, 0x47, 0xe4, 0x48, 0x66, 0x8b, + 0x07, 0x52, 0xa3, 0x03, 0x93, 0x28, 0xad, 0xcc, + 0xa3, 0x86, 0xad, 0x63, 0x04, 0x35, 0x6c, 0x49, + 0xd5, 0x28, 0x0e, 0x00, 0x47, 0xf4, 0xd4, 0x32, + 0x27, 0x19, 0xb3, 0x29, 0xe7, 0xbc, 0xbb, 0xce, + 0x3e, 0x3e, 0xd5, 0x67, 0x20, 0xe4, 0x0b, 0x75, + 0x95, 0x24, 0xe0, 0x6c, 0xb6, 0x29, 0x0c, 0x14, + 0xfd +}; +static const u8 key74[] __initconst = { + 0xf0, 0x41, 0x5b, 0x00, 0x56, 0xc4, 0xac, 0xf6, + 0xa2, 0x4c, 0x33, 0x41, 0x16, 0x09, 0x1b, 0x8e, + 0x4d, 0xe8, 0x8c, 0xd9, 0x48, 0xab, 0x3e, 0x60, + 0xcb, 0x49, 0x3e, 0xaf, 0x2b, 0x8b, 0xc8, 0xf0 +}; +enum { nonce74 = 0xcbdb0ffd0e923384ULL }; + +static const struct chacha20_testvec chacha20_testvecs[] __initconst = { + { input01, output01, key01, nonce01, sizeof(input01) }, + { input02, output02, key02, nonce02, sizeof(input02) }, + { input03, output03, key03, nonce03, sizeof(input03) }, + { input04, output04, key04, nonce04, sizeof(input04) }, + { input05, output05, key05, nonce05, sizeof(input05) }, + { input06, output06, key06, nonce06, sizeof(input06) }, + { input07, output07, key07, nonce07, sizeof(input07) }, + { input08, output08, key08, nonce08, sizeof(input08) }, + { input09, output09, key09, nonce09, sizeof(input09) }, + { input10, output10, key10, nonce10, sizeof(input10) }, + { input11, output11, key11, nonce11, sizeof(input11) }, + { input12, output12, key12, nonce12, sizeof(input12) }, + { input13, output13, key13, nonce13, sizeof(input13) }, + { input14, output14, key14, nonce14, sizeof(input14) }, + { input15, output15, key15, nonce15, sizeof(input15) }, + { input16, output16, key16, nonce16, sizeof(input16) }, + { input17, output17, key17, nonce17, sizeof(input17) }, + { input18, output18, key18, nonce18, sizeof(input18) }, + { input19, output19, key19, nonce19, sizeof(input19) }, + { input20, output20, key20, nonce20, sizeof(input20) }, + { input21, output21, key21, nonce21, sizeof(input21) }, + { input22, output22, key22, nonce22, sizeof(input22) }, + { input23, output23, key23, nonce23, sizeof(input23) }, + { input24, output24, key24, nonce24, sizeof(input24) }, + { input25, output25, key25, nonce25, sizeof(input25) }, + { input26, output26, key26, nonce26, sizeof(input26) }, + { input27, output27, key27, nonce27, sizeof(input27) }, + { input28, output28, key28, nonce28, sizeof(input28) }, + { input29, output29, key29, nonce29, sizeof(input29) }, + { input30, output30, key30, nonce30, sizeof(input30) }, + { input31, output31, key31, nonce31, sizeof(input31) }, + { input32, output32, key32, nonce32, sizeof(input32) }, + { input33, output33, key33, nonce33, sizeof(input33) }, + { input34, output34, key34, nonce34, sizeof(input34) }, + { input35, output35, key35, nonce35, sizeof(input35) }, + { input36, output36, key36, nonce36, sizeof(input36) }, + { input37, output37, key37, nonce37, sizeof(input37) }, + { input38, output38, key38, nonce38, sizeof(input38) }, + { input39, output39, key39, nonce39, sizeof(input39) }, + { input40, output40, key40, nonce40, sizeof(input40) }, + { input41, output41, key41, nonce41, sizeof(input41) }, + { input42, output42, key42, nonce42, sizeof(input42) }, + { input43, output43, key43, nonce43, sizeof(input43) }, + { input44, output44, key44, nonce44, sizeof(input44) }, + { input45, output45, key45, nonce45, sizeof(input45) }, + { input46, output46, key46, nonce46, sizeof(input46) }, + { input47, output47, key47, nonce47, sizeof(input47) }, + { input48, output48, key48, nonce48, sizeof(input48) }, + { input49, output49, key49, nonce49, sizeof(input49) }, + { input50, output50, key50, nonce50, sizeof(input50) }, + { input51, output51, key51, nonce51, sizeof(input51) }, + { input52, output52, key52, nonce52, sizeof(input52) }, + { input53, output53, key53, nonce53, sizeof(input53) }, + { input54, output54, key54, nonce54, sizeof(input54) }, + { input55, output55, key55, nonce55, sizeof(input55) }, + { input56, output56, key56, nonce56, sizeof(input56) }, + { input57, output57, key57, nonce57, sizeof(input57) }, + { input58, output58, key58, nonce58, sizeof(input58) }, + { input59, output59, key59, nonce59, sizeof(input59) }, + { input60, output60, key60, nonce60, sizeof(input60) }, + { input61, output61, key61, nonce61, sizeof(input61) }, + { input62, output62, key62, nonce62, sizeof(input62) }, + { input63, output63, key63, nonce63, sizeof(input63) }, + { input64, output64, key64, nonce64, sizeof(input64) }, + { input65, output65, key65, nonce65, sizeof(input65) }, + { input66, output66, key66, nonce66, sizeof(input66) }, + { input67, output67, key67, nonce67, sizeof(input67) }, + { input68, output68, key68, nonce68, sizeof(input68) }, + { input69, output69, key69, nonce69, sizeof(input69) }, + { input70, output70, key70, nonce70, sizeof(input70) }, + { input71, output71, key71, nonce71, sizeof(input71) }, + { input72, output72, key72, nonce72, sizeof(input72) }, + { input73, output73, key73, nonce73, sizeof(input73) }, + { input74, output74, key74, nonce74, sizeof(input74) } +}; + +static const struct hchacha20_testvec hchacha20_testvecs[] __initconst = {{ + .key = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f }, + .nonce = { 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x4a, + 0x00, 0x00, 0x00, 0x00, 0x31, 0x41, 0x59, 0x27 }, + .output = { 0x82, 0x41, 0x3b, 0x42, 0x27, 0xb2, 0x7b, 0xfe, + 0xd3, 0x0e, 0x42, 0x50, 0x8a, 0x87, 0x7d, 0x73, + 0xa0, 0xf9, 0xe4, 0xd5, 0x8a, 0x74, 0xa8, 0x53, + 0xc1, 0x2e, 0xc4, 0x13, 0x26, 0xd3, 0xec, 0xdc } +}}; + +static bool __init chacha20_selftest(void) +{ + enum { + MAXIMUM_TEST_BUFFER_LEN = 1UL << 10, + OUTRAGEOUSLY_HUGE_BUFFER_LEN = PAGE_SIZE * 35 + 17 /* 143k */ + }; + size_t i, j, k; + u32 derived_key[CHACHA20_KEY_WORDS]; + u8 *offset_input = NULL, *computed_output = NULL, *massive_input = NULL; + u8 offset_key[CHACHA20_KEY_SIZE + 1] + __aligned(__alignof__(unsigned long)); + struct chacha20_ctx state; + bool success = true; + simd_context_t simd_context; + + offset_input = kmalloc(MAXIMUM_TEST_BUFFER_LEN + 1, GFP_KERNEL); + computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN + 1, GFP_KERNEL); + massive_input = vzalloc(OUTRAGEOUSLY_HUGE_BUFFER_LEN); + if (!computed_output || !offset_input || !massive_input) { + pr_err("chacha20 self-test malloc: FAIL\n"); + success = false; + goto out; + } + + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20_testvecs); ++i) { + /* Boring case */ + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu: FAIL\n", i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + + /* Unaligned case */ + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + memcpy(offset_input + 1, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen); + memcpy(offset_key + 1, chacha20_testvecs[i].key, + CHACHA20_KEY_SIZE); + chacha20_init(&state, offset_key + 1, chacha20_testvecs[i].nonce); + chacha20(&state, computed_output + 1, offset_input + 1, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output + 1, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (unaligned): FAIL\n", + i + 1); + success = false; + } + if (computed_output[0]) { + pr_err("chacha20 self-test %zu (unaligned, zero check): FAIL\n", + i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen + 1; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + + /* Chunked case */ + if (chacha20_testvecs[i].ilen <= CHACHA20_BLOCK_SIZE) + goto next_test; + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output, chacha20_testvecs[i].input, + CHACHA20_BLOCK_SIZE, &simd_context); + chacha20(&state, computed_output + CHACHA20_BLOCK_SIZE, + chacha20_testvecs[i].input + CHACHA20_BLOCK_SIZE, + chacha20_testvecs[i].ilen - CHACHA20_BLOCK_SIZE, + &simd_context); + if (memcmp(computed_output, chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (chunked): FAIL\n", + i + 1); + success = false; + } + for (k = chacha20_testvecs[i].ilen; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (chunked, zero check): FAIL\n", + i + 1); + success = false; + break; + } + } + +next_test: + /* Sliding unaligned case */ + if (chacha20_testvecs[i].ilen > CHACHA20_BLOCK_SIZE + 1 || + !chacha20_testvecs[i].ilen) + continue; + for (j = 1; j < CHACHA20_BLOCK_SIZE; ++j) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN + 1); + memset(&state, 0, sizeof(state)); + memcpy(offset_input + j, chacha20_testvecs[i].input, + chacha20_testvecs[i].ilen); + chacha20_init(&state, chacha20_testvecs[i].key, + chacha20_testvecs[i].nonce); + chacha20(&state, computed_output + j, offset_input + j, + chacha20_testvecs[i].ilen, &simd_context); + if (memcmp(computed_output + j, + chacha20_testvecs[i].output, + chacha20_testvecs[i].ilen)) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu): FAIL\n", + i + 1, j); + success = false; + } + for (k = j; k < j; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu, zero check): FAIL\n", + i + 1, j); + success = false; + break; + } + } + for (k = chacha20_testvecs[i].ilen + j; + k < MAXIMUM_TEST_BUFFER_LEN + 1; ++k) { + if (computed_output[k]) { + pr_err("chacha20 self-test %zu (unaligned, slide %zu, zero check): FAIL\n", + i + 1, j); + success = false; + break; + } + } + } + } + for (i = 0; i < ARRAY_SIZE(hchacha20_testvecs); ++i) { + memset(&derived_key, 0, sizeof(derived_key)); + hchacha20(derived_key, hchacha20_testvecs[i].nonce, + hchacha20_testvecs[i].key, &simd_context); + cpu_to_le32_array(derived_key, ARRAY_SIZE(derived_key)); + if (memcmp(derived_key, hchacha20_testvecs[i].output, + CHACHA20_KEY_SIZE)) { + pr_err("hchacha20 self-test %zu: FAIL\n", i + 1); + success = false; + } + } + memset(&state, 0, sizeof(state)); + chacha20_init(&state, chacha20_testvecs[0].key, + chacha20_testvecs[0].nonce); + chacha20(&state, massive_input, massive_input, + OUTRAGEOUSLY_HUGE_BUFFER_LEN, &simd_context); + chacha20_init(&state, chacha20_testvecs[0].key, + chacha20_testvecs[0].nonce); + chacha20(&state, massive_input, massive_input, + OUTRAGEOUSLY_HUGE_BUFFER_LEN, DONT_USE_SIMD); + for (k = 0; k < OUTRAGEOUSLY_HUGE_BUFFER_LEN; ++k) { + if (massive_input[k]) { + pr_err("chacha20 self-test massive: FAIL\n"); + success = false; + break; + } + } + + simd_put(&simd_context); + +out: + kfree(offset_input); + kfree(computed_output); + vfree(massive_input); + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c b/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c new file mode 100644 index 0000000000000..c58ac6e69d541 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/chacha20poly1305.c @@ -0,0 +1,9076 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct chacha20poly1305_testvec { + const u8 *input, *output, *assoc, *nonce, *key; + size_t ilen, alen, nlen; + bool failure; +}; + +/* The first of these are the ChaCha20-Poly1305 AEAD test vectors from RFC7539 + * 2.8.2. After they are generated by reference implementations. And the final + * marked ones are taken from wycheproof, but we only do these for the encrypt + * side, because mostly we're stressing the primitives rather than the actual + * chapoly construction. This also requires adding a 96-bit nonce construction, + * just for the purpose of the tests. + */ + +static const u8 enc_input001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 enc_output001[] __initconst = { + 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, + 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, + 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, + 0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2, + 0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, 0x43, 0xee, + 0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0, + 0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00, + 0xd4, 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf, + 0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce, + 0x98, 0xc8, 0xa8, 0x4a, 0xbd, 0x0b, 0x94, 0x81, + 0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd, + 0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, 0x55, + 0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61, + 0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38, + 0x36, 0x06, 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0, + 0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4, + 0xb9, 0x16, 0x6c, 0x76, 0x7b, 0x80, 0x4d, 0x46, + 0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9, + 0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e, + 0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e, + 0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15, + 0x5b, 0x00, 0x47, 0x71, 0x8c, 0xbc, 0x54, 0x6a, + 0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea, + 0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, 0x27, 0x1a, + 0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99, + 0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e, + 0xce, 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10, + 0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10, + 0x49, 0xe6, 0x17, 0xd9, 0x1d, 0x36, 0x10, 0x94, + 0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30, + 0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, 0xdf, + 0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29, + 0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70, + 0x9b, 0xee, 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb, + 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, + 0x38 +}; +static const u8 enc_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 enc_nonce001[] __initconst = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 +}; +static const u8 enc_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const u8 enc_input002[] __initconst = { }; +static const u8 enc_output002[] __initconst = { + 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, + 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 +}; +static const u8 enc_assoc002[] __initconst = { }; +static const u8 enc_nonce002[] __initconst = { + 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e +}; +static const u8 enc_key002[] __initconst = { + 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, + 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, + 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, + 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 +}; + +static const u8 enc_input003[] __initconst = { }; +static const u8 enc_output003[] __initconst = { + 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, + 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 +}; +static const u8 enc_assoc003[] __initconst = { + 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b +}; +static const u8 enc_nonce003[] __initconst = { + 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d +}; +static const u8 enc_key003[] __initconst = { + 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, + 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, + 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, + 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d +}; + +static const u8 enc_input004[] __initconst = { + 0xa4 +}; +static const u8 enc_output004[] __initconst = { + 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, + 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, + 0x89 +}; +static const u8 enc_assoc004[] __initconst = { + 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 +}; +static const u8 enc_nonce004[] __initconst = { + 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 +}; +static const u8 enc_key004[] __initconst = { + 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, + 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, + 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, + 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e +}; + +static const u8 enc_input005[] __initconst = { + 0x2d +}; +static const u8 enc_output005[] __initconst = { + 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, + 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, + 0xac +}; +static const u8 enc_assoc005[] __initconst = { }; +static const u8 enc_nonce005[] __initconst = { + 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 +}; +static const u8 enc_key005[] __initconst = { + 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, + 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, + 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, + 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 +}; + +static const u8 enc_input006[] __initconst = { + 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, + 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, + 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, + 0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50, + 0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, 0x17, 0xec, + 0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb, + 0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66, + 0x3b, 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb, + 0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b, + 0xc7, 0x42, 0xce, 0x2f, 0x0c, 0xa6, 0x85, 0x6e, + 0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3, + 0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, 0xf0, + 0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb, + 0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41, + 0xd5, 0xdc, 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc, + 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, + 0x8f +}; +static const u8 enc_output006[] __initconst = { + 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, + 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, + 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, + 0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda, + 0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, 0x4b, 0x11, + 0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8, + 0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc, + 0x93, 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3, + 0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5, + 0xbc, 0xc4, 0xcb, 0x7b, 0x3a, 0x8e, 0x7f, 0x02, + 0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93, + 0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, 0x78, + 0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1, + 0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66, + 0x3e, 0x6c, 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc, + 0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0, + 0x8c, 0x9d, 0x84, 0x43, 0x6b, 0xc1, 0xf7, 0x8d, + 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, + 0xeb +}; +static const u8 enc_assoc006[] __initconst = { + 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b +}; +static const u8 enc_nonce006[] __initconst = { + 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c +}; +static const u8 enc_key006[] __initconst = { + 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, + 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, + 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, + 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 +}; + +static const u8 enc_input007[] __initconst = { + 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, + 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, + 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, + 0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17, + 0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, 0x8c, 0x1c, + 0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1, + 0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51, + 0x9d, 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1, + 0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86, + 0xfc, 0x44, 0xb3, 0x4f, 0xf3, 0xea, 0x67, 0x5a, + 0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a, + 0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, 0x98, + 0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36, + 0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34, + 0xaa, 0x2f, 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57, + 0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84, + 0x52, 0x4f, 0xc2, 0x4a, 0x40, 0x3b, 0x3c, 0xd4, + 0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80, + 0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82, + 0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5, + 0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d, + 0xac, 0x2f, 0x3d, 0x71, 0x85, 0x7b, 0xcf, 0x3c, + 0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf, + 0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, 0xa0, 0xfc, + 0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3, + 0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14, + 0x55, 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81, + 0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77, + 0x91, 0xe1, 0xce, 0xa2, 0x7e, 0x7f, 0x42, 0xe3, + 0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2, + 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, + 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 +}; +static const u8 enc_output007[] __initconst = { + 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, + 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, + 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, + 0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb, + 0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, 0x7c, 0xc0, + 0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21, + 0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70, + 0x82, 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac, + 0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99, + 0x19, 0x66, 0xd0, 0xf6, 0x88, 0x2c, 0x77, 0xd9, + 0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f, + 0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, 0xf7, + 0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53, + 0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12, + 0x47, 0x52, 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6, + 0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0, + 0x31, 0xde, 0x1f, 0x9f, 0x2f, 0x05, 0x38, 0x54, + 0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6, + 0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e, + 0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb, + 0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30, + 0x6b, 0xb2, 0x03, 0xc4, 0x1c, 0x04, 0xf8, 0x0f, + 0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2, + 0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, 0xd5, 0x2e, + 0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34, + 0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39, + 0xa9, 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7, + 0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9, + 0x15, 0x2a, 0xd0, 0xa0, 0x7a, 0x87, 0x34, 0x82, + 0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04, + 0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, 0x34, + 0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef, + 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, + 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 +}; +static const u8 enc_assoc007[] __initconst = { }; +static const u8 enc_nonce007[] __initconst = { + 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 +}; +static const u8 enc_key007[] __initconst = { + 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, + 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, + 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, + 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 +}; + +static const u8 enc_input008[] __initconst = { + 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, + 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, + 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, + 0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb, + 0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, 0xe1, 0x12, + 0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa, + 0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6, + 0x83, 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4, + 0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91, + 0x43, 0x5c, 0x92, 0x49, 0x62, 0x61, 0x7b, 0xeb, + 0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47, + 0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, 0x15, + 0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f, + 0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a, + 0xad, 0xaa, 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3, + 0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97, + 0x05, 0x2a, 0xbc, 0x7c, 0x7b, 0x25, 0xf8, 0x80, + 0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e, + 0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f, + 0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10, + 0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a, + 0x26, 0xfa, 0xfe, 0x41, 0x32, 0x83, 0x10, 0xe0, + 0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35, + 0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, 0x93, 0x4d, + 0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d, + 0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57, + 0x88, 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4, + 0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f, + 0xcc, 0x8a, 0x24, 0x9b, 0xdf, 0x6d, 0xf0, 0x39, + 0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda, + 0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, 0x17, + 0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43, + 0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19, + 0x59, 0xbc, 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09, + 0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21, + 0x97, 0xbf, 0x89, 0x71, 0xa5, 0xb0, 0x6e, 0x07, + 0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f, + 0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b, + 0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a, + 0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed, + 0xd2, 0x15, 0x8f, 0x5e, 0x91, 0xdb, 0x33, 0xf2, + 0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca, + 0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, 0x90, 0xff, + 0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b, + 0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b, + 0xec, 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b, + 0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6, + 0x8e, 0x5f, 0xd4, 0xb9, 0xb7, 0x0f, 0x21, 0x04, + 0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48, + 0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, 0x8b, + 0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13, + 0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8, + 0x3c, 0x85, 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f, + 0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0, + 0xe6, 0x58, 0xb5, 0x8f, 0xc5, 0x29, 0xa2, 0x92, + 0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a, + 0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41, + 0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17, + 0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30, + 0xf3, 0xf7, 0x30, 0x3c, 0x96, 0xe6, 0x6a, 0x20, + 0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49, + 0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, 0xf8, 0x5a, + 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, + 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 +}; +static const u8 enc_output008[] __initconst = { + 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, + 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, + 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, + 0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d, + 0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, 0xd1, 0x2c, + 0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6, + 0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4, + 0xa8, 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5, + 0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84, + 0x03, 0x73, 0x1e, 0x8c, 0x49, 0xac, 0x20, 0xdd, + 0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed, + 0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, 0xab, + 0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13, + 0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49, + 0x43, 0xea, 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6, + 0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8, + 0xda, 0xd4, 0xb7, 0xeb, 0xe8, 0x5c, 0x09, 0xa2, + 0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94, + 0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18, + 0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60, + 0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8, + 0xce, 0xc7, 0xbe, 0x5c, 0xd2, 0x95, 0xa8, 0x4b, + 0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f, + 0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, 0x92, 0x9c, + 0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20, + 0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff, + 0xb3, 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9, + 0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c, + 0x16, 0x52, 0xd9, 0xf3, 0xf7, 0x98, 0x2e, 0xc9, + 0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6, + 0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, 0xea, + 0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e, + 0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82, + 0xec, 0x1e, 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1, + 0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70, + 0x38, 0x4a, 0x8c, 0x49, 0xc5, 0x43, 0xa0, 0xa1, + 0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c, + 0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7, + 0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc, + 0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc, + 0x2c, 0x0e, 0xa8, 0x51, 0x4d, 0x80, 0x0d, 0xa3, + 0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb, + 0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, 0xf5, 0x97, + 0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f, + 0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39, + 0xda, 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f, + 0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d, + 0x71, 0x52, 0xa7, 0xb8, 0xc0, 0xa5, 0xc6, 0xa2, + 0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d, + 0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, 0x96, + 0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b, + 0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20, + 0x50, 0xba, 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95, + 0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb, + 0x66, 0x76, 0x44, 0xdc, 0x03, 0x74, 0x48, 0x35, + 0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62, + 0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9, + 0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6, + 0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8, + 0x35, 0x7f, 0xdc, 0x40, 0x2c, 0xe9, 0xbc, 0x8a, + 0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93, + 0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, 0x50, 0x14, + 0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99, + 0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86, + 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, + 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 +}; +static const u8 enc_assoc008[] __initconst = { }; +static const u8 enc_nonce008[] __initconst = { + 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 +}; +static const u8 enc_key008[] __initconst = { + 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, + 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, + 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, + 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba +}; + +static const u8 enc_input009[] __initconst = { + 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, + 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, + 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, + 0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09, + 0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, 0xe1, 0xe5, + 0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85, + 0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44, + 0x0f, 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97, + 0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77, + 0x8b, 0x15, 0xad, 0x10, 0xa0, 0x2b, 0x7b, 0x41, + 0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c, + 0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, 0x00, + 0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82, + 0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f, + 0x01, 0x9e, 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e, + 0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55, + 0x10, 0x9a, 0xdf, 0x67, 0x22, 0x8b, 0x43, 0xab, + 0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17, + 0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e, + 0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f, + 0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82, + 0x6e, 0x16, 0x92, 0xb1, 0x12, 0x17, 0x07, 0xc3, + 0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f, + 0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, 0x9c, 0xb0, + 0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08, + 0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b, + 0xd5, 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85, + 0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28, + 0x10, 0x79, 0xf1, 0x3c, 0xbf, 0x1a, 0x41, 0x5c, + 0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62, + 0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, 0xa2, + 0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3, + 0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62, + 0x31, 0x10, 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40, + 0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f, + 0x32, 0x1d, 0x0a, 0x8e, 0x79, 0xd8, 0xa4, 0x1b, + 0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91, + 0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5, + 0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c, + 0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4, + 0x83, 0xaa, 0x66, 0x89, 0x67, 0x7e, 0xc0, 0x49, + 0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04, + 0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, 0xfa, 0x03, + 0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa, + 0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec, + 0x9c, 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6, + 0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69, + 0x5c, 0x31, 0x95, 0x42, 0xa6, 0x2c, 0xd1, 0x36, + 0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8, + 0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, 0xbf, + 0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe, + 0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82, + 0x76, 0x26, 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab, + 0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d, + 0xa1, 0x4f, 0x80, 0xd8, 0x3f, 0x94, 0xfb, 0xd3, + 0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5, + 0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34, + 0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49, + 0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f, + 0x4b, 0x08, 0x6e, 0xb1, 0x12, 0x22, 0x10, 0x9d, + 0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42, + 0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, 0xf7, 0xef, + 0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27, + 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, + 0x65 +}; +static const u8 enc_output009[] __initconst = { + 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, + 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, + 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, + 0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd, + 0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, 0xd0, 0x28, + 0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe, + 0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06, + 0xfa, 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5, + 0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7, + 0xeb, 0x05, 0x48, 0x0d, 0x7c, 0x35, 0x4a, 0x09, + 0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a, + 0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, 0x00, + 0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62, + 0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb, + 0xc1, 0xb0, 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2, + 0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28, + 0x5b, 0x83, 0xcc, 0x18, 0x91, 0x88, 0xb0, 0x2e, + 0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a, + 0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6, + 0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83, + 0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9, + 0xb2, 0x55, 0xcb, 0x3c, 0x10, 0xf0, 0x24, 0x8a, + 0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79, + 0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, 0xb0, 0x0a, + 0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea, + 0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b, + 0x19, 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52, + 0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb, + 0x64, 0x89, 0xba, 0x26, 0xf9, 0xc7, 0xe1, 0x89, + 0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad, + 0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, 0x19, + 0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71, + 0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d, + 0x49, 0x00, 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54, + 0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a, + 0xe9, 0x7a, 0x7a, 0xcf, 0xfc, 0x8a, 0x4e, 0x4d, + 0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95, + 0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42, + 0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16, + 0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6, + 0xe7, 0x6b, 0x2e, 0x8e, 0x4c, 0x3d, 0xe2, 0xaf, + 0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d, + 0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, 0xf1, 0x9f, + 0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b, + 0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e, + 0x8d, 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4, + 0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c, + 0x5d, 0x12, 0x86, 0xdb, 0x6f, 0x1c, 0x33, 0xc4, + 0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1, + 0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, 0xfb, + 0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff, + 0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2, + 0xd7, 0xa2, 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06, + 0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66, + 0xc5, 0x54, 0xc2, 0xfc, 0x06, 0xda, 0x05, 0x90, + 0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55, + 0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc, + 0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8, + 0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62, + 0x75, 0x3f, 0x09, 0xd5, 0xf5, 0xd9, 0x26, 0xba, + 0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2, + 0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, 0x81, 0x89, + 0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06, + 0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90, + 0x42, 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf, + 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, + 0xae +}; +static const u8 enc_assoc009[] __initconst = { + 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, + 0xef +}; +static const u8 enc_nonce009[] __initconst = { + 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 +}; +static const u8 enc_key009[] __initconst = { + 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, + 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, + 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, + 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b +}; + +static const u8 enc_input010[] __initconst = { + 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, + 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, + 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, + 0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc, + 0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, 0xe7, 0x16, + 0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7, + 0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4, + 0x95, 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d, + 0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5, + 0xc7, 0x45, 0x50, 0xf6, 0xa2, 0x1a, 0xb5, 0x46, + 0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82, + 0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, 0x2b, + 0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a, + 0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf, + 0xba, 0xd7, 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca, + 0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95, + 0xae, 0xa6, 0x8d, 0x04, 0xcc, 0xee, 0xf7, 0x09, + 0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3, + 0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3, + 0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f, + 0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58, + 0x84, 0x6e, 0xf9, 0x3d, 0xdf, 0x25, 0xea, 0xad, + 0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde, + 0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, 0xce, 0x44, + 0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a, + 0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9, + 0x60, 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26, + 0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc, + 0x91, 0x78, 0x53, 0x98, 0x86, 0x5b, 0x9c, 0x74, + 0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b, + 0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, 0x93, + 0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37, + 0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f, + 0x14, 0x12, 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d, + 0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca, + 0x10, 0x68, 0xaf, 0x7e, 0xb7, 0x33, 0x54, 0x73, + 0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f, + 0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1, + 0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9, + 0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76, + 0xd7, 0x01, 0xa0, 0x1a, 0xc8, 0x4e, 0xaa, 0xac, + 0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7, + 0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, 0xfb, 0xce, + 0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30, + 0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb, + 0x1e, 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa, + 0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd, + 0x0a, 0x97, 0xd0, 0xe4, 0x37, 0x83, 0x61, 0x5f, + 0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb, + 0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, 0x34, + 0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e, + 0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f, + 0x8a, 0x08, 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53, + 0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41, + 0x01, 0x39, 0x0a, 0x24, 0x3c, 0x7e, 0xbe, 0x4e, + 0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d, + 0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27, + 0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e, + 0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8, + 0xd4, 0x79, 0x9d, 0x80, 0x15, 0x8e, 0x53, 0x2a, + 0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12, + 0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, 0xf1, 0xc3, + 0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66, + 0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0, + 0x28, 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c, + 0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4, + 0x50, 0xff, 0x84, 0x5c, 0x47, 0x0c, 0x6a, 0x49, + 0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90, + 0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, 0x11, + 0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c, + 0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b, + 0x1e, 0xd5, 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74, + 0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c, + 0xae, 0x6c, 0x1d, 0x9a, 0x30, 0x04, 0x4d, 0x27, + 0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1, + 0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27, + 0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88, + 0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27, + 0x2e, 0x76, 0x1e, 0x1a, 0x63, 0x65, 0xf5, 0x3b, + 0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39, + 0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, 0xd8, 0xc7, + 0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc, + 0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe, + 0xc9, 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5, + 0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf, + 0x06, 0xdb, 0xdf, 0x96, 0x45, 0x58, 0xda, 0x05, + 0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73, + 0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, 0xda, + 0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe, + 0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71, + 0x36, 0x83, 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed, + 0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d, + 0xff, 0xde, 0xb1, 0xef, 0x61, 0x5a, 0x45, 0x33, + 0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f, + 0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a, + 0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa, + 0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e, + 0x2d, 0x3f, 0x1b, 0x64, 0xaf, 0x8d, 0x06, 0x0e, + 0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87, + 0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, 0x94, 0xd5, + 0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4, + 0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38, + 0xe6, 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34, + 0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f, + 0x71, 0x7a, 0x38, 0x6b, 0x98, 0xfb, 0x49, 0x36, + 0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69, + 0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, 0x44, + 0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5, + 0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce, + 0xe5, 0x91, 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd, + 0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27, + 0x3c, 0xd3, 0x0e, 0x71, 0xf2, 0xff, 0xf5, 0x2f, + 0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8, + 0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a, + 0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5, + 0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca, + 0x87, 0xa0, 0xae, 0xc9, 0xa6, 0x62, 0x1b, 0x6e, + 0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92, + 0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, 0x1f, 0x13, + 0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf, + 0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6, + 0xb6, 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3, + 0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b, + 0xf2, 0x17, 0x59, 0x08, 0x04, 0x58, 0x81, 0x9d, + 0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f, + 0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, 0x40, + 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, + 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f +}; +static const u8 enc_output010[] __initconst = { + 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, + 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, + 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, + 0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd, + 0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, 0x9f, 0xa6, + 0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5, + 0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96, + 0xeb, 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02, + 0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30, + 0x41, 0x24, 0xce, 0x68, 0x61, 0x49, 0x86, 0x57, + 0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53, + 0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, 0x65, + 0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71, + 0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9, + 0xde, 0xa4, 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18, + 0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce, + 0x2f, 0x43, 0x68, 0xd6, 0x06, 0xe2, 0x74, 0x6a, + 0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69, + 0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2, + 0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95, + 0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49, + 0xde, 0x9c, 0xbc, 0xee, 0x14, 0x3f, 0x81, 0x5e, + 0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a, + 0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, 0x33, 0x0a, + 0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e, + 0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19, + 0xce, 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b, + 0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75, + 0x01, 0x81, 0xe6, 0x4b, 0x57, 0x7c, 0xdd, 0x6d, + 0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d, + 0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, 0x2f, + 0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a, + 0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d, + 0xbd, 0x47, 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5, + 0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c, + 0x39, 0xd2, 0x97, 0xc1, 0xcb, 0xeb, 0xf4, 0x77, + 0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46, + 0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43, + 0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe, + 0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8, + 0x2e, 0xca, 0xfa, 0xdc, 0x59, 0xd5, 0xc3, 0x76, + 0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47, + 0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, 0x4c, 0xe8, + 0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32, + 0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59, + 0x5c, 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae, + 0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a, + 0x3b, 0x3a, 0x4d, 0xae, 0xeb, 0xbd, 0x22, 0xc3, + 0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74, + 0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, 0x75, + 0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2, + 0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e, + 0xfe, 0x5b, 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2, + 0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9, + 0xb4, 0x9b, 0xf8, 0xef, 0xbd, 0x1c, 0x92, 0xc1, + 0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07, + 0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79, + 0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71, + 0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad, + 0xc2, 0xdd, 0x64, 0x5d, 0x79, 0xb6, 0xf5, 0x7a, + 0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c, + 0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, 0x98, 0xa9, + 0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79, + 0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27, + 0xba, 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90, + 0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe, + 0xeb, 0xcd, 0x01, 0x05, 0x44, 0x72, 0xdb, 0x99, + 0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1, + 0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, 0xe9, + 0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0, + 0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28, + 0x70, 0x51, 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e, + 0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20, + 0xc7, 0x42, 0x25, 0x3e, 0x9a, 0x14, 0xd7, 0x60, + 0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47, + 0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68, + 0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe, + 0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33, + 0x3a, 0x61, 0x2e, 0xc7, 0xff, 0xa4, 0x2a, 0xa8, + 0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38, + 0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, 0x28, 0xa7, + 0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04, + 0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c, + 0xdf, 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f, + 0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c, + 0xec, 0xd7, 0x05, 0x60, 0x97, 0xbb, 0x74, 0x77, + 0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54, + 0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, 0xa5, + 0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4, + 0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2, + 0xd5, 0x13, 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e, + 0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27, + 0xa8, 0x0a, 0x91, 0x01, 0x68, 0x71, 0x8a, 0x3f, + 0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92, + 0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55, + 0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe, + 0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04, + 0x9f, 0x66, 0x02, 0xb9, 0x88, 0x10, 0xd9, 0xc4, + 0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56, + 0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, 0xa4, 0x02, + 0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2, + 0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8, + 0xf3, 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27, + 0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47, + 0xe5, 0xdf, 0x5f, 0x01, 0xaa, 0xb0, 0xd4, 0x10, + 0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43, + 0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, 0xe0, + 0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee, + 0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47, + 0xfb, 0xbf, 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6, + 0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d, + 0xd5, 0xd0, 0x20, 0x60, 0x03, 0xab, 0x3f, 0x8c, + 0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3, + 0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b, + 0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09, + 0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d, + 0x5c, 0xa9, 0x11, 0xd4, 0x7d, 0xaf, 0x9e, 0xf1, + 0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd, + 0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, 0x40, 0xf4, + 0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63, + 0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87, + 0x3c, 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd, + 0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e, + 0x4d, 0x38, 0xb2, 0xc0, 0xb8, 0x95, 0x01, 0x7a, + 0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c, + 0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, 0x38, + 0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a, + 0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5, + 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, + 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 +}; +static const u8 enc_assoc010[] __initconst = { + 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, + 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 +}; +static const u8 enc_nonce010[] __initconst = { + 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 +}; +static const u8 enc_key010[] __initconst = { + 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, + 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, + 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, + 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 +}; + +static const u8 enc_input011[] __initconst = { + 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, + 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, + 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, + 0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee, + 0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, 0x4c, 0x30, + 0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20, + 0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f, + 0x15, 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e, + 0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66, + 0x1b, 0x00, 0x64, 0xa5, 0x93, 0x8d, 0x06, 0x46, + 0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35, + 0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, 0xb6, + 0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0, + 0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15, + 0x26, 0x12, 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13, + 0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7, + 0x40, 0xff, 0x5e, 0xce, 0x48, 0x9a, 0x60, 0xe3, + 0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37, + 0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc, + 0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95, + 0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8, + 0x08, 0x63, 0x6a, 0x36, 0xd3, 0x3c, 0xb8, 0xac, + 0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45, + 0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, 0xbd, 0xaf, + 0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d, + 0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc, + 0x52, 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45, + 0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a, + 0xa8, 0xaf, 0xb5, 0xe3, 0xbb, 0x77, 0x52, 0xec, + 0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e, + 0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, 0x10, + 0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8, + 0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66, + 0x4f, 0xcc, 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0, + 0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62, + 0x35, 0x9c, 0x88, 0x59, 0x09, 0xdd, 0x82, 0x1b, + 0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4, + 0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96, + 0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7, + 0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74, + 0x05, 0x1b, 0x1d, 0xe0, 0xcd, 0x16, 0xb0, 0xa8, + 0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b, + 0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, 0xce, 0x70, + 0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95, + 0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3, + 0x46, 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9, + 0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d, + 0xe0, 0x93, 0xa0, 0xbe, 0x09, 0x1c, 0x2b, 0x4e, + 0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32, + 0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, 0xc5, + 0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80, + 0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3, + 0x8e, 0xdc, 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad, + 0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d, + 0xa5, 0x7f, 0x21, 0x10, 0xf1, 0x1f, 0x13, 0x20, + 0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17, + 0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6, + 0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d, + 0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82, + 0xa1, 0xf9, 0x4e, 0x54, 0x87, 0x89, 0xc9, 0x0c, + 0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9, + 0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, 0x0d, 0xbb, + 0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96, + 0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9, + 0x7e, 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f, + 0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40, + 0xd4, 0x4c, 0x6b, 0xd2, 0x56, 0x62, 0xb0, 0xcc, + 0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce, + 0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, 0x71, + 0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f, + 0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35, + 0xea, 0x92, 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90, + 0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8, + 0x91, 0xf8, 0x46, 0x15, 0x72, 0x63, 0x70, 0x01, + 0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1, + 0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe, + 0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4, + 0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf, + 0x6b, 0xa8, 0x9e, 0xf4, 0x16, 0x96, 0x36, 0xb9, + 0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f, + 0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, 0xc6, 0x04, + 0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7, + 0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15, + 0xf7, 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc, + 0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0, + 0x33, 0x82, 0x4c, 0x79, 0x3c, 0xfd, 0xb1, 0xae, + 0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb, + 0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, 0xed, + 0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51, + 0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52, + 0x87, 0xd8, 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84, + 0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5, + 0x85, 0x1c, 0x1f, 0x6b, 0x47, 0xa0, 0xc4, 0xe4, + 0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e, + 0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74, + 0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f, + 0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13, + 0xc8, 0xac, 0x25, 0x96, 0x23, 0xd8, 0x09, 0xea, + 0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b, + 0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, 0x0a, 0xef, + 0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09, + 0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe, + 0x9f, 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1, + 0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9, + 0x49, 0xce, 0x30, 0x8e, 0x44, 0xb5, 0x76, 0x15, + 0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a, + 0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, 0xab, + 0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36, + 0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd, + 0x55, 0x25, 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde, + 0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd, + 0x6d, 0x16, 0x7a, 0x73, 0x38, 0x46, 0xe5, 0x47, + 0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5, + 0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69, + 0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21, + 0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98, + 0x7c, 0x5f, 0x7d, 0x92, 0x88, 0xb9, 0x94, 0x07, + 0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57, + 0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, 0xb3, 0xbd, + 0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03, + 0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11, + 0xa3, 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96, + 0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91, + 0xf5, 0xb6, 0x18, 0xfe, 0xc2, 0x83, 0x18, 0x7d, + 0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0, + 0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, 0xf9, + 0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42, + 0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a, + 0x6d, 0x5f, 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18, + 0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc, + 0xb7, 0xdd, 0x2a, 0x20, 0x77, 0xf5, 0xf9, 0xce, + 0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc, + 0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0, + 0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf, + 0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7, + 0xfc, 0x98, 0x48, 0x8c, 0x0d, 0x82, 0xc9, 0x80, + 0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c, + 0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, 0x67, 0x82, + 0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9, + 0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20, + 0xfb, 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58, + 0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6, + 0x27, 0x55, 0x7b, 0x19, 0xe2, 0xfb, 0x64, 0xfc, + 0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50, + 0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, 0x86, + 0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a, + 0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80, + 0x3a, 0x6e, 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec, + 0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08, + 0x86, 0x53, 0xbe, 0xbd, 0x40, 0x41, 0x38, 0x1c, + 0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde, + 0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d, + 0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17, + 0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f, + 0xca, 0x6c, 0xe7, 0x9b, 0x04, 0x62, 0x0e, 0x26, + 0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96, + 0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, 0xfe, 0x97, + 0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6, + 0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55, + 0x1a, 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e, + 0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88, + 0x69, 0x99, 0x4b, 0xee, 0xbe, 0x9a, 0x99, 0xb5, + 0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b, + 0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, 0x15, + 0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1, + 0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4, + 0x94, 0x2a, 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3, + 0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf, + 0x1a, 0x88, 0x59, 0xdc, 0x74, 0x12, 0xb4, 0x8e, + 0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb, + 0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76, + 0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5, + 0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c, + 0x88, 0xf8, 0xd6, 0x1b, 0xa6, 0x7d, 0x9e, 0xde, + 0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f, + 0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, 0x59, 0x51, + 0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9, + 0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99, + 0xf5, 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6, + 0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04, + 0x1e, 0x35, 0xd4, 0x80, 0x20, 0xf4, 0x9c, 0x31, + 0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a, + 0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, 0x56, + 0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e, + 0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78, + 0x93, 0x9a, 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a, + 0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7, + 0xd0, 0x13, 0x9d, 0x32, 0x40, 0x5e, 0xcf, 0xfb, + 0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6, + 0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8, + 0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc, + 0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84, + 0x55, 0xde, 0x08, 0x91, 0x16, 0x3a, 0x0c, 0x86, + 0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76, + 0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, 0x8c, 0x2a, + 0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73, + 0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8, + 0x29, 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6, + 0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2, + 0xd0, 0x57, 0xc1, 0x7c, 0xd2, 0x6a, 0xd2, 0x56, + 0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb, + 0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, 0xab, + 0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76, + 0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69, + 0xe2, 0xb0, 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d, + 0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc, + 0x0e, 0x2c, 0x9c, 0xb1, 0xa1, 0x63, 0x17, 0x22, + 0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39, + 0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6, + 0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9, + 0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f, + 0x3b, 0xaa, 0xe0, 0x52, 0x85, 0xc5, 0x65, 0xc1, + 0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83, + 0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, 0xd4, 0xfc, + 0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4, + 0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59, + 0x6c, 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68, + 0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef, + 0x03, 0xf2, 0xeb, 0x18, 0x57, 0x36, 0xe8, 0xa1, + 0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3, + 0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, 0x44, + 0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09, + 0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8, + 0x83, 0x19, 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a, + 0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d, + 0xf4, 0xed, 0x27, 0x03, 0x65, 0xa2, 0xa1, 0xae, + 0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2, + 0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10, + 0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a, + 0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34, + 0x18, 0x0d, 0xc9, 0x9f, 0x7b, 0x0c, 0x9b, 0x8f, + 0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9, + 0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, 0x28, 0x2b, + 0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d, + 0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57, + 0xb6, 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03, + 0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87, + 0x61, 0x79, 0x6c, 0x95, 0x0e, 0x9c, 0xdd, 0xca, + 0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53, + 0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, 0x8f, + 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, + 0x10, 0x1e, 0xbf, 0xec, 0xa8 +}; +static const u8 enc_output011[] __initconst = { + 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, + 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, + 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, + 0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73, + 0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, 0x86, 0x5e, + 0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9, + 0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e, + 0xcb, 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd, + 0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57, + 0x66, 0xef, 0x72, 0x4c, 0x42, 0x6e, 0x16, 0x19, + 0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f, + 0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, 0x45, + 0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e, + 0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39, + 0x62, 0xc6, 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03, + 0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f, + 0xf2, 0x13, 0x77, 0xf2, 0x8d, 0xb9, 0x47, 0xd0, + 0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce, + 0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb, + 0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52, + 0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21, + 0xf5, 0x47, 0xaa, 0x18, 0x21, 0x5c, 0xc9, 0x9a, + 0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35, + 0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, 0x8b, 0x91, + 0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b, + 0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e, + 0x75, 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19, + 0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07, + 0x07, 0x30, 0xa7, 0x19, 0x0c, 0xbf, 0xe6, 0x18, + 0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96, + 0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, 0x68, + 0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4, + 0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57, + 0xfe, 0x05, 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c, + 0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23, + 0x1a, 0xa0, 0x4f, 0x69, 0x56, 0x4c, 0xe1, 0xc8, + 0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6, + 0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40, + 0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab, + 0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb, + 0x9d, 0x9a, 0x1f, 0xec, 0x76, 0x99, 0xe7, 0xea, + 0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8, + 0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, 0xc2, 0x31, + 0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0, + 0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc, + 0xea, 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94, + 0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1, + 0x28, 0xbf, 0x94, 0x67, 0x22, 0xc3, 0x36, 0x46, + 0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6, + 0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, 0xa7, + 0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71, + 0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a, + 0x63, 0x33, 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33, + 0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38, + 0x08, 0xe6, 0x02, 0xd3, 0x25, 0x2c, 0x47, 0x23, + 0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb, + 0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65, + 0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73, + 0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8, + 0xa9, 0x3c, 0xf6, 0xdc, 0x12, 0x5c, 0xe1, 0xbb, + 0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a, + 0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, 0xee, 0xca, + 0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5, + 0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71, + 0x2e, 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8, + 0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d, + 0xf2, 0x4d, 0x54, 0xd4, 0xfc, 0x3a, 0x05, 0xe6, + 0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d, + 0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, 0xe7, + 0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5, + 0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8, + 0xa6, 0xa3, 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd, + 0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29, + 0x74, 0x33, 0xd0, 0x9e, 0x83, 0x86, 0x72, 0x22, + 0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5, + 0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67, + 0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11, + 0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e, + 0xd0, 0x27, 0xd7, 0x54, 0x3b, 0x67, 0x73, 0x09, + 0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4, + 0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, 0x55, 0x4f, + 0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa, + 0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec, + 0x53, 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b, + 0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d, + 0x5e, 0x74, 0xac, 0x72, 0xc1, 0x97, 0xe5, 0x1b, + 0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48, + 0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, 0xc3, + 0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63, + 0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd, + 0x72, 0x73, 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78, + 0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed, + 0xb9, 0xcc, 0x86, 0x30, 0xdb, 0x2b, 0xd3, 0x82, + 0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f, + 0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3, + 0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9, + 0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72, + 0x84, 0xb6, 0xc6, 0xeb, 0x0c, 0x27, 0x07, 0x74, + 0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40, + 0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, 0xec, 0x4b, + 0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a, + 0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5, + 0x1d, 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98, + 0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71, + 0xb7, 0x67, 0x81, 0x23, 0x96, 0xae, 0xb9, 0x7e, + 0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4, + 0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, 0x46, + 0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e, + 0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f, + 0x9d, 0x66, 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93, + 0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0, + 0x64, 0xed, 0x07, 0x12, 0xb3, 0xe6, 0xe4, 0xe5, + 0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61, + 0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64, + 0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85, + 0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20, + 0x72, 0xc9, 0x28, 0x7b, 0x79, 0x00, 0xa1, 0xa6, + 0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc, + 0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, 0x3f, 0xc8, + 0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50, + 0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4, + 0xac, 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80, + 0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0, + 0x5d, 0x7a, 0xca, 0x4d, 0xa0, 0x57, 0xbd, 0x2a, + 0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35, + 0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, 0x43, + 0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12, + 0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7, + 0x0a, 0x8d, 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34, + 0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42, + 0x20, 0x51, 0x65, 0x5d, 0x09, 0xc3, 0xaa, 0xc0, + 0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95, + 0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74, + 0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5, + 0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12, + 0x1a, 0xdf, 0x48, 0xf3, 0xae, 0xb3, 0xe6, 0xe6, + 0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86, + 0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, 0xd4, 0x97, + 0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45, + 0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19, + 0xf3, 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86, + 0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c, + 0x48, 0x18, 0x54, 0x13, 0x4e, 0xcf, 0xfd, 0xba, + 0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29, + 0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, 0xf6, + 0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6, + 0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09, + 0x81, 0xd4, 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31, + 0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99, + 0x6d, 0xa3, 0xf4, 0xd7, 0x38, 0xee, 0x1a, 0x2b, + 0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca, + 0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00, + 0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93, + 0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3, + 0x3e, 0x28, 0x2d, 0x3b, 0x93, 0x8a, 0xcc, 0x07, + 0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda, + 0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, 0x4a, 0x90, + 0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b, + 0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a, + 0xba, 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6, + 0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c, + 0xde, 0xbc, 0xff, 0x64, 0x7e, 0x4e, 0x59, 0x57, + 0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15, + 0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, 0x7e, + 0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51, + 0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75, + 0x93, 0xac, 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19, + 0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08, + 0xa5, 0x13, 0xf9, 0x0e, 0x7e, 0x78, 0x6e, 0x14, + 0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba, + 0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff, + 0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90, + 0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e, + 0x53, 0x6e, 0xf4, 0x26, 0x57, 0x93, 0x15, 0x93, + 0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad, + 0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, 0x04, 0xd2, + 0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac, + 0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d, + 0x00, 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06, + 0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c, + 0x36, 0x5d, 0x23, 0xa2, 0xcc, 0x57, 0x40, 0x91, + 0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17, + 0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, 0x20, + 0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7, + 0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf, + 0x56, 0x9f, 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c, + 0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2, + 0x9c, 0xdf, 0x39, 0xbb, 0x6f, 0xa2, 0x8c, 0x7e, + 0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a, + 0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05, + 0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58, + 0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8, + 0xb0, 0x86, 0xd9, 0xbf, 0x7b, 0x35, 0x51, 0x4d, + 0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71, + 0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, 0x27, 0xc3, + 0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe, + 0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62, + 0x68, 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16, + 0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66, + 0x1f, 0x6e, 0x93, 0xf8, 0xc5, 0x51, 0xeb, 0xa4, + 0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2, + 0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, 0x35, + 0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3, + 0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4, + 0xb7, 0xab, 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f, + 0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe, + 0x64, 0xb9, 0xbb, 0xd8, 0x5e, 0x3a, 0x1a, 0x56, + 0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b, + 0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37, + 0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3, + 0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f, + 0xbd, 0x62, 0xac, 0xa3, 0x47, 0x27, 0xcf, 0x5f, + 0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0, + 0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, 0x03, 0x70, + 0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd, + 0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f, + 0x8d, 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e, + 0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67, + 0x3d, 0xe0, 0x74, 0x4f, 0x03, 0xf2, 0x70, 0x51, + 0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23, + 0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, 0xc3, + 0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5, + 0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09, + 0x45, 0xa4, 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7, + 0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed, + 0x28, 0xe0, 0xa1, 0xf8, 0x26, 0xcf, 0xcd, 0xcb, + 0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6, + 0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5, + 0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96, + 0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe, + 0x55, 0xfb, 0x0c, 0x7e, 0x67, 0xe2, 0x61, 0x44, + 0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6, + 0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, 0xd8, 0x3e, + 0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0, + 0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79, + 0xd4, 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f, + 0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d, + 0xd5, 0x4e, 0x99, 0x21, 0x65, 0x4d, 0xf5, 0x82, + 0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47, + 0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, 0x93, + 0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6, + 0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69, + 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, + 0x2b, 0xdf, 0xcd, 0xf9, 0x3c +}; +static const u8 enc_assoc011[] __initconst = { + 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 +}; +static const u8 enc_nonce011[] __initconst = { + 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa +}; +static const u8 enc_key011[] __initconst = { + 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, + 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, + 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, + 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 +}; + +static const u8 enc_input012[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 enc_output012[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd6 +}; +static const u8 enc_assoc012[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 enc_nonce012[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 enc_key012[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +/* wycheproof - rfc7539 */ +static const u8 enc_input013[] __initconst = { + 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39, + 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, + 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, + 0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, + 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, + 0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, + 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73, + 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, + 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, + 0x74, 0x2e +}; +static const u8 enc_output013[] __initconst = { + 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, + 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, + 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, + 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, + 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12, + 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, + 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, + 0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, + 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, + 0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58, + 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, + 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc, + 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, + 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, + 0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, + 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, + 0x06, 0x91 +}; +static const u8 enc_assoc013[] __initconst = { + 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7 +}; +static const u8 enc_nonce013[] __initconst = { + 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47 +}; +static const u8 enc_key013[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input014[] __initconst = { }; +static const u8 enc_output014[] __initconst = { + 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, + 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d +}; +static const u8 enc_assoc014[] __initconst = { }; +static const u8 enc_nonce014[] __initconst = { + 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, + 0xea, 0x12, 0x37, 0x9d +}; +static const u8 enc_key014[] __initconst = { + 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, + 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, + 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, + 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 +}; + +/* wycheproof - misc */ +static const u8 enc_input015[] __initconst = { }; +static const u8 enc_output015[] __initconst = { + 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, + 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 +}; +static const u8 enc_assoc015[] __initconst = { + 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 +}; +static const u8 enc_nonce015[] __initconst = { + 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, + 0xa3, 0xc6, 0xf6, 0x89 +}; +static const u8 enc_key015[] __initconst = { + 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, + 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, + 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, + 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 +}; + +/* wycheproof - misc */ +static const u8 enc_input016[] __initconst = { + 0x2a +}; +static const u8 enc_output016[] __initconst = { + 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, + 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, + 0x22 +}; +static const u8 enc_assoc016[] __initconst = { }; +static const u8 enc_nonce016[] __initconst = { + 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, + 0xee, 0xab, 0x60, 0xf1 +}; +static const u8 enc_key016[] __initconst = { + 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, + 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, + 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, + 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 +}; + +/* wycheproof - misc */ +static const u8 enc_input017[] __initconst = { + 0x51 +}; +static const u8 enc_output017[] __initconst = { + 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, + 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, + 0xb9 +}; +static const u8 enc_assoc017[] __initconst = { + 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 +}; +static const u8 enc_nonce017[] __initconst = { + 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, + 0x78, 0x2f, 0x44, 0x03 +}; +static const u8 enc_key017[] __initconst = { + 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, + 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, + 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, + 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee +}; + +/* wycheproof - misc */ +static const u8 enc_input018[] __initconst = { + 0x5c, 0x60 +}; +static const u8 enc_output018[] __initconst = { + 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, + 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, + 0xe5, 0xe2 +}; +static const u8 enc_assoc018[] __initconst = { }; +static const u8 enc_nonce018[] __initconst = { + 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, + 0x7e, 0x03, 0xf2, 0xdb +}; +static const u8 enc_key018[] __initconst = { + 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, + 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, + 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, + 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 +}; + +/* wycheproof - misc */ +static const u8 enc_input019[] __initconst = { + 0xdd, 0xf2 +}; +static const u8 enc_output019[] __initconst = { + 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, + 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, + 0x37, 0xa2 +}; +static const u8 enc_assoc019[] __initconst = { + 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf +}; +static const u8 enc_nonce019[] __initconst = { + 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, + 0xb6, 0x04, 0x0a, 0xc6 +}; +static const u8 enc_key019[] __initconst = { + 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, + 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, + 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, + 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 +}; + +/* wycheproof - misc */ +static const u8 enc_input020[] __initconst = { + 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 +}; +static const u8 enc_output020[] __initconst = { + 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, + 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, + 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 +}; +static const u8 enc_assoc020[] __initconst = { }; +static const u8 enc_nonce020[] __initconst = { + 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, + 0x6d, 0xc5, 0x5b, 0xb7 +}; +static const u8 enc_key020[] __initconst = { + 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, + 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, + 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, + 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 +}; + +/* wycheproof - misc */ +static const u8 enc_input021[] __initconst = { + 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 +}; +static const u8 enc_output021[] __initconst = { + 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, + 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, + 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba +}; +static const u8 enc_assoc021[] __initconst = { + 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 +}; +static const u8 enc_nonce021[] __initconst = { + 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, + 0xde, 0x94, 0x83, 0x96 +}; +static const u8 enc_key021[] __initconst = { + 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, + 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, + 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, + 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f +}; + +/* wycheproof - misc */ +static const u8 enc_input022[] __initconst = { + 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed +}; +static const u8 enc_output022[] __initconst = { + 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, + 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, + 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 +}; +static const u8 enc_assoc022[] __initconst = { }; +static const u8 enc_nonce022[] __initconst = { + 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, + 0x36, 0x18, 0x23, 0xd3 +}; +static const u8 enc_key022[] __initconst = { + 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, + 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, + 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, + 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 +}; + +/* wycheproof - misc */ +static const u8 enc_input023[] __initconst = { + 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf +}; +static const u8 enc_output023[] __initconst = { + 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, + 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, + 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 +}; +static const u8 enc_assoc023[] __initconst = { + 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d +}; +static const u8 enc_nonce023[] __initconst = { + 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, + 0x66, 0x48, 0x4a, 0x78 +}; +static const u8 enc_key023[] __initconst = { + 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, + 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, + 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, + 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd +}; + +/* wycheproof - misc */ +static const u8 enc_input024[] __initconst = { + 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, + 0x36, 0x8d, 0x14, 0xe0 +}; +static const u8 enc_output024[] __initconst = { + 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, + 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, + 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, + 0x6d, 0xcb, 0x29, 0xb4 +}; +static const u8 enc_assoc024[] __initconst = { }; +static const u8 enc_nonce024[] __initconst = { + 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, + 0x07, 0x53, 0x86, 0x56 +}; +static const u8 enc_key024[] __initconst = { + 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, + 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, + 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, + 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe +}; + +/* wycheproof - misc */ +static const u8 enc_input025[] __initconst = { + 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, + 0x59, 0xb1, 0xa0, 0xda +}; +static const u8 enc_output025[] __initconst = { + 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, + 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, + 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, + 0xcf, 0x05, 0x07, 0x2f +}; +static const u8 enc_assoc025[] __initconst = { + 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 +}; +static const u8 enc_nonce025[] __initconst = { + 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, + 0xd9, 0x06, 0xe9, 0xce +}; +static const u8 enc_key025[] __initconst = { + 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, + 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, + 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, + 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input026[] __initconst = { + 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, + 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 +}; +static const u8 enc_output026[] __initconst = { + 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, + 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, + 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, + 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 +}; +static const u8 enc_assoc026[] __initconst = { }; +static const u8 enc_nonce026[] __initconst = { + 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, + 0xd4, 0x61, 0xd2, 0x3c +}; +static const u8 enc_key026[] __initconst = { + 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, + 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, + 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, + 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb +}; + +/* wycheproof - misc */ +static const u8 enc_input027[] __initconst = { + 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, + 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 +}; +static const u8 enc_output027[] __initconst = { + 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, + 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, + 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, + 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 +}; +static const u8 enc_assoc027[] __initconst = { + 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 +}; +static const u8 enc_nonce027[] __initconst = { + 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, + 0xc5, 0x03, 0x5b, 0x6a +}; +static const u8 enc_key027[] __initconst = { + 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, + 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, + 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, + 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d +}; + +/* wycheproof - misc */ +static const u8 enc_input028[] __initconst = { + 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, + 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 +}; +static const u8 enc_output028[] __initconst = { + 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, + 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, + 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, + 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c +}; +static const u8 enc_assoc028[] __initconst = { }; +static const u8 enc_nonce028[] __initconst = { + 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, + 0x47, 0x40, 0xad, 0x9b +}; +static const u8 enc_key028[] __initconst = { + 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, + 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, + 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, + 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a +}; + +/* wycheproof - misc */ +static const u8 enc_input029[] __initconst = { + 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, + 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 +}; +static const u8 enc_output029[] __initconst = { + 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, + 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, + 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, + 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d +}; +static const u8 enc_assoc029[] __initconst = { + 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff +}; +static const u8 enc_nonce029[] __initconst = { + 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, + 0x07, 0x1f, 0x52, 0x66 +}; +static const u8 enc_key029[] __initconst = { + 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, + 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, + 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, + 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e +}; + +/* wycheproof - misc */ +static const u8 enc_input030[] __initconst = { + 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, + 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, + 0x1f +}; +static const u8 enc_output030[] __initconst = { + 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, + 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, + 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, + 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, + 0x13 +}; +static const u8 enc_assoc030[] __initconst = { }; +static const u8 enc_nonce030[] __initconst = { + 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, + 0xdc, 0x03, 0x44, 0x5d +}; +static const u8 enc_key030[] __initconst = { + 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, + 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, + 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, + 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 +}; + +/* wycheproof - misc */ +static const u8 enc_input031[] __initconst = { + 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, + 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, + 0x9c +}; +static const u8 enc_output031[] __initconst = { + 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, + 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, + 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, + 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, + 0xeb +}; +static const u8 enc_assoc031[] __initconst = { + 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 +}; +static const u8 enc_nonce031[] __initconst = { + 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, + 0x63, 0x3d, 0x99, 0x3d +}; +static const u8 enc_key031[] __initconst = { + 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, + 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, + 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, + 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 +}; + +/* wycheproof - misc */ +static const u8 enc_input032[] __initconst = { + 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, + 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, + 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 +}; +static const u8 enc_output032[] __initconst = { + 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, + 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, + 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, + 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, + 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a +}; +static const u8 enc_assoc032[] __initconst = { }; +static const u8 enc_nonce032[] __initconst = { + 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, + 0x2c, 0x65, 0xf3, 0x1b +}; +static const u8 enc_key032[] __initconst = { + 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, + 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, + 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, + 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 +}; + +/* wycheproof - misc */ +static const u8 enc_input033[] __initconst = { + 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, + 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, + 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 +}; +static const u8 enc_output033[] __initconst = { + 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, + 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, + 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, + 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, + 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 +}; +static const u8 enc_assoc033[] __initconst = { + 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 +}; +static const u8 enc_nonce033[] __initconst = { + 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, + 0x78, 0x34, 0xed, 0x55 +}; +static const u8 enc_key033[] __initconst = { + 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, + 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, + 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, + 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 +}; + +/* wycheproof - misc */ +static const u8 enc_input034[] __initconst = { + 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, + 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, + 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, + 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 +}; +static const u8 enc_output034[] __initconst = { + 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, + 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, + 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, + 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, + 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, + 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 +}; +static const u8 enc_assoc034[] __initconst = { }; +static const u8 enc_nonce034[] __initconst = { + 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, + 0x37, 0x1a, 0x6f, 0xd2 +}; +static const u8 enc_key034[] __initconst = { + 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, + 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, + 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, + 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 +}; + +/* wycheproof - misc */ +static const u8 enc_input035[] __initconst = { + 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, + 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, + 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, + 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb +}; +static const u8 enc_output035[] __initconst = { + 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, + 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, + 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, + 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, + 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, + 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f +}; +static const u8 enc_assoc035[] __initconst = { + 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 +}; +static const u8 enc_nonce035[] __initconst = { + 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, + 0xb8, 0x1a, 0x1f, 0x8b +}; +static const u8 enc_key035[] __initconst = { + 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, + 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, + 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, + 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 +}; + +/* wycheproof - misc */ +static const u8 enc_input036[] __initconst = { + 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, + 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, + 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, + 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 +}; +static const u8 enc_output036[] __initconst = { + 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, + 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, + 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, + 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, + 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, + 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea +}; +static const u8 enc_assoc036[] __initconst = { }; +static const u8 enc_nonce036[] __initconst = { + 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, + 0x2b, 0x7e, 0x6e, 0x6a +}; +static const u8 enc_key036[] __initconst = { + 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, + 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, + 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, + 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 +}; + +/* wycheproof - misc */ +static const u8 enc_input037[] __initconst = { + 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, + 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, + 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, + 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e +}; +static const u8 enc_output037[] __initconst = { + 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, + 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, + 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, + 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, + 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, + 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 +}; +static const u8 enc_assoc037[] __initconst = { + 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 +}; +static const u8 enc_nonce037[] __initconst = { + 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, + 0x34, 0x0d, 0xd1, 0xb8 +}; +static const u8 enc_key037[] __initconst = { + 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, + 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, + 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, + 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 +}; + +/* wycheproof - misc */ +static const u8 enc_input038[] __initconst = { + 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, + 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, + 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, + 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, + 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, + 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a +}; +static const u8 enc_output038[] __initconst = { + 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, + 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, + 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, + 0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, + 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24, + 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, + 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, + 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 +}; +static const u8 enc_assoc038[] __initconst = { }; +static const u8 enc_nonce038[] __initconst = { + 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, + 0x0b, 0xef, 0x55, 0xd2 +}; +static const u8 enc_key038[] __initconst = { + 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, + 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, + 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, + 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 +}; + +/* wycheproof - misc */ +static const u8 enc_input039[] __initconst = { + 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, + 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, + 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, + 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, + 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, + 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 +}; +static const u8 enc_output039[] __initconst = { + 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, + 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, + 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, + 0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, + 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3, + 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, + 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, + 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 +}; +static const u8 enc_assoc039[] __initconst = { + 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 +}; +static const u8 enc_nonce039[] __initconst = { + 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, + 0x31, 0xcd, 0x4d, 0x95 +}; +static const u8 enc_key039[] __initconst = { + 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, + 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, + 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, + 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac +}; + +/* wycheproof - misc */ +static const u8 enc_input040[] __initconst = { + 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, + 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, + 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, + 0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, + 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88, + 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, + 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, + 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 +}; +static const u8 enc_output040[] __initconst = { + 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, + 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, + 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, + 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, + 0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc, + 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, + 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, + 0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50, + 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, + 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 +}; +static const u8 enc_assoc040[] __initconst = { }; +static const u8 enc_nonce040[] __initconst = { + 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, + 0x23, 0xcc, 0x06, 0x5b +}; +static const u8 enc_key040[] __initconst = { + 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, + 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, + 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, + 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input041[] __initconst = { + 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, + 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, + 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, + 0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, + 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec, + 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, + 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, + 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 +}; +static const u8 enc_output041[] __initconst = { + 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, + 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, + 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, + 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, + 0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84, + 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, + 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, + 0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e, + 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, + 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec +}; +static const u8 enc_assoc041[] __initconst = { + 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 +}; +static const u8 enc_nonce041[] __initconst = { + 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, + 0xe4, 0xeb, 0xb1, 0x9c +}; +static const u8 enc_key041[] __initconst = { + 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, + 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, + 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, + 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e +}; + +/* wycheproof - misc */ +static const u8 enc_input042[] __initconst = { + 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, + 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, + 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, + 0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64, + 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, + 0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, + 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, + 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, + 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6, + 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, + 0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4, + 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, + 0x92 +}; +static const u8 enc_output042[] __initconst = { + 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, + 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, + 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, + 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, + 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45, + 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, + 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, + 0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, + 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, + 0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a, + 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, + 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3, + 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, + 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, + 0xb0 +}; +static const u8 enc_assoc042[] __initconst = { }; +static const u8 enc_nonce042[] __initconst = { + 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, + 0xac, 0xde, 0x27, 0x99 +}; +static const u8 enc_key042[] __initconst = { + 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, + 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, + 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, + 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input043[] __initconst = { + 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, + 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, + 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, + 0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5, + 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, + 0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, + 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, + 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, + 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3, + 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, + 0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92, + 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, + 0x76 +}; +static const u8 enc_output043[] __initconst = { + 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, + 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, + 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, + 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, + 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37, + 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, + 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, + 0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, + 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, + 0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07, + 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, + 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54, + 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, + 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, + 0xf6 +}; +static const u8 enc_assoc043[] __initconst = { + 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 +}; +static const u8 enc_nonce043[] __initconst = { + 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, + 0xe5, 0x22, 0x94, 0x60 +}; +static const u8 enc_key043[] __initconst = { + 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, + 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, + 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, + 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e +}; + +/* wycheproof - misc */ +static const u8 enc_input044[] __initconst = { + 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, + 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 +}; +static const u8 enc_output044[] __initconst = { + 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, + 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, + 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, + 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 +}; +static const u8 enc_assoc044[] __initconst = { + 0x02 +}; +static const u8 enc_nonce044[] __initconst = { + 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, + 0x02, 0xd5, 0x06, 0x56 +}; +static const u8 enc_key044[] __initconst = { + 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, + 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, + 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, + 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c +}; + +/* wycheproof - misc */ +static const u8 enc_input045[] __initconst = { + 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, + 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 +}; +static const u8 enc_output045[] __initconst = { + 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, + 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, + 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, + 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 +}; +static const u8 enc_assoc045[] __initconst = { + 0xb6, 0x48 +}; +static const u8 enc_nonce045[] __initconst = { + 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, + 0x5b, 0x3a, 0xa7, 0x13 +}; +static const u8 enc_key045[] __initconst = { + 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, + 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, + 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, + 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc +}; + +/* wycheproof - misc */ +static const u8 enc_input046[] __initconst = { + 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, + 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 +}; +static const u8 enc_output046[] __initconst = { + 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, + 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, + 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, + 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 +}; +static const u8 enc_assoc046[] __initconst = { + 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, + 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 +}; +static const u8 enc_nonce046[] __initconst = { + 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, + 0xa4, 0x65, 0x96, 0xdf +}; +static const u8 enc_key046[] __initconst = { + 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, + 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, + 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, + 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f +}; + +/* wycheproof - misc */ +static const u8 enc_input047[] __initconst = { + 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, + 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 +}; +static const u8 enc_output047[] __initconst = { + 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, + 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, + 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, + 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f +}; +static const u8 enc_assoc047[] __initconst = { + 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, + 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b +}; +static const u8 enc_nonce047[] __initconst = { + 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, + 0x80, 0x92, 0x66, 0xd9 +}; +static const u8 enc_key047[] __initconst = { + 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, + 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, + 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, + 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 +}; + +/* wycheproof - misc */ +static const u8 enc_input048[] __initconst = { + 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, + 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 +}; +static const u8 enc_output048[] __initconst = { + 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, + 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, + 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, + 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 +}; +static const u8 enc_assoc048[] __initconst = { + 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, + 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, + 0x0e +}; +static const u8 enc_nonce048[] __initconst = { + 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, + 0xfc, 0x10, 0x68, 0xc3 +}; +static const u8 enc_key048[] __initconst = { + 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, + 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, + 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, + 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input049[] __initconst = { + 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, + 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 +}; +static const u8 enc_output049[] __initconst = { + 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, + 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, + 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, + 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 +}; +static const u8 enc_assoc049[] __initconst = { + 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, + 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, + 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, + 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda +}; +static const u8 enc_nonce049[] __initconst = { + 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, + 0xad, 0x14, 0xd5, 0x3e +}; +static const u8 enc_key049[] __initconst = { + 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, + 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, + 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, + 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 +}; + +/* wycheproof - misc */ +static const u8 enc_input050[] __initconst = { + 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, + 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c +}; +static const u8 enc_output050[] __initconst = { + 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, + 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, + 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, + 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 +}; +static const u8 enc_assoc050[] __initconst = { + 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, + 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, + 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, + 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 +}; +static const u8 enc_nonce050[] __initconst = { + 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, + 0x6a, 0x7b, 0x76, 0x51 +}; +static const u8 enc_key050[] __initconst = { + 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, + 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, + 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, + 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 +}; + +/* wycheproof - misc */ +static const u8 enc_input051[] __initconst = { + 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, + 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 +}; +static const u8 enc_output051[] __initconst = { + 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, + 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, + 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, + 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 +}; +static const u8 enc_assoc051[] __initconst = { + 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, + 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, + 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, + 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, + 0x73 +}; +static const u8 enc_nonce051[] __initconst = { + 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, + 0xa8, 0xfa, 0x89, 0x49 +}; +static const u8 enc_key051[] __initconst = { + 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, + 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, + 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, + 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 +}; + +/* wycheproof - misc */ +static const u8 enc_input052[] __initconst = { + 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, + 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed +}; +static const u8 enc_output052[] __initconst = { + 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, + 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, + 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, + 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 +}; +static const u8 enc_assoc052[] __initconst = { + 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, + 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, + 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, + 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, + 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, + 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 +}; +static const u8 enc_nonce052[] __initconst = { + 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, + 0x0e, 0xc3, 0x5d, 0x12 +}; +static const u8 enc_key052[] __initconst = { + 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, + 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, + 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, + 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 +}; + +/* wycheproof - misc */ +static const u8 enc_input053[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe +}; +static const u8 enc_output053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe6, 0xd3, 0xd7, 0x32, 0x4a, 0x1c, 0xbb, 0xa7, + 0x77, 0xbb, 0xb0, 0xec, 0xdd, 0xa3, 0x78, 0x07 +}; +static const u8 enc_assoc053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce053[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key053[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input054[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe, + 0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, 0xfe, + 0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b, + 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, + 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd +}; +static const u8 enc_output054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x06, 0x2d, 0xe6, 0x79, 0x5f, 0x27, 0x4f, 0xd2, + 0xa3, 0x05, 0xd7, 0x69, 0x80, 0xbc, 0x9c, 0xce +}; +static const u8 enc_assoc054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce054[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key054[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input055[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, + 0x55, 0xd3, 0x04, 0x84, 0x64, 0x43, 0xfe, 0xe8, + 0xdf, 0x99, 0x47, 0x03, 0x03, 0xfb, 0x3b, 0x7b, + 0x80, 0xe0, 0x30, 0xbe, 0xeb, 0xd3, 0x29, 0xbe, + 0xe3, 0xbc, 0xdb, 0x5b, 0x1e, 0xde, 0xfc, 0xfe, + 0x8b, 0xcd, 0xa1, 0xb6, 0xa1, 0x5c, 0x8c, 0x2b, + 0x08, 0x69, 0xff, 0xd2, 0xec, 0x5e, 0x26, 0xe5, + 0x53, 0xb7, 0xb2, 0x27, 0xfe, 0x87, 0xfd, 0xbd, + 0x7a, 0xda, 0x44, 0x42, 0x42, 0x69, 0xbf, 0xfa, + 0x55, 0x27, 0xf2, 0x70, 0xac, 0xf6, 0x85, 0x02, + 0xb7, 0x4c, 0x5a, 0xe2, 0xe6, 0x0c, 0x05, 0x80, + 0x98, 0x1a, 0x49, 0x38, 0x45, 0x93, 0x92, 0xc4, + 0x9b, 0xb2, 0xf2, 0x84, 0xb6, 0x46, 0xef, 0xc7, + 0xf3, 0xf0, 0xb1, 0x36, 0x1d, 0xc3, 0x48, 0xed, + 0x77, 0xd3, 0x0b, 0xc5, 0x76, 0x92, 0xed, 0x38, + 0xfb, 0xac, 0x01, 0x88, 0x38, 0x04, 0x88, 0xc7 +}; +static const u8 enc_output055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0xb4, 0x79, 0x02, 0xba, 0xae, 0xaf, 0xb3, + 0x42, 0x03, 0x05, 0x15, 0x29, 0xaf, 0x28, 0x2e +}; +static const u8 enc_assoc055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce055[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key055[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input056[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output056[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb3, 0x89, 0x1c, 0x84, 0x9c, 0xb5, 0x2c, 0x27, + 0x74, 0x7e, 0xdf, 0xcf, 0x31, 0x21, 0x3b, 0xb6 +}; +static const u8 enc_assoc056[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce056[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key056[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input057[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41, + 0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, 0x01, + 0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4, + 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, + 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42 +}; +static const u8 enc_output057[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf0, 0xc1, 0x2d, 0x26, 0xef, 0x03, 0x02, 0x9b, + 0x62, 0xc0, 0x08, 0xda, 0x27, 0xc5, 0xdc, 0x68 +}; +static const u8 enc_assoc057[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce057[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key057[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input058[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x7f, 0x6b, 0xe8, 0x7c, + 0xaa, 0x2c, 0xfb, 0x7b, 0x9b, 0xbc, 0x01, 0x17, + 0x20, 0x66, 0xb8, 0xfc, 0xfc, 0x04, 0xc4, 0x84, + 0x7f, 0x1f, 0xcf, 0x41, 0x14, 0x2c, 0xd6, 0x41, + 0x1c, 0x43, 0x24, 0xa4, 0xe1, 0x21, 0x03, 0x01, + 0x74, 0x32, 0x5e, 0x49, 0x5e, 0xa3, 0x73, 0xd4, + 0xf7, 0x96, 0x00, 0x2d, 0x13, 0xa1, 0xd9, 0x1a, + 0xac, 0x48, 0x4d, 0xd8, 0x01, 0x78, 0x02, 0x42, + 0x85, 0x25, 0xbb, 0xbd, 0xbd, 0x96, 0x40, 0x05, + 0xaa, 0xd8, 0x0d, 0x8f, 0x53, 0x09, 0x7a, 0xfd, + 0x48, 0xb3, 0xa5, 0x1d, 0x19, 0xf3, 0xfa, 0x7f, + 0x67, 0xe5, 0xb6, 0xc7, 0xba, 0x6c, 0x6d, 0x3b, + 0x64, 0x4d, 0x0d, 0x7b, 0x49, 0xb9, 0x10, 0x38, + 0x0c, 0x0f, 0x4e, 0xc9, 0xe2, 0x3c, 0xb7, 0x12, + 0x88, 0x2c, 0xf4, 0x3a, 0x89, 0x6d, 0x12, 0xc7, + 0x04, 0x53, 0xfe, 0x77, 0xc7, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output058[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xee, 0x65, 0x78, 0x30, 0x01, 0xc2, 0x56, 0x91, + 0xfa, 0x28, 0xd0, 0xf5, 0xf1, 0xc1, 0xd7, 0x62 +}; +static const u8 enc_assoc058[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce058[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key058[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input059[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e +}; +static const u8 enc_output059[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x79, 0xba, 0x7a, 0x29, 0xf5, 0xa7, 0xbb, 0x75, + 0x79, 0x7a, 0xf8, 0x7a, 0x61, 0x01, 0x29, 0xa4 +}; +static const u8 enc_assoc059[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce059[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key059[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input060[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e, + 0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, 0x7e, + 0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab, + 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, + 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d +}; +static const u8 enc_output060[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x36, 0xb1, 0x74, 0x38, 0x19, 0xe1, 0xb9, 0xba, + 0x15, 0x51, 0xe8, 0xed, 0x92, 0x2a, 0x95, 0x9a +}; +static const u8 enc_assoc060[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce060[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key060[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input061[] __initconst = { + 0x25, 0x6d, 0x40, 0x08, 0x80, 0x94, 0x17, 0x03, + 0x55, 0xd3, 0x04, 0x04, 0x64, 0x43, 0xfe, 0x68, + 0xdf, 0x99, 0x47, 0x83, 0x03, 0xfb, 0x3b, 0xfb, + 0x80, 0xe0, 0x30, 0x3e, 0xeb, 0xd3, 0x29, 0x3e, + 0xe3, 0xbc, 0xdb, 0xdb, 0x1e, 0xde, 0xfc, 0x7e, + 0x8b, 0xcd, 0xa1, 0x36, 0xa1, 0x5c, 0x8c, 0xab, + 0x08, 0x69, 0xff, 0x52, 0xec, 0x5e, 0x26, 0x65, + 0x53, 0xb7, 0xb2, 0xa7, 0xfe, 0x87, 0xfd, 0x3d, + 0x7a, 0xda, 0x44, 0xc2, 0x42, 0x69, 0xbf, 0x7a, + 0x55, 0x27, 0xf2, 0xf0, 0xac, 0xf6, 0x85, 0x82, + 0xb7, 0x4c, 0x5a, 0x62, 0xe6, 0x0c, 0x05, 0x00, + 0x98, 0x1a, 0x49, 0xb8, 0x45, 0x93, 0x92, 0x44, + 0x9b, 0xb2, 0xf2, 0x04, 0xb6, 0x46, 0xef, 0x47, + 0xf3, 0xf0, 0xb1, 0xb6, 0x1d, 0xc3, 0x48, 0x6d, + 0x77, 0xd3, 0x0b, 0x45, 0x76, 0x92, 0xed, 0xb8, + 0xfb, 0xac, 0x01, 0x08, 0x38, 0x04, 0x88, 0x47 +}; +static const u8 enc_output061[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0xfe, 0xac, 0x49, 0x55, 0x55, 0x4e, 0x80, 0x6f, + 0x3a, 0x19, 0x02, 0xe2, 0x44, 0x32, 0xc0, 0x8a +}; +static const u8 enc_assoc061[] __initconst = { + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_nonce061[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key061[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input062[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1 +}; +static const u8 enc_output062[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0x20, 0xa3, 0x79, 0x8d, 0xf1, 0x29, 0x2c, 0x59, + 0x72, 0xbf, 0x97, 0x41, 0xae, 0xc3, 0x8a, 0x19 +}; +static const u8 enc_assoc062[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce062[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key062[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input063[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1, + 0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, 0x81, + 0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54, + 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, + 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2 +}; +static const u8 enc_output063[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xc0, 0x3d, 0x9f, 0x67, 0x35, 0x4a, 0x97, 0xb2, + 0xf0, 0x74, 0xf7, 0x55, 0x15, 0x57, 0xe4, 0x9c +}; +static const u8 enc_assoc063[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce063[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key063[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input064[] __initconst = { + 0xda, 0x92, 0xbf, 0xf7, 0x7f, 0x6b, 0xe8, 0xfc, + 0xaa, 0x2c, 0xfb, 0xfb, 0x9b, 0xbc, 0x01, 0x97, + 0x20, 0x66, 0xb8, 0x7c, 0xfc, 0x04, 0xc4, 0x04, + 0x7f, 0x1f, 0xcf, 0xc1, 0x14, 0x2c, 0xd6, 0xc1, + 0x1c, 0x43, 0x24, 0x24, 0xe1, 0x21, 0x03, 0x81, + 0x74, 0x32, 0x5e, 0xc9, 0x5e, 0xa3, 0x73, 0x54, + 0xf7, 0x96, 0x00, 0xad, 0x13, 0xa1, 0xd9, 0x9a, + 0xac, 0x48, 0x4d, 0x58, 0x01, 0x78, 0x02, 0xc2, + 0x85, 0x25, 0xbb, 0x3d, 0xbd, 0x96, 0x40, 0x85, + 0xaa, 0xd8, 0x0d, 0x0f, 0x53, 0x09, 0x7a, 0x7d, + 0x48, 0xb3, 0xa5, 0x9d, 0x19, 0xf3, 0xfa, 0xff, + 0x67, 0xe5, 0xb6, 0x47, 0xba, 0x6c, 0x6d, 0xbb, + 0x64, 0x4d, 0x0d, 0xfb, 0x49, 0xb9, 0x10, 0xb8, + 0x0c, 0x0f, 0x4e, 0x49, 0xe2, 0x3c, 0xb7, 0x92, + 0x88, 0x2c, 0xf4, 0xba, 0x89, 0x6d, 0x12, 0x47, + 0x04, 0x53, 0xfe, 0xf7, 0xc7, 0xfb, 0x77, 0xb8 +}; +static const u8 enc_output064[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xc8, 0x6d, 0xa8, 0xdd, 0x65, 0x22, 0x86, 0xd5, + 0x02, 0x13, 0xd3, 0x28, 0xd6, 0x3e, 0x40, 0x06 +}; +static const u8 enc_assoc064[] __initconst = { + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_nonce064[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key064[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input065[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output065[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0xbe, 0xde, 0x90, 0x83, 0xce, 0xb3, 0x6d, 0xdf, + 0xe5, 0xfa, 0x81, 0x1f, 0x95, 0x47, 0x1c, 0x67 +}; +static const u8 enc_assoc065[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce065[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key065[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input066[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41, + 0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, 0x01, + 0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4, + 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, + 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42 +}; +static const u8 enc_output066[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x30, 0x08, 0x74, 0xbb, 0x06, 0x92, 0xb6, 0x89, + 0xde, 0xad, 0x9a, 0xe1, 0x5b, 0x06, 0x73, 0x90 +}; +static const u8 enc_assoc066[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce066[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key066[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input067[] __initconst = { + 0x5a, 0x92, 0xbf, 0x77, 0xff, 0x6b, 0xe8, 0x7c, + 0x2a, 0x2c, 0xfb, 0x7b, 0x1b, 0xbc, 0x01, 0x17, + 0xa0, 0x66, 0xb8, 0xfc, 0x7c, 0x04, 0xc4, 0x84, + 0xff, 0x1f, 0xcf, 0x41, 0x94, 0x2c, 0xd6, 0x41, + 0x9c, 0x43, 0x24, 0xa4, 0x61, 0x21, 0x03, 0x01, + 0xf4, 0x32, 0x5e, 0x49, 0xde, 0xa3, 0x73, 0xd4, + 0x77, 0x96, 0x00, 0x2d, 0x93, 0xa1, 0xd9, 0x1a, + 0x2c, 0x48, 0x4d, 0xd8, 0x81, 0x78, 0x02, 0x42, + 0x05, 0x25, 0xbb, 0xbd, 0x3d, 0x96, 0x40, 0x05, + 0x2a, 0xd8, 0x0d, 0x8f, 0xd3, 0x09, 0x7a, 0xfd, + 0xc8, 0xb3, 0xa5, 0x1d, 0x99, 0xf3, 0xfa, 0x7f, + 0xe7, 0xe5, 0xb6, 0xc7, 0x3a, 0x6c, 0x6d, 0x3b, + 0xe4, 0x4d, 0x0d, 0x7b, 0xc9, 0xb9, 0x10, 0x38, + 0x8c, 0x0f, 0x4e, 0xc9, 0x62, 0x3c, 0xb7, 0x12, + 0x08, 0x2c, 0xf4, 0x3a, 0x09, 0x6d, 0x12, 0xc7, + 0x84, 0x53, 0xfe, 0x77, 0x47, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output067[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x99, 0xca, 0xd8, 0x5f, 0x45, 0xca, 0x40, 0x94, + 0x2d, 0x0d, 0x4d, 0x5e, 0x95, 0x0a, 0xde, 0x22 +}; +static const u8 enc_assoc067[] __initconst = { + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, + 0x7f, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce067[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key067[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input068[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41 +}; +static const u8 enc_output068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x8b, 0xbe, 0x14, 0x52, 0x72, 0xe7, 0xc2, 0xd9, + 0xa1, 0x89, 0x1a, 0x3a, 0xb0, 0x98, 0x3d, 0x9d +}; +static const u8 enc_assoc068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce068[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key068[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input069[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41, + 0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, 0x01, + 0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4, + 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, + 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42 +}; +static const u8 enc_output069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x41, 0x86, 0x19, 0x13, 0xa8, 0xf6, 0xde, + 0x7f, 0x61, 0xe2, 0x25, 0x63, 0x1b, 0xc3, 0x82 +}; +static const u8 enc_assoc069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce069[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key069[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input070[] __initconst = { + 0x25, 0x6d, 0x40, 0x88, 0x7f, 0x6b, 0xe8, 0x7c, + 0x55, 0xd3, 0x04, 0x84, 0x9b, 0xbc, 0x01, 0x17, + 0xdf, 0x99, 0x47, 0x03, 0xfc, 0x04, 0xc4, 0x84, + 0x80, 0xe0, 0x30, 0xbe, 0x14, 0x2c, 0xd6, 0x41, + 0xe3, 0xbc, 0xdb, 0x5b, 0xe1, 0x21, 0x03, 0x01, + 0x8b, 0xcd, 0xa1, 0xb6, 0x5e, 0xa3, 0x73, 0xd4, + 0x08, 0x69, 0xff, 0xd2, 0x13, 0xa1, 0xd9, 0x1a, + 0x53, 0xb7, 0xb2, 0x27, 0x01, 0x78, 0x02, 0x42, + 0x7a, 0xda, 0x44, 0x42, 0xbd, 0x96, 0x40, 0x05, + 0x55, 0x27, 0xf2, 0x70, 0x53, 0x09, 0x7a, 0xfd, + 0xb7, 0x4c, 0x5a, 0xe2, 0x19, 0xf3, 0xfa, 0x7f, + 0x98, 0x1a, 0x49, 0x38, 0xba, 0x6c, 0x6d, 0x3b, + 0x9b, 0xb2, 0xf2, 0x84, 0x49, 0xb9, 0x10, 0x38, + 0xf3, 0xf0, 0xb1, 0x36, 0xe2, 0x3c, 0xb7, 0x12, + 0x77, 0xd3, 0x0b, 0xc5, 0x89, 0x6d, 0x12, 0xc7, + 0xfb, 0xac, 0x01, 0x88, 0xc7, 0xfb, 0x77, 0x38 +}; +static const u8 enc_output070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x84, 0x28, 0xbc, 0xf0, 0x23, 0xec, 0x6b, 0xf3, + 0x1f, 0xd9, 0xef, 0xb2, 0x03, 0xff, 0x08, 0x71 +}; +static const u8 enc_assoc070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce070[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key070[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input071[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe +}; +static const u8 enc_output071[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x13, 0x9f, 0xdf, 0x64, 0x74, 0xea, 0x24, 0xf5, + 0x49, 0xb0, 0x75, 0x82, 0x5f, 0x2c, 0x76, 0x20 +}; +static const u8 enc_assoc071[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce071[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key071[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input072[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe, + 0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, 0xfe, + 0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b, + 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, + 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd +}; +static const u8 enc_output072[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xbb, 0xad, 0x8d, 0x86, 0x3b, 0x83, 0x5a, 0x8e, + 0x86, 0x64, 0xfd, 0x1d, 0x45, 0x66, 0xb6, 0xb4 +}; +static const u8 enc_assoc072[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce072[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key072[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input073[] __initconst = { + 0xda, 0x92, 0xbf, 0x77, 0x80, 0x94, 0x17, 0x83, + 0xaa, 0x2c, 0xfb, 0x7b, 0x64, 0x43, 0xfe, 0xe8, + 0x20, 0x66, 0xb8, 0xfc, 0x03, 0xfb, 0x3b, 0x7b, + 0x7f, 0x1f, 0xcf, 0x41, 0xeb, 0xd3, 0x29, 0xbe, + 0x1c, 0x43, 0x24, 0xa4, 0x1e, 0xde, 0xfc, 0xfe, + 0x74, 0x32, 0x5e, 0x49, 0xa1, 0x5c, 0x8c, 0x2b, + 0xf7, 0x96, 0x00, 0x2d, 0xec, 0x5e, 0x26, 0xe5, + 0xac, 0x48, 0x4d, 0xd8, 0xfe, 0x87, 0xfd, 0xbd, + 0x85, 0x25, 0xbb, 0xbd, 0x42, 0x69, 0xbf, 0xfa, + 0xaa, 0xd8, 0x0d, 0x8f, 0xac, 0xf6, 0x85, 0x02, + 0x48, 0xb3, 0xa5, 0x1d, 0xe6, 0x0c, 0x05, 0x80, + 0x67, 0xe5, 0xb6, 0xc7, 0x45, 0x93, 0x92, 0xc4, + 0x64, 0x4d, 0x0d, 0x7b, 0xb6, 0x46, 0xef, 0xc7, + 0x0c, 0x0f, 0x4e, 0xc9, 0x1d, 0xc3, 0x48, 0xed, + 0x88, 0x2c, 0xf4, 0x3a, 0x76, 0x92, 0xed, 0x38, + 0x04, 0x53, 0xfe, 0x77, 0x38, 0x04, 0x88, 0xc7 +}; +static const u8 enc_output073[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x42, 0xf2, 0x35, 0x42, 0x97, 0x84, 0x9a, 0x51, + 0x1d, 0x53, 0xe5, 0x57, 0x17, 0x72, 0xf7, 0x1f +}; +static const u8 enc_assoc073[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_nonce073[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0xee, 0x32, 0x00 +}; +static const u8 enc_key073[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input074[] __initconst = { + 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, + 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, + 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, + 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, + 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0, + 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, + 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, + 0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, + 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, + 0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49, + 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, + 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58, + 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, + 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, + 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, + 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a +}; +static const u8 enc_output074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, + 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca +}; +static const u8 enc_assoc074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x02, 0x50, 0x6e +}; +static const u8 enc_key074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input075[] __initconst = { + 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, + 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, + 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, + 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, + 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f, + 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, + 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, + 0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, + 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, + 0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8, + 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, + 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b, + 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, + 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, + 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, + 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 +}; +static const u8 enc_output075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, + 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 +}; +static const u8 enc_assoc075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x03, 0x18, 0xa5 +}; +static const u8 enc_key075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input076[] __initconst = { + 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, + 0xde, 0x22, 0xff, 0x5b, 0x8a, 0xdd, 0x95, 0x02, + 0xce, 0x03, 0xa0, 0xfa, 0xf5, 0x99, 0x2a, 0x09, + 0x52, 0x2c, 0xdd, 0x12, 0x06, 0xd2, 0x20, 0xb8, + 0xf8, 0xbd, 0x07, 0xd1, 0xf1, 0xf5, 0xa1, 0xbd, + 0x9a, 0x71, 0xd1, 0x1c, 0x7f, 0x57, 0x9b, 0x85, + 0x58, 0x18, 0xc0, 0x8d, 0x4d, 0xe0, 0x36, 0x39, + 0x31, 0x83, 0xb7, 0xf5, 0x90, 0xb3, 0x35, 0xae, + 0xd8, 0xde, 0x5b, 0x57, 0xb1, 0x3c, 0x5f, 0xed, + 0xe2, 0x44, 0x1c, 0x3e, 0x18, 0x4a, 0xa9, 0xd4, + 0x6e, 0x61, 0x59, 0x85, 0x06, 0xb3, 0xe1, 0x1c, + 0x43, 0xc6, 0x2c, 0xbc, 0xac, 0xec, 0xed, 0x33, + 0x19, 0x08, 0x75, 0xb0, 0x12, 0x21, 0x8b, 0x19, + 0x30, 0xfb, 0x7c, 0x38, 0xec, 0x45, 0xac, 0x11, + 0xc3, 0x53, 0xd0, 0xcf, 0x93, 0x8d, 0xcc, 0xb9, + 0xef, 0xad, 0x8f, 0xed, 0xbe, 0x46, 0xda, 0xa5 +}; +static const u8 enc_output076[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4b, 0x0b, 0xda, 0x8a, 0xd0, 0x43, 0x83, 0x0d, + 0x83, 0x19, 0xab, 0x82, 0xc5, 0x0c, 0x76, 0x63 +}; +static const u8 enc_assoc076[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce076[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb4, 0xf0 +}; +static const u8 enc_key076[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input077[] __initconst = { + 0x86, 0xcb, 0xac, 0xae, 0x4d, 0x3f, 0x74, 0xae, + 0x01, 0x21, 0x3e, 0x05, 0x51, 0xcc, 0x15, 0x16, + 0x0e, 0xa1, 0xbe, 0x84, 0x08, 0xe3, 0xd5, 0xd7, + 0x4f, 0x01, 0x46, 0x49, 0x95, 0xa6, 0x9e, 0x61, + 0x76, 0xcb, 0x9e, 0x02, 0xb2, 0x24, 0x7e, 0xd2, + 0x99, 0x89, 0x2f, 0x91, 0x82, 0xa4, 0x5c, 0xaf, + 0x4c, 0x69, 0x40, 0x56, 0x11, 0x76, 0x6e, 0xdf, + 0xaf, 0xdc, 0x28, 0x55, 0x19, 0xea, 0x30, 0x48, + 0x0c, 0x44, 0xf0, 0x5e, 0x78, 0x1e, 0xac, 0xf8, + 0xfc, 0xec, 0xc7, 0x09, 0x0a, 0xbb, 0x28, 0xfa, + 0x5f, 0xd5, 0x85, 0xac, 0x8c, 0xda, 0x7e, 0x87, + 0x72, 0xe5, 0x94, 0xe4, 0xce, 0x6c, 0x88, 0x32, + 0x81, 0x93, 0x2e, 0x0f, 0x89, 0xf8, 0x77, 0xa1, + 0xf0, 0x4d, 0x9c, 0x32, 0xb0, 0x6c, 0xf9, 0x0b, + 0x0e, 0x76, 0x2b, 0x43, 0x0c, 0x4d, 0x51, 0x7c, + 0x97, 0x10, 0x70, 0x68, 0xf4, 0x98, 0xef, 0x7f +}; +static const u8 enc_output077[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4b, 0xc9, 0x8f, 0x72, 0xc4, 0x94, 0xc2, 0xa4, + 0x3c, 0x2b, 0x15, 0xa1, 0x04, 0x3f, 0x1c, 0xfa +}; +static const u8 enc_assoc077[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce077[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xfb, 0x66 +}; +static const u8 enc_key077[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input078[] __initconst = { + 0xfa, 0xb1, 0xcd, 0xdf, 0x4f, 0xe1, 0x98, 0xef, + 0x63, 0xad, 0xd8, 0x81, 0xd6, 0xea, 0xd6, 0xc5, + 0x76, 0x37, 0xbb, 0xe9, 0x20, 0x18, 0xca, 0x7c, + 0x0b, 0x96, 0xfb, 0xa0, 0x87, 0x1e, 0x93, 0x2d, + 0xb1, 0xfb, 0xf9, 0x07, 0x61, 0xbe, 0x25, 0xdf, + 0x8d, 0xfa, 0xf9, 0x31, 0xce, 0x57, 0x57, 0xe6, + 0x17, 0xb3, 0xd7, 0xa9, 0xf0, 0xbf, 0x0f, 0xfe, + 0x5d, 0x59, 0x1a, 0x33, 0xc1, 0x43, 0xb8, 0xf5, + 0x3f, 0xd0, 0xb5, 0xa1, 0x96, 0x09, 0xfd, 0x62, + 0xe5, 0xc2, 0x51, 0xa4, 0x28, 0x1a, 0x20, 0x0c, + 0xfd, 0xc3, 0x4f, 0x28, 0x17, 0x10, 0x40, 0x6f, + 0x4e, 0x37, 0x62, 0x54, 0x46, 0xff, 0x6e, 0xf2, + 0x24, 0x91, 0x3d, 0xeb, 0x0d, 0x89, 0xaf, 0x33, + 0x71, 0x28, 0xe3, 0xd1, 0x55, 0xd1, 0x6d, 0x3e, + 0xc3, 0x24, 0x60, 0x41, 0x43, 0x21, 0x43, 0xe9, + 0xab, 0x3a, 0x6d, 0x2c, 0xcc, 0x2f, 0x4d, 0x62 +}; +static const u8 enc_output078[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf7, 0xe9, 0xe1, 0x51, 0xb0, 0x25, 0x33, 0xc7, + 0x46, 0x58, 0xbf, 0xc7, 0x73, 0x7c, 0x68, 0x0d +}; +static const u8 enc_assoc078[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce078[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x38, 0xbb, 0x90 +}; +static const u8 enc_key078[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input079[] __initconst = { + 0x22, 0x72, 0x02, 0xbe, 0x7f, 0x35, 0x15, 0xe9, + 0xd1, 0xc0, 0x2e, 0xea, 0x2f, 0x19, 0x50, 0xb6, + 0x48, 0x1b, 0x04, 0x8a, 0x4c, 0x91, 0x50, 0x6c, + 0xb4, 0x0d, 0x50, 0x4e, 0x6c, 0x94, 0x9f, 0x82, + 0xd1, 0x97, 0xc2, 0x5a, 0xd1, 0x7d, 0xc7, 0x21, + 0x65, 0x11, 0x25, 0x78, 0x2a, 0xc7, 0xa7, 0x12, + 0x47, 0xfe, 0xae, 0xf3, 0x2f, 0x1f, 0x25, 0x0c, + 0xe4, 0xbb, 0x8f, 0x79, 0xac, 0xaa, 0x17, 0x9d, + 0x45, 0xa7, 0xb0, 0x54, 0x5f, 0x09, 0x24, 0x32, + 0x5e, 0xfa, 0x87, 0xd5, 0xe4, 0x41, 0xd2, 0x84, + 0x78, 0xc6, 0x1f, 0x22, 0x23, 0xee, 0x67, 0xc3, + 0xb4, 0x1f, 0x43, 0x94, 0x53, 0x5e, 0x2a, 0x24, + 0x36, 0x9a, 0x2e, 0x16, 0x61, 0x3c, 0x45, 0x94, + 0x90, 0xc1, 0x4f, 0xb1, 0xd7, 0x55, 0xfe, 0x53, + 0xfb, 0xe1, 0xee, 0x45, 0xb1, 0xb2, 0x1f, 0x71, + 0x62, 0xe2, 0xfc, 0xaa, 0x74, 0x2a, 0xbe, 0xfd +}; +static const u8 enc_output079[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x79, 0x5b, 0xcf, 0xf6, 0x47, 0xc5, 0x53, 0xc2, + 0xe4, 0xeb, 0x6e, 0x0e, 0xaf, 0xd9, 0xe0, 0x4e +}; +static const u8 enc_assoc079[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce079[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x70, 0x48, 0x4a +}; +static const u8 enc_key079[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input080[] __initconst = { + 0xfa, 0xe5, 0x83, 0x45, 0xc1, 0x6c, 0xb0, 0xf5, + 0xcc, 0x53, 0x7f, 0x2b, 0x1b, 0x34, 0x69, 0xc9, + 0x69, 0x46, 0x3b, 0x3e, 0xa7, 0x1b, 0xcf, 0x6b, + 0x98, 0xd6, 0x69, 0xa8, 0xe6, 0x0e, 0x04, 0xfc, + 0x08, 0xd5, 0xfd, 0x06, 0x9c, 0x36, 0x26, 0x38, + 0xe3, 0x40, 0x0e, 0xf4, 0xcb, 0x24, 0x2e, 0x27, + 0xe2, 0x24, 0x5e, 0x68, 0xcb, 0x9e, 0xc5, 0x83, + 0xda, 0x53, 0x40, 0xb1, 0x2e, 0xdf, 0x42, 0x3b, + 0x73, 0x26, 0xad, 0x20, 0xfe, 0xeb, 0x57, 0xda, + 0xca, 0x2e, 0x04, 0x67, 0xa3, 0x28, 0x99, 0xb4, + 0x2d, 0xf8, 0xe5, 0x6d, 0x84, 0xe0, 0x06, 0xbc, + 0x8a, 0x7a, 0xcc, 0x73, 0x1e, 0x7c, 0x1f, 0x6b, + 0xec, 0xb5, 0x71, 0x9f, 0x70, 0x77, 0xf0, 0xd4, + 0xf4, 0xc6, 0x1a, 0xb1, 0x1e, 0xba, 0xc1, 0x00, + 0x18, 0x01, 0xce, 0x33, 0xc4, 0xe4, 0xa7, 0x7d, + 0x83, 0x1d, 0x3c, 0xe3, 0x4e, 0x84, 0x10, 0xe1 +}; +static const u8 enc_output080[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x19, 0x46, 0xd6, 0x53, 0x96, 0x0f, 0x94, 0x7a, + 0x74, 0xd3, 0xe8, 0x09, 0x3c, 0xf4, 0x85, 0x02 +}; +static const u8 enc_assoc080[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce080[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x2f, 0x40 +}; +static const u8 enc_key080[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input081[] __initconst = { + 0xeb, 0xb2, 0x16, 0xdd, 0xd7, 0xca, 0x70, 0x92, + 0x15, 0xf5, 0x03, 0xdf, 0x9c, 0xe6, 0x3c, 0x5c, + 0xd2, 0x19, 0x4e, 0x7d, 0x90, 0x99, 0xe8, 0xa9, + 0x0b, 0x2a, 0xfa, 0xad, 0x5e, 0xba, 0x35, 0x06, + 0x99, 0x25, 0xa6, 0x03, 0xfd, 0xbc, 0x34, 0x1a, + 0xae, 0xd4, 0x15, 0x05, 0xb1, 0x09, 0x41, 0xfa, + 0x38, 0x56, 0xa7, 0xe2, 0x47, 0xb1, 0x04, 0x07, + 0x09, 0x74, 0x6c, 0xfc, 0x20, 0x96, 0xca, 0xa6, + 0x31, 0xb2, 0xff, 0xf4, 0x1c, 0x25, 0x05, 0x06, + 0xd8, 0x89, 0xc1, 0xc9, 0x06, 0x71, 0xad, 0xe8, + 0x53, 0xee, 0x63, 0x94, 0xc1, 0x91, 0x92, 0xa5, + 0xcf, 0x37, 0x10, 0xd1, 0x07, 0x30, 0x99, 0xe5, + 0xbc, 0x94, 0x65, 0x82, 0xfc, 0x0f, 0xab, 0x9f, + 0x54, 0x3c, 0x71, 0x6a, 0xe2, 0x48, 0x6a, 0x86, + 0x83, 0xfd, 0xca, 0x39, 0xd2, 0xe1, 0x4f, 0x23, + 0xd0, 0x0a, 0x58, 0x26, 0x64, 0xf4, 0xec, 0xb1 +}; +static const u8 enc_output081[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x36, 0xc3, 0x00, 0x29, 0x85, 0xdd, 0x21, 0xba, + 0xf8, 0x95, 0xd6, 0x33, 0x57, 0x3f, 0x12, 0xc0 +}; +static const u8 enc_assoc081[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce081[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x35 +}; +static const u8 enc_key081[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input082[] __initconst = { + 0x40, 0x8a, 0xe6, 0xef, 0x1c, 0x7e, 0xf0, 0xfb, + 0x2c, 0x2d, 0x61, 0x08, 0x16, 0xfc, 0x78, 0x49, + 0xef, 0xa5, 0x8f, 0x78, 0x27, 0x3f, 0x5f, 0x16, + 0x6e, 0xa6, 0x5f, 0x81, 0xb5, 0x75, 0x74, 0x7d, + 0x03, 0x5b, 0x30, 0x40, 0xfe, 0xde, 0x1e, 0xb9, + 0x45, 0x97, 0x88, 0x66, 0x97, 0x88, 0x40, 0x8e, + 0x00, 0x41, 0x3b, 0x3e, 0x37, 0x6d, 0x15, 0x2d, + 0x20, 0x4a, 0xa2, 0xb7, 0xa8, 0x35, 0x58, 0xfc, + 0xd4, 0x8a, 0x0e, 0xf7, 0xa2, 0x6b, 0x1c, 0xd6, + 0xd3, 0x5d, 0x23, 0xb3, 0xf5, 0xdf, 0xe0, 0xca, + 0x77, 0xa4, 0xce, 0x32, 0xb9, 0x4a, 0xbf, 0x83, + 0xda, 0x2a, 0xef, 0xca, 0xf0, 0x68, 0x38, 0x08, + 0x79, 0xe8, 0x9f, 0xb0, 0xa3, 0x82, 0x95, 0x95, + 0xcf, 0x44, 0xc3, 0x85, 0x2a, 0xe2, 0xcc, 0x66, + 0x2b, 0x68, 0x9f, 0x93, 0x55, 0xd9, 0xc1, 0x83, + 0x80, 0x1f, 0x6a, 0xcc, 0x31, 0x3f, 0x89, 0x07 +}; +static const u8 enc_output082[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x65, 0x14, 0x51, 0x8e, 0x0a, 0x26, 0x41, 0x42, + 0xe0, 0xb7, 0x35, 0x1f, 0x96, 0x7f, 0xc2, 0xae +}; +static const u8 enc_assoc082[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce082[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xf7, 0xd5 +}; +static const u8 enc_key082[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input083[] __initconst = { + 0x0a, 0x0a, 0x24, 0x49, 0x9b, 0xca, 0xde, 0x58, + 0xcf, 0x15, 0x76, 0xc3, 0x12, 0xac, 0xa9, 0x84, + 0x71, 0x8c, 0xb4, 0xcc, 0x7e, 0x01, 0x53, 0xf5, + 0xa9, 0x01, 0x58, 0x10, 0x85, 0x96, 0x44, 0xdf, + 0xc0, 0x21, 0x17, 0x4e, 0x0b, 0x06, 0x0a, 0x39, + 0x74, 0x48, 0xde, 0x8b, 0x48, 0x4a, 0x86, 0x03, + 0xbe, 0x68, 0x0a, 0x69, 0x34, 0xc0, 0x90, 0x6f, + 0x30, 0xdd, 0x17, 0xea, 0xe2, 0xd4, 0xc5, 0xfa, + 0xa7, 0x77, 0xf8, 0xca, 0x53, 0x37, 0x0e, 0x08, + 0x33, 0x1b, 0x88, 0xc3, 0x42, 0xba, 0xc9, 0x59, + 0x78, 0x7b, 0xbb, 0x33, 0x93, 0x0e, 0x3b, 0x56, + 0xbe, 0x86, 0xda, 0x7f, 0x2a, 0x6e, 0xb1, 0xf9, + 0x40, 0x89, 0xd1, 0xd1, 0x81, 0x07, 0x4d, 0x43, + 0x02, 0xf8, 0xe0, 0x55, 0x2d, 0x0d, 0xe1, 0xfa, + 0xb3, 0x06, 0xa2, 0x1b, 0x42, 0xd4, 0xc3, 0xba, + 0x6e, 0x6f, 0x0c, 0xbc, 0xc8, 0x1e, 0x87, 0x7a +}; +static const u8 enc_output083[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4c, 0x19, 0x4d, 0xa6, 0xa9, 0x9f, 0xd6, 0x5b, + 0x40, 0xe9, 0xca, 0xd7, 0x98, 0xf4, 0x4b, 0x19 +}; +static const u8 enc_assoc083[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce083[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xfc, 0xe4 +}; +static const u8 enc_key083[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input084[] __initconst = { + 0x4a, 0x0a, 0xaf, 0xf8, 0x49, 0x47, 0x29, 0x18, + 0x86, 0x91, 0x70, 0x13, 0x40, 0xf3, 0xce, 0x2b, + 0x8a, 0x78, 0xee, 0xd3, 0xa0, 0xf0, 0x65, 0x99, + 0x4b, 0x72, 0x48, 0x4e, 0x79, 0x91, 0xd2, 0x5c, + 0x29, 0xaa, 0x07, 0x5e, 0xb1, 0xfc, 0x16, 0xde, + 0x93, 0xfe, 0x06, 0x90, 0x58, 0x11, 0x2a, 0xb2, + 0x84, 0xa3, 0xed, 0x18, 0x78, 0x03, 0x26, 0xd1, + 0x25, 0x8a, 0x47, 0x22, 0x2f, 0xa6, 0x33, 0xd8, + 0xb2, 0x9f, 0x3b, 0xd9, 0x15, 0x0b, 0x23, 0x9b, + 0x15, 0x46, 0xc2, 0xbb, 0x9b, 0x9f, 0x41, 0x0f, + 0xeb, 0xea, 0xd3, 0x96, 0x00, 0x0e, 0xe4, 0x77, + 0x70, 0x15, 0x32, 0xc3, 0xd0, 0xf5, 0xfb, 0xf8, + 0x95, 0xd2, 0x80, 0x19, 0x6d, 0x2f, 0x73, 0x7c, + 0x5e, 0x9f, 0xec, 0x50, 0xd9, 0x2b, 0xb0, 0xdf, + 0x5d, 0x7e, 0x51, 0x3b, 0xe5, 0xb8, 0xea, 0x97, + 0x13, 0x10, 0xd5, 0xbf, 0x16, 0xba, 0x7a, 0xee +}; +static const u8 enc_output084[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc8, 0xae, 0x77, 0x88, 0xcd, 0x28, 0x74, 0xab, + 0xc1, 0x38, 0x54, 0x1e, 0x11, 0xfd, 0x05, 0x87 +}; +static const u8 enc_assoc084[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce084[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x01, 0x84, 0x86, 0xa8 +}; +static const u8 enc_key084[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input085[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x78, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x9c, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xd4, 0xd2, 0x06, 0x61, 0x6f, 0x92, 0x93, 0xf6, + 0x5b, 0x45, 0xdb, 0xbc, 0x74, 0xe7, 0xc2, 0xed, + 0xfb, 0xcb, 0xbf, 0x1c, 0xfb, 0x67, 0x9b, 0xb7, + 0x39, 0xa5, 0x86, 0x2d, 0xe2, 0xbc, 0xb9, 0x37, + 0xf7, 0x4d, 0x5b, 0xf8, 0x67, 0x1c, 0x5a, 0x8a, + 0x50, 0x92, 0xf6, 0x1d, 0x54, 0xc9, 0xaa, 0x5b +}; +static const u8 enc_output085[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x93, 0x3a, 0x51, 0x63, 0xc7, 0xf6, 0x23, 0x68, + 0x32, 0x7b, 0x3f, 0xbc, 0x10, 0x36, 0xc9, 0x43 +}; +static const u8 enc_assoc085[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce085[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key085[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input086[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output086[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const u8 enc_assoc086[] __initconst = { + 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, + 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 +}; +static const u8 enc_nonce086[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key086[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input087[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, + 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 +}; +static const u8 enc_nonce087[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key087[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input088[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output088[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_assoc088[] __initconst = { + 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, + 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 +}; +static const u8 enc_nonce088[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key088[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input089[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output089[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_assoc089[] __initconst = { + 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, + 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 +}; +static const u8 enc_nonce089[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key089[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input090[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_assoc090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, + 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 +}; +static const u8 enc_nonce090[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key090[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input091[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output091[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc091[] __initconst = { + 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, + 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 +}; +static const u8 enc_nonce091[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key091[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input092[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output092[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc092[] __initconst = { + 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, + 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 +}; +static const u8 enc_nonce092[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key092[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input093[] __initconst = { + 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, + 0x3d, 0xb7, 0x66, 0x4a, 0x34, 0xae, 0x6b, 0x44, + 0x4d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x5b, 0x8b, 0x94, 0x50, 0x9e, 0x2b, 0x74, 0xa3, + 0x6d, 0x34, 0x6e, 0x33, 0xd5, 0x72, 0x65, 0x9b, + 0xa9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x83, 0xdc, 0xe9, 0xf3, 0x07, 0x3e, 0xfa, 0xdb, + 0x7d, 0x23, 0xb8, 0x7a, 0xce, 0x35, 0x16, 0x8c +}; +static const u8 enc_output093[] __initconst = { + 0x00, 0x39, 0xe2, 0xfd, 0x2f, 0xd3, 0x12, 0x14, + 0x9e, 0x98, 0x98, 0x80, 0x88, 0x48, 0x13, 0xe7, + 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96, + 0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00, + 0xca, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x3b, 0x0e, 0x86, 0x9a, 0xaa, 0x8e, 0xa4, 0x96, + 0x32, 0xff, 0xff, 0x37, 0xb9, 0xe8, 0xce, 0x00, + 0xa5, 0x19, 0xac, 0x1a, 0x35, 0xb4, 0xa5, 0x77, + 0x87, 0x51, 0x0a, 0xf7, 0x8d, 0x8d, 0x20, 0x0a +}; +static const u8 enc_assoc093[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce093[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key093[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input094[] __initconst = { + 0xd3, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xe5, 0xda, 0x78, 0x76, 0x6f, 0xa1, 0x92, 0x90, + 0xc0, 0x31, 0xf7, 0x52, 0x08, 0x50, 0x67, 0x45, + 0xae, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x49, 0x6d, 0xde, 0xb0, 0x55, 0x09, 0xc6, 0xef, + 0xff, 0xab, 0x75, 0xeb, 0x2d, 0xf4, 0xab, 0x09, + 0x76, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x01, 0x49, 0xef, 0x50, 0x4b, 0x71, 0xb1, 0x20, + 0xca, 0x4f, 0xf3, 0x95, 0x19, 0xc2, 0xc2, 0x10 +}; +static const u8 enc_output094[] __initconst = { + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x62, 0x18, 0xb2, 0x7f, 0x83, 0xb8, 0xb4, 0x66, + 0x02, 0xf6, 0xe1, 0xd8, 0x34, 0x20, 0x7b, 0x02, + 0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29, + 0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02, + 0xce, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2a, 0x64, 0x16, 0xce, 0xdb, 0x1c, 0xdd, 0x29, + 0x6e, 0xf5, 0xd7, 0xd6, 0x92, 0xda, 0xff, 0x02, + 0x30, 0x2f, 0xe8, 0x2a, 0xb0, 0xa0, 0x9a, 0xf6, + 0x44, 0x00, 0xd0, 0x15, 0xae, 0x83, 0xd9, 0xcc +}; +static const u8 enc_assoc094[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce094[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key094[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input095[] __initconst = { + 0xe9, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x6d, 0xf1, 0x39, 0x4e, 0xdc, 0x53, 0x9b, 0x5b, + 0x3a, 0x09, 0x57, 0xbe, 0x0f, 0xb8, 0x59, 0x46, + 0x80, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xd1, 0x76, 0x9f, 0xe8, 0x06, 0xbb, 0xfe, 0xb6, + 0xf5, 0x90, 0x95, 0x0f, 0x2e, 0xac, 0x9e, 0x0a, + 0x58, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x99, 0x52, 0xae, 0x08, 0x18, 0xc3, 0x89, 0x79, + 0xc0, 0x74, 0x13, 0x71, 0x1a, 0x9a, 0xf7, 0x13 +}; +static const u8 enc_output095[] __initconst = { + 0xe9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xea, 0x33, 0xf3, 0x47, 0x30, 0x4a, 0xbd, 0xad, + 0xf8, 0xce, 0x41, 0x34, 0x33, 0xc8, 0x45, 0x01, + 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70, + 0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01, + 0xe0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb2, 0x7f, 0x57, 0x96, 0x88, 0xae, 0xe5, 0x70, + 0x64, 0xce, 0x37, 0x32, 0x91, 0x82, 0xca, 0x01, + 0x98, 0xa7, 0xe8, 0x36, 0xe0, 0xee, 0x4d, 0x02, + 0x35, 0x00, 0xd0, 0x55, 0x7e, 0xc2, 0xcb, 0xe0 +}; +static const u8 enc_assoc095[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce095[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key095[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input096[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x64, 0xf9, 0x0f, 0x5b, 0x26, 0x92, 0xb8, 0x60, + 0xd4, 0x59, 0x6f, 0xf4, 0xb3, 0x40, 0x2c, 0x5c, + 0x00, 0xb9, 0xbb, 0x53, 0x70, 0x7a, 0xa6, 0x67, + 0xd3, 0x56, 0xfe, 0x50, 0xc7, 0x19, 0x96, 0x94, + 0x03, 0x35, 0x61, 0xe7, 0xca, 0xca, 0x6d, 0x94, + 0x1d, 0xc3, 0xcd, 0x69, 0x14, 0xad, 0x69, 0x04 +}; +static const u8 enc_output096[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xe3, 0x3b, 0xc5, 0x52, 0xca, 0x8b, 0x9e, 0x96, + 0x16, 0x9e, 0x79, 0x7e, 0x8f, 0x30, 0x30, 0x1b, + 0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52, + 0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f, + 0x60, 0x3c, 0xa9, 0x99, 0x44, 0xdf, 0x76, 0x52, + 0x8c, 0x9d, 0x6f, 0x54, 0xab, 0x83, 0x3d, 0x0f, + 0x6a, 0xb8, 0xdc, 0xe2, 0xc5, 0x9d, 0xa4, 0x73, + 0x71, 0x30, 0xb0, 0x25, 0x2f, 0x68, 0xa8, 0xd8 +}; +static const u8 enc_assoc096[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce096[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key096[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input097[] __initconst = { + 0x68, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xb0, 0x8f, 0x25, 0x67, 0x5b, 0x9b, 0xcb, 0xf6, + 0xe3, 0x84, 0x07, 0xde, 0x2e, 0xc7, 0x5a, 0x47, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x2d, 0x2a, 0xf7, 0xcd, 0x6b, 0x08, 0x05, 0x01, + 0xd3, 0x1b, 0xa5, 0x4f, 0xb2, 0xeb, 0x75, 0x96, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x65, 0x0e, 0xc6, 0x2d, 0x75, 0x70, 0x72, 0xce, + 0xe6, 0xff, 0x23, 0x31, 0x86, 0xdd, 0x1c, 0x8f +}; +static const u8 enc_output097[] __initconst = { + 0x68, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x37, 0x4d, 0xef, 0x6e, 0xb7, 0x82, 0xed, 0x00, + 0x21, 0x43, 0x11, 0x54, 0x12, 0xb7, 0x46, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7, + 0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4e, 0x23, 0x3f, 0xb3, 0xe5, 0x1d, 0x1e, 0xc7, + 0x42, 0x45, 0x07, 0x72, 0x0d, 0xc5, 0x21, 0x9d, + 0x04, 0x4d, 0xea, 0x60, 0x88, 0x80, 0x41, 0x2b, + 0xfd, 0xff, 0xcf, 0x35, 0x57, 0x9e, 0x9b, 0x26 +}; +static const u8 enc_assoc097[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce097[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key097[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input098[] __initconst = { + 0x6d, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xa1, 0x61, 0xb5, 0xab, 0x04, 0x09, 0x00, 0x62, + 0x9e, 0xfe, 0xff, 0x78, 0xd7, 0xd8, 0x6b, 0x45, + 0x9f, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xc6, 0xf8, 0x07, 0x8c, 0xc8, 0xef, 0x12, 0xa0, + 0xff, 0x65, 0x7d, 0x6d, 0x08, 0xdb, 0x10, 0xb8, + 0x47, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x8e, 0xdc, 0x36, 0x6c, 0xd6, 0x97, 0x65, 0x6f, + 0xca, 0x81, 0xfb, 0x13, 0x3c, 0xed, 0x79, 0xa1 +}; +static const u8 enc_output098[] __initconst = { + 0x6d, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x26, 0xa3, 0x7f, 0xa2, 0xe8, 0x10, 0x26, 0x94, + 0x5c, 0x39, 0xe9, 0xf2, 0xeb, 0xa8, 0x77, 0x02, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66, + 0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa5, 0xf1, 0xcf, 0xf2, 0x46, 0xfa, 0x09, 0x66, + 0x6e, 0x3b, 0xdf, 0x50, 0xb7, 0xf5, 0x44, 0xb3, + 0x1e, 0x6b, 0xea, 0x63, 0x14, 0x54, 0x2e, 0x2e, + 0xf9, 0xff, 0xcf, 0x45, 0x0b, 0x2e, 0x98, 0x2b +}; +static const u8 enc_assoc098[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce098[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key098[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input099[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xfc, 0x01, 0xb8, 0x91, 0xe5, 0xf0, 0xf9, 0x12, + 0x8d, 0x7d, 0x1c, 0x57, 0x91, 0x92, 0xb6, 0x98, + 0x63, 0x41, 0x44, 0x15, 0xb6, 0x99, 0x68, 0x95, + 0x9a, 0x72, 0x91, 0xb7, 0xa5, 0xaf, 0x13, 0x48, + 0x60, 0xcd, 0x9e, 0xa1, 0x0c, 0x29, 0xa3, 0x66, + 0x54, 0xe7, 0xa2, 0x8e, 0x76, 0x1b, 0xec, 0xd8 +}; +static const u8 enc_output099[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x7b, 0xc3, 0x72, 0x98, 0x09, 0xe9, 0xdf, 0xe4, + 0x4f, 0xba, 0x0a, 0xdd, 0xad, 0xe2, 0xaa, 0xdf, + 0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0, + 0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3, + 0x03, 0xc4, 0x56, 0xdf, 0x82, 0x3c, 0xb8, 0xa0, + 0xc5, 0xb9, 0x00, 0xb3, 0xc9, 0x35, 0xb8, 0xd3, + 0xed, 0x20, 0x17, 0xc8, 0xdb, 0xa4, 0x77, 0x56, + 0x29, 0x04, 0x9d, 0x78, 0x6e, 0x3b, 0xce, 0xb1 +}; +static const u8 enc_assoc099[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce099[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key099[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input100[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x6b, 0x6d, 0xc9, 0xd2, 0x1a, 0x81, 0x9e, 0x70, + 0xb5, 0x77, 0xf4, 0x41, 0x37, 0xd3, 0xd6, 0xbd, + 0x13, 0x35, 0xf5, 0xeb, 0x44, 0x49, 0x40, 0x77, + 0xb2, 0x64, 0x49, 0xa5, 0x4b, 0x6c, 0x7c, 0x75, + 0x10, 0xb9, 0x2f, 0x5f, 0xfe, 0xf9, 0x8b, 0x84, + 0x7c, 0xf1, 0x7a, 0x9c, 0x98, 0xd8, 0x83, 0xe5 +}; +static const u8 enc_output100[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xec, 0xaf, 0x03, 0xdb, 0xf6, 0x98, 0xb8, 0x86, + 0x77, 0xb0, 0xe2, 0xcb, 0x0b, 0xa3, 0xca, 0xfa, + 0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42, + 0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee, + 0x73, 0xb0, 0xe7, 0x21, 0x70, 0xec, 0x90, 0x42, + 0xed, 0xaf, 0xd8, 0xa1, 0x27, 0xf6, 0xd7, 0xee, + 0x07, 0x3f, 0x17, 0xcb, 0x67, 0x78, 0x64, 0x59, + 0x25, 0x04, 0x9d, 0x88, 0x22, 0xcb, 0xca, 0xb6 +}; +static const u8 enc_assoc100[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce100[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key100[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input101[] __initconst = { + 0xff, 0xcb, 0x2b, 0x11, 0x06, 0xf8, 0x23, 0x4c, + 0x5e, 0x99, 0xd4, 0xdb, 0x4c, 0x70, 0x48, 0xde, + 0x32, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x16, 0xe9, 0x88, 0x4a, 0x11, 0x4f, 0x0e, 0x92, + 0x66, 0xce, 0xa3, 0x88, 0x5f, 0xe3, 0x6b, 0x9f, + 0xd6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xce, 0xbe, 0xf5, 0xe9, 0x88, 0x5a, 0x80, 0xea, + 0x76, 0xd9, 0x75, 0xc1, 0x44, 0xa4, 0x18, 0x88 +}; +static const u8 enc_output101[] __initconst = { + 0xff, 0xa0, 0xfc, 0x3e, 0x80, 0x32, 0xc3, 0xd5, + 0xfd, 0xb6, 0x2a, 0x11, 0xf0, 0x96, 0x30, 0x7d, + 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7, + 0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04, + 0xb5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x6c, 0x9a, 0x80, 0x25, 0xea, 0xde, 0xa7, + 0x39, 0x05, 0x32, 0x8c, 0x33, 0x79, 0xc0, 0x04, + 0x8b, 0x9b, 0xb4, 0xb4, 0x86, 0x12, 0x89, 0x65, + 0x8c, 0x69, 0x6a, 0x83, 0x40, 0x15, 0x04, 0x05 +}; +static const u8 enc_assoc101[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce101[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key101[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input102[] __initconst = { + 0x6f, 0x9e, 0x70, 0xed, 0x3b, 0x8b, 0xac, 0xa0, + 0x26, 0xe4, 0x6a, 0x5a, 0x09, 0x43, 0x15, 0x8d, + 0x21, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x0c, 0x61, 0x2c, 0x5e, 0x8d, 0x89, 0xa8, 0x73, + 0xdb, 0xca, 0xad, 0x5b, 0x73, 0x46, 0x42, 0x9b, + 0xc5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xd4, 0x36, 0x51, 0xfd, 0x14, 0x9c, 0x26, 0x0b, + 0xcb, 0xdd, 0x7b, 0x12, 0x68, 0x01, 0x31, 0x8c +}; +static const u8 enc_output102[] __initconst = { + 0x6f, 0xf5, 0xa7, 0xc2, 0xbd, 0x41, 0x4c, 0x39, + 0x85, 0xcb, 0x94, 0x90, 0xb5, 0xa5, 0x6d, 0x2e, + 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46, + 0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00, + 0xa6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x6c, 0xe4, 0x3e, 0x94, 0xb9, 0x2c, 0x78, 0x46, + 0x84, 0x01, 0x3c, 0x5f, 0x1f, 0xdc, 0xe9, 0x00, + 0x8b, 0x3b, 0xbd, 0x51, 0x64, 0x44, 0x59, 0x56, + 0x8d, 0x81, 0xca, 0x1f, 0xa7, 0x2c, 0xe4, 0x04 +}; +static const u8 enc_assoc102[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce102[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key102[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input103[] __initconst = { + 0x41, 0x2b, 0x08, 0x0a, 0x3e, 0x19, 0xc1, 0x0d, + 0x44, 0xa1, 0xaf, 0x1e, 0xab, 0xde, 0xb4, 0xce, + 0x35, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x6b, 0x83, 0x94, 0x33, 0x09, 0x21, 0x48, 0x6c, + 0xa1, 0x1d, 0x29, 0x1c, 0x3e, 0x97, 0xee, 0x9a, + 0xd1, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xb3, 0xd4, 0xe9, 0x90, 0x90, 0x34, 0xc6, 0x14, + 0xb1, 0x0a, 0xff, 0x55, 0x25, 0xd0, 0x9d, 0x8d +}; +static const u8 enc_output103[] __initconst = { + 0x41, 0x40, 0xdf, 0x25, 0xb8, 0xd3, 0x21, 0x94, + 0xe7, 0x8e, 0x51, 0xd4, 0x17, 0x38, 0xcc, 0x6d, + 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59, + 0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01, + 0xb2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x06, 0x86, 0xf9, 0x3d, 0x84, 0x98, 0x59, + 0xfe, 0xd6, 0xb8, 0x18, 0x52, 0x0d, 0x45, 0x01, + 0x86, 0xfb, 0xab, 0x2b, 0x4a, 0x94, 0xf4, 0x7a, + 0xa5, 0x6f, 0x0a, 0xea, 0x65, 0xd1, 0x10, 0x08 +}; +static const u8 enc_assoc103[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce103[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key103[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input104[] __initconst = { + 0xb2, 0x47, 0xa7, 0x47, 0x23, 0x49, 0x1a, 0xac, + 0xac, 0xaa, 0xd7, 0x09, 0xc9, 0x1e, 0x93, 0x2b, + 0x31, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x9a, 0xde, 0x04, 0xe7, 0x5b, 0xb7, 0x01, 0xd9, + 0x66, 0x06, 0x01, 0xb3, 0x47, 0x65, 0xde, 0x98, + 0xd5, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x42, 0x89, 0x79, 0x44, 0xc2, 0xa2, 0x8f, 0xa1, + 0x76, 0x11, 0xd7, 0xfa, 0x5c, 0x22, 0xad, 0x8f +}; +static const u8 enc_output104[] __initconst = { + 0xb2, 0x2c, 0x70, 0x68, 0xa5, 0x83, 0xfa, 0x35, + 0x0f, 0x85, 0x29, 0xc3, 0x75, 0xf8, 0xeb, 0x88, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec, + 0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfa, 0x5b, 0x16, 0x2d, 0x6f, 0x12, 0xd1, 0xec, + 0x39, 0xcd, 0x90, 0xb7, 0x2b, 0xff, 0x75, 0x03, + 0xa0, 0x19, 0xac, 0x2e, 0xd6, 0x67, 0xe1, 0x7d, + 0xa1, 0x6f, 0x0a, 0xfa, 0x19, 0x61, 0x0d, 0x0d +}; +static const u8 enc_assoc104[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce104[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key104[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input105[] __initconst = { + 0x74, 0x0f, 0x9e, 0x49, 0xf6, 0x10, 0xef, 0xa5, + 0x85, 0xb6, 0x59, 0xca, 0x6e, 0xd8, 0xb4, 0x99, + 0x2d, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x41, 0x2d, 0x96, 0xaf, 0xbe, 0x80, 0xec, 0x3e, + 0x79, 0xd4, 0x51, 0xb0, 0x0a, 0x2d, 0xb2, 0x9a, + 0xc9, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x99, 0x7a, 0xeb, 0x0c, 0x27, 0x95, 0x62, 0x46, + 0x69, 0xc3, 0x87, 0xf9, 0x11, 0x6a, 0xc1, 0x8d +}; +static const u8 enc_output105[] __initconst = { + 0x74, 0x64, 0x49, 0x66, 0x70, 0xda, 0x0f, 0x3c, + 0x26, 0x99, 0xa7, 0x00, 0xd2, 0x3e, 0xcc, 0x3a, + 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b, + 0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01, + 0xaa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x21, 0xa8, 0x84, 0x65, 0x8a, 0x25, 0x3c, 0x0b, + 0x26, 0x1f, 0xc0, 0xb4, 0x66, 0xb7, 0x19, 0x01, + 0x73, 0x6e, 0x18, 0x18, 0x16, 0x96, 0xa5, 0x88, + 0x9c, 0x31, 0x59, 0xfa, 0xab, 0xab, 0x20, 0xfd +}; +static const u8 enc_assoc105[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce105[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key105[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input106[] __initconst = { + 0xad, 0xba, 0x5d, 0x10, 0x5b, 0xc8, 0xaa, 0x06, + 0x2c, 0x23, 0x36, 0xcb, 0x88, 0x9d, 0xdb, 0xd5, + 0x37, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x17, 0x7c, 0x5f, 0xfe, 0x28, 0x75, 0xf4, 0x68, + 0xf6, 0xc2, 0x96, 0x57, 0x48, 0xf3, 0x59, 0x9a, + 0xd3, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xcf, 0x2b, 0x22, 0x5d, 0xb1, 0x60, 0x7a, 0x10, + 0xe6, 0xd5, 0x40, 0x1e, 0x53, 0xb4, 0x2a, 0x8d +}; +static const u8 enc_output106[] __initconst = { + 0xad, 0xd1, 0x8a, 0x3f, 0xdd, 0x02, 0x4a, 0x9f, + 0x8f, 0x0c, 0xc8, 0x01, 0x34, 0x7b, 0xa3, 0x76, + 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d, + 0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01, + 0xb0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x77, 0xf9, 0x4d, 0x34, 0x1c, 0xd0, 0x24, 0x5d, + 0xa9, 0x09, 0x07, 0x53, 0x24, 0x69, 0xf2, 0x01, + 0xba, 0xd5, 0x8f, 0x10, 0xa9, 0x1e, 0x6a, 0x88, + 0x9a, 0xba, 0x32, 0xfd, 0x17, 0xd8, 0x33, 0x1a +}; +static const u8 enc_assoc106[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce106[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key106[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input107[] __initconst = { + 0xfe, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xc0, 0x01, 0xed, 0xc5, 0xda, 0x44, 0x2e, 0x71, + 0x9b, 0xce, 0x9a, 0xbe, 0x27, 0x3a, 0xf1, 0x44, + 0xb4, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x48, 0x02, 0x5f, 0x41, 0xfa, 0x4e, 0x33, 0x6c, + 0x78, 0x69, 0x57, 0xa2, 0xa7, 0xc4, 0x93, 0x0a, + 0x6c, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x00, 0x26, 0x6e, 0xa1, 0xe4, 0x36, 0x44, 0xa3, + 0x4d, 0x8d, 0xd1, 0xdc, 0x93, 0xf2, 0xfa, 0x13 +}; +static const u8 enc_output107[] __initconst = { + 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x47, 0xc3, 0x27, 0xcc, 0x36, 0x5d, 0x08, 0x87, + 0x59, 0x09, 0x8c, 0x34, 0x1b, 0x4a, 0xed, 0x03, + 0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa, + 0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01, + 0xd4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x2b, 0x0b, 0x97, 0x3f, 0x74, 0x5b, 0x28, 0xaa, + 0xe9, 0x37, 0xf5, 0x9f, 0x18, 0xea, 0xc7, 0x01, + 0xd6, 0x8c, 0xe1, 0x74, 0x07, 0x9a, 0xdd, 0x02, + 0x8d, 0xd0, 0x5c, 0xf8, 0x14, 0x63, 0x04, 0x88 +}; +static const u8 enc_assoc107[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce107[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key107[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input108[] __initconst = { + 0xb5, 0x13, 0xb0, 0x6a, 0xb9, 0xac, 0x14, 0x43, + 0x5a, 0xcb, 0x8a, 0xa3, 0xa3, 0x7a, 0xfd, 0xb6, + 0x54, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x61, 0x95, 0x01, 0x93, 0xb1, 0xbf, 0x03, 0x11, + 0xff, 0x11, 0x79, 0x89, 0xae, 0xd9, 0xa9, 0x99, + 0xb0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xb9, 0xc2, 0x7c, 0x30, 0x28, 0xaa, 0x8d, 0x69, + 0xef, 0x06, 0xaf, 0xc0, 0xb5, 0x9e, 0xda, 0x8e +}; +static const u8 enc_output108[] __initconst = { + 0xb5, 0x78, 0x67, 0x45, 0x3f, 0x66, 0xf4, 0xda, + 0xf9, 0xe4, 0x74, 0x69, 0x1f, 0x9c, 0x85, 0x15, + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24, + 0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02, + 0xd3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x10, 0x13, 0x59, 0x85, 0x1a, 0xd3, 0x24, + 0xa0, 0xda, 0xe8, 0x8d, 0xc2, 0x43, 0x02, 0x02, + 0xaa, 0x48, 0xa3, 0x88, 0x7d, 0x4b, 0x05, 0x96, + 0x99, 0xc2, 0xfd, 0xf9, 0xc6, 0x78, 0x7e, 0x0a +}; +static const u8 enc_assoc108[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce108[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key108[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input109[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xd4, 0xf1, 0x09, 0xe8, 0x14, 0xce, 0xa8, 0x5a, + 0x08, 0xc0, 0x11, 0xd8, 0x50, 0xdd, 0x1d, 0xcb, + 0xcf, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x53, 0x40, 0xb8, 0x5a, 0x9a, 0xa0, 0x82, 0x96, + 0xb7, 0x7a, 0x5f, 0xc3, 0x96, 0x1f, 0x66, 0x0f, + 0x17, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x1b, 0x64, 0x89, 0xba, 0x84, 0xd8, 0xf5, 0x59, + 0x82, 0x9e, 0xd9, 0xbd, 0xa2, 0x29, 0x0f, 0x16 +}; +static const u8 enc_output109[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x53, 0x33, 0xc3, 0xe1, 0xf8, 0xd7, 0x8e, 0xac, + 0xca, 0x07, 0x07, 0x52, 0x6c, 0xad, 0x01, 0x8c, + 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50, + 0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04, + 0xaf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x30, 0x49, 0x70, 0x24, 0x14, 0xb5, 0x99, 0x50, + 0x26, 0x24, 0xfd, 0xfe, 0x29, 0x31, 0x32, 0x04, + 0xb9, 0x36, 0xa8, 0x17, 0xf2, 0x21, 0x1a, 0xf1, + 0x29, 0xe2, 0xcf, 0x16, 0x0f, 0xd4, 0x2b, 0xcb +}; +static const u8 enc_assoc109[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce109[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key109[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input110[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xdf, 0x4c, 0x62, 0x03, 0x2d, 0x41, 0x19, 0xb5, + 0x88, 0x47, 0x7e, 0x99, 0x92, 0x5a, 0x56, 0xd9, + 0xd6, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xfa, 0x84, 0xf0, 0x64, 0x55, 0x36, 0x42, 0x1b, + 0x2b, 0xb9, 0x24, 0x6e, 0xc2, 0x19, 0xed, 0x0b, + 0x0e, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xb2, 0xa0, 0xc1, 0x84, 0x4b, 0x4e, 0x35, 0xd4, + 0x1e, 0x5d, 0xa2, 0x10, 0xf6, 0x2f, 0x84, 0x12 +}; +static const u8 enc_output110[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x58, 0x8e, 0xa8, 0x0a, 0xc1, 0x58, 0x3f, 0x43, + 0x4a, 0x80, 0x68, 0x13, 0xae, 0x2a, 0x4a, 0x9e, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd, + 0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00, + 0xb6, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x99, 0x8d, 0x38, 0x1a, 0xdb, 0x23, 0x59, 0xdd, + 0xba, 0xe7, 0x86, 0x53, 0x7d, 0x37, 0xb9, 0x00, + 0x9f, 0x7a, 0xc4, 0x35, 0x1f, 0x6b, 0x91, 0xe6, + 0x30, 0x97, 0xa7, 0x13, 0x11, 0x5d, 0x05, 0xbe +}; +static const u8 enc_assoc110[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce110[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key110[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input111[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x13, 0xf8, 0x0a, 0x00, 0x6d, 0xc1, 0xbb, 0xda, + 0xd6, 0x39, 0xa9, 0x2f, 0xc7, 0xec, 0xa6, 0x55, + 0xf7, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x63, 0x48, 0xb8, 0xfd, 0x29, 0xbf, 0x96, 0xd5, + 0x63, 0xa5, 0x17, 0xe2, 0x7d, 0x7b, 0xfc, 0x0f, + 0x2f, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x2b, 0x6c, 0x89, 0x1d, 0x37, 0xc7, 0xe1, 0x1a, + 0x56, 0x41, 0x91, 0x9c, 0x49, 0x4d, 0x95, 0x16 +}; +static const u8 enc_output111[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x94, 0x3a, 0xc0, 0x09, 0x81, 0xd8, 0x9d, 0x2c, + 0x14, 0xfe, 0xbf, 0xa5, 0xfb, 0x9c, 0xba, 0x12, + 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13, + 0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04, + 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x41, 0x70, 0x83, 0xa7, 0xaa, 0x8d, 0x13, + 0xf2, 0xfb, 0xb5, 0xdf, 0xc2, 0x55, 0xa8, 0x04, + 0x9a, 0x18, 0xa8, 0x28, 0x07, 0x02, 0x69, 0xf4, + 0x47, 0x00, 0xd0, 0x09, 0xe7, 0x17, 0x1c, 0xc9 +}; +static const u8 enc_assoc111[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce111[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key111[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input112[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x82, 0xe5, 0x9b, 0x45, 0x82, 0x91, 0x50, 0x38, + 0xf9, 0x33, 0x81, 0x1e, 0x65, 0x2d, 0xc6, 0x6a, + 0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xb6, 0x71, 0xc8, 0xca, 0xc2, 0x70, 0xc2, 0x65, + 0xa0, 0xac, 0x2f, 0x53, 0x57, 0x99, 0x88, 0x0a, + 0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xfe, 0x55, 0xf9, 0x2a, 0xdc, 0x08, 0xb5, 0xaa, + 0x95, 0x48, 0xa9, 0x2d, 0x63, 0xaf, 0xe1, 0x13 +}; +static const u8 enc_output112[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x05, 0x27, 0x51, 0x4c, 0x6e, 0x88, 0x76, 0xce, + 0x3b, 0xf4, 0x97, 0x94, 0x59, 0x5d, 0xda, 0x2d, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3, + 0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd5, 0x78, 0x00, 0xb4, 0x4c, 0x65, 0xd9, 0xa3, + 0x31, 0xf2, 0x8d, 0x6e, 0xe8, 0xb7, 0xdc, 0x01, + 0xb4, 0x36, 0xa8, 0x2b, 0x93, 0xd5, 0x55, 0xf7, + 0x43, 0x00, 0xd0, 0x19, 0x9b, 0xa7, 0x18, 0xce +}; +static const u8 enc_assoc112[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce112[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key112[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input113[] __initconst = { + 0xff, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0xf1, 0xd1, 0x28, 0x87, 0xb7, 0x21, 0x69, 0x86, + 0xa1, 0x2d, 0x79, 0x09, 0x8b, 0x6d, 0xe6, 0x0f, + 0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xa7, 0xc7, 0x58, 0x99, 0xf3, 0xe6, 0x0a, 0xf1, + 0xfc, 0xb6, 0xc7, 0x30, 0x7d, 0x87, 0x59, 0x0f, + 0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0xef, 0xe3, 0x69, 0x79, 0xed, 0x9e, 0x7d, 0x3e, + 0xc9, 0x52, 0x41, 0x4e, 0x49, 0xb1, 0x30, 0x16 +}; +static const u8 enc_output113[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x76, 0x13, 0xe2, 0x8e, 0x5b, 0x38, 0x4f, 0x70, + 0x63, 0xea, 0x6f, 0x83, 0xb7, 0x1d, 0xfa, 0x48, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37, + 0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc4, 0xce, 0x90, 0xe7, 0x7d, 0xf3, 0x11, 0x37, + 0x6d, 0xe8, 0x65, 0x0d, 0xc2, 0xa9, 0x0d, 0x04, + 0xce, 0x54, 0xa8, 0x2e, 0x1f, 0xa9, 0x42, 0xfa, + 0x3f, 0x00, 0xd0, 0x29, 0x4f, 0x37, 0x15, 0xd3 +}; +static const u8 enc_assoc113[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce113[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key113[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input114[] __initconst = { + 0xcb, 0xf1, 0xda, 0x9e, 0x0b, 0xa9, 0x37, 0x73, + 0x74, 0xe6, 0x9e, 0x1c, 0x0e, 0x60, 0x0c, 0xfc, + 0x34, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0xbe, 0x3f, 0xa6, 0x6b, 0x6c, 0xe7, 0x80, 0x8a, + 0xa3, 0xe4, 0x59, 0x49, 0xf9, 0x44, 0x64, 0x9f, + 0xd0, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0x66, 0x68, 0xdb, 0xc8, 0xf5, 0xf2, 0x0e, 0xf2, + 0xb3, 0xf3, 0x8f, 0x00, 0xe2, 0x03, 0x17, 0x88 +}; +static const u8 enc_output114[] __initconst = { + 0xcb, 0x9a, 0x0d, 0xb1, 0x8d, 0x63, 0xd7, 0xea, + 0xd7, 0xc9, 0x60, 0xd6, 0xb2, 0x86, 0x74, 0x5f, + 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf, + 0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04, + 0xb3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xde, 0xba, 0xb4, 0xa1, 0x58, 0x42, 0x50, 0xbf, + 0xfc, 0x2f, 0xc8, 0x4d, 0x95, 0xde, 0xcf, 0x04, + 0x23, 0x83, 0xab, 0x0b, 0x79, 0x92, 0x05, 0x69, + 0x9b, 0x51, 0x0a, 0xa7, 0x09, 0xbf, 0x31, 0xf1 +}; +static const u8 enc_assoc114[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce114[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key114[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input115[] __initconst = { + 0x8f, 0x27, 0x86, 0x94, 0xc4, 0xe9, 0xda, 0xeb, + 0xd5, 0x8d, 0x3e, 0x5b, 0x96, 0x6e, 0x8b, 0x68, + 0x42, 0x3d, 0x35, 0xf6, 0x13, 0xe6, 0xd9, 0x09, + 0x3d, 0x38, 0xe9, 0x75, 0xc3, 0x8f, 0xe3, 0xb8, + 0x06, 0x53, 0xe7, 0xa3, 0x31, 0x71, 0x88, 0x33, + 0xac, 0xc3, 0xb9, 0xad, 0xff, 0x1c, 0x31, 0x98, + 0xa6, 0xf6, 0x37, 0x81, 0x71, 0xea, 0xe4, 0x39, + 0x6e, 0xa1, 0x5d, 0xc2, 0x40, 0xd1, 0xab, 0xf4, + 0xde, 0x04, 0x9a, 0x00, 0xa8, 0x64, 0x06, 0x4b, + 0xbc, 0xd4, 0x6f, 0xe4, 0xe4, 0x5b, 0x42, 0x8f +}; +static const u8 enc_output115[] __initconst = { + 0x8f, 0x4c, 0x51, 0xbb, 0x42, 0x23, 0x3a, 0x72, + 0x76, 0xa2, 0xc0, 0x91, 0x2a, 0x88, 0xf3, 0xcb, + 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06, + 0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03, + 0xc5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0xd6, 0xf5, 0x69, 0x05, 0xd4, 0x58, 0x06, + 0xf3, 0x08, 0x28, 0xa9, 0x93, 0x86, 0x9a, 0x03, + 0x8b, 0xfb, 0xab, 0x17, 0xa9, 0xe0, 0xb8, 0x74, + 0x8b, 0x51, 0x0a, 0xe7, 0xd9, 0xfd, 0x23, 0x05 +}; +static const u8 enc_assoc115[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce115[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key115[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input116[] __initconst = { + 0xd5, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x9a, 0x22, 0xd7, 0x0a, 0x48, 0xe2, 0x4f, 0xdd, + 0xcd, 0xd4, 0x41, 0x9d, 0xe6, 0x4c, 0x8f, 0x44, + 0xfc, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x77, 0xb5, 0xc9, 0x07, 0xd9, 0xc9, 0xe1, 0xea, + 0x51, 0x85, 0x1a, 0x20, 0x4a, 0xad, 0x9f, 0x0a, + 0x24, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x3f, 0x91, 0xf8, 0xe7, 0xc7, 0xb1, 0x96, 0x25, + 0x64, 0x61, 0x9c, 0x5e, 0x7e, 0x9b, 0xf6, 0x13 +}; +static const u8 enc_output116[] __initconst = { + 0xd5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x1d, 0xe0, 0x1d, 0x03, 0xa4, 0xfb, 0x69, 0x2b, + 0x0f, 0x13, 0x57, 0x17, 0xda, 0x3c, 0x93, 0x03, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c, + 0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01, + 0x9c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x14, 0xbc, 0x01, 0x79, 0x57, 0xdc, 0xfa, 0x2c, + 0xc0, 0xdb, 0xb8, 0x1d, 0xf5, 0x83, 0xcb, 0x01, + 0x49, 0xbc, 0x6e, 0x9f, 0xc5, 0x1c, 0x4d, 0x50, + 0x30, 0x36, 0x64, 0x4d, 0x84, 0x27, 0x73, 0xd2 +}; +static const u8 enc_assoc116[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce116[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key116[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input117[] __initconst = { + 0xdb, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x75, 0xd5, 0x64, 0x3a, 0xa5, 0xaf, 0x93, 0x4d, + 0x8c, 0xce, 0x39, 0x2c, 0xc3, 0xee, 0xdb, 0x47, + 0xc0, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0x60, 0x1b, 0x5a, 0xd2, 0x06, 0x7f, 0x28, 0x06, + 0x6a, 0x8f, 0x32, 0x81, 0x71, 0x5b, 0xa8, 0x08, + 0x18, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x28, 0x3f, 0x6b, 0x32, 0x18, 0x07, 0x5f, 0xc9, + 0x5f, 0x6b, 0xb4, 0xff, 0x45, 0x6d, 0xc1, 0x11 +}; +static const u8 enc_output117[] __initconst = { + 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf2, 0x17, 0xae, 0x33, 0x49, 0xb6, 0xb5, 0xbb, + 0x4e, 0x09, 0x2f, 0xa6, 0xff, 0x9e, 0xc7, 0x00, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0, + 0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03, + 0xa0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x03, 0x12, 0x92, 0xac, 0x88, 0x6a, 0x33, 0xc0, + 0xfb, 0xd1, 0x90, 0xbc, 0xce, 0x75, 0xfc, 0x03, + 0x63, 0xda, 0x6e, 0xa2, 0x51, 0xf0, 0x39, 0x53, + 0x2c, 0x36, 0x64, 0x5d, 0x38, 0xb7, 0x6f, 0xd7 +}; +static const u8 enc_assoc117[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce117[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key117[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - edge case intermediate sums in poly1305 */ +static const u8 enc_input118[] __initconst = { + 0x93, 0x94, 0x28, 0xd0, 0x79, 0x35, 0x1f, 0x66, + 0x5c, 0xd0, 0x01, 0x35, 0x43, 0x19, 0x87, 0x5c, + 0x62, 0x48, 0x39, 0x60, 0x42, 0x16, 0xe4, 0x03, + 0xeb, 0xcc, 0x6a, 0xf5, 0x59, 0xec, 0x8b, 0x43, + 0x97, 0x7a, 0xed, 0x35, 0xcb, 0x5a, 0x2f, 0xca, + 0xa0, 0x34, 0x6e, 0xfb, 0x93, 0x65, 0x54, 0x64, + 0xd8, 0xc8, 0xc3, 0xfa, 0x1a, 0x9e, 0x47, 0x4a, + 0xbe, 0x52, 0xd0, 0x2c, 0x81, 0x87, 0xe9, 0x0f, + 0x4f, 0x2d, 0x90, 0x96, 0x52, 0x4f, 0xa1, 0xb2, + 0xb0, 0x23, 0xb8, 0xb2, 0x88, 0x22, 0x27, 0x73, + 0x90, 0xec, 0xf2, 0x1a, 0x04, 0xe6, 0x30, 0x85, + 0x8b, 0xb6, 0x56, 0x52, 0xb5, 0xb1, 0x80, 0x16 +}; +static const u8 enc_output118[] __initconst = { + 0x93, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xe5, 0x8a, 0xf3, 0x69, 0xae, 0x0f, 0xc2, 0xf5, + 0x29, 0x0b, 0x7c, 0x7f, 0x65, 0x9c, 0x97, 0x04, + 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c, + 0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04, + 0xf7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xbb, 0xc1, 0x0b, 0x84, 0x94, 0x8b, 0x5c, 0x8c, + 0x2f, 0x0c, 0x72, 0x11, 0x3e, 0xa9, 0xbd, 0x04, + 0x73, 0xeb, 0x27, 0x24, 0xb5, 0xc4, 0x05, 0xf0, + 0x4d, 0x00, 0xd0, 0xf1, 0x58, 0x40, 0xa1, 0xc1 +}; +static const u8 enc_assoc118[] __initconst = { + 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce118[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x06, 0x4c, 0x2d, 0x52 +}; +static const u8 enc_key118[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +static const struct chacha20poly1305_testvec +chacha20poly1305_enc_vectors[] __initconst = { + { enc_input001, enc_output001, enc_assoc001, enc_nonce001, enc_key001, + sizeof(enc_input001), sizeof(enc_assoc001), sizeof(enc_nonce001) }, + { enc_input002, enc_output002, enc_assoc002, enc_nonce002, enc_key002, + sizeof(enc_input002), sizeof(enc_assoc002), sizeof(enc_nonce002) }, + { enc_input003, enc_output003, enc_assoc003, enc_nonce003, enc_key003, + sizeof(enc_input003), sizeof(enc_assoc003), sizeof(enc_nonce003) }, + { enc_input004, enc_output004, enc_assoc004, enc_nonce004, enc_key004, + sizeof(enc_input004), sizeof(enc_assoc004), sizeof(enc_nonce004) }, + { enc_input005, enc_output005, enc_assoc005, enc_nonce005, enc_key005, + sizeof(enc_input005), sizeof(enc_assoc005), sizeof(enc_nonce005) }, + { enc_input006, enc_output006, enc_assoc006, enc_nonce006, enc_key006, + sizeof(enc_input006), sizeof(enc_assoc006), sizeof(enc_nonce006) }, + { enc_input007, enc_output007, enc_assoc007, enc_nonce007, enc_key007, + sizeof(enc_input007), sizeof(enc_assoc007), sizeof(enc_nonce007) }, + { enc_input008, enc_output008, enc_assoc008, enc_nonce008, enc_key008, + sizeof(enc_input008), sizeof(enc_assoc008), sizeof(enc_nonce008) }, + { enc_input009, enc_output009, enc_assoc009, enc_nonce009, enc_key009, + sizeof(enc_input009), sizeof(enc_assoc009), sizeof(enc_nonce009) }, + { enc_input010, enc_output010, enc_assoc010, enc_nonce010, enc_key010, + sizeof(enc_input010), sizeof(enc_assoc010), sizeof(enc_nonce010) }, + { enc_input011, enc_output011, enc_assoc011, enc_nonce011, enc_key011, + sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) }, + { enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012, + sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) }, + { enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013, + sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) }, + { enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014, + sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) }, + { enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015, + sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) }, + { enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016, + sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) }, + { enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017, + sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) }, + { enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018, + sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) }, + { enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019, + sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) }, + { enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020, + sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) }, + { enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021, + sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) }, + { enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022, + sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) }, + { enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023, + sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) }, + { enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024, + sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) }, + { enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025, + sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) }, + { enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026, + sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) }, + { enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027, + sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) }, + { enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028, + sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) }, + { enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029, + sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) }, + { enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030, + sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) }, + { enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031, + sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) }, + { enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032, + sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) }, + { enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033, + sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) }, + { enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034, + sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) }, + { enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035, + sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) }, + { enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036, + sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) }, + { enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037, + sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) }, + { enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038, + sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) }, + { enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039, + sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) }, + { enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040, + sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) }, + { enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041, + sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) }, + { enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042, + sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) }, + { enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043, + sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) }, + { enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044, + sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) }, + { enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045, + sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) }, + { enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046, + sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) }, + { enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047, + sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) }, + { enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048, + sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) }, + { enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049, + sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) }, + { enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050, + sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) }, + { enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051, + sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) }, + { enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052, + sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) }, + { enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053, + sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) }, + { enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054, + sizeof(enc_input054), sizeof(enc_assoc054), sizeof(enc_nonce054) }, + { enc_input055, enc_output055, enc_assoc055, enc_nonce055, enc_key055, + sizeof(enc_input055), sizeof(enc_assoc055), sizeof(enc_nonce055) }, + { enc_input056, enc_output056, enc_assoc056, enc_nonce056, enc_key056, + sizeof(enc_input056), sizeof(enc_assoc056), sizeof(enc_nonce056) }, + { enc_input057, enc_output057, enc_assoc057, enc_nonce057, enc_key057, + sizeof(enc_input057), sizeof(enc_assoc057), sizeof(enc_nonce057) }, + { enc_input058, enc_output058, enc_assoc058, enc_nonce058, enc_key058, + sizeof(enc_input058), sizeof(enc_assoc058), sizeof(enc_nonce058) }, + { enc_input059, enc_output059, enc_assoc059, enc_nonce059, enc_key059, + sizeof(enc_input059), sizeof(enc_assoc059), sizeof(enc_nonce059) }, + { enc_input060, enc_output060, enc_assoc060, enc_nonce060, enc_key060, + sizeof(enc_input060), sizeof(enc_assoc060), sizeof(enc_nonce060) }, + { enc_input061, enc_output061, enc_assoc061, enc_nonce061, enc_key061, + sizeof(enc_input061), sizeof(enc_assoc061), sizeof(enc_nonce061) }, + { enc_input062, enc_output062, enc_assoc062, enc_nonce062, enc_key062, + sizeof(enc_input062), sizeof(enc_assoc062), sizeof(enc_nonce062) }, + { enc_input063, enc_output063, enc_assoc063, enc_nonce063, enc_key063, + sizeof(enc_input063), sizeof(enc_assoc063), sizeof(enc_nonce063) }, + { enc_input064, enc_output064, enc_assoc064, enc_nonce064, enc_key064, + sizeof(enc_input064), sizeof(enc_assoc064), sizeof(enc_nonce064) }, + { enc_input065, enc_output065, enc_assoc065, enc_nonce065, enc_key065, + sizeof(enc_input065), sizeof(enc_assoc065), sizeof(enc_nonce065) }, + { enc_input066, enc_output066, enc_assoc066, enc_nonce066, enc_key066, + sizeof(enc_input066), sizeof(enc_assoc066), sizeof(enc_nonce066) }, + { enc_input067, enc_output067, enc_assoc067, enc_nonce067, enc_key067, + sizeof(enc_input067), sizeof(enc_assoc067), sizeof(enc_nonce067) }, + { enc_input068, enc_output068, enc_assoc068, enc_nonce068, enc_key068, + sizeof(enc_input068), sizeof(enc_assoc068), sizeof(enc_nonce068) }, + { enc_input069, enc_output069, enc_assoc069, enc_nonce069, enc_key069, + sizeof(enc_input069), sizeof(enc_assoc069), sizeof(enc_nonce069) }, + { enc_input070, enc_output070, enc_assoc070, enc_nonce070, enc_key070, + sizeof(enc_input070), sizeof(enc_assoc070), sizeof(enc_nonce070) }, + { enc_input071, enc_output071, enc_assoc071, enc_nonce071, enc_key071, + sizeof(enc_input071), sizeof(enc_assoc071), sizeof(enc_nonce071) }, + { enc_input072, enc_output072, enc_assoc072, enc_nonce072, enc_key072, + sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) }, + { enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073, + sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) }, + { enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074, + sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) }, + { enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075, + sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) }, + { enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076, + sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) }, + { enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077, + sizeof(enc_input077), sizeof(enc_assoc077), sizeof(enc_nonce077) }, + { enc_input078, enc_output078, enc_assoc078, enc_nonce078, enc_key078, + sizeof(enc_input078), sizeof(enc_assoc078), sizeof(enc_nonce078) }, + { enc_input079, enc_output079, enc_assoc079, enc_nonce079, enc_key079, + sizeof(enc_input079), sizeof(enc_assoc079), sizeof(enc_nonce079) }, + { enc_input080, enc_output080, enc_assoc080, enc_nonce080, enc_key080, + sizeof(enc_input080), sizeof(enc_assoc080), sizeof(enc_nonce080) }, + { enc_input081, enc_output081, enc_assoc081, enc_nonce081, enc_key081, + sizeof(enc_input081), sizeof(enc_assoc081), sizeof(enc_nonce081) }, + { enc_input082, enc_output082, enc_assoc082, enc_nonce082, enc_key082, + sizeof(enc_input082), sizeof(enc_assoc082), sizeof(enc_nonce082) }, + { enc_input083, enc_output083, enc_assoc083, enc_nonce083, enc_key083, + sizeof(enc_input083), sizeof(enc_assoc083), sizeof(enc_nonce083) }, + { enc_input084, enc_output084, enc_assoc084, enc_nonce084, enc_key084, + sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) }, + { enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085, + sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) }, + { enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086, + sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) }, + { enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087, + sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) }, + { enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088, + sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) }, + { enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089, + sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) }, + { enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090, + sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) }, + { enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091, + sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) }, + { enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092, + sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) }, + { enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093, + sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) }, + { enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094, + sizeof(enc_input094), sizeof(enc_assoc094), sizeof(enc_nonce094) }, + { enc_input095, enc_output095, enc_assoc095, enc_nonce095, enc_key095, + sizeof(enc_input095), sizeof(enc_assoc095), sizeof(enc_nonce095) }, + { enc_input096, enc_output096, enc_assoc096, enc_nonce096, enc_key096, + sizeof(enc_input096), sizeof(enc_assoc096), sizeof(enc_nonce096) }, + { enc_input097, enc_output097, enc_assoc097, enc_nonce097, enc_key097, + sizeof(enc_input097), sizeof(enc_assoc097), sizeof(enc_nonce097) }, + { enc_input098, enc_output098, enc_assoc098, enc_nonce098, enc_key098, + sizeof(enc_input098), sizeof(enc_assoc098), sizeof(enc_nonce098) }, + { enc_input099, enc_output099, enc_assoc099, enc_nonce099, enc_key099, + sizeof(enc_input099), sizeof(enc_assoc099), sizeof(enc_nonce099) }, + { enc_input100, enc_output100, enc_assoc100, enc_nonce100, enc_key100, + sizeof(enc_input100), sizeof(enc_assoc100), sizeof(enc_nonce100) }, + { enc_input101, enc_output101, enc_assoc101, enc_nonce101, enc_key101, + sizeof(enc_input101), sizeof(enc_assoc101), sizeof(enc_nonce101) }, + { enc_input102, enc_output102, enc_assoc102, enc_nonce102, enc_key102, + sizeof(enc_input102), sizeof(enc_assoc102), sizeof(enc_nonce102) }, + { enc_input103, enc_output103, enc_assoc103, enc_nonce103, enc_key103, + sizeof(enc_input103), sizeof(enc_assoc103), sizeof(enc_nonce103) }, + { enc_input104, enc_output104, enc_assoc104, enc_nonce104, enc_key104, + sizeof(enc_input104), sizeof(enc_assoc104), sizeof(enc_nonce104) }, + { enc_input105, enc_output105, enc_assoc105, enc_nonce105, enc_key105, + sizeof(enc_input105), sizeof(enc_assoc105), sizeof(enc_nonce105) }, + { enc_input106, enc_output106, enc_assoc106, enc_nonce106, enc_key106, + sizeof(enc_input106), sizeof(enc_assoc106), sizeof(enc_nonce106) }, + { enc_input107, enc_output107, enc_assoc107, enc_nonce107, enc_key107, + sizeof(enc_input107), sizeof(enc_assoc107), sizeof(enc_nonce107) }, + { enc_input108, enc_output108, enc_assoc108, enc_nonce108, enc_key108, + sizeof(enc_input108), sizeof(enc_assoc108), sizeof(enc_nonce108) }, + { enc_input109, enc_output109, enc_assoc109, enc_nonce109, enc_key109, + sizeof(enc_input109), sizeof(enc_assoc109), sizeof(enc_nonce109) }, + { enc_input110, enc_output110, enc_assoc110, enc_nonce110, enc_key110, + sizeof(enc_input110), sizeof(enc_assoc110), sizeof(enc_nonce110) }, + { enc_input111, enc_output111, enc_assoc111, enc_nonce111, enc_key111, + sizeof(enc_input111), sizeof(enc_assoc111), sizeof(enc_nonce111) }, + { enc_input112, enc_output112, enc_assoc112, enc_nonce112, enc_key112, + sizeof(enc_input112), sizeof(enc_assoc112), sizeof(enc_nonce112) }, + { enc_input113, enc_output113, enc_assoc113, enc_nonce113, enc_key113, + sizeof(enc_input113), sizeof(enc_assoc113), sizeof(enc_nonce113) }, + { enc_input114, enc_output114, enc_assoc114, enc_nonce114, enc_key114, + sizeof(enc_input114), sizeof(enc_assoc114), sizeof(enc_nonce114) }, + { enc_input115, enc_output115, enc_assoc115, enc_nonce115, enc_key115, + sizeof(enc_input115), sizeof(enc_assoc115), sizeof(enc_nonce115) }, + { enc_input116, enc_output116, enc_assoc116, enc_nonce116, enc_key116, + sizeof(enc_input116), sizeof(enc_assoc116), sizeof(enc_nonce116) }, + { enc_input117, enc_output117, enc_assoc117, enc_nonce117, enc_key117, + sizeof(enc_input117), sizeof(enc_assoc117), sizeof(enc_nonce117) }, + { enc_input118, enc_output118, enc_assoc118, enc_nonce118, enc_key118, + sizeof(enc_input118), sizeof(enc_assoc118), sizeof(enc_nonce118) } +}; + +static const u8 dec_input001[] __initconst = { + 0x64, 0xa0, 0x86, 0x15, 0x75, 0x86, 0x1a, 0xf4, + 0x60, 0xf0, 0x62, 0xc7, 0x9b, 0xe6, 0x43, 0xbd, + 0x5e, 0x80, 0x5c, 0xfd, 0x34, 0x5c, 0xf3, 0x89, + 0xf1, 0x08, 0x67, 0x0a, 0xc7, 0x6c, 0x8c, 0xb2, + 0x4c, 0x6c, 0xfc, 0x18, 0x75, 0x5d, 0x43, 0xee, + 0xa0, 0x9e, 0xe9, 0x4e, 0x38, 0x2d, 0x26, 0xb0, + 0xbd, 0xb7, 0xb7, 0x3c, 0x32, 0x1b, 0x01, 0x00, + 0xd4, 0xf0, 0x3b, 0x7f, 0x35, 0x58, 0x94, 0xcf, + 0x33, 0x2f, 0x83, 0x0e, 0x71, 0x0b, 0x97, 0xce, + 0x98, 0xc8, 0xa8, 0x4a, 0xbd, 0x0b, 0x94, 0x81, + 0x14, 0xad, 0x17, 0x6e, 0x00, 0x8d, 0x33, 0xbd, + 0x60, 0xf9, 0x82, 0xb1, 0xff, 0x37, 0xc8, 0x55, + 0x97, 0x97, 0xa0, 0x6e, 0xf4, 0xf0, 0xef, 0x61, + 0xc1, 0x86, 0x32, 0x4e, 0x2b, 0x35, 0x06, 0x38, + 0x36, 0x06, 0x90, 0x7b, 0x6a, 0x7c, 0x02, 0xb0, + 0xf9, 0xf6, 0x15, 0x7b, 0x53, 0xc8, 0x67, 0xe4, + 0xb9, 0x16, 0x6c, 0x76, 0x7b, 0x80, 0x4d, 0x46, + 0xa5, 0x9b, 0x52, 0x16, 0xcd, 0xe7, 0xa4, 0xe9, + 0x90, 0x40, 0xc5, 0xa4, 0x04, 0x33, 0x22, 0x5e, + 0xe2, 0x82, 0xa1, 0xb0, 0xa0, 0x6c, 0x52, 0x3e, + 0xaf, 0x45, 0x34, 0xd7, 0xf8, 0x3f, 0xa1, 0x15, + 0x5b, 0x00, 0x47, 0x71, 0x8c, 0xbc, 0x54, 0x6a, + 0x0d, 0x07, 0x2b, 0x04, 0xb3, 0x56, 0x4e, 0xea, + 0x1b, 0x42, 0x22, 0x73, 0xf5, 0x48, 0x27, 0x1a, + 0x0b, 0xb2, 0x31, 0x60, 0x53, 0xfa, 0x76, 0x99, + 0x19, 0x55, 0xeb, 0xd6, 0x31, 0x59, 0x43, 0x4e, + 0xce, 0xbb, 0x4e, 0x46, 0x6d, 0xae, 0x5a, 0x10, + 0x73, 0xa6, 0x72, 0x76, 0x27, 0x09, 0x7a, 0x10, + 0x49, 0xe6, 0x17, 0xd9, 0x1d, 0x36, 0x10, 0x94, + 0xfa, 0x68, 0xf0, 0xff, 0x77, 0x98, 0x71, 0x30, + 0x30, 0x5b, 0xea, 0xba, 0x2e, 0xda, 0x04, 0xdf, + 0x99, 0x7b, 0x71, 0x4d, 0x6c, 0x6f, 0x2c, 0x29, + 0xa6, 0xad, 0x5c, 0xb4, 0x02, 0x2b, 0x02, 0x70, + 0x9b, 0xee, 0xad, 0x9d, 0x67, 0x89, 0x0c, 0xbb, + 0x22, 0x39, 0x23, 0x36, 0xfe, 0xa1, 0x85, 0x1f, + 0x38 +}; +static const u8 dec_output001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 dec_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 dec_nonce001[] __initconst = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08 +}; +static const u8 dec_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const u8 dec_input002[] __initconst = { + 0xea, 0xe0, 0x1e, 0x9e, 0x2c, 0x91, 0xaa, 0xe1, + 0xdb, 0x5d, 0x99, 0x3f, 0x8a, 0xf7, 0x69, 0x92 +}; +static const u8 dec_output002[] __initconst = { }; +static const u8 dec_assoc002[] __initconst = { }; +static const u8 dec_nonce002[] __initconst = { + 0xca, 0xbf, 0x33, 0x71, 0x32, 0x45, 0x77, 0x8e +}; +static const u8 dec_key002[] __initconst = { + 0x4c, 0xf5, 0x96, 0x83, 0x38, 0xe6, 0xae, 0x7f, + 0x2d, 0x29, 0x25, 0x76, 0xd5, 0x75, 0x27, 0x86, + 0x91, 0x9a, 0x27, 0x7a, 0xfb, 0x46, 0xc5, 0xef, + 0x94, 0x81, 0x79, 0x57, 0x14, 0x59, 0x40, 0x68 +}; + +static const u8 dec_input003[] __initconst = { + 0xdd, 0x6b, 0x3b, 0x82, 0xce, 0x5a, 0xbd, 0xd6, + 0xa9, 0x35, 0x83, 0xd8, 0x8c, 0x3d, 0x85, 0x77 +}; +static const u8 dec_output003[] __initconst = { }; +static const u8 dec_assoc003[] __initconst = { + 0x33, 0x10, 0x41, 0x12, 0x1f, 0xf3, 0xd2, 0x6b +}; +static const u8 dec_nonce003[] __initconst = { + 0x3d, 0x86, 0xb5, 0x6b, 0xc8, 0xa3, 0x1f, 0x1d +}; +static const u8 dec_key003[] __initconst = { + 0x2d, 0xb0, 0x5d, 0x40, 0xc8, 0xed, 0x44, 0x88, + 0x34, 0xd1, 0x13, 0xaf, 0x57, 0xa1, 0xeb, 0x3a, + 0x2a, 0x80, 0x51, 0x36, 0xec, 0x5b, 0xbc, 0x08, + 0x93, 0x84, 0x21, 0xb5, 0x13, 0x88, 0x3c, 0x0d +}; + +static const u8 dec_input004[] __initconst = { + 0xb7, 0x1b, 0xb0, 0x73, 0x59, 0xb0, 0x84, 0xb2, + 0x6d, 0x8e, 0xab, 0x94, 0x31, 0xa1, 0xae, 0xac, + 0x89 +}; +static const u8 dec_output004[] __initconst = { + 0xa4 +}; +static const u8 dec_assoc004[] __initconst = { + 0x6a, 0xe2, 0xad, 0x3f, 0x88, 0x39, 0x5a, 0x40 +}; +static const u8 dec_nonce004[] __initconst = { + 0xd2, 0x32, 0x1f, 0x29, 0x28, 0xc6, 0xc4, 0xc4 +}; +static const u8 dec_key004[] __initconst = { + 0x4b, 0x28, 0x4b, 0xa3, 0x7b, 0xbe, 0xe9, 0xf8, + 0x31, 0x80, 0x82, 0xd7, 0xd8, 0xe8, 0xb5, 0xa1, + 0xe2, 0x18, 0x18, 0x8a, 0x9c, 0xfa, 0xa3, 0x3d, + 0x25, 0x71, 0x3e, 0x40, 0xbc, 0x54, 0x7a, 0x3e +}; + +static const u8 dec_input005[] __initconst = { + 0xbf, 0xe1, 0x5b, 0x0b, 0xdb, 0x6b, 0xf5, 0x5e, + 0x6c, 0x5d, 0x84, 0x44, 0x39, 0x81, 0xc1, 0x9c, + 0xac +}; +static const u8 dec_output005[] __initconst = { + 0x2d +}; +static const u8 dec_assoc005[] __initconst = { }; +static const u8 dec_nonce005[] __initconst = { + 0x20, 0x1c, 0xaa, 0x5f, 0x9c, 0xbf, 0x92, 0x30 +}; +static const u8 dec_key005[] __initconst = { + 0x66, 0xca, 0x9c, 0x23, 0x2a, 0x4b, 0x4b, 0x31, + 0x0e, 0x92, 0x89, 0x8b, 0xf4, 0x93, 0xc7, 0x87, + 0x98, 0xa3, 0xd8, 0x39, 0xf8, 0xf4, 0xa7, 0x01, + 0xc0, 0x2e, 0x0a, 0xa6, 0x7e, 0x5a, 0x78, 0x87 +}; + +static const u8 dec_input006[] __initconst = { + 0x8b, 0x06, 0xd3, 0x31, 0xb0, 0x93, 0x45, 0xb1, + 0x75, 0x6e, 0x26, 0xf9, 0x67, 0xbc, 0x90, 0x15, + 0x81, 0x2c, 0xb5, 0xf0, 0xc6, 0x2b, 0xc7, 0x8c, + 0x56, 0xd1, 0xbf, 0x69, 0x6c, 0x07, 0xa0, 0xda, + 0x65, 0x27, 0xc9, 0x90, 0x3d, 0xef, 0x4b, 0x11, + 0x0f, 0x19, 0x07, 0xfd, 0x29, 0x92, 0xd9, 0xc8, + 0xf7, 0x99, 0x2e, 0x4a, 0xd0, 0xb8, 0x2c, 0xdc, + 0x93, 0xf5, 0x9e, 0x33, 0x78, 0xd1, 0x37, 0xc3, + 0x66, 0xd7, 0x5e, 0xbc, 0x44, 0xbf, 0x53, 0xa5, + 0xbc, 0xc4, 0xcb, 0x7b, 0x3a, 0x8e, 0x7f, 0x02, + 0xbd, 0xbb, 0xe7, 0xca, 0xa6, 0x6c, 0x6b, 0x93, + 0x21, 0x93, 0x10, 0x61, 0xe7, 0x69, 0xd0, 0x78, + 0xf3, 0x07, 0x5a, 0x1a, 0x8f, 0x73, 0xaa, 0xb1, + 0x4e, 0xd3, 0xda, 0x4f, 0xf3, 0x32, 0xe1, 0x66, + 0x3e, 0x6c, 0xc6, 0x13, 0xba, 0x06, 0x5b, 0xfc, + 0x6a, 0xe5, 0x6f, 0x60, 0xfb, 0x07, 0x40, 0xb0, + 0x8c, 0x9d, 0x84, 0x43, 0x6b, 0xc1, 0xf7, 0x8d, + 0x8d, 0x31, 0xf7, 0x7a, 0x39, 0x4d, 0x8f, 0x9a, + 0xeb +}; +static const u8 dec_output006[] __initconst = { + 0x33, 0x2f, 0x94, 0xc1, 0xa4, 0xef, 0xcc, 0x2a, + 0x5b, 0xa6, 0xe5, 0x8f, 0x1d, 0x40, 0xf0, 0x92, + 0x3c, 0xd9, 0x24, 0x11, 0xa9, 0x71, 0xf9, 0x37, + 0x14, 0x99, 0xfa, 0xbe, 0xe6, 0x80, 0xde, 0x50, + 0xc9, 0x96, 0xd4, 0xb0, 0xec, 0x9e, 0x17, 0xec, + 0xd2, 0x5e, 0x72, 0x99, 0xfc, 0x0a, 0xe1, 0xcb, + 0x48, 0xd2, 0x85, 0xdd, 0x2f, 0x90, 0xe0, 0x66, + 0x3b, 0xe6, 0x20, 0x74, 0xbe, 0x23, 0x8f, 0xcb, + 0xb4, 0xe4, 0xda, 0x48, 0x40, 0xa6, 0xd1, 0x1b, + 0xc7, 0x42, 0xce, 0x2f, 0x0c, 0xa6, 0x85, 0x6e, + 0x87, 0x37, 0x03, 0xb1, 0x7c, 0x25, 0x96, 0xa3, + 0x05, 0xd8, 0xb0, 0xf4, 0xed, 0xea, 0xc2, 0xf0, + 0x31, 0x98, 0x6c, 0xd1, 0x14, 0x25, 0xc0, 0xcb, + 0x01, 0x74, 0xd0, 0x82, 0xf4, 0x36, 0xf5, 0x41, + 0xd5, 0xdc, 0xca, 0xc5, 0xbb, 0x98, 0xfe, 0xfc, + 0x69, 0x21, 0x70, 0xd8, 0xa4, 0x4b, 0xc8, 0xde, + 0x8f +}; +static const u8 dec_assoc006[] __initconst = { + 0x70, 0xd3, 0x33, 0xf3, 0x8b, 0x18, 0x0b +}; +static const u8 dec_nonce006[] __initconst = { + 0xdf, 0x51, 0x84, 0x82, 0x42, 0x0c, 0x75, 0x9c +}; +static const u8 dec_key006[] __initconst = { + 0x68, 0x7b, 0x8d, 0x8e, 0xe3, 0xc4, 0xdd, 0xae, + 0xdf, 0x72, 0x7f, 0x53, 0x72, 0x25, 0x1e, 0x78, + 0x91, 0xcb, 0x69, 0x76, 0x1f, 0x49, 0x93, 0xf9, + 0x6f, 0x21, 0xcc, 0x39, 0x9c, 0xad, 0xb1, 0x01 +}; + +static const u8 dec_input007[] __initconst = { + 0x85, 0x04, 0xc2, 0xed, 0x8d, 0xfd, 0x97, 0x5c, + 0xd2, 0xb7, 0xe2, 0xc1, 0x6b, 0xa3, 0xba, 0xf8, + 0xc9, 0x50, 0xc3, 0xc6, 0xa5, 0xe3, 0xa4, 0x7c, + 0xc3, 0x23, 0x49, 0x5e, 0xa9, 0xb9, 0x32, 0xeb, + 0x8a, 0x7c, 0xca, 0xe5, 0xec, 0xfb, 0x7c, 0xc0, + 0xcb, 0x7d, 0xdc, 0x2c, 0x9d, 0x92, 0x55, 0x21, + 0x0a, 0xc8, 0x43, 0x63, 0x59, 0x0a, 0x31, 0x70, + 0x82, 0x67, 0x41, 0x03, 0xf8, 0xdf, 0xf2, 0xac, + 0xa7, 0x02, 0xd4, 0xd5, 0x8a, 0x2d, 0xc8, 0x99, + 0x19, 0x66, 0xd0, 0xf6, 0x88, 0x2c, 0x77, 0xd9, + 0xd4, 0x0d, 0x6c, 0xbd, 0x98, 0xde, 0xe7, 0x7f, + 0xad, 0x7e, 0x8a, 0xfb, 0xe9, 0x4b, 0xe5, 0xf7, + 0xe5, 0x50, 0xa0, 0x90, 0x3f, 0xd6, 0x22, 0x53, + 0xe3, 0xfe, 0x1b, 0xcc, 0x79, 0x3b, 0xec, 0x12, + 0x47, 0x52, 0xa7, 0xd6, 0x04, 0xe3, 0x52, 0xe6, + 0x93, 0x90, 0x91, 0x32, 0x73, 0x79, 0xb8, 0xd0, + 0x31, 0xde, 0x1f, 0x9f, 0x2f, 0x05, 0x38, 0x54, + 0x2f, 0x35, 0x04, 0x39, 0xe0, 0xa7, 0xba, 0xc6, + 0x52, 0xf6, 0x37, 0x65, 0x4c, 0x07, 0xa9, 0x7e, + 0xb3, 0x21, 0x6f, 0x74, 0x8c, 0xc9, 0xde, 0xdb, + 0x65, 0x1b, 0x9b, 0xaa, 0x60, 0xb1, 0x03, 0x30, + 0x6b, 0xb2, 0x03, 0xc4, 0x1c, 0x04, 0xf8, 0x0f, + 0x64, 0xaf, 0x46, 0xe4, 0x65, 0x99, 0x49, 0xe2, + 0xea, 0xce, 0x78, 0x00, 0xd8, 0x8b, 0xd5, 0x2e, + 0xcf, 0xfc, 0x40, 0x49, 0xe8, 0x58, 0xdc, 0x34, + 0x9c, 0x8c, 0x61, 0xbf, 0x0a, 0x8e, 0xec, 0x39, + 0xa9, 0x30, 0x05, 0x5a, 0xd2, 0x56, 0x01, 0xc7, + 0xda, 0x8f, 0x4e, 0xbb, 0x43, 0xa3, 0x3a, 0xf9, + 0x15, 0x2a, 0xd0, 0xa0, 0x7a, 0x87, 0x34, 0x82, + 0xfe, 0x8a, 0xd1, 0x2d, 0x5e, 0xc7, 0xbf, 0x04, + 0x53, 0x5f, 0x3b, 0x36, 0xd4, 0x25, 0x5c, 0x34, + 0x7a, 0x8d, 0xd5, 0x05, 0xce, 0x72, 0xca, 0xef, + 0x7a, 0x4b, 0xbc, 0xb0, 0x10, 0x5c, 0x96, 0x42, + 0x3a, 0x00, 0x98, 0xcd, 0x15, 0xe8, 0xb7, 0x53 +}; +static const u8 dec_output007[] __initconst = { + 0x9b, 0x18, 0xdb, 0xdd, 0x9a, 0x0f, 0x3e, 0xa5, + 0x15, 0x17, 0xde, 0xdf, 0x08, 0x9d, 0x65, 0x0a, + 0x67, 0x30, 0x12, 0xe2, 0x34, 0x77, 0x4b, 0xc1, + 0xd9, 0xc6, 0x1f, 0xab, 0xc6, 0x18, 0x50, 0x17, + 0xa7, 0x9d, 0x3c, 0xa6, 0xc5, 0x35, 0x8c, 0x1c, + 0xc0, 0xa1, 0x7c, 0x9f, 0x03, 0x89, 0xca, 0xe1, + 0xe6, 0xe9, 0xd4, 0xd3, 0x88, 0xdb, 0xb4, 0x51, + 0x9d, 0xec, 0xb4, 0xfc, 0x52, 0xee, 0x6d, 0xf1, + 0x75, 0x42, 0xc6, 0xfd, 0xbd, 0x7a, 0x8e, 0x86, + 0xfc, 0x44, 0xb3, 0x4f, 0xf3, 0xea, 0x67, 0x5a, + 0x41, 0x13, 0xba, 0xb0, 0xdc, 0xe1, 0xd3, 0x2a, + 0x7c, 0x22, 0xb3, 0xca, 0xac, 0x6a, 0x37, 0x98, + 0x3e, 0x1d, 0x40, 0x97, 0xf7, 0x9b, 0x1d, 0x36, + 0x6b, 0xb3, 0x28, 0xbd, 0x60, 0x82, 0x47, 0x34, + 0xaa, 0x2f, 0x7d, 0xe9, 0xa8, 0x70, 0x81, 0x57, + 0xd4, 0xb9, 0x77, 0x0a, 0x9d, 0x29, 0xa7, 0x84, + 0x52, 0x4f, 0xc2, 0x4a, 0x40, 0x3b, 0x3c, 0xd4, + 0xc9, 0x2a, 0xdb, 0x4a, 0x53, 0xc4, 0xbe, 0x80, + 0xe9, 0x51, 0x7f, 0x8f, 0xc7, 0xa2, 0xce, 0x82, + 0x5c, 0x91, 0x1e, 0x74, 0xd9, 0xd0, 0xbd, 0xd5, + 0xf3, 0xfd, 0xda, 0x4d, 0x25, 0xb4, 0xbb, 0x2d, + 0xac, 0x2f, 0x3d, 0x71, 0x85, 0x7b, 0xcf, 0x3c, + 0x7b, 0x3e, 0x0e, 0x22, 0x78, 0x0c, 0x29, 0xbf, + 0xe4, 0xf4, 0x57, 0xb3, 0xcb, 0x49, 0xa0, 0xfc, + 0x1e, 0x05, 0x4e, 0x16, 0xbc, 0xd5, 0xa8, 0xa3, + 0xee, 0x05, 0x35, 0xc6, 0x7c, 0xab, 0x60, 0x14, + 0x55, 0x1a, 0x8e, 0xc5, 0x88, 0x5d, 0xd5, 0x81, + 0xc2, 0x81, 0xa5, 0xc4, 0x60, 0xdb, 0xaf, 0x77, + 0x91, 0xe1, 0xce, 0xa2, 0x7e, 0x7f, 0x42, 0xe3, + 0xb0, 0x13, 0x1c, 0x1f, 0x25, 0x60, 0x21, 0xe2, + 0x40, 0x5f, 0x99, 0xb7, 0x73, 0xec, 0x9b, 0x2b, + 0xf0, 0x65, 0x11, 0xc8, 0xd0, 0x0a, 0x9f, 0xd3 +}; +static const u8 dec_assoc007[] __initconst = { }; +static const u8 dec_nonce007[] __initconst = { + 0xde, 0x7b, 0xef, 0xc3, 0x65, 0x1b, 0x68, 0xb0 +}; +static const u8 dec_key007[] __initconst = { + 0x8d, 0xb8, 0x91, 0x48, 0xf0, 0xe7, 0x0a, 0xbd, + 0xf9, 0x3f, 0xcd, 0xd9, 0xa0, 0x1e, 0x42, 0x4c, + 0xe7, 0xde, 0x25, 0x3d, 0xa3, 0xd7, 0x05, 0x80, + 0x8d, 0xf2, 0x82, 0xac, 0x44, 0x16, 0x51, 0x01 +}; + +static const u8 dec_input008[] __initconst = { + 0x14, 0xf6, 0x41, 0x37, 0xa6, 0xd4, 0x27, 0xcd, + 0xdb, 0x06, 0x3e, 0x9a, 0x4e, 0xab, 0xd5, 0xb1, + 0x1e, 0x6b, 0xd2, 0xbc, 0x11, 0xf4, 0x28, 0x93, + 0x63, 0x54, 0xef, 0xbb, 0x5e, 0x1d, 0x3a, 0x1d, + 0x37, 0x3c, 0x0a, 0x6c, 0x1e, 0xc2, 0xd1, 0x2c, + 0xb5, 0xa3, 0xb5, 0x7b, 0xb8, 0x8f, 0x25, 0xa6, + 0x1b, 0x61, 0x1c, 0xec, 0x28, 0x58, 0x26, 0xa4, + 0xa8, 0x33, 0x28, 0x25, 0x5c, 0x45, 0x05, 0xe5, + 0x6c, 0x99, 0xe5, 0x45, 0xc4, 0xa2, 0x03, 0x84, + 0x03, 0x73, 0x1e, 0x8c, 0x49, 0xac, 0x20, 0xdd, + 0x8d, 0xb3, 0xc4, 0xf5, 0xe7, 0x4f, 0xf1, 0xed, + 0xa1, 0x98, 0xde, 0xa4, 0x96, 0xdd, 0x2f, 0xab, + 0xab, 0x97, 0xcf, 0x3e, 0xd2, 0x9e, 0xb8, 0x13, + 0x07, 0x28, 0x29, 0x19, 0xaf, 0xfd, 0xf2, 0x49, + 0x43, 0xea, 0x49, 0x26, 0x91, 0xc1, 0x07, 0xd6, + 0xbb, 0x81, 0x75, 0x35, 0x0d, 0x24, 0x7f, 0xc8, + 0xda, 0xd4, 0xb7, 0xeb, 0xe8, 0x5c, 0x09, 0xa2, + 0x2f, 0xdc, 0x28, 0x7d, 0x3a, 0x03, 0xfa, 0x94, + 0xb5, 0x1d, 0x17, 0x99, 0x36, 0xc3, 0x1c, 0x18, + 0x34, 0xe3, 0x9f, 0xf5, 0x55, 0x7c, 0xb0, 0x60, + 0x9d, 0xff, 0xac, 0xd4, 0x61, 0xf2, 0xad, 0xf8, + 0xce, 0xc7, 0xbe, 0x5c, 0xd2, 0x95, 0xa8, 0x4b, + 0x77, 0x13, 0x19, 0x59, 0x26, 0xc9, 0xb7, 0x8f, + 0x6a, 0xcb, 0x2d, 0x37, 0x91, 0xea, 0x92, 0x9c, + 0x94, 0x5b, 0xda, 0x0b, 0xce, 0xfe, 0x30, 0x20, + 0xf8, 0x51, 0xad, 0xf2, 0xbe, 0xe7, 0xc7, 0xff, + 0xb3, 0x33, 0x91, 0x6a, 0xc9, 0x1a, 0x41, 0xc9, + 0x0f, 0xf3, 0x10, 0x0e, 0xfd, 0x53, 0xff, 0x6c, + 0x16, 0x52, 0xd9, 0xf3, 0xf7, 0x98, 0x2e, 0xc9, + 0x07, 0x31, 0x2c, 0x0c, 0x72, 0xd7, 0xc5, 0xc6, + 0x08, 0x2a, 0x7b, 0xda, 0xbd, 0x7e, 0x02, 0xea, + 0x1a, 0xbb, 0xf2, 0x04, 0x27, 0x61, 0x28, 0x8e, + 0xf5, 0x04, 0x03, 0x1f, 0x4c, 0x07, 0x55, 0x82, + 0xec, 0x1e, 0xd7, 0x8b, 0x2f, 0x65, 0x56, 0xd1, + 0xd9, 0x1e, 0x3c, 0xe9, 0x1f, 0x5e, 0x98, 0x70, + 0x38, 0x4a, 0x8c, 0x49, 0xc5, 0x43, 0xa0, 0xa1, + 0x8b, 0x74, 0x9d, 0x4c, 0x62, 0x0d, 0x10, 0x0c, + 0xf4, 0x6c, 0x8f, 0xe0, 0xaa, 0x9a, 0x8d, 0xb7, + 0xe0, 0xbe, 0x4c, 0x87, 0xf1, 0x98, 0x2f, 0xcc, + 0xed, 0xc0, 0x52, 0x29, 0xdc, 0x83, 0xf8, 0xfc, + 0x2c, 0x0e, 0xa8, 0x51, 0x4d, 0x80, 0x0d, 0xa3, + 0xfe, 0xd8, 0x37, 0xe7, 0x41, 0x24, 0xfc, 0xfb, + 0x75, 0xe3, 0x71, 0x7b, 0x57, 0x45, 0xf5, 0x97, + 0x73, 0x65, 0x63, 0x14, 0x74, 0xb8, 0x82, 0x9f, + 0xf8, 0x60, 0x2f, 0x8a, 0xf2, 0x4e, 0xf1, 0x39, + 0xda, 0x33, 0x91, 0xf8, 0x36, 0xe0, 0x8d, 0x3f, + 0x1f, 0x3b, 0x56, 0xdc, 0xa0, 0x8f, 0x3c, 0x9d, + 0x71, 0x52, 0xa7, 0xb8, 0xc0, 0xa5, 0xc6, 0xa2, + 0x73, 0xda, 0xf4, 0x4b, 0x74, 0x5b, 0x00, 0x3d, + 0x99, 0xd7, 0x96, 0xba, 0xe6, 0xe1, 0xa6, 0x96, + 0x38, 0xad, 0xb3, 0xc0, 0xd2, 0xba, 0x91, 0x6b, + 0xf9, 0x19, 0xdd, 0x3b, 0xbe, 0xbe, 0x9c, 0x20, + 0x50, 0xba, 0xa1, 0xd0, 0xce, 0x11, 0xbd, 0x95, + 0xd8, 0xd1, 0xdd, 0x33, 0x85, 0x74, 0xdc, 0xdb, + 0x66, 0x76, 0x44, 0xdc, 0x03, 0x74, 0x48, 0x35, + 0x98, 0xb1, 0x18, 0x47, 0x94, 0x7d, 0xff, 0x62, + 0xe4, 0x58, 0x78, 0xab, 0xed, 0x95, 0x36, 0xd9, + 0x84, 0x91, 0x82, 0x64, 0x41, 0xbb, 0x58, 0xe6, + 0x1c, 0x20, 0x6d, 0x15, 0x6b, 0x13, 0x96, 0xe8, + 0x35, 0x7f, 0xdc, 0x40, 0x2c, 0xe9, 0xbc, 0x8a, + 0x4f, 0x92, 0xec, 0x06, 0x2d, 0x50, 0xdf, 0x93, + 0x5d, 0x65, 0x5a, 0xa8, 0xfc, 0x20, 0x50, 0x14, + 0xa9, 0x8a, 0x7e, 0x1d, 0x08, 0x1f, 0xe2, 0x99, + 0xd0, 0xbe, 0xfb, 0x3a, 0x21, 0x9d, 0xad, 0x86, + 0x54, 0xfd, 0x0d, 0x98, 0x1c, 0x5a, 0x6f, 0x1f, + 0x9a, 0x40, 0xcd, 0xa2, 0xff, 0x6a, 0xf1, 0x54 +}; +static const u8 dec_output008[] __initconst = { + 0xc3, 0x09, 0x94, 0x62, 0xe6, 0x46, 0x2e, 0x10, + 0xbe, 0x00, 0xe4, 0xfc, 0xf3, 0x40, 0xa3, 0xe2, + 0x0f, 0xc2, 0x8b, 0x28, 0xdc, 0xba, 0xb4, 0x3c, + 0xe4, 0x21, 0x58, 0x61, 0xcd, 0x8b, 0xcd, 0xfb, + 0xac, 0x94, 0xa1, 0x45, 0xf5, 0x1c, 0xe1, 0x12, + 0xe0, 0x3b, 0x67, 0x21, 0x54, 0x5e, 0x8c, 0xaa, + 0xcf, 0xdb, 0xb4, 0x51, 0xd4, 0x13, 0xda, 0xe6, + 0x83, 0x89, 0xb6, 0x92, 0xe9, 0x21, 0x76, 0xa4, + 0x93, 0x7d, 0x0e, 0xfd, 0x96, 0x36, 0x03, 0x91, + 0x43, 0x5c, 0x92, 0x49, 0x62, 0x61, 0x7b, 0xeb, + 0x43, 0x89, 0xb8, 0x12, 0x20, 0x43, 0xd4, 0x47, + 0x06, 0x84, 0xee, 0x47, 0xe9, 0x8a, 0x73, 0x15, + 0x0f, 0x72, 0xcf, 0xed, 0xce, 0x96, 0xb2, 0x7f, + 0x21, 0x45, 0x76, 0xeb, 0x26, 0x28, 0x83, 0x6a, + 0xad, 0xaa, 0xa6, 0x81, 0xd8, 0x55, 0xb1, 0xa3, + 0x85, 0xb3, 0x0c, 0xdf, 0xf1, 0x69, 0x2d, 0x97, + 0x05, 0x2a, 0xbc, 0x7c, 0x7b, 0x25, 0xf8, 0x80, + 0x9d, 0x39, 0x25, 0xf3, 0x62, 0xf0, 0x66, 0x5e, + 0xf4, 0xa0, 0xcf, 0xd8, 0xfd, 0x4f, 0xb1, 0x1f, + 0x60, 0x3a, 0x08, 0x47, 0xaf, 0xe1, 0xf6, 0x10, + 0x77, 0x09, 0xa7, 0x27, 0x8f, 0x9a, 0x97, 0x5a, + 0x26, 0xfa, 0xfe, 0x41, 0x32, 0x83, 0x10, 0xe0, + 0x1d, 0xbf, 0x64, 0x0d, 0xf4, 0x1c, 0x32, 0x35, + 0xe5, 0x1b, 0x36, 0xef, 0xd4, 0x4a, 0x93, 0x4d, + 0x00, 0x7c, 0xec, 0x02, 0x07, 0x8b, 0x5d, 0x7d, + 0x1b, 0x0e, 0xd1, 0xa6, 0xa5, 0x5d, 0x7d, 0x57, + 0x88, 0xa8, 0xcc, 0x81, 0xb4, 0x86, 0x4e, 0xb4, + 0x40, 0xe9, 0x1d, 0xc3, 0xb1, 0x24, 0x3e, 0x7f, + 0xcc, 0x8a, 0x24, 0x9b, 0xdf, 0x6d, 0xf0, 0x39, + 0x69, 0x3e, 0x4c, 0xc0, 0x96, 0xe4, 0x13, 0xda, + 0x90, 0xda, 0xf4, 0x95, 0x66, 0x8b, 0x17, 0x17, + 0xfe, 0x39, 0x43, 0x25, 0xaa, 0xda, 0xa0, 0x43, + 0x3c, 0xb1, 0x41, 0x02, 0xa3, 0xf0, 0xa7, 0x19, + 0x59, 0xbc, 0x1d, 0x7d, 0x6c, 0x6d, 0x91, 0x09, + 0x5c, 0xb7, 0x5b, 0x01, 0xd1, 0x6f, 0x17, 0x21, + 0x97, 0xbf, 0x89, 0x71, 0xa5, 0xb0, 0x6e, 0x07, + 0x45, 0xfd, 0x9d, 0xea, 0x07, 0xf6, 0x7a, 0x9f, + 0x10, 0x18, 0x22, 0x30, 0x73, 0xac, 0xd4, 0x6b, + 0x72, 0x44, 0xed, 0xd9, 0x19, 0x9b, 0x2d, 0x4a, + 0x41, 0xdd, 0xd1, 0x85, 0x5e, 0x37, 0x19, 0xed, + 0xd2, 0x15, 0x8f, 0x5e, 0x91, 0xdb, 0x33, 0xf2, + 0xe4, 0xdb, 0xff, 0x98, 0xfb, 0xa3, 0xb5, 0xca, + 0x21, 0x69, 0x08, 0xe7, 0x8a, 0xdf, 0x90, 0xff, + 0x3e, 0xe9, 0x20, 0x86, 0x3c, 0xe9, 0xfc, 0x0b, + 0xfe, 0x5c, 0x61, 0xaa, 0x13, 0x92, 0x7f, 0x7b, + 0xec, 0xe0, 0x6d, 0xa8, 0x23, 0x22, 0xf6, 0x6b, + 0x77, 0xc4, 0xfe, 0x40, 0x07, 0x3b, 0xb6, 0xf6, + 0x8e, 0x5f, 0xd4, 0xb9, 0xb7, 0x0f, 0x21, 0x04, + 0xef, 0x83, 0x63, 0x91, 0x69, 0x40, 0xa3, 0x48, + 0x5c, 0xd2, 0x60, 0xf9, 0x4f, 0x6c, 0x47, 0x8b, + 0x3b, 0xb1, 0x9f, 0x8e, 0xee, 0x16, 0x8a, 0x13, + 0xfc, 0x46, 0x17, 0xc3, 0xc3, 0x32, 0x56, 0xf8, + 0x3c, 0x85, 0x3a, 0xb6, 0x3e, 0xaa, 0x89, 0x4f, + 0xb3, 0xdf, 0x38, 0xfd, 0xf1, 0xe4, 0x3a, 0xc0, + 0xe6, 0x58, 0xb5, 0x8f, 0xc5, 0x29, 0xa2, 0x92, + 0x4a, 0xb6, 0xa0, 0x34, 0x7f, 0xab, 0xb5, 0x8a, + 0x90, 0xa1, 0xdb, 0x4d, 0xca, 0xb6, 0x2c, 0x41, + 0x3c, 0xf7, 0x2b, 0x21, 0xc3, 0xfd, 0xf4, 0x17, + 0x5c, 0xb5, 0x33, 0x17, 0x68, 0x2b, 0x08, 0x30, + 0xf3, 0xf7, 0x30, 0x3c, 0x96, 0xe6, 0x6a, 0x20, + 0x97, 0xe7, 0x4d, 0x10, 0x5f, 0x47, 0x5f, 0x49, + 0x96, 0x09, 0xf0, 0x27, 0x91, 0xc8, 0xf8, 0x5a, + 0x2e, 0x79, 0xb5, 0xe2, 0xb8, 0xe8, 0xb9, 0x7b, + 0xd5, 0x10, 0xcb, 0xff, 0x5d, 0x14, 0x73, 0xf3 +}; +static const u8 dec_assoc008[] __initconst = { }; +static const u8 dec_nonce008[] __initconst = { + 0x0e, 0x0d, 0x57, 0xbb, 0x7b, 0x40, 0x54, 0x02 +}; +static const u8 dec_key008[] __initconst = { + 0xf2, 0xaa, 0x4f, 0x99, 0xfd, 0x3e, 0xa8, 0x53, + 0xc1, 0x44, 0xe9, 0x81, 0x18, 0xdc, 0xf5, 0xf0, + 0x3e, 0x44, 0x15, 0x59, 0xe0, 0xc5, 0x44, 0x86, + 0xc3, 0x91, 0xa8, 0x75, 0xc0, 0x12, 0x46, 0xba +}; + +static const u8 dec_input009[] __initconst = { + 0xfd, 0x81, 0x8d, 0xd0, 0x3d, 0xb4, 0xd5, 0xdf, + 0xd3, 0x42, 0x47, 0x5a, 0x6d, 0x19, 0x27, 0x66, + 0x4b, 0x2e, 0x0c, 0x27, 0x9c, 0x96, 0x4c, 0x72, + 0x02, 0xa3, 0x65, 0xc3, 0xb3, 0x6f, 0x2e, 0xbd, + 0x63, 0x8a, 0x4a, 0x5d, 0x29, 0xa2, 0xd0, 0x28, + 0x48, 0xc5, 0x3d, 0x98, 0xa3, 0xbc, 0xe0, 0xbe, + 0x3b, 0x3f, 0xe6, 0x8a, 0xa4, 0x7f, 0x53, 0x06, + 0xfa, 0x7f, 0x27, 0x76, 0x72, 0x31, 0xa1, 0xf5, + 0xd6, 0x0c, 0x52, 0x47, 0xba, 0xcd, 0x4f, 0xd7, + 0xeb, 0x05, 0x48, 0x0d, 0x7c, 0x35, 0x4a, 0x09, + 0xc9, 0x76, 0x71, 0x02, 0xa3, 0xfb, 0xb7, 0x1a, + 0x65, 0xb7, 0xed, 0x98, 0xc6, 0x30, 0x8a, 0x00, + 0xae, 0xa1, 0x31, 0xe5, 0xb5, 0x9e, 0x6d, 0x62, + 0xda, 0xda, 0x07, 0x0f, 0x38, 0x38, 0xd3, 0xcb, + 0xc1, 0xb0, 0xad, 0xec, 0x72, 0xec, 0xb1, 0xa2, + 0x7b, 0x59, 0xf3, 0x3d, 0x2b, 0xef, 0xcd, 0x28, + 0x5b, 0x83, 0xcc, 0x18, 0x91, 0x88, 0xb0, 0x2e, + 0xf9, 0x29, 0x31, 0x18, 0xf9, 0x4e, 0xe9, 0x0a, + 0x91, 0x92, 0x9f, 0xae, 0x2d, 0xad, 0xf4, 0xe6, + 0x1a, 0xe2, 0xa4, 0xee, 0x47, 0x15, 0xbf, 0x83, + 0x6e, 0xd7, 0x72, 0x12, 0x3b, 0x2d, 0x24, 0xe9, + 0xb2, 0x55, 0xcb, 0x3c, 0x10, 0xf0, 0x24, 0x8a, + 0x4a, 0x02, 0xea, 0x90, 0x25, 0xf0, 0xb4, 0x79, + 0x3a, 0xef, 0x6e, 0xf5, 0x52, 0xdf, 0xb0, 0x0a, + 0xcd, 0x24, 0x1c, 0xd3, 0x2e, 0x22, 0x74, 0xea, + 0x21, 0x6f, 0xe9, 0xbd, 0xc8, 0x3e, 0x36, 0x5b, + 0x19, 0xf1, 0xca, 0x99, 0x0a, 0xb4, 0xa7, 0x52, + 0x1a, 0x4e, 0xf2, 0xad, 0x8d, 0x56, 0x85, 0xbb, + 0x64, 0x89, 0xba, 0x26, 0xf9, 0xc7, 0xe1, 0x89, + 0x19, 0x22, 0x77, 0xc3, 0xa8, 0xfc, 0xff, 0xad, + 0xfe, 0xb9, 0x48, 0xae, 0x12, 0x30, 0x9f, 0x19, + 0xfb, 0x1b, 0xef, 0x14, 0x87, 0x8a, 0x78, 0x71, + 0xf3, 0xf4, 0xb7, 0x00, 0x9c, 0x1d, 0xb5, 0x3d, + 0x49, 0x00, 0x0c, 0x06, 0xd4, 0x50, 0xf9, 0x54, + 0x45, 0xb2, 0x5b, 0x43, 0xdb, 0x6d, 0xcf, 0x1a, + 0xe9, 0x7a, 0x7a, 0xcf, 0xfc, 0x8a, 0x4e, 0x4d, + 0x0b, 0x07, 0x63, 0x28, 0xd8, 0xe7, 0x08, 0x95, + 0xdf, 0xa6, 0x72, 0x93, 0x2e, 0xbb, 0xa0, 0x42, + 0x89, 0x16, 0xf1, 0xd9, 0x0c, 0xf9, 0xa1, 0x16, + 0xfd, 0xd9, 0x03, 0xb4, 0x3b, 0x8a, 0xf5, 0xf6, + 0xe7, 0x6b, 0x2e, 0x8e, 0x4c, 0x3d, 0xe2, 0xaf, + 0x08, 0x45, 0x03, 0xff, 0x09, 0xb6, 0xeb, 0x2d, + 0xc6, 0x1b, 0x88, 0x94, 0xac, 0x3e, 0xf1, 0x9f, + 0x0e, 0x0e, 0x2b, 0xd5, 0x00, 0x4d, 0x3f, 0x3b, + 0x53, 0xae, 0xaf, 0x1c, 0x33, 0x5f, 0x55, 0x6e, + 0x8d, 0xaf, 0x05, 0x7a, 0x10, 0x34, 0xc9, 0xf4, + 0x66, 0xcb, 0x62, 0x12, 0xa6, 0xee, 0xe8, 0x1c, + 0x5d, 0x12, 0x86, 0xdb, 0x6f, 0x1c, 0x33, 0xc4, + 0x1c, 0xda, 0x82, 0x2d, 0x3b, 0x59, 0xfe, 0xb1, + 0xa4, 0x59, 0x41, 0x86, 0xd0, 0xef, 0xae, 0xfb, + 0xda, 0x6d, 0x11, 0xb8, 0xca, 0xe9, 0x6e, 0xff, + 0xf7, 0xa9, 0xd9, 0x70, 0x30, 0xfc, 0x53, 0xe2, + 0xd7, 0xa2, 0x4e, 0xc7, 0x91, 0xd9, 0x07, 0x06, + 0xaa, 0xdd, 0xb0, 0x59, 0x28, 0x1d, 0x00, 0x66, + 0xc5, 0x54, 0xc2, 0xfc, 0x06, 0xda, 0x05, 0x90, + 0x52, 0x1d, 0x37, 0x66, 0xee, 0xf0, 0xb2, 0x55, + 0x8a, 0x5d, 0xd2, 0x38, 0x86, 0x94, 0x9b, 0xfc, + 0x10, 0x4c, 0xa1, 0xb9, 0x64, 0x3e, 0x44, 0xb8, + 0x5f, 0xb0, 0x0c, 0xec, 0xe0, 0xc9, 0xe5, 0x62, + 0x75, 0x3f, 0x09, 0xd5, 0xf5, 0xd9, 0x26, 0xba, + 0x9e, 0xd2, 0xf4, 0xb9, 0x48, 0x0a, 0xbc, 0xa2, + 0xd6, 0x7c, 0x36, 0x11, 0x7d, 0x26, 0x81, 0x89, + 0xcf, 0xa4, 0xad, 0x73, 0x0e, 0xee, 0xcc, 0x06, + 0xa9, 0xdb, 0xb1, 0xfd, 0xfb, 0x09, 0x7f, 0x90, + 0x42, 0x37, 0x2f, 0xe1, 0x9c, 0x0f, 0x6f, 0xcf, + 0x43, 0xb5, 0xd9, 0x90, 0xe1, 0x85, 0xf5, 0xa8, + 0xae +}; +static const u8 dec_output009[] __initconst = { + 0xe6, 0xc3, 0xdb, 0x63, 0x55, 0x15, 0xe3, 0x5b, + 0xb7, 0x4b, 0x27, 0x8b, 0x5a, 0xdd, 0xc2, 0xe8, + 0x3a, 0x6b, 0xd7, 0x81, 0x96, 0x35, 0x97, 0xca, + 0xd7, 0x68, 0xe8, 0xef, 0xce, 0xab, 0xda, 0x09, + 0x6e, 0xd6, 0x8e, 0xcb, 0x55, 0xb5, 0xe1, 0xe5, + 0x57, 0xfd, 0xc4, 0xe3, 0xe0, 0x18, 0x4f, 0x85, + 0xf5, 0x3f, 0x7e, 0x4b, 0x88, 0xc9, 0x52, 0x44, + 0x0f, 0xea, 0xaf, 0x1f, 0x71, 0x48, 0x9f, 0x97, + 0x6d, 0xb9, 0x6f, 0x00, 0xa6, 0xde, 0x2b, 0x77, + 0x8b, 0x15, 0xad, 0x10, 0xa0, 0x2b, 0x7b, 0x41, + 0x90, 0x03, 0x2d, 0x69, 0xae, 0xcc, 0x77, 0x7c, + 0xa5, 0x9d, 0x29, 0x22, 0xc2, 0xea, 0xb4, 0x00, + 0x1a, 0xd2, 0x7a, 0x98, 0x8a, 0xf9, 0xf7, 0x82, + 0xb0, 0xab, 0xd8, 0xa6, 0x94, 0x8d, 0x58, 0x2f, + 0x01, 0x9e, 0x00, 0x20, 0xfc, 0x49, 0xdc, 0x0e, + 0x03, 0xe8, 0x45, 0x10, 0xd6, 0xa8, 0xda, 0x55, + 0x10, 0x9a, 0xdf, 0x67, 0x22, 0x8b, 0x43, 0xab, + 0x00, 0xbb, 0x02, 0xc8, 0xdd, 0x7b, 0x97, 0x17, + 0xd7, 0x1d, 0x9e, 0x02, 0x5e, 0x48, 0xde, 0x8e, + 0xcf, 0x99, 0x07, 0x95, 0x92, 0x3c, 0x5f, 0x9f, + 0xc5, 0x8a, 0xc0, 0x23, 0xaa, 0xd5, 0x8c, 0x82, + 0x6e, 0x16, 0x92, 0xb1, 0x12, 0x17, 0x07, 0xc3, + 0xfb, 0x36, 0xf5, 0x6c, 0x35, 0xd6, 0x06, 0x1f, + 0x9f, 0xa7, 0x94, 0xa2, 0x38, 0x63, 0x9c, 0xb0, + 0x71, 0xb3, 0xa5, 0xd2, 0xd8, 0xba, 0x9f, 0x08, + 0x01, 0xb3, 0xff, 0x04, 0x97, 0x73, 0x45, 0x1b, + 0xd5, 0xa9, 0x9c, 0x80, 0xaf, 0x04, 0x9a, 0x85, + 0xdb, 0x32, 0x5b, 0x5d, 0x1a, 0xc1, 0x36, 0x28, + 0x10, 0x79, 0xf1, 0x3c, 0xbf, 0x1a, 0x41, 0x5c, + 0x4e, 0xdf, 0xb2, 0x7c, 0x79, 0x3b, 0x7a, 0x62, + 0x3d, 0x4b, 0xc9, 0x9b, 0x2a, 0x2e, 0x7c, 0xa2, + 0xb1, 0x11, 0x98, 0xa7, 0x34, 0x1a, 0x00, 0xf3, + 0xd1, 0xbc, 0x18, 0x22, 0xba, 0x02, 0x56, 0x62, + 0x31, 0x10, 0x11, 0x6d, 0xe0, 0x54, 0x9d, 0x40, + 0x1f, 0x26, 0x80, 0x41, 0xca, 0x3f, 0x68, 0x0f, + 0x32, 0x1d, 0x0a, 0x8e, 0x79, 0xd8, 0xa4, 0x1b, + 0x29, 0x1c, 0x90, 0x8e, 0xc5, 0xe3, 0xb4, 0x91, + 0x37, 0x9a, 0x97, 0x86, 0x99, 0xd5, 0x09, 0xc5, + 0xbb, 0xa3, 0x3f, 0x21, 0x29, 0x82, 0x14, 0x5c, + 0xab, 0x25, 0xfb, 0xf2, 0x4f, 0x58, 0x26, 0xd4, + 0x83, 0xaa, 0x66, 0x89, 0x67, 0x7e, 0xc0, 0x49, + 0xe1, 0x11, 0x10, 0x7f, 0x7a, 0xda, 0x29, 0x04, + 0xff, 0xf0, 0xcb, 0x09, 0x7c, 0x9d, 0xfa, 0x03, + 0x6f, 0x81, 0x09, 0x31, 0x60, 0xfb, 0x08, 0xfa, + 0x74, 0xd3, 0x64, 0x44, 0x7c, 0x55, 0x85, 0xec, + 0x9c, 0x6e, 0x25, 0xb7, 0x6c, 0xc5, 0x37, 0xb6, + 0x83, 0x87, 0x72, 0x95, 0x8b, 0x9d, 0xe1, 0x69, + 0x5c, 0x31, 0x95, 0x42, 0xa6, 0x2c, 0xd1, 0x36, + 0x47, 0x1f, 0xec, 0x54, 0xab, 0xa2, 0x1c, 0xd8, + 0x00, 0xcc, 0xbc, 0x0d, 0x65, 0xe2, 0x67, 0xbf, + 0xbc, 0xea, 0xee, 0x9e, 0xe4, 0x36, 0x95, 0xbe, + 0x73, 0xd9, 0xa6, 0xd9, 0x0f, 0xa0, 0xcc, 0x82, + 0x76, 0x26, 0xad, 0x5b, 0x58, 0x6c, 0x4e, 0xab, + 0x29, 0x64, 0xd3, 0xd9, 0xa9, 0x08, 0x8c, 0x1d, + 0xa1, 0x4f, 0x80, 0xd8, 0x3f, 0x94, 0xfb, 0xd3, + 0x7b, 0xfc, 0xd1, 0x2b, 0xc3, 0x21, 0xeb, 0xe5, + 0x1c, 0x84, 0x23, 0x7f, 0x4b, 0xfa, 0xdb, 0x34, + 0x18, 0xa2, 0xc2, 0xe5, 0x13, 0xfe, 0x6c, 0x49, + 0x81, 0xd2, 0x73, 0xe7, 0xe2, 0xd7, 0xe4, 0x4f, + 0x4b, 0x08, 0x6e, 0xb1, 0x12, 0x22, 0x10, 0x9d, + 0xac, 0x51, 0x1e, 0x17, 0xd9, 0x8a, 0x0b, 0x42, + 0x88, 0x16, 0x81, 0x37, 0x7c, 0x6a, 0xf7, 0xef, + 0x2d, 0xe3, 0xd9, 0xf8, 0x5f, 0xe0, 0x53, 0x27, + 0x74, 0xb9, 0xe2, 0xd6, 0x1c, 0x80, 0x2c, 0x52, + 0x65 +}; +static const u8 dec_assoc009[] __initconst = { + 0x5a, 0x27, 0xff, 0xeb, 0xdf, 0x84, 0xb2, 0x9e, + 0xef +}; +static const u8 dec_nonce009[] __initconst = { + 0xef, 0x2d, 0x63, 0xee, 0x6b, 0x80, 0x8b, 0x78 +}; +static const u8 dec_key009[] __initconst = { + 0xea, 0xbc, 0x56, 0x99, 0xe3, 0x50, 0xff, 0xc5, + 0xcc, 0x1a, 0xd7, 0xc1, 0x57, 0x72, 0xea, 0x86, + 0x5b, 0x89, 0x88, 0x61, 0x3d, 0x2f, 0x9b, 0xb2, + 0xe7, 0x9c, 0xec, 0x74, 0x6e, 0x3e, 0xf4, 0x3b +}; + +static const u8 dec_input010[] __initconst = { + 0xe5, 0x26, 0xa4, 0x3d, 0xbd, 0x33, 0xd0, 0x4b, + 0x6f, 0x05, 0xa7, 0x6e, 0x12, 0x7a, 0xd2, 0x74, + 0xa6, 0xdd, 0xbd, 0x95, 0xeb, 0xf9, 0xa4, 0xf1, + 0x59, 0x93, 0x91, 0x70, 0xd9, 0xfe, 0x9a, 0xcd, + 0x53, 0x1f, 0x3a, 0xab, 0xa6, 0x7c, 0x9f, 0xa6, + 0x9e, 0xbd, 0x99, 0xd9, 0xb5, 0x97, 0x44, 0xd5, + 0x14, 0x48, 0x4d, 0x9d, 0xc0, 0xd0, 0x05, 0x96, + 0xeb, 0x4c, 0x78, 0x55, 0x09, 0x08, 0x01, 0x02, + 0x30, 0x90, 0x7b, 0x96, 0x7a, 0x7b, 0x5f, 0x30, + 0x41, 0x24, 0xce, 0x68, 0x61, 0x49, 0x86, 0x57, + 0x82, 0xdd, 0x53, 0x1c, 0x51, 0x28, 0x2b, 0x53, + 0x6e, 0x2d, 0xc2, 0x20, 0x4c, 0xdd, 0x8f, 0x65, + 0x10, 0x20, 0x50, 0xdd, 0x9d, 0x50, 0xe5, 0x71, + 0x40, 0x53, 0x69, 0xfc, 0x77, 0x48, 0x11, 0xb9, + 0xde, 0xa4, 0x8d, 0x58, 0xe4, 0xa6, 0x1a, 0x18, + 0x47, 0x81, 0x7e, 0xfc, 0xdd, 0xf6, 0xef, 0xce, + 0x2f, 0x43, 0x68, 0xd6, 0x06, 0xe2, 0x74, 0x6a, + 0xad, 0x90, 0xf5, 0x37, 0xf3, 0x3d, 0x82, 0x69, + 0x40, 0xe9, 0x6b, 0xa7, 0x3d, 0xa8, 0x1e, 0xd2, + 0x02, 0x7c, 0xb7, 0x9b, 0xe4, 0xda, 0x8f, 0x95, + 0x06, 0xc5, 0xdf, 0x73, 0xa3, 0x20, 0x9a, 0x49, + 0xde, 0x9c, 0xbc, 0xee, 0x14, 0x3f, 0x81, 0x5e, + 0xf8, 0x3b, 0x59, 0x3c, 0xe1, 0x68, 0x12, 0x5a, + 0x3a, 0x76, 0x3a, 0x3f, 0xf7, 0x87, 0x33, 0x0a, + 0x01, 0xb8, 0xd4, 0xed, 0xb6, 0xbe, 0x94, 0x5e, + 0x70, 0x40, 0x56, 0x67, 0x1f, 0x50, 0x44, 0x19, + 0xce, 0x82, 0x70, 0x10, 0x87, 0x13, 0x20, 0x0b, + 0x4c, 0x5a, 0xb6, 0xf6, 0xa7, 0xae, 0x81, 0x75, + 0x01, 0x81, 0xe6, 0x4b, 0x57, 0x7c, 0xdd, 0x6d, + 0xf8, 0x1c, 0x29, 0x32, 0xf7, 0xda, 0x3c, 0x2d, + 0xf8, 0x9b, 0x25, 0x6e, 0x00, 0xb4, 0xf7, 0x2f, + 0xf7, 0x04, 0xf7, 0xa1, 0x56, 0xac, 0x4f, 0x1a, + 0x64, 0xb8, 0x47, 0x55, 0x18, 0x7b, 0x07, 0x4d, + 0xbd, 0x47, 0x24, 0x80, 0x5d, 0xa2, 0x70, 0xc5, + 0xdd, 0x8e, 0x82, 0xd4, 0xeb, 0xec, 0xb2, 0x0c, + 0x39, 0xd2, 0x97, 0xc1, 0xcb, 0xeb, 0xf4, 0x77, + 0x59, 0xb4, 0x87, 0xef, 0xcb, 0x43, 0x2d, 0x46, + 0x54, 0xd1, 0xa7, 0xd7, 0x15, 0x99, 0x0a, 0x43, + 0xa1, 0xe0, 0x99, 0x33, 0x71, 0xc1, 0xed, 0xfe, + 0x72, 0x46, 0x33, 0x8e, 0x91, 0x08, 0x9f, 0xc8, + 0x2e, 0xca, 0xfa, 0xdc, 0x59, 0xd5, 0xc3, 0x76, + 0x84, 0x9f, 0xa3, 0x37, 0x68, 0xc3, 0xf0, 0x47, + 0x2c, 0x68, 0xdb, 0x5e, 0xc3, 0x49, 0x4c, 0xe8, + 0x92, 0x85, 0xe2, 0x23, 0xd3, 0x3f, 0xad, 0x32, + 0xe5, 0x2b, 0x82, 0xd7, 0x8f, 0x99, 0x0a, 0x59, + 0x5c, 0x45, 0xd9, 0xb4, 0x51, 0x52, 0xc2, 0xae, + 0xbf, 0x80, 0xcf, 0xc9, 0xc9, 0x51, 0x24, 0x2a, + 0x3b, 0x3a, 0x4d, 0xae, 0xeb, 0xbd, 0x22, 0xc3, + 0x0e, 0x0f, 0x59, 0x25, 0x92, 0x17, 0xe9, 0x74, + 0xc7, 0x8b, 0x70, 0x70, 0x36, 0x55, 0x95, 0x75, + 0x4b, 0xad, 0x61, 0x2b, 0x09, 0xbc, 0x82, 0xf2, + 0x6e, 0x94, 0x43, 0xae, 0xc3, 0xd5, 0xcd, 0x8e, + 0xfe, 0x5b, 0x9a, 0x88, 0x43, 0x01, 0x75, 0xb2, + 0x23, 0x09, 0xf7, 0x89, 0x83, 0xe7, 0xfa, 0xf9, + 0xb4, 0x9b, 0xf8, 0xef, 0xbd, 0x1c, 0x92, 0xc1, + 0xda, 0x7e, 0xfe, 0x05, 0xba, 0x5a, 0xcd, 0x07, + 0x6a, 0x78, 0x9e, 0x5d, 0xfb, 0x11, 0x2f, 0x79, + 0x38, 0xb6, 0xc2, 0x5b, 0x6b, 0x51, 0xb4, 0x71, + 0xdd, 0xf7, 0x2a, 0xe4, 0xf4, 0x72, 0x76, 0xad, + 0xc2, 0xdd, 0x64, 0x5d, 0x79, 0xb6, 0xf5, 0x7a, + 0x77, 0x20, 0x05, 0x3d, 0x30, 0x06, 0xd4, 0x4c, + 0x0a, 0x2c, 0x98, 0x5a, 0xb9, 0xd4, 0x98, 0xa9, + 0x3f, 0xc6, 0x12, 0xea, 0x3b, 0x4b, 0xc5, 0x79, + 0x64, 0x63, 0x6b, 0x09, 0x54, 0x3b, 0x14, 0x27, + 0xba, 0x99, 0x80, 0xc8, 0x72, 0xa8, 0x12, 0x90, + 0x29, 0xba, 0x40, 0x54, 0x97, 0x2b, 0x7b, 0xfe, + 0xeb, 0xcd, 0x01, 0x05, 0x44, 0x72, 0xdb, 0x99, + 0xe4, 0x61, 0xc9, 0x69, 0xd6, 0xb9, 0x28, 0xd1, + 0x05, 0x3e, 0xf9, 0x0b, 0x49, 0x0a, 0x49, 0xe9, + 0x8d, 0x0e, 0xa7, 0x4a, 0x0f, 0xaf, 0x32, 0xd0, + 0xe0, 0xb2, 0x3a, 0x55, 0x58, 0xfe, 0x5c, 0x28, + 0x70, 0x51, 0x23, 0xb0, 0x7b, 0x6a, 0x5f, 0x1e, + 0xb8, 0x17, 0xd7, 0x94, 0x15, 0x8f, 0xee, 0x20, + 0xc7, 0x42, 0x25, 0x3e, 0x9a, 0x14, 0xd7, 0x60, + 0x72, 0x39, 0x47, 0x48, 0xa9, 0xfe, 0xdd, 0x47, + 0x0a, 0xb1, 0xe6, 0x60, 0x28, 0x8c, 0x11, 0x68, + 0xe1, 0xff, 0xd7, 0xce, 0xc8, 0xbe, 0xb3, 0xfe, + 0x27, 0x30, 0x09, 0x70, 0xd7, 0xfa, 0x02, 0x33, + 0x3a, 0x61, 0x2e, 0xc7, 0xff, 0xa4, 0x2a, 0xa8, + 0x6e, 0xb4, 0x79, 0x35, 0x6d, 0x4c, 0x1e, 0x38, + 0xf8, 0xee, 0xd4, 0x84, 0x4e, 0x6e, 0x28, 0xa7, + 0xce, 0xc8, 0xc1, 0xcf, 0x80, 0x05, 0xf3, 0x04, + 0xef, 0xc8, 0x18, 0x28, 0x2e, 0x8d, 0x5e, 0x0c, + 0xdf, 0xb8, 0x5f, 0x96, 0xe8, 0xc6, 0x9c, 0x2f, + 0xe5, 0xa6, 0x44, 0xd7, 0xe7, 0x99, 0x44, 0x0c, + 0xec, 0xd7, 0x05, 0x60, 0x97, 0xbb, 0x74, 0x77, + 0x58, 0xd5, 0xbb, 0x48, 0xde, 0x5a, 0xb2, 0x54, + 0x7f, 0x0e, 0x46, 0x70, 0x6a, 0x6f, 0x78, 0xa5, + 0x08, 0x89, 0x05, 0x4e, 0x7e, 0xa0, 0x69, 0xb4, + 0x40, 0x60, 0x55, 0x77, 0x75, 0x9b, 0x19, 0xf2, + 0xd5, 0x13, 0x80, 0x77, 0xf9, 0x4b, 0x3f, 0x1e, + 0xee, 0xe6, 0x76, 0x84, 0x7b, 0x8c, 0xe5, 0x27, + 0xa8, 0x0a, 0x91, 0x01, 0x68, 0x71, 0x8a, 0x3f, + 0x06, 0xab, 0xf6, 0xa9, 0xa5, 0xe6, 0x72, 0x92, + 0xe4, 0x67, 0xe2, 0xa2, 0x46, 0x35, 0x84, 0x55, + 0x7d, 0xca, 0xa8, 0x85, 0xd0, 0xf1, 0x3f, 0xbe, + 0xd7, 0x34, 0x64, 0xfc, 0xae, 0xe3, 0xe4, 0x04, + 0x9f, 0x66, 0x02, 0xb9, 0x88, 0x10, 0xd9, 0xc4, + 0x4c, 0x31, 0x43, 0x7a, 0x93, 0xe2, 0x9b, 0x56, + 0x43, 0x84, 0xdc, 0xdc, 0xde, 0x1d, 0xa4, 0x02, + 0x0e, 0xc2, 0xef, 0xc3, 0xf8, 0x78, 0xd1, 0xb2, + 0x6b, 0x63, 0x18, 0xc9, 0xa9, 0xe5, 0x72, 0xd8, + 0xf3, 0xb9, 0xd1, 0x8a, 0xc7, 0x1a, 0x02, 0x27, + 0x20, 0x77, 0x10, 0xe5, 0xc8, 0xd4, 0x4a, 0x47, + 0xe5, 0xdf, 0x5f, 0x01, 0xaa, 0xb0, 0xd4, 0x10, + 0xbb, 0x69, 0xe3, 0x36, 0xc8, 0xe1, 0x3d, 0x43, + 0xfb, 0x86, 0xcd, 0xcc, 0xbf, 0xf4, 0x88, 0xe0, + 0x20, 0xca, 0xb7, 0x1b, 0xf1, 0x2f, 0x5c, 0xee, + 0xd4, 0xd3, 0xa3, 0xcc, 0xa4, 0x1e, 0x1c, 0x47, + 0xfb, 0xbf, 0xfc, 0xa2, 0x41, 0x55, 0x9d, 0xf6, + 0x5a, 0x5e, 0x65, 0x32, 0x34, 0x7b, 0x52, 0x8d, + 0xd5, 0xd0, 0x20, 0x60, 0x03, 0xab, 0x3f, 0x8c, + 0xd4, 0x21, 0xea, 0x2a, 0xd9, 0xc4, 0xd0, 0xd3, + 0x65, 0xd8, 0x7a, 0x13, 0x28, 0x62, 0x32, 0x4b, + 0x2c, 0x87, 0x93, 0xa8, 0xb4, 0x52, 0x45, 0x09, + 0x44, 0xec, 0xec, 0xc3, 0x17, 0xdb, 0x9a, 0x4d, + 0x5c, 0xa9, 0x11, 0xd4, 0x7d, 0xaf, 0x9e, 0xf1, + 0x2d, 0xb2, 0x66, 0xc5, 0x1d, 0xed, 0xb7, 0xcd, + 0x0b, 0x25, 0x5e, 0x30, 0x47, 0x3f, 0x40, 0xf4, + 0xa1, 0xa0, 0x00, 0x94, 0x10, 0xc5, 0x6a, 0x63, + 0x1a, 0xd5, 0x88, 0x92, 0x8e, 0x82, 0x39, 0x87, + 0x3c, 0x78, 0x65, 0x58, 0x42, 0x75, 0x5b, 0xdd, + 0x77, 0x3e, 0x09, 0x4e, 0x76, 0x5b, 0xe6, 0x0e, + 0x4d, 0x38, 0xb2, 0xc0, 0xb8, 0x95, 0x01, 0x7a, + 0x10, 0xe0, 0xfb, 0x07, 0xf2, 0xab, 0x2d, 0x8c, + 0x32, 0xed, 0x2b, 0xc0, 0x46, 0xc2, 0xf5, 0x38, + 0x83, 0xf0, 0x17, 0xec, 0xc1, 0x20, 0x6a, 0x9a, + 0x0b, 0x00, 0xa0, 0x98, 0x22, 0x50, 0x23, 0xd5, + 0x80, 0x6b, 0xf6, 0x1f, 0xc3, 0xcc, 0x97, 0xc9, + 0x24, 0x9f, 0xf3, 0xaf, 0x43, 0x14, 0xd5, 0xa0 +}; +static const u8 dec_output010[] __initconst = { + 0x42, 0x93, 0xe4, 0xeb, 0x97, 0xb0, 0x57, 0xbf, + 0x1a, 0x8b, 0x1f, 0xe4, 0x5f, 0x36, 0x20, 0x3c, + 0xef, 0x0a, 0xa9, 0x48, 0x5f, 0x5f, 0x37, 0x22, + 0x3a, 0xde, 0xe3, 0xae, 0xbe, 0xad, 0x07, 0xcc, + 0xb1, 0xf6, 0xf5, 0xf9, 0x56, 0xdd, 0xe7, 0x16, + 0x1e, 0x7f, 0xdf, 0x7a, 0x9e, 0x75, 0xb7, 0xc7, + 0xbe, 0xbe, 0x8a, 0x36, 0x04, 0xc0, 0x10, 0xf4, + 0x95, 0x20, 0x03, 0xec, 0xdc, 0x05, 0xa1, 0x7d, + 0xc4, 0xa9, 0x2c, 0x82, 0xd0, 0xbc, 0x8b, 0xc5, + 0xc7, 0x45, 0x50, 0xf6, 0xa2, 0x1a, 0xb5, 0x46, + 0x3b, 0x73, 0x02, 0xa6, 0x83, 0x4b, 0x73, 0x82, + 0x58, 0x5e, 0x3b, 0x65, 0x2f, 0x0e, 0xfd, 0x2b, + 0x59, 0x16, 0xce, 0xa1, 0x60, 0x9c, 0xe8, 0x3a, + 0x99, 0xed, 0x8d, 0x5a, 0xcf, 0xf6, 0x83, 0xaf, + 0xba, 0xd7, 0x73, 0x73, 0x40, 0x97, 0x3d, 0xca, + 0xef, 0x07, 0x57, 0xe6, 0xd9, 0x70, 0x0e, 0x95, + 0xae, 0xa6, 0x8d, 0x04, 0xcc, 0xee, 0xf7, 0x09, + 0x31, 0x77, 0x12, 0xa3, 0x23, 0x97, 0x62, 0xb3, + 0x7b, 0x32, 0xfb, 0x80, 0x14, 0x48, 0x81, 0xc3, + 0xe5, 0xea, 0x91, 0x39, 0x52, 0x81, 0xa2, 0x4f, + 0xe4, 0xb3, 0x09, 0xff, 0xde, 0x5e, 0xe9, 0x58, + 0x84, 0x6e, 0xf9, 0x3d, 0xdf, 0x25, 0xea, 0xad, + 0xae, 0xe6, 0x9a, 0xd1, 0x89, 0x55, 0xd3, 0xde, + 0x6c, 0x52, 0xdb, 0x70, 0xfe, 0x37, 0xce, 0x44, + 0x0a, 0xa8, 0x25, 0x5f, 0x92, 0xc1, 0x33, 0x4a, + 0x4f, 0x9b, 0x62, 0x35, 0xff, 0xce, 0xc0, 0xa9, + 0x60, 0xce, 0x52, 0x00, 0x97, 0x51, 0x35, 0x26, + 0x2e, 0xb9, 0x36, 0xa9, 0x87, 0x6e, 0x1e, 0xcc, + 0x91, 0x78, 0x53, 0x98, 0x86, 0x5b, 0x9c, 0x74, + 0x7d, 0x88, 0x33, 0xe1, 0xdf, 0x37, 0x69, 0x2b, + 0xbb, 0xf1, 0x4d, 0xf4, 0xd1, 0xf1, 0x39, 0x93, + 0x17, 0x51, 0x19, 0xe3, 0x19, 0x1e, 0x76, 0x37, + 0x25, 0xfb, 0x09, 0x27, 0x6a, 0xab, 0x67, 0x6f, + 0x14, 0x12, 0x64, 0xe7, 0xc4, 0x07, 0xdf, 0x4d, + 0x17, 0xbb, 0x6d, 0xe0, 0xe9, 0xb9, 0xab, 0xca, + 0x10, 0x68, 0xaf, 0x7e, 0xb7, 0x33, 0x54, 0x73, + 0x07, 0x6e, 0xf7, 0x81, 0x97, 0x9c, 0x05, 0x6f, + 0x84, 0x5f, 0xd2, 0x42, 0xfb, 0x38, 0xcf, 0xd1, + 0x2f, 0x14, 0x30, 0x88, 0x98, 0x4d, 0x5a, 0xa9, + 0x76, 0xd5, 0x4f, 0x3e, 0x70, 0x6c, 0x85, 0x76, + 0xd7, 0x01, 0xa0, 0x1a, 0xc8, 0x4e, 0xaa, 0xac, + 0x78, 0xfe, 0x46, 0xde, 0x6a, 0x05, 0x46, 0xa7, + 0x43, 0x0c, 0xb9, 0xde, 0xb9, 0x68, 0xfb, 0xce, + 0x42, 0x99, 0x07, 0x4d, 0x0b, 0x3b, 0x5a, 0x30, + 0x35, 0xa8, 0xf9, 0x3a, 0x73, 0xef, 0x0f, 0xdb, + 0x1e, 0x16, 0x42, 0xc4, 0xba, 0xae, 0x58, 0xaa, + 0xf8, 0xe5, 0x75, 0x2f, 0x1b, 0x15, 0x5c, 0xfd, + 0x0a, 0x97, 0xd0, 0xe4, 0x37, 0x83, 0x61, 0x5f, + 0x43, 0xa6, 0xc7, 0x3f, 0x38, 0x59, 0xe6, 0xeb, + 0xa3, 0x90, 0xc3, 0xaa, 0xaa, 0x5a, 0xd3, 0x34, + 0xd4, 0x17, 0xc8, 0x65, 0x3e, 0x57, 0xbc, 0x5e, + 0xdd, 0x9e, 0xb7, 0xf0, 0x2e, 0x5b, 0xb2, 0x1f, + 0x8a, 0x08, 0x0d, 0x45, 0x91, 0x0b, 0x29, 0x53, + 0x4f, 0x4c, 0x5a, 0x73, 0x56, 0xfe, 0xaf, 0x41, + 0x01, 0x39, 0x0a, 0x24, 0x3c, 0x7e, 0xbe, 0x4e, + 0x53, 0xf3, 0xeb, 0x06, 0x66, 0x51, 0x28, 0x1d, + 0xbd, 0x41, 0x0a, 0x01, 0xab, 0x16, 0x47, 0x27, + 0x47, 0x47, 0xf7, 0xcb, 0x46, 0x0a, 0x70, 0x9e, + 0x01, 0x9c, 0x09, 0xe1, 0x2a, 0x00, 0x1a, 0xd8, + 0xd4, 0x79, 0x9d, 0x80, 0x15, 0x8e, 0x53, 0x2a, + 0x65, 0x83, 0x78, 0x3e, 0x03, 0x00, 0x07, 0x12, + 0x1f, 0x33, 0x3e, 0x7b, 0x13, 0x37, 0xf1, 0xc3, + 0xef, 0xb7, 0xc1, 0x20, 0x3c, 0x3e, 0x67, 0x66, + 0x5d, 0x88, 0xa7, 0x7d, 0x33, 0x50, 0x77, 0xb0, + 0x28, 0x8e, 0xe7, 0x2c, 0x2e, 0x7a, 0xf4, 0x3c, + 0x8d, 0x74, 0x83, 0xaf, 0x8e, 0x87, 0x0f, 0xe4, + 0x50, 0xff, 0x84, 0x5c, 0x47, 0x0c, 0x6a, 0x49, + 0xbf, 0x42, 0x86, 0x77, 0x15, 0x48, 0xa5, 0x90, + 0x5d, 0x93, 0xd6, 0x2a, 0x11, 0xd5, 0xd5, 0x11, + 0xaa, 0xce, 0xe7, 0x6f, 0xa5, 0xb0, 0x09, 0x2c, + 0x8d, 0xd3, 0x92, 0xf0, 0x5a, 0x2a, 0xda, 0x5b, + 0x1e, 0xd5, 0x9a, 0xc4, 0xc4, 0xf3, 0x49, 0x74, + 0x41, 0xca, 0xe8, 0xc1, 0xf8, 0x44, 0xd6, 0x3c, + 0xae, 0x6c, 0x1d, 0x9a, 0x30, 0x04, 0x4d, 0x27, + 0x0e, 0xb1, 0x5f, 0x59, 0xa2, 0x24, 0xe8, 0xe1, + 0x98, 0xc5, 0x6a, 0x4c, 0xfe, 0x41, 0xd2, 0x27, + 0x42, 0x52, 0xe1, 0xe9, 0x7d, 0x62, 0xe4, 0x88, + 0x0f, 0xad, 0xb2, 0x70, 0xcb, 0x9d, 0x4c, 0x27, + 0x2e, 0x76, 0x1e, 0x1a, 0x63, 0x65, 0xf5, 0x3b, + 0xf8, 0x57, 0x69, 0xeb, 0x5b, 0x38, 0x26, 0x39, + 0x33, 0x25, 0x45, 0x3e, 0x91, 0xb8, 0xd8, 0xc7, + 0xd5, 0x42, 0xc0, 0x22, 0x31, 0x74, 0xf4, 0xbc, + 0x0c, 0x23, 0xf1, 0xca, 0xc1, 0x8d, 0xd7, 0xbe, + 0xc9, 0x62, 0xe4, 0x08, 0x1a, 0xcf, 0x36, 0xd5, + 0xfe, 0x55, 0x21, 0x59, 0x91, 0x87, 0x87, 0xdf, + 0x06, 0xdb, 0xdf, 0x96, 0x45, 0x58, 0xda, 0x05, + 0xcd, 0x50, 0x4d, 0xd2, 0x7d, 0x05, 0x18, 0x73, + 0x6a, 0x8d, 0x11, 0x85, 0xa6, 0x88, 0xe8, 0xda, + 0xe6, 0x30, 0x33, 0xa4, 0x89, 0x31, 0x75, 0xbe, + 0x69, 0x43, 0x84, 0x43, 0x50, 0x87, 0xdd, 0x71, + 0x36, 0x83, 0xc3, 0x78, 0x74, 0x24, 0x0a, 0xed, + 0x7b, 0xdb, 0xa4, 0x24, 0x0b, 0xb9, 0x7e, 0x5d, + 0xff, 0xde, 0xb1, 0xef, 0x61, 0x5a, 0x45, 0x33, + 0xf6, 0x17, 0x07, 0x08, 0x98, 0x83, 0x92, 0x0f, + 0x23, 0x6d, 0xe6, 0xaa, 0x17, 0x54, 0xad, 0x6a, + 0xc8, 0xdb, 0x26, 0xbe, 0xb8, 0xb6, 0x08, 0xfa, + 0x68, 0xf1, 0xd7, 0x79, 0x6f, 0x18, 0xb4, 0x9e, + 0x2d, 0x3f, 0x1b, 0x64, 0xaf, 0x8d, 0x06, 0x0e, + 0x49, 0x28, 0xe0, 0x5d, 0x45, 0x68, 0x13, 0x87, + 0xfa, 0xde, 0x40, 0x7b, 0xd2, 0xc3, 0x94, 0xd5, + 0xe1, 0xd9, 0xc2, 0xaf, 0x55, 0x89, 0xeb, 0xb4, + 0x12, 0x59, 0xa8, 0xd4, 0xc5, 0x29, 0x66, 0x38, + 0xe6, 0xac, 0x22, 0x22, 0xd9, 0x64, 0x9b, 0x34, + 0x0a, 0x32, 0x9f, 0xc2, 0xbf, 0x17, 0x6c, 0x3f, + 0x71, 0x7a, 0x38, 0x6b, 0x98, 0xfb, 0x49, 0x36, + 0x89, 0xc9, 0xe2, 0xd6, 0xc7, 0x5d, 0xd0, 0x69, + 0x5f, 0x23, 0x35, 0xc9, 0x30, 0xe2, 0xfd, 0x44, + 0x58, 0x39, 0xd7, 0x97, 0xfb, 0x5c, 0x00, 0xd5, + 0x4f, 0x7a, 0x1a, 0x95, 0x8b, 0x62, 0x4b, 0xce, + 0xe5, 0x91, 0x21, 0x7b, 0x30, 0x00, 0xd6, 0xdd, + 0x6d, 0x02, 0x86, 0x49, 0x0f, 0x3c, 0x1a, 0x27, + 0x3c, 0xd3, 0x0e, 0x71, 0xf2, 0xff, 0xf5, 0x2f, + 0x87, 0xac, 0x67, 0x59, 0x81, 0xa3, 0xf7, 0xf8, + 0xd6, 0x11, 0x0c, 0x84, 0xa9, 0x03, 0xee, 0x2a, + 0xc4, 0xf3, 0x22, 0xab, 0x7c, 0xe2, 0x25, 0xf5, + 0x67, 0xa3, 0xe4, 0x11, 0xe0, 0x59, 0xb3, 0xca, + 0x87, 0xa0, 0xae, 0xc9, 0xa6, 0x62, 0x1b, 0x6e, + 0x4d, 0x02, 0x6b, 0x07, 0x9d, 0xfd, 0xd0, 0x92, + 0x06, 0xe1, 0xb2, 0x9a, 0x4a, 0x1f, 0x1f, 0x13, + 0x49, 0x99, 0x97, 0x08, 0xde, 0x7f, 0x98, 0xaf, + 0x51, 0x98, 0xee, 0x2c, 0xcb, 0xf0, 0x0b, 0xc6, + 0xb6, 0xb7, 0x2d, 0x9a, 0xb1, 0xac, 0xa6, 0xe3, + 0x15, 0x77, 0x9d, 0x6b, 0x1a, 0xe4, 0xfc, 0x8b, + 0xf2, 0x17, 0x59, 0x08, 0x04, 0x58, 0x81, 0x9d, + 0x1b, 0x1b, 0x69, 0x55, 0xc2, 0xb4, 0x3c, 0x1f, + 0x50, 0xf1, 0x7f, 0x77, 0x90, 0x4c, 0x66, 0x40, + 0x5a, 0xc0, 0x33, 0x1f, 0xcb, 0x05, 0x6d, 0x5c, + 0x06, 0x87, 0x52, 0xa2, 0x8f, 0x26, 0xd5, 0x4f +}; +static const u8 dec_assoc010[] __initconst = { + 0xd2, 0xa1, 0x70, 0xdb, 0x7a, 0xf8, 0xfa, 0x27, + 0xba, 0x73, 0x0f, 0xbf, 0x3d, 0x1e, 0x82, 0xb2 +}; +static const u8 dec_nonce010[] __initconst = { + 0xdb, 0x92, 0x0f, 0x7f, 0x17, 0x54, 0x0c, 0x30 +}; +static const u8 dec_key010[] __initconst = { + 0x47, 0x11, 0xeb, 0x86, 0x2b, 0x2c, 0xab, 0x44, + 0x34, 0xda, 0x7f, 0x57, 0x03, 0x39, 0x0c, 0xaf, + 0x2c, 0x14, 0xfd, 0x65, 0x23, 0xe9, 0x8e, 0x74, + 0xd5, 0x08, 0x68, 0x08, 0xe7, 0xb4, 0x72, 0xd7 +}; + +static const u8 dec_input011[] __initconst = { + 0x6a, 0xfc, 0x4b, 0x25, 0xdf, 0xc0, 0xe4, 0xe8, + 0x17, 0x4d, 0x4c, 0xc9, 0x7e, 0xde, 0x3a, 0xcc, + 0x3c, 0xba, 0x6a, 0x77, 0x47, 0xdb, 0xe3, 0x74, + 0x7a, 0x4d, 0x5f, 0x8d, 0x37, 0x55, 0x80, 0x73, + 0x90, 0x66, 0x5d, 0x3a, 0x7d, 0x5d, 0x86, 0x5e, + 0x8d, 0xfd, 0x83, 0xff, 0x4e, 0x74, 0x6f, 0xf9, + 0xe6, 0x70, 0x17, 0x70, 0x3e, 0x96, 0xa7, 0x7e, + 0xcb, 0xab, 0x8f, 0x58, 0x24, 0x9b, 0x01, 0xfd, + 0xcb, 0xe6, 0x4d, 0x9b, 0xf0, 0x88, 0x94, 0x57, + 0x66, 0xef, 0x72, 0x4c, 0x42, 0x6e, 0x16, 0x19, + 0x15, 0xea, 0x70, 0x5b, 0xac, 0x13, 0xdb, 0x9f, + 0x18, 0xe2, 0x3c, 0x26, 0x97, 0xbc, 0xdc, 0x45, + 0x8c, 0x6c, 0x24, 0x69, 0x9c, 0xf7, 0x65, 0x1e, + 0x18, 0x59, 0x31, 0x7c, 0xe4, 0x73, 0xbc, 0x39, + 0x62, 0xc6, 0x5c, 0x9f, 0xbf, 0xfa, 0x90, 0x03, + 0xc9, 0x72, 0x26, 0xb6, 0x1b, 0xc2, 0xb7, 0x3f, + 0xf2, 0x13, 0x77, 0xf2, 0x8d, 0xb9, 0x47, 0xd0, + 0x53, 0xdd, 0xc8, 0x91, 0x83, 0x8b, 0xb1, 0xce, + 0xa3, 0xfe, 0xcd, 0xd9, 0xdd, 0x92, 0x7b, 0xdb, + 0xb8, 0xfb, 0xc9, 0x2d, 0x01, 0x59, 0x39, 0x52, + 0xad, 0x1b, 0xec, 0xcf, 0xd7, 0x70, 0x13, 0x21, + 0xf5, 0x47, 0xaa, 0x18, 0x21, 0x5c, 0xc9, 0x9a, + 0xd2, 0x6b, 0x05, 0x9c, 0x01, 0xa1, 0xda, 0x35, + 0x5d, 0xb3, 0x70, 0xe6, 0xa9, 0x80, 0x8b, 0x91, + 0xb7, 0xb3, 0x5f, 0x24, 0x9a, 0xb7, 0xd1, 0x6b, + 0xa1, 0x1c, 0x50, 0xba, 0x49, 0xe0, 0xee, 0x2e, + 0x75, 0xac, 0x69, 0xc0, 0xeb, 0x03, 0xdd, 0x19, + 0xe5, 0xf6, 0x06, 0xdd, 0xc3, 0xd7, 0x2b, 0x07, + 0x07, 0x30, 0xa7, 0x19, 0x0c, 0xbf, 0xe6, 0x18, + 0xcc, 0xb1, 0x01, 0x11, 0x85, 0x77, 0x1d, 0x96, + 0xa7, 0xa3, 0x00, 0x84, 0x02, 0xa2, 0x83, 0x68, + 0xda, 0x17, 0x27, 0xc8, 0x7f, 0x23, 0xb7, 0xf4, + 0x13, 0x85, 0xcf, 0xdd, 0x7a, 0x7d, 0x24, 0x57, + 0xfe, 0x05, 0x93, 0xf5, 0x74, 0xce, 0xed, 0x0c, + 0x20, 0x98, 0x8d, 0x92, 0x30, 0xa1, 0x29, 0x23, + 0x1a, 0xa0, 0x4f, 0x69, 0x56, 0x4c, 0xe1, 0xc8, + 0xce, 0xf6, 0x9a, 0x0c, 0xa4, 0xfa, 0x04, 0xf6, + 0x62, 0x95, 0xf2, 0xfa, 0xc7, 0x40, 0x68, 0x40, + 0x8f, 0x41, 0xda, 0xb4, 0x26, 0x6f, 0x70, 0xab, + 0x40, 0x61, 0xa4, 0x0e, 0x75, 0xfb, 0x86, 0xeb, + 0x9d, 0x9a, 0x1f, 0xec, 0x76, 0x99, 0xe7, 0xea, + 0xaa, 0x1e, 0x2d, 0xb5, 0xd4, 0xa6, 0x1a, 0xb8, + 0x61, 0x0a, 0x1d, 0x16, 0x5b, 0x98, 0xc2, 0x31, + 0x40, 0xe7, 0x23, 0x1d, 0x66, 0x99, 0xc8, 0xc0, + 0xd7, 0xce, 0xf3, 0x57, 0x40, 0x04, 0x3f, 0xfc, + 0xea, 0xb3, 0xfc, 0xd2, 0xd3, 0x99, 0xa4, 0x94, + 0x69, 0xa0, 0xef, 0xd1, 0x85, 0xb3, 0xa6, 0xb1, + 0x28, 0xbf, 0x94, 0x67, 0x22, 0xc3, 0x36, 0x46, + 0xf8, 0xd2, 0x0f, 0x5f, 0xf4, 0x59, 0x80, 0xe6, + 0x2d, 0x43, 0x08, 0x7d, 0x19, 0x09, 0x97, 0xa7, + 0x4c, 0x3d, 0x8d, 0xba, 0x65, 0x62, 0xa3, 0x71, + 0x33, 0x29, 0x62, 0xdb, 0xc1, 0x33, 0x34, 0x1a, + 0x63, 0x33, 0x16, 0xb6, 0x64, 0x7e, 0xab, 0x33, + 0xf0, 0xe6, 0x26, 0x68, 0xba, 0x1d, 0x2e, 0x38, + 0x08, 0xe6, 0x02, 0xd3, 0x25, 0x2c, 0x47, 0x23, + 0x58, 0x34, 0x0f, 0x9d, 0x63, 0x4f, 0x63, 0xbb, + 0x7f, 0x3b, 0x34, 0x38, 0xa7, 0xb5, 0x8d, 0x65, + 0xd9, 0x9f, 0x79, 0x55, 0x3e, 0x4d, 0xe7, 0x73, + 0xd8, 0xf6, 0x98, 0x97, 0x84, 0x60, 0x9c, 0xc8, + 0xa9, 0x3c, 0xf6, 0xdc, 0x12, 0x5c, 0xe1, 0xbb, + 0x0b, 0x8b, 0x98, 0x9c, 0x9d, 0x26, 0x7c, 0x4a, + 0xe6, 0x46, 0x36, 0x58, 0x21, 0x4a, 0xee, 0xca, + 0xd7, 0x3b, 0xc2, 0x6c, 0x49, 0x2f, 0xe5, 0xd5, + 0x03, 0x59, 0x84, 0x53, 0xcb, 0xfe, 0x92, 0x71, + 0x2e, 0x7c, 0x21, 0xcc, 0x99, 0x85, 0x7f, 0xb8, + 0x74, 0x90, 0x13, 0x42, 0x3f, 0xe0, 0x6b, 0x1d, + 0xf2, 0x4d, 0x54, 0xd4, 0xfc, 0x3a, 0x05, 0xe6, + 0x74, 0xaf, 0xa6, 0xa0, 0x2a, 0x20, 0x23, 0x5d, + 0x34, 0x5c, 0xd9, 0x3e, 0x4e, 0xfa, 0x93, 0xe7, + 0xaa, 0xe9, 0x6f, 0x08, 0x43, 0x67, 0x41, 0xc5, + 0xad, 0xfb, 0x31, 0x95, 0x82, 0x73, 0x32, 0xd8, + 0xa6, 0xa3, 0xed, 0x0e, 0x2d, 0xf6, 0x5f, 0xfd, + 0x80, 0xa6, 0x7a, 0xe0, 0xdf, 0x78, 0x15, 0x29, + 0x74, 0x33, 0xd0, 0x9e, 0x83, 0x86, 0x72, 0x22, + 0x57, 0x29, 0xb9, 0x9e, 0x5d, 0xd3, 0x1a, 0xb5, + 0x96, 0x72, 0x41, 0x3d, 0xf1, 0x64, 0x43, 0x67, + 0xee, 0xaa, 0x5c, 0xd3, 0x9a, 0x96, 0x13, 0x11, + 0x5d, 0xf3, 0x0c, 0x87, 0x82, 0x1e, 0x41, 0x9e, + 0xd0, 0x27, 0xd7, 0x54, 0x3b, 0x67, 0x73, 0x09, + 0x91, 0xe9, 0xd5, 0x36, 0xa7, 0xb5, 0x55, 0xe4, + 0xf3, 0x21, 0x51, 0x49, 0x22, 0x07, 0x55, 0x4f, + 0x44, 0x4b, 0xd2, 0x15, 0x93, 0x17, 0x2a, 0xfa, + 0x4d, 0x4a, 0x57, 0xdb, 0x4c, 0xa6, 0xeb, 0xec, + 0x53, 0x25, 0x6c, 0x21, 0xed, 0x00, 0x4c, 0x3b, + 0xca, 0x14, 0x57, 0xa9, 0xd6, 0x6a, 0xcd, 0x8d, + 0x5e, 0x74, 0xac, 0x72, 0xc1, 0x97, 0xe5, 0x1b, + 0x45, 0x4e, 0xda, 0xfc, 0xcc, 0x40, 0xe8, 0x48, + 0x88, 0x0b, 0xa3, 0xe3, 0x8d, 0x83, 0x42, 0xc3, + 0x23, 0xfd, 0x68, 0xb5, 0x8e, 0xf1, 0x9d, 0x63, + 0x77, 0xe9, 0xa3, 0x8e, 0x8c, 0x26, 0x6b, 0xbd, + 0x72, 0x73, 0x35, 0x0c, 0x03, 0xf8, 0x43, 0x78, + 0x52, 0x71, 0x15, 0x1f, 0x71, 0x5d, 0x6e, 0xed, + 0xb9, 0xcc, 0x86, 0x30, 0xdb, 0x2b, 0xd3, 0x82, + 0x88, 0x23, 0x71, 0x90, 0x53, 0x5c, 0xa9, 0x2f, + 0x76, 0x01, 0xb7, 0x9a, 0xfe, 0x43, 0x55, 0xa3, + 0x04, 0x9b, 0x0e, 0xe4, 0x59, 0xdf, 0xc9, 0xe9, + 0xb1, 0xea, 0x29, 0x28, 0x3c, 0x5c, 0xae, 0x72, + 0x84, 0xb6, 0xc6, 0xeb, 0x0c, 0x27, 0x07, 0x74, + 0x90, 0x0d, 0x31, 0xb0, 0x00, 0x77, 0xe9, 0x40, + 0x70, 0x6f, 0x68, 0xa7, 0xfd, 0x06, 0xec, 0x4b, + 0xc0, 0xb7, 0xac, 0xbc, 0x33, 0xb7, 0x6d, 0x0a, + 0xbd, 0x12, 0x1b, 0x59, 0xcb, 0xdd, 0x32, 0xf5, + 0x1d, 0x94, 0x57, 0x76, 0x9e, 0x0c, 0x18, 0x98, + 0x71, 0xd7, 0x2a, 0xdb, 0x0b, 0x7b, 0xa7, 0x71, + 0xb7, 0x67, 0x81, 0x23, 0x96, 0xae, 0xb9, 0x7e, + 0x32, 0x43, 0x92, 0x8a, 0x19, 0xa0, 0xc4, 0xd4, + 0x3b, 0x57, 0xf9, 0x4a, 0x2c, 0xfb, 0x51, 0x46, + 0xbb, 0xcb, 0x5d, 0xb3, 0xef, 0x13, 0x93, 0x6e, + 0x68, 0x42, 0x54, 0x57, 0xd3, 0x6a, 0x3a, 0x8f, + 0x9d, 0x66, 0xbf, 0xbd, 0x36, 0x23, 0xf5, 0x93, + 0x83, 0x7b, 0x9c, 0xc0, 0xdd, 0xc5, 0x49, 0xc0, + 0x64, 0xed, 0x07, 0x12, 0xb3, 0xe6, 0xe4, 0xe5, + 0x38, 0x95, 0x23, 0xb1, 0xa0, 0x3b, 0x1a, 0x61, + 0xda, 0x17, 0xac, 0xc3, 0x58, 0xdd, 0x74, 0x64, + 0x22, 0x11, 0xe8, 0x32, 0x1d, 0x16, 0x93, 0x85, + 0x99, 0xa5, 0x9c, 0x34, 0x55, 0xb1, 0xe9, 0x20, + 0x72, 0xc9, 0x28, 0x7b, 0x79, 0x00, 0xa1, 0xa6, + 0xa3, 0x27, 0x40, 0x18, 0x8a, 0x54, 0xe0, 0xcc, + 0xe8, 0x4e, 0x8e, 0x43, 0x96, 0xe7, 0x3f, 0xc8, + 0xe9, 0xb2, 0xf9, 0xc9, 0xda, 0x04, 0x71, 0x50, + 0x47, 0xe4, 0xaa, 0xce, 0xa2, 0x30, 0xc8, 0xe4, + 0xac, 0xc7, 0x0d, 0x06, 0x2e, 0xe6, 0xe8, 0x80, + 0x36, 0x29, 0x9e, 0x01, 0xb8, 0xc3, 0xf0, 0xa0, + 0x5d, 0x7a, 0xca, 0x4d, 0xa0, 0x57, 0xbd, 0x2a, + 0x45, 0xa7, 0x7f, 0x9c, 0x93, 0x07, 0x8f, 0x35, + 0x67, 0x92, 0xe3, 0xe9, 0x7f, 0xa8, 0x61, 0x43, + 0x9e, 0x25, 0x4f, 0x33, 0x76, 0x13, 0x6e, 0x12, + 0xb9, 0xdd, 0xa4, 0x7c, 0x08, 0x9f, 0x7c, 0xe7, + 0x0a, 0x8d, 0x84, 0x06, 0xa4, 0x33, 0x17, 0x34, + 0x5e, 0x10, 0x7c, 0xc0, 0xa8, 0x3d, 0x1f, 0x42, + 0x20, 0x51, 0x65, 0x5d, 0x09, 0xc3, 0xaa, 0xc0, + 0xc8, 0x0d, 0xf0, 0x79, 0xbc, 0x20, 0x1b, 0x95, + 0xe7, 0x06, 0x7d, 0x47, 0x20, 0x03, 0x1a, 0x74, + 0xdd, 0xe2, 0xd4, 0xae, 0x38, 0x71, 0x9b, 0xf5, + 0x80, 0xec, 0x08, 0x4e, 0x56, 0xba, 0x76, 0x12, + 0x1a, 0xdf, 0x48, 0xf3, 0xae, 0xb3, 0xe6, 0xe6, + 0xbe, 0xc0, 0x91, 0x2e, 0x01, 0xb3, 0x01, 0x86, + 0xa2, 0xb9, 0x52, 0xd1, 0x21, 0xae, 0xd4, 0x97, + 0x1d, 0xef, 0x41, 0x12, 0x95, 0x3d, 0x48, 0x45, + 0x1c, 0x56, 0x32, 0x8f, 0xb8, 0x43, 0xbb, 0x19, + 0xf3, 0xca, 0xe9, 0xeb, 0x6d, 0x84, 0xbe, 0x86, + 0x06, 0xe2, 0x36, 0xb2, 0x62, 0x9d, 0xd3, 0x4c, + 0x48, 0x18, 0x54, 0x13, 0x4e, 0xcf, 0xfd, 0xba, + 0x84, 0xb9, 0x30, 0x53, 0xcf, 0xfb, 0xb9, 0x29, + 0x8f, 0xdc, 0x9f, 0xef, 0x60, 0x0b, 0x64, 0xf6, + 0x8b, 0xee, 0xa6, 0x91, 0xc2, 0x41, 0x6c, 0xf6, + 0xfa, 0x79, 0x67, 0x4b, 0xc1, 0x3f, 0xaf, 0x09, + 0x81, 0xd4, 0x5d, 0xcb, 0x09, 0xdf, 0x36, 0x31, + 0xc0, 0x14, 0x3c, 0x7c, 0x0e, 0x65, 0x95, 0x99, + 0x6d, 0xa3, 0xf4, 0xd7, 0x38, 0xee, 0x1a, 0x2b, + 0x37, 0xe2, 0xa4, 0x3b, 0x4b, 0xd0, 0x65, 0xca, + 0xf8, 0xc3, 0xe8, 0x15, 0x20, 0xef, 0xf2, 0x00, + 0xfd, 0x01, 0x09, 0xc5, 0xc8, 0x17, 0x04, 0x93, + 0xd0, 0x93, 0x03, 0x55, 0xc5, 0xfe, 0x32, 0xa3, + 0x3e, 0x28, 0x2d, 0x3b, 0x93, 0x8a, 0xcc, 0x07, + 0x72, 0x80, 0x8b, 0x74, 0x16, 0x24, 0xbb, 0xda, + 0x94, 0x39, 0x30, 0x8f, 0xb1, 0xcd, 0x4a, 0x90, + 0x92, 0x7c, 0x14, 0x8f, 0x95, 0x4e, 0xac, 0x9b, + 0xd8, 0x8f, 0x1a, 0x87, 0xa4, 0x32, 0x27, 0x8a, + 0xba, 0xf7, 0x41, 0xcf, 0x84, 0x37, 0x19, 0xe6, + 0x06, 0xf5, 0x0e, 0xcf, 0x36, 0xf5, 0x9e, 0x6c, + 0xde, 0xbc, 0xff, 0x64, 0x7e, 0x4e, 0x59, 0x57, + 0x48, 0xfe, 0x14, 0xf7, 0x9c, 0x93, 0x5d, 0x15, + 0xad, 0xcc, 0x11, 0xb1, 0x17, 0x18, 0xb2, 0x7e, + 0xcc, 0xab, 0xe9, 0xce, 0x7d, 0x77, 0x5b, 0x51, + 0x1b, 0x1e, 0x20, 0xa8, 0x32, 0x06, 0x0e, 0x75, + 0x93, 0xac, 0xdb, 0x35, 0x37, 0x1f, 0xe9, 0x19, + 0x1d, 0xb4, 0x71, 0x97, 0xd6, 0x4e, 0x2c, 0x08, + 0xa5, 0x13, 0xf9, 0x0e, 0x7e, 0x78, 0x6e, 0x14, + 0xe0, 0xa9, 0xb9, 0x96, 0x4c, 0x80, 0x82, 0xba, + 0x17, 0xb3, 0x9d, 0x69, 0xb0, 0x84, 0x46, 0xff, + 0xf9, 0x52, 0x79, 0x94, 0x58, 0x3a, 0x62, 0x90, + 0x15, 0x35, 0x71, 0x10, 0x37, 0xed, 0xa1, 0x8e, + 0x53, 0x6e, 0xf4, 0x26, 0x57, 0x93, 0x15, 0x93, + 0xf6, 0x81, 0x2c, 0x5a, 0x10, 0xda, 0x92, 0xad, + 0x2f, 0xdb, 0x28, 0x31, 0x2d, 0x55, 0x04, 0xd2, + 0x06, 0x28, 0x8c, 0x1e, 0xdc, 0xea, 0x54, 0xac, + 0xff, 0xb7, 0x6c, 0x30, 0x15, 0xd4, 0xb4, 0x0d, + 0x00, 0x93, 0x57, 0xdd, 0xd2, 0x07, 0x07, 0x06, + 0xd9, 0x43, 0x9b, 0xcd, 0x3a, 0xf4, 0x7d, 0x4c, + 0x36, 0x5d, 0x23, 0xa2, 0xcc, 0x57, 0x40, 0x91, + 0xe9, 0x2c, 0x2f, 0x2c, 0xd5, 0x30, 0x9b, 0x17, + 0xb0, 0xc9, 0xf7, 0xa7, 0x2f, 0xd1, 0x93, 0x20, + 0x6b, 0xc6, 0xc1, 0xe4, 0x6f, 0xcb, 0xd1, 0xe7, + 0x09, 0x0f, 0x9e, 0xdc, 0xaa, 0x9f, 0x2f, 0xdf, + 0x56, 0x9f, 0xd4, 0x33, 0x04, 0xaf, 0xd3, 0x6c, + 0x58, 0x61, 0xf0, 0x30, 0xec, 0xf2, 0x7f, 0xf2, + 0x9c, 0xdf, 0x39, 0xbb, 0x6f, 0xa2, 0x8c, 0x7e, + 0xc4, 0x22, 0x51, 0x71, 0xc0, 0x4d, 0x14, 0x1a, + 0xc4, 0xcd, 0x04, 0xd9, 0x87, 0x08, 0x50, 0x05, + 0xcc, 0xaf, 0xf6, 0xf0, 0x8f, 0x92, 0x54, 0x58, + 0xc2, 0xc7, 0x09, 0x7a, 0x59, 0x02, 0x05, 0xe8, + 0xb0, 0x86, 0xd9, 0xbf, 0x7b, 0x35, 0x51, 0x4d, + 0xaf, 0x08, 0x97, 0x2c, 0x65, 0xda, 0x2a, 0x71, + 0x3a, 0xa8, 0x51, 0xcc, 0xf2, 0x73, 0x27, 0xc3, + 0xfd, 0x62, 0xcf, 0xe3, 0xb2, 0xca, 0xcb, 0xbe, + 0x1a, 0x0a, 0xa1, 0x34, 0x7b, 0x77, 0xc4, 0x62, + 0x68, 0x78, 0x5f, 0x94, 0x07, 0x04, 0x65, 0x16, + 0x4b, 0x61, 0xcb, 0xff, 0x75, 0x26, 0x50, 0x66, + 0x1f, 0x6e, 0x93, 0xf8, 0xc5, 0x51, 0xeb, 0xa4, + 0x4a, 0x48, 0x68, 0x6b, 0xe2, 0x5e, 0x44, 0xb2, + 0x50, 0x2c, 0x6c, 0xae, 0x79, 0x4e, 0x66, 0x35, + 0x81, 0x50, 0xac, 0xbc, 0x3f, 0xb1, 0x0c, 0xf3, + 0x05, 0x3c, 0x4a, 0xa3, 0x6c, 0x2a, 0x79, 0xb4, + 0xb7, 0xab, 0xca, 0xc7, 0x9b, 0x8e, 0xcd, 0x5f, + 0x11, 0x03, 0xcb, 0x30, 0xa3, 0xab, 0xda, 0xfe, + 0x64, 0xb9, 0xbb, 0xd8, 0x5e, 0x3a, 0x1a, 0x56, + 0xe5, 0x05, 0x48, 0x90, 0x1e, 0x61, 0x69, 0x1b, + 0x22, 0xe6, 0x1a, 0x3c, 0x75, 0xad, 0x1f, 0x37, + 0x28, 0xdc, 0xe4, 0x6d, 0xbd, 0x42, 0xdc, 0xd3, + 0xc8, 0xb6, 0x1c, 0x48, 0xfe, 0x94, 0x77, 0x7f, + 0xbd, 0x62, 0xac, 0xa3, 0x47, 0x27, 0xcf, 0x5f, + 0xd9, 0xdb, 0xaf, 0xec, 0xf7, 0x5e, 0xc1, 0xb0, + 0x9d, 0x01, 0x26, 0x99, 0x7e, 0x8f, 0x03, 0x70, + 0xb5, 0x42, 0xbe, 0x67, 0x28, 0x1b, 0x7c, 0xbd, + 0x61, 0x21, 0x97, 0xcc, 0x5c, 0xe1, 0x97, 0x8f, + 0x8d, 0xde, 0x2b, 0xaa, 0xa7, 0x71, 0x1d, 0x1e, + 0x02, 0x73, 0x70, 0x58, 0x32, 0x5b, 0x1d, 0x67, + 0x3d, 0xe0, 0x74, 0x4f, 0x03, 0xf2, 0x70, 0x51, + 0x79, 0xf1, 0x61, 0x70, 0x15, 0x74, 0x9d, 0x23, + 0x89, 0xde, 0xac, 0xfd, 0xde, 0xd0, 0x1f, 0xc3, + 0x87, 0x44, 0x35, 0x4b, 0xe5, 0xb0, 0x60, 0xc5, + 0x22, 0xe4, 0x9e, 0xca, 0xeb, 0xd5, 0x3a, 0x09, + 0x45, 0xa4, 0xdb, 0xfa, 0x3f, 0xeb, 0x1b, 0xc7, + 0xc8, 0x14, 0x99, 0x51, 0x92, 0x10, 0xed, 0xed, + 0x28, 0xe0, 0xa1, 0xf8, 0x26, 0xcf, 0xcd, 0xcb, + 0x63, 0xa1, 0x3b, 0xe3, 0xdf, 0x7e, 0xfe, 0xa6, + 0xf0, 0x81, 0x9a, 0xbf, 0x55, 0xde, 0x54, 0xd5, + 0x56, 0x60, 0x98, 0x10, 0x68, 0xf4, 0x38, 0x96, + 0x8e, 0x6f, 0x1d, 0x44, 0x7f, 0xd6, 0x2f, 0xfe, + 0x55, 0xfb, 0x0c, 0x7e, 0x67, 0xe2, 0x61, 0x44, + 0xed, 0xf2, 0x35, 0x30, 0x5d, 0xe9, 0xc7, 0xd6, + 0x6d, 0xe0, 0xa0, 0xed, 0xf3, 0xfc, 0xd8, 0x3e, + 0x0a, 0x7b, 0xcd, 0xaf, 0x65, 0x68, 0x18, 0xc0, + 0xec, 0x04, 0x1c, 0x74, 0x6d, 0xe2, 0x6e, 0x79, + 0xd4, 0x11, 0x2b, 0x62, 0xd5, 0x27, 0xad, 0x4f, + 0x01, 0x59, 0x73, 0xcc, 0x6a, 0x53, 0xfb, 0x2d, + 0xd5, 0x4e, 0x99, 0x21, 0x65, 0x4d, 0xf5, 0x82, + 0xf7, 0xd8, 0x42, 0xce, 0x6f, 0x3d, 0x36, 0x47, + 0xf1, 0x05, 0x16, 0xe8, 0x1b, 0x6a, 0x8f, 0x93, + 0xf2, 0x8f, 0x37, 0x40, 0x12, 0x28, 0xa3, 0xe6, + 0xb9, 0x17, 0x4a, 0x1f, 0xb1, 0xd1, 0x66, 0x69, + 0x86, 0xc4, 0xfc, 0x97, 0xae, 0x3f, 0x8f, 0x1e, + 0x2b, 0xdf, 0xcd, 0xf9, 0x3c +}; +static const u8 dec_output011[] __initconst = { + 0x7a, 0x57, 0xf2, 0xc7, 0x06, 0x3f, 0x50, 0x7b, + 0x36, 0x1a, 0x66, 0x5c, 0xb9, 0x0e, 0x5e, 0x3b, + 0x45, 0x60, 0xbe, 0x9a, 0x31, 0x9f, 0xff, 0x5d, + 0x66, 0x34, 0xb4, 0xdc, 0xfb, 0x9d, 0x8e, 0xee, + 0x6a, 0x33, 0xa4, 0x07, 0x3c, 0xf9, 0x4c, 0x30, + 0xa1, 0x24, 0x52, 0xf9, 0x50, 0x46, 0x88, 0x20, + 0x02, 0x32, 0x3a, 0x0e, 0x99, 0x63, 0xaf, 0x1f, + 0x15, 0x28, 0x2a, 0x05, 0xff, 0x57, 0x59, 0x5e, + 0x18, 0xa1, 0x1f, 0xd0, 0x92, 0x5c, 0x88, 0x66, + 0x1b, 0x00, 0x64, 0xa5, 0x93, 0x8d, 0x06, 0x46, + 0xb0, 0x64, 0x8b, 0x8b, 0xef, 0x99, 0x05, 0x35, + 0x85, 0xb3, 0xf3, 0x33, 0xbb, 0xec, 0x66, 0xb6, + 0x3d, 0x57, 0x42, 0xe3, 0xb4, 0xc6, 0xaa, 0xb0, + 0x41, 0x2a, 0xb9, 0x59, 0xa9, 0xf6, 0x3e, 0x15, + 0x26, 0x12, 0x03, 0x21, 0x4c, 0x74, 0x43, 0x13, + 0x2a, 0x03, 0x27, 0x09, 0xb4, 0xfb, 0xe7, 0xb7, + 0x40, 0xff, 0x5e, 0xce, 0x48, 0x9a, 0x60, 0xe3, + 0x8b, 0x80, 0x8c, 0x38, 0x2d, 0xcb, 0x93, 0x37, + 0x74, 0x05, 0x52, 0x6f, 0x73, 0x3e, 0xc3, 0xbc, + 0xca, 0x72, 0x0a, 0xeb, 0xf1, 0x3b, 0xa0, 0x95, + 0xdc, 0x8a, 0xc4, 0xa9, 0xdc, 0xca, 0x44, 0xd8, + 0x08, 0x63, 0x6a, 0x36, 0xd3, 0x3c, 0xb8, 0xac, + 0x46, 0x7d, 0xfd, 0xaa, 0xeb, 0x3e, 0x0f, 0x45, + 0x8f, 0x49, 0xda, 0x2b, 0xf2, 0x12, 0xbd, 0xaf, + 0x67, 0x8a, 0x63, 0x48, 0x4b, 0x55, 0x5f, 0x6d, + 0x8c, 0xb9, 0x76, 0x34, 0x84, 0xae, 0xc2, 0xfc, + 0x52, 0x64, 0x82, 0xf7, 0xb0, 0x06, 0xf0, 0x45, + 0x73, 0x12, 0x50, 0x30, 0x72, 0xea, 0x78, 0x9a, + 0xa8, 0xaf, 0xb5, 0xe3, 0xbb, 0x77, 0x52, 0xec, + 0x59, 0x84, 0xbf, 0x6b, 0x8f, 0xce, 0x86, 0x5e, + 0x1f, 0x23, 0xe9, 0xfb, 0x08, 0x86, 0xf7, 0x10, + 0xb9, 0xf2, 0x44, 0x96, 0x44, 0x63, 0xa9, 0xa8, + 0x78, 0x00, 0x23, 0xd6, 0xc7, 0xe7, 0x6e, 0x66, + 0x4f, 0xcc, 0xee, 0x15, 0xb3, 0xbd, 0x1d, 0xa0, + 0xe5, 0x9c, 0x1b, 0x24, 0x2c, 0x4d, 0x3c, 0x62, + 0x35, 0x9c, 0x88, 0x59, 0x09, 0xdd, 0x82, 0x1b, + 0xcf, 0x0a, 0x83, 0x6b, 0x3f, 0xae, 0x03, 0xc4, + 0xb4, 0xdd, 0x7e, 0x5b, 0x28, 0x76, 0x25, 0x96, + 0xd9, 0xc9, 0x9d, 0x5f, 0x86, 0xfa, 0xf6, 0xd7, + 0xd2, 0xe6, 0x76, 0x1d, 0x0f, 0xa1, 0xdc, 0x74, + 0x05, 0x1b, 0x1d, 0xe0, 0xcd, 0x16, 0xb0, 0xa8, + 0x8a, 0x34, 0x7b, 0x15, 0x11, 0x77, 0xe5, 0x7b, + 0x7e, 0x20, 0xf7, 0xda, 0x38, 0xda, 0xce, 0x70, + 0xe9, 0xf5, 0x6c, 0xd9, 0xbe, 0x0c, 0x4c, 0x95, + 0x4c, 0xc2, 0x9b, 0x34, 0x55, 0x55, 0xe1, 0xf3, + 0x46, 0x8e, 0x48, 0x74, 0x14, 0x4f, 0x9d, 0xc9, + 0xf5, 0xe8, 0x1a, 0xf0, 0x11, 0x4a, 0xc1, 0x8d, + 0xe0, 0x93, 0xa0, 0xbe, 0x09, 0x1c, 0x2b, 0x4e, + 0x0f, 0xb2, 0x87, 0x8b, 0x84, 0xfe, 0x92, 0x32, + 0x14, 0xd7, 0x93, 0xdf, 0xe7, 0x44, 0xbc, 0xc5, + 0xae, 0x53, 0x69, 0xd8, 0xb3, 0x79, 0x37, 0x80, + 0xe3, 0x17, 0x5c, 0xec, 0x53, 0x00, 0x9a, 0xe3, + 0x8e, 0xdc, 0x38, 0xb8, 0x66, 0xf0, 0xd3, 0xad, + 0x1d, 0x02, 0x96, 0x86, 0x3e, 0x9d, 0x3b, 0x5d, + 0xa5, 0x7f, 0x21, 0x10, 0xf1, 0x1f, 0x13, 0x20, + 0xf9, 0x57, 0x87, 0x20, 0xf5, 0x5f, 0xf1, 0x17, + 0x48, 0x0a, 0x51, 0x5a, 0xcd, 0x19, 0x03, 0xa6, + 0x5a, 0xd1, 0x12, 0x97, 0xe9, 0x48, 0xe2, 0x1d, + 0x83, 0x75, 0x50, 0xd9, 0x75, 0x7d, 0x6a, 0x82, + 0xa1, 0xf9, 0x4e, 0x54, 0x87, 0x89, 0xc9, 0x0c, + 0xb7, 0x5b, 0x6a, 0x91, 0xc1, 0x9c, 0xb2, 0xa9, + 0xdc, 0x9a, 0xa4, 0x49, 0x0a, 0x6d, 0x0d, 0xbb, + 0xde, 0x86, 0x44, 0xdd, 0x5d, 0x89, 0x2b, 0x96, + 0x0f, 0x23, 0x95, 0xad, 0xcc, 0xa2, 0xb3, 0xb9, + 0x7e, 0x74, 0x38, 0xba, 0x9f, 0x73, 0xae, 0x5f, + 0xf8, 0x68, 0xa2, 0xe0, 0xa9, 0xce, 0xbd, 0x40, + 0xd4, 0x4c, 0x6b, 0xd2, 0x56, 0x62, 0xb0, 0xcc, + 0x63, 0x7e, 0x5b, 0xd3, 0xae, 0xd1, 0x75, 0xce, + 0xbb, 0xb4, 0x5b, 0xa8, 0xf8, 0xb4, 0xac, 0x71, + 0x75, 0xaa, 0xc9, 0x9f, 0xbb, 0x6c, 0xad, 0x0f, + 0x55, 0x5d, 0xe8, 0x85, 0x7d, 0xf9, 0x21, 0x35, + 0xea, 0x92, 0x85, 0x2b, 0x00, 0xec, 0x84, 0x90, + 0x0a, 0x63, 0x96, 0xe4, 0x6b, 0xa9, 0x77, 0xb8, + 0x91, 0xf8, 0x46, 0x15, 0x72, 0x63, 0x70, 0x01, + 0x40, 0xa3, 0xa5, 0x76, 0x62, 0x2b, 0xbf, 0xf1, + 0xe5, 0x8d, 0x9f, 0xa3, 0xfa, 0x9b, 0x03, 0xbe, + 0xfe, 0x65, 0x6f, 0xa2, 0x29, 0x0d, 0x54, 0xb4, + 0x71, 0xce, 0xa9, 0xd6, 0x3d, 0x88, 0xf9, 0xaf, + 0x6b, 0xa8, 0x9e, 0xf4, 0x16, 0x96, 0x36, 0xb9, + 0x00, 0xdc, 0x10, 0xab, 0xb5, 0x08, 0x31, 0x1f, + 0x00, 0xb1, 0x3c, 0xd9, 0x38, 0x3e, 0xc6, 0x04, + 0xa7, 0x4e, 0xe8, 0xae, 0xed, 0x98, 0xc2, 0xf7, + 0xb9, 0x00, 0x5f, 0x8c, 0x60, 0xd1, 0xe5, 0x15, + 0xf7, 0xae, 0x1e, 0x84, 0x88, 0xd1, 0xf6, 0xbc, + 0x3a, 0x89, 0x35, 0x22, 0x83, 0x7c, 0xca, 0xf0, + 0x33, 0x82, 0x4c, 0x79, 0x3c, 0xfd, 0xb1, 0xae, + 0x52, 0x62, 0x55, 0xd2, 0x41, 0x60, 0xc6, 0xbb, + 0xfa, 0x0e, 0x59, 0xd6, 0xa8, 0xfe, 0x5d, 0xed, + 0x47, 0x3d, 0xe0, 0xea, 0x1f, 0x6e, 0x43, 0x51, + 0xec, 0x10, 0x52, 0x56, 0x77, 0x42, 0x6b, 0x52, + 0x87, 0xd8, 0xec, 0xe0, 0xaa, 0x76, 0xa5, 0x84, + 0x2a, 0x22, 0x24, 0xfd, 0x92, 0x40, 0x88, 0xd5, + 0x85, 0x1c, 0x1f, 0x6b, 0x47, 0xa0, 0xc4, 0xe4, + 0xef, 0xf4, 0xea, 0xd7, 0x59, 0xac, 0x2a, 0x9e, + 0x8c, 0xfa, 0x1f, 0x42, 0x08, 0xfe, 0x4f, 0x74, + 0xa0, 0x26, 0xf5, 0xb3, 0x84, 0xf6, 0x58, 0x5f, + 0x26, 0x66, 0x3e, 0xd7, 0xe4, 0x22, 0x91, 0x13, + 0xc8, 0xac, 0x25, 0x96, 0x23, 0xd8, 0x09, 0xea, + 0x45, 0x75, 0x23, 0xb8, 0x5f, 0xc2, 0x90, 0x8b, + 0x09, 0xc4, 0xfc, 0x47, 0x6c, 0x6d, 0x0a, 0xef, + 0x69, 0xa4, 0x38, 0x19, 0xcf, 0x7d, 0xf9, 0x09, + 0x73, 0x9b, 0x60, 0x5a, 0xf7, 0x37, 0xb5, 0xfe, + 0x9f, 0xe3, 0x2b, 0x4c, 0x0d, 0x6e, 0x19, 0xf1, + 0xd6, 0xc0, 0x70, 0xf3, 0x9d, 0x22, 0x3c, 0xf9, + 0x49, 0xce, 0x30, 0x8e, 0x44, 0xb5, 0x76, 0x15, + 0x8f, 0x52, 0xfd, 0xa5, 0x04, 0xb8, 0x55, 0x6a, + 0x36, 0x59, 0x7c, 0xc4, 0x48, 0xb8, 0xd7, 0xab, + 0x05, 0x66, 0xe9, 0x5e, 0x21, 0x6f, 0x6b, 0x36, + 0x29, 0xbb, 0xe9, 0xe3, 0xa2, 0x9a, 0xa8, 0xcd, + 0x55, 0x25, 0x11, 0xba, 0x5a, 0x58, 0xa0, 0xde, + 0xae, 0x19, 0x2a, 0x48, 0x5a, 0xff, 0x36, 0xcd, + 0x6d, 0x16, 0x7a, 0x73, 0x38, 0x46, 0xe5, 0x47, + 0x59, 0xc8, 0xa2, 0xf6, 0xe2, 0x6c, 0x83, 0xc5, + 0x36, 0x2c, 0x83, 0x7d, 0xb4, 0x01, 0x05, 0x69, + 0xe7, 0xaf, 0x5c, 0xc4, 0x64, 0x82, 0x12, 0x21, + 0xef, 0xf7, 0xd1, 0x7d, 0xb8, 0x8d, 0x8c, 0x98, + 0x7c, 0x5f, 0x7d, 0x92, 0x88, 0xb9, 0x94, 0x07, + 0x9c, 0xd8, 0xe9, 0x9c, 0x17, 0x38, 0xe3, 0x57, + 0x6c, 0xe0, 0xdc, 0xa5, 0x92, 0x42, 0xb3, 0xbd, + 0x50, 0xa2, 0x7e, 0xb5, 0xb1, 0x52, 0x72, 0x03, + 0x97, 0xd8, 0xaa, 0x9a, 0x1e, 0x75, 0x41, 0x11, + 0xa3, 0x4f, 0xcc, 0xd4, 0xe3, 0x73, 0xad, 0x96, + 0xdc, 0x47, 0x41, 0x9f, 0xb0, 0xbe, 0x79, 0x91, + 0xf5, 0xb6, 0x18, 0xfe, 0xc2, 0x83, 0x18, 0x7d, + 0x73, 0xd9, 0x4f, 0x83, 0x84, 0x03, 0xb3, 0xf0, + 0x77, 0x66, 0x3d, 0x83, 0x63, 0x2e, 0x2c, 0xf9, + 0xdd, 0xa6, 0x1f, 0x89, 0x82, 0xb8, 0x23, 0x42, + 0xeb, 0xe2, 0xca, 0x70, 0x82, 0x61, 0x41, 0x0a, + 0x6d, 0x5f, 0x75, 0xc5, 0xe2, 0xc4, 0x91, 0x18, + 0x44, 0x22, 0xfa, 0x34, 0x10, 0xf5, 0x20, 0xdc, + 0xb7, 0xdd, 0x2a, 0x20, 0x77, 0xf5, 0xf9, 0xce, + 0xdb, 0xa0, 0x0a, 0x52, 0x2a, 0x4e, 0xdd, 0xcc, + 0x97, 0xdf, 0x05, 0xe4, 0x5e, 0xb7, 0xaa, 0xf0, + 0xe2, 0x80, 0xff, 0xba, 0x1a, 0x0f, 0xac, 0xdf, + 0x02, 0x32, 0xe6, 0xf7, 0xc7, 0x17, 0x13, 0xb7, + 0xfc, 0x98, 0x48, 0x8c, 0x0d, 0x82, 0xc9, 0x80, + 0x7a, 0xe2, 0x0a, 0xc5, 0xb4, 0xde, 0x7c, 0x3c, + 0x79, 0x81, 0x0e, 0x28, 0x65, 0x79, 0x67, 0x82, + 0x69, 0x44, 0x66, 0x09, 0xf7, 0x16, 0x1a, 0xf9, + 0x7d, 0x80, 0xa1, 0x79, 0x14, 0xa9, 0xc8, 0x20, + 0xfb, 0xa2, 0x46, 0xbe, 0x08, 0x35, 0x17, 0x58, + 0xc1, 0x1a, 0xda, 0x2a, 0x6b, 0x2e, 0x1e, 0xe6, + 0x27, 0x55, 0x7b, 0x19, 0xe2, 0xfb, 0x64, 0xfc, + 0x5e, 0x15, 0x54, 0x3c, 0xe7, 0xc2, 0x11, 0x50, + 0x30, 0xb8, 0x72, 0x03, 0x0b, 0x1a, 0x9f, 0x86, + 0x27, 0x11, 0x5c, 0x06, 0x2b, 0xbd, 0x75, 0x1a, + 0x0a, 0xda, 0x01, 0xfa, 0x5c, 0x4a, 0xc1, 0x80, + 0x3a, 0x6e, 0x30, 0xc8, 0x2c, 0xeb, 0x56, 0xec, + 0x89, 0xfa, 0x35, 0x7b, 0xb2, 0xf0, 0x97, 0x08, + 0x86, 0x53, 0xbe, 0xbd, 0x40, 0x41, 0x38, 0x1c, + 0xb4, 0x8b, 0x79, 0x2e, 0x18, 0x96, 0x94, 0xde, + 0xe8, 0xca, 0xe5, 0x9f, 0x92, 0x9f, 0x15, 0x5d, + 0x56, 0x60, 0x5c, 0x09, 0xf9, 0x16, 0xf4, 0x17, + 0x0f, 0xf6, 0x4c, 0xda, 0xe6, 0x67, 0x89, 0x9f, + 0xca, 0x6c, 0xe7, 0x9b, 0x04, 0x62, 0x0e, 0x26, + 0xa6, 0x52, 0xbd, 0x29, 0xff, 0xc7, 0xa4, 0x96, + 0xe6, 0x6a, 0x02, 0xa5, 0x2e, 0x7b, 0xfe, 0x97, + 0x68, 0x3e, 0x2e, 0x5f, 0x3b, 0x0f, 0x36, 0xd6, + 0x98, 0x19, 0x59, 0x48, 0xd2, 0xc6, 0xe1, 0x55, + 0x1a, 0x6e, 0xd6, 0xed, 0x2c, 0xba, 0xc3, 0x9e, + 0x64, 0xc9, 0x95, 0x86, 0x35, 0x5e, 0x3e, 0x88, + 0x69, 0x99, 0x4b, 0xee, 0xbe, 0x9a, 0x99, 0xb5, + 0x6e, 0x58, 0xae, 0xdd, 0x22, 0xdb, 0xdd, 0x6b, + 0xfc, 0xaf, 0x90, 0xa3, 0x3d, 0xa4, 0xc1, 0x15, + 0x92, 0x18, 0x8d, 0xd2, 0x4b, 0x7b, 0x06, 0xd1, + 0x37, 0xb5, 0xe2, 0x7c, 0x2c, 0xf0, 0x25, 0xe4, + 0x94, 0x2a, 0xbd, 0xe3, 0x82, 0x70, 0x78, 0xa3, + 0x82, 0x10, 0x5a, 0x90, 0xd7, 0xa4, 0xfa, 0xaf, + 0x1a, 0x88, 0x59, 0xdc, 0x74, 0x12, 0xb4, 0x8e, + 0xd7, 0x19, 0x46, 0xf4, 0x84, 0x69, 0x9f, 0xbb, + 0x70, 0xa8, 0x4c, 0x52, 0x81, 0xa9, 0xff, 0x76, + 0x1c, 0xae, 0xd8, 0x11, 0x3d, 0x7f, 0x7d, 0xc5, + 0x12, 0x59, 0x28, 0x18, 0xc2, 0xa2, 0xb7, 0x1c, + 0x88, 0xf8, 0xd6, 0x1b, 0xa6, 0x7d, 0x9e, 0xde, + 0x29, 0xf8, 0xed, 0xff, 0xeb, 0x92, 0x24, 0x4f, + 0x05, 0xaa, 0xd9, 0x49, 0xba, 0x87, 0x59, 0x51, + 0xc9, 0x20, 0x5c, 0x9b, 0x74, 0xcf, 0x03, 0xd9, + 0x2d, 0x34, 0xc7, 0x5b, 0xa5, 0x40, 0xb2, 0x99, + 0xf5, 0xcb, 0xb4, 0xf6, 0xb7, 0x72, 0x4a, 0xd6, + 0xbd, 0xb0, 0xf3, 0x93, 0xe0, 0x1b, 0xa8, 0x04, + 0x1e, 0x35, 0xd4, 0x80, 0x20, 0xf4, 0x9c, 0x31, + 0x6b, 0x45, 0xb9, 0x15, 0xb0, 0x5e, 0xdd, 0x0a, + 0x33, 0x9c, 0x83, 0xcd, 0x58, 0x89, 0x50, 0x56, + 0xbb, 0x81, 0x00, 0x91, 0x32, 0xf3, 0x1b, 0x3e, + 0xcf, 0x45, 0xe1, 0xf9, 0xe1, 0x2c, 0x26, 0x78, + 0x93, 0x9a, 0x60, 0x46, 0xc9, 0xb5, 0x5e, 0x6a, + 0x28, 0x92, 0x87, 0x3f, 0x63, 0x7b, 0xdb, 0xf7, + 0xd0, 0x13, 0x9d, 0x32, 0x40, 0x5e, 0xcf, 0xfb, + 0x79, 0x68, 0x47, 0x4c, 0xfd, 0x01, 0x17, 0xe6, + 0x97, 0x93, 0x78, 0xbb, 0xa6, 0x27, 0xa3, 0xe8, + 0x1a, 0xe8, 0x94, 0x55, 0x7d, 0x08, 0xe5, 0xdc, + 0x66, 0xa3, 0x69, 0xc8, 0xca, 0xc5, 0xa1, 0x84, + 0x55, 0xde, 0x08, 0x91, 0x16, 0x3a, 0x0c, 0x86, + 0xab, 0x27, 0x2b, 0x64, 0x34, 0x02, 0x6c, 0x76, + 0x8b, 0xc6, 0xaf, 0xcc, 0xe1, 0xd6, 0x8c, 0x2a, + 0x18, 0x3d, 0xa6, 0x1b, 0x37, 0x75, 0x45, 0x73, + 0xc2, 0x75, 0xd7, 0x53, 0x78, 0x3a, 0xd6, 0xe8, + 0x29, 0xd2, 0x4a, 0xa8, 0x1e, 0x82, 0xf6, 0xb6, + 0x81, 0xde, 0x21, 0xed, 0x2b, 0x56, 0xbb, 0xf2, + 0xd0, 0x57, 0xc1, 0x7c, 0xd2, 0x6a, 0xd2, 0x56, + 0xf5, 0x13, 0x5f, 0x1c, 0x6a, 0x0b, 0x74, 0xfb, + 0xe9, 0xfe, 0x9e, 0xea, 0x95, 0xb2, 0x46, 0xab, + 0x0a, 0xfc, 0xfd, 0xf3, 0xbb, 0x04, 0x2b, 0x76, + 0x1b, 0xa4, 0x74, 0xb0, 0xc1, 0x78, 0xc3, 0x69, + 0xe2, 0xb0, 0x01, 0xe1, 0xde, 0x32, 0x4c, 0x8d, + 0x1a, 0xb3, 0x38, 0x08, 0xd5, 0xfc, 0x1f, 0xdc, + 0x0e, 0x2c, 0x9c, 0xb1, 0xa1, 0x63, 0x17, 0x22, + 0xf5, 0x6c, 0x93, 0x70, 0x74, 0x00, 0xf8, 0x39, + 0x01, 0x94, 0xd1, 0x32, 0x23, 0x56, 0x5d, 0xa6, + 0x02, 0x76, 0x76, 0x93, 0xce, 0x2f, 0x19, 0xe9, + 0x17, 0x52, 0xae, 0x6e, 0x2c, 0x6d, 0x61, 0x7f, + 0x3b, 0xaa, 0xe0, 0x52, 0x85, 0xc5, 0x65, 0xc1, + 0xbb, 0x8e, 0x5b, 0x21, 0xd5, 0xc9, 0x78, 0x83, + 0x07, 0x97, 0x4c, 0x62, 0x61, 0x41, 0xd4, 0xfc, + 0xc9, 0x39, 0xe3, 0x9b, 0xd0, 0xcc, 0x75, 0xc4, + 0x97, 0xe6, 0xdd, 0x2a, 0x5f, 0xa6, 0xe8, 0x59, + 0x6c, 0x98, 0xb9, 0x02, 0xe2, 0xa2, 0xd6, 0x68, + 0xee, 0x3b, 0x1d, 0xe3, 0x4d, 0x5b, 0x30, 0xef, + 0x03, 0xf2, 0xeb, 0x18, 0x57, 0x36, 0xe8, 0xa1, + 0xf4, 0x47, 0xfb, 0xcb, 0x8f, 0xcb, 0xc8, 0xf3, + 0x4f, 0x74, 0x9d, 0x9d, 0xb1, 0x8d, 0x14, 0x44, + 0xd9, 0x19, 0xb4, 0x54, 0x4f, 0x75, 0x19, 0x09, + 0xa0, 0x75, 0xbc, 0x3b, 0x82, 0xc6, 0x3f, 0xb8, + 0x83, 0x19, 0x6e, 0xd6, 0x37, 0xfe, 0x6e, 0x8a, + 0x4e, 0xe0, 0x4a, 0xab, 0x7b, 0xc8, 0xb4, 0x1d, + 0xf4, 0xed, 0x27, 0x03, 0x65, 0xa2, 0xa1, 0xae, + 0x11, 0xe7, 0x98, 0x78, 0x48, 0x91, 0xd2, 0xd2, + 0xd4, 0x23, 0x78, 0x50, 0xb1, 0x5b, 0x85, 0x10, + 0x8d, 0xca, 0x5f, 0x0f, 0x71, 0xae, 0x72, 0x9a, + 0xf6, 0x25, 0x19, 0x60, 0x06, 0xf7, 0x10, 0x34, + 0x18, 0x0d, 0xc9, 0x9f, 0x7b, 0x0c, 0x9b, 0x8f, + 0x91, 0x1b, 0x9f, 0xcd, 0x10, 0xee, 0x75, 0xf9, + 0x97, 0x66, 0xfc, 0x4d, 0x33, 0x6e, 0x28, 0x2b, + 0x92, 0x85, 0x4f, 0xab, 0x43, 0x8d, 0x8f, 0x7d, + 0x86, 0xa7, 0xc7, 0xd8, 0xd3, 0x0b, 0x8b, 0x57, + 0xb6, 0x1d, 0x95, 0x0d, 0xe9, 0xbc, 0xd9, 0x03, + 0xd9, 0x10, 0x19, 0xc3, 0x46, 0x63, 0x55, 0x87, + 0x61, 0x79, 0x6c, 0x95, 0x0e, 0x9c, 0xdd, 0xca, + 0xc3, 0xf3, 0x64, 0xf0, 0x7d, 0x76, 0xb7, 0x53, + 0x67, 0x2b, 0x1e, 0x44, 0x56, 0x81, 0xea, 0x8f, + 0x5c, 0x42, 0x16, 0xb8, 0x28, 0xeb, 0x1b, 0x61, + 0x10, 0x1e, 0xbf, 0xec, 0xa8 +}; +static const u8 dec_assoc011[] __initconst = { + 0xd6, 0x31, 0xda, 0x5d, 0x42, 0x5e, 0xd7 +}; +static const u8 dec_nonce011[] __initconst = { + 0xfd, 0x87, 0xd4, 0xd8, 0x62, 0xfd, 0xec, 0xaa +}; +static const u8 dec_key011[] __initconst = { + 0x35, 0x4e, 0xb5, 0x70, 0x50, 0x42, 0x8a, 0x85, + 0xf2, 0xfb, 0xed, 0x7b, 0xd0, 0x9e, 0x97, 0xca, + 0xfa, 0x98, 0x66, 0x63, 0xee, 0x37, 0xcc, 0x52, + 0xfe, 0xd1, 0xdf, 0x95, 0x15, 0x34, 0x29, 0x38 +}; + +static const u8 dec_input012[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd6 +}; +static const u8 dec_output012[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 dec_assoc012[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 dec_nonce012[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 dec_key012[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +static const u8 dec_input013[] __initconst = { + 0x52, 0x34, 0xb3, 0x65, 0x3b, 0xb7, 0xe5, 0xd3, + 0xab, 0x49, 0x17, 0x60, 0xd2, 0x52, 0x56, 0xdf, + 0xdf, 0x34, 0x56, 0x82, 0xe2, 0xbe, 0xe5, 0xe1, + 0x28, 0xd1, 0x4e, 0x5f, 0x4f, 0x01, 0x7d, 0x3f, + 0x99, 0x6b, 0x30, 0x6e, 0x1a, 0x7c, 0x4c, 0x8e, + 0x62, 0x81, 0xae, 0x86, 0x3f, 0x6b, 0xd0, 0xb5, + 0xa9, 0xcf, 0x50, 0xf1, 0x02, 0x12, 0xa0, 0x0b, + 0x24, 0xe9, 0xe6, 0x72, 0x89, 0x2c, 0x52, 0x1b, + 0x34, 0x38, 0xf8, 0x75, 0x5f, 0xa0, 0x74, 0xe2, + 0x99, 0xdd, 0xa6, 0x4b, 0x14, 0x50, 0x4e, 0xf1, + 0xbe, 0xd6, 0x9e, 0xdb, 0xb2, 0x24, 0x27, 0x74, + 0x12, 0x4a, 0x78, 0x78, 0x17, 0xa5, 0x58, 0x8e, + 0x2f, 0xf9, 0xf4, 0x8d, 0xee, 0x03, 0x88, 0xae, + 0xb8, 0x29, 0xa1, 0x2f, 0x4b, 0xee, 0x92, 0xbd, + 0x87, 0xb3, 0xce, 0x34, 0x21, 0x57, 0x46, 0x04, + 0x49, 0x0c, 0x80, 0xf2, 0x01, 0x13, 0xa1, 0x55, + 0xb3, 0xff, 0x44, 0x30, 0x3c, 0x1c, 0xd0, 0xef, + 0xbc, 0x18, 0x74, 0x26, 0xad, 0x41, 0x5b, 0x5b, + 0x3e, 0x9a, 0x7a, 0x46, 0x4f, 0x16, 0xd6, 0x74, + 0x5a, 0xb7, 0x3a, 0x28, 0x31, 0xd8, 0xae, 0x26, + 0xac, 0x50, 0x53, 0x86, 0xf2, 0x56, 0xd7, 0x3f, + 0x29, 0xbc, 0x45, 0x68, 0x8e, 0xcb, 0x98, 0x64, + 0xdd, 0xc9, 0xba, 0xb8, 0x4b, 0x7b, 0x82, 0xdd, + 0x14, 0xa7, 0xcb, 0x71, 0x72, 0x00, 0x5c, 0xad, + 0x7b, 0x6a, 0x89, 0xa4, 0x3d, 0xbf, 0xb5, 0x4b, + 0x3e, 0x7c, 0x5a, 0xcf, 0xb8, 0xa1, 0xc5, 0x6e, + 0xc8, 0xb6, 0x31, 0x57, 0x7b, 0xdf, 0xa5, 0x7e, + 0xb1, 0xd6, 0x42, 0x2a, 0x31, 0x36, 0xd1, 0xd0, + 0x3f, 0x7a, 0xe5, 0x94, 0xd6, 0x36, 0xa0, 0x6f, + 0xb7, 0x40, 0x7d, 0x37, 0xc6, 0x55, 0x7c, 0x50, + 0x40, 0x6d, 0x29, 0x89, 0xe3, 0x5a, 0xae, 0x97, + 0xe7, 0x44, 0x49, 0x6e, 0xbd, 0x81, 0x3d, 0x03, + 0x93, 0x06, 0x12, 0x06, 0xe2, 0x41, 0x12, 0x4a, + 0xf1, 0x6a, 0xa4, 0x58, 0xa2, 0xfb, 0xd2, 0x15, + 0xba, 0xc9, 0x79, 0xc9, 0xce, 0x5e, 0x13, 0xbb, + 0xf1, 0x09, 0x04, 0xcc, 0xfd, 0xe8, 0x51, 0x34, + 0x6a, 0xe8, 0x61, 0x88, 0xda, 0xed, 0x01, 0x47, + 0x84, 0xf5, 0x73, 0x25, 0xf9, 0x1c, 0x42, 0x86, + 0x07, 0xf3, 0x5b, 0x1a, 0x01, 0xb3, 0xeb, 0x24, + 0x32, 0x8d, 0xf6, 0xed, 0x7c, 0x4b, 0xeb, 0x3c, + 0x36, 0x42, 0x28, 0xdf, 0xdf, 0xb6, 0xbe, 0xd9, + 0x8c, 0x52, 0xd3, 0x2b, 0x08, 0x90, 0x8c, 0xe7, + 0x98, 0x31, 0xe2, 0x32, 0x8e, 0xfc, 0x11, 0x48, + 0x00, 0xa8, 0x6a, 0x42, 0x4a, 0x02, 0xc6, 0x4b, + 0x09, 0xf1, 0xe3, 0x49, 0xf3, 0x45, 0x1f, 0x0e, + 0xbc, 0x56, 0xe2, 0xe4, 0xdf, 0xfb, 0xeb, 0x61, + 0xfa, 0x24, 0xc1, 0x63, 0x75, 0xbb, 0x47, 0x75, + 0xaf, 0xe1, 0x53, 0x16, 0x96, 0x21, 0x85, 0x26, + 0x11, 0xb3, 0x76, 0xe3, 0x23, 0xa1, 0x6b, 0x74, + 0x37, 0xd0, 0xde, 0x06, 0x90, 0x71, 0x5d, 0x43, + 0x88, 0x9b, 0x00, 0x54, 0xa6, 0x75, 0x2f, 0xa1, + 0xc2, 0x0b, 0x73, 0x20, 0x1d, 0xb6, 0x21, 0x79, + 0x57, 0x3f, 0xfa, 0x09, 0xbe, 0x8a, 0x33, 0xc3, + 0x52, 0xf0, 0x1d, 0x82, 0x31, 0xd1, 0x55, 0xb5, + 0x6c, 0x99, 0x25, 0xcf, 0x5c, 0x32, 0xce, 0xe9, + 0x0d, 0xfa, 0x69, 0x2c, 0xd5, 0x0d, 0xc5, 0x6d, + 0x86, 0xd0, 0x0c, 0x3b, 0x06, 0x50, 0x79, 0xe8, + 0xc3, 0xae, 0x04, 0xe6, 0xcd, 0x51, 0xe4, 0x26, + 0x9b, 0x4f, 0x7e, 0xa6, 0x0f, 0xab, 0xd8, 0xe5, + 0xde, 0xa9, 0x00, 0x95, 0xbe, 0xa3, 0x9d, 0x5d, + 0xb2, 0x09, 0x70, 0x18, 0x1c, 0xf0, 0xac, 0x29, + 0x23, 0x02, 0x29, 0x28, 0xd2, 0x74, 0x35, 0x57, + 0x62, 0x0f, 0x24, 0xea, 0x5e, 0x33, 0xc2, 0x92, + 0xf3, 0x78, 0x4d, 0x30, 0x1e, 0xa1, 0x99, 0xa9, + 0x82, 0xb0, 0x42, 0x31, 0x8d, 0xad, 0x8a, 0xbc, + 0xfc, 0xd4, 0x57, 0x47, 0x3e, 0xb4, 0x50, 0xdd, + 0x6e, 0x2c, 0x80, 0x4d, 0x22, 0xf1, 0xfb, 0x57, + 0xc4, 0xdd, 0x17, 0xe1, 0x8a, 0x36, 0x4a, 0xb3, + 0x37, 0xca, 0xc9, 0x4e, 0xab, 0xd5, 0x69, 0xc4, + 0xf4, 0xbc, 0x0b, 0x3b, 0x44, 0x4b, 0x29, 0x9c, + 0xee, 0xd4, 0x35, 0x22, 0x21, 0xb0, 0x1f, 0x27, + 0x64, 0xa8, 0x51, 0x1b, 0xf0, 0x9f, 0x19, 0x5c, + 0xfb, 0x5a, 0x64, 0x74, 0x70, 0x45, 0x09, 0xf5, + 0x64, 0xfe, 0x1a, 0x2d, 0xc9, 0x14, 0x04, 0x14, + 0xcf, 0xd5, 0x7d, 0x60, 0xaf, 0x94, 0x39, 0x94, + 0xe2, 0x7d, 0x79, 0x82, 0xd0, 0x65, 0x3b, 0x6b, + 0x9c, 0x19, 0x84, 0xb4, 0x6d, 0xb3, 0x0c, 0x99, + 0xc0, 0x56, 0xa8, 0xbd, 0x73, 0xce, 0x05, 0x84, + 0x3e, 0x30, 0xaa, 0xc4, 0x9b, 0x1b, 0x04, 0x2a, + 0x9f, 0xd7, 0x43, 0x2b, 0x23, 0xdf, 0xbf, 0xaa, + 0xd5, 0xc2, 0x43, 0x2d, 0x70, 0xab, 0xdc, 0x75, + 0xad, 0xac, 0xf7, 0xc0, 0xbe, 0x67, 0xb2, 0x74, + 0xed, 0x67, 0x10, 0x4a, 0x92, 0x60, 0xc1, 0x40, + 0x50, 0x19, 0x8a, 0x8a, 0x8c, 0x09, 0x0e, 0x72, + 0xe1, 0x73, 0x5e, 0xe8, 0x41, 0x85, 0x63, 0x9f, + 0x3f, 0xd7, 0x7d, 0xc4, 0xfb, 0x22, 0x5d, 0x92, + 0x6c, 0xb3, 0x1e, 0xe2, 0x50, 0x2f, 0x82, 0xa8, + 0x28, 0xc0, 0xb5, 0xd7, 0x5f, 0x68, 0x0d, 0x2c, + 0x2d, 0xaf, 0x7e, 0xfa, 0x2e, 0x08, 0x0f, 0x1f, + 0x70, 0x9f, 0xe9, 0x19, 0x72, 0x55, 0xf8, 0xfb, + 0x51, 0xd2, 0x33, 0x5d, 0xa0, 0xd3, 0x2b, 0x0a, + 0x6c, 0xbc, 0x4e, 0xcf, 0x36, 0x4d, 0xdc, 0x3b, + 0xe9, 0x3e, 0x81, 0x7c, 0x61, 0xdb, 0x20, 0x2d, + 0x3a, 0xc3, 0xb3, 0x0c, 0x1e, 0x00, 0xb9, 0x7c, + 0xf5, 0xca, 0x10, 0x5f, 0x3a, 0x71, 0xb3, 0xe4, + 0x20, 0xdb, 0x0c, 0x2a, 0x98, 0x63, 0x45, 0x00, + 0x58, 0xf6, 0x68, 0xe4, 0x0b, 0xda, 0x13, 0x3b, + 0x60, 0x5c, 0x76, 0xdb, 0xb9, 0x97, 0x71, 0xe4, + 0xd9, 0xb7, 0xdb, 0xbd, 0x68, 0xc7, 0x84, 0x84, + 0xaa, 0x7c, 0x68, 0x62, 0x5e, 0x16, 0xfc, 0xba, + 0x72, 0xaa, 0x9a, 0xa9, 0xeb, 0x7c, 0x75, 0x47, + 0x97, 0x7e, 0xad, 0xe2, 0xd9, 0x91, 0xe8, 0xe4, + 0xa5, 0x31, 0xd7, 0x01, 0x8e, 0xa2, 0x11, 0x88, + 0x95, 0xb9, 0xf2, 0x9b, 0xd3, 0x7f, 0x1b, 0x81, + 0x22, 0xf7, 0x98, 0x60, 0x0a, 0x64, 0xa6, 0xc1, + 0xf6, 0x49, 0xc7, 0xe3, 0x07, 0x4d, 0x94, 0x7a, + 0xcf, 0x6e, 0x68, 0x0c, 0x1b, 0x3f, 0x6e, 0x2e, + 0xee, 0x92, 0xfa, 0x52, 0xb3, 0x59, 0xf8, 0xf1, + 0x8f, 0x6a, 0x66, 0xa3, 0x82, 0x76, 0x4a, 0x07, + 0x1a, 0xc7, 0xdd, 0xf5, 0xda, 0x9c, 0x3c, 0x24, + 0xbf, 0xfd, 0x42, 0xa1, 0x10, 0x64, 0x6a, 0x0f, + 0x89, 0xee, 0x36, 0xa5, 0xce, 0x99, 0x48, 0x6a, + 0xf0, 0x9f, 0x9e, 0x69, 0xa4, 0x40, 0x20, 0xe9, + 0x16, 0x15, 0xf7, 0xdb, 0x75, 0x02, 0xcb, 0xe9, + 0x73, 0x8b, 0x3b, 0x49, 0x2f, 0xf0, 0xaf, 0x51, + 0x06, 0x5c, 0xdf, 0x27, 0x27, 0x49, 0x6a, 0xd1, + 0xcc, 0xc7, 0xb5, 0x63, 0xb5, 0xfc, 0xb8, 0x5c, + 0x87, 0x7f, 0x84, 0xb4, 0xcc, 0x14, 0xa9, 0x53, + 0xda, 0xa4, 0x56, 0xf8, 0xb6, 0x1b, 0xcc, 0x40, + 0x27, 0x52, 0x06, 0x5a, 0x13, 0x81, 0xd7, 0x3a, + 0xd4, 0x3b, 0xfb, 0x49, 0x65, 0x31, 0x33, 0xb2, + 0xfa, 0xcd, 0xad, 0x58, 0x4e, 0x2b, 0xae, 0xd2, + 0x20, 0xfb, 0x1a, 0x48, 0xb4, 0x3f, 0x9a, 0xd8, + 0x7a, 0x35, 0x4a, 0xc8, 0xee, 0x88, 0x5e, 0x07, + 0x66, 0x54, 0xb9, 0xec, 0x9f, 0xa3, 0xe3, 0xb9, + 0x37, 0xaa, 0x49, 0x76, 0x31, 0xda, 0x74, 0x2d, + 0x3c, 0xa4, 0x65, 0x10, 0x32, 0x38, 0xf0, 0xde, + 0xd3, 0x99, 0x17, 0xaa, 0x71, 0xaa, 0x8f, 0x0f, + 0x8c, 0xaf, 0xa2, 0xf8, 0x5d, 0x64, 0xba, 0x1d, + 0xa3, 0xef, 0x96, 0x73, 0xe8, 0xa1, 0x02, 0x8d, + 0x0c, 0x6d, 0xb8, 0x06, 0x90, 0xb8, 0x08, 0x56, + 0x2c, 0xa7, 0x06, 0xc9, 0xc2, 0x38, 0xdb, 0x7c, + 0x63, 0xb1, 0x57, 0x8e, 0xea, 0x7c, 0x79, 0xf3, + 0x49, 0x1d, 0xfe, 0x9f, 0xf3, 0x6e, 0xb1, 0x1d, + 0xba, 0x19, 0x80, 0x1a, 0x0a, 0xd3, 0xb0, 0x26, + 0x21, 0x40, 0xb1, 0x7c, 0xf9, 0x4d, 0x8d, 0x10, + 0xc1, 0x7e, 0xf4, 0xf6, 0x3c, 0xa8, 0xfd, 0x7c, + 0xa3, 0x92, 0xb2, 0x0f, 0xaa, 0xcc, 0xa6, 0x11, + 0xfe, 0x04, 0xe3, 0xd1, 0x7a, 0x32, 0x89, 0xdf, + 0x0d, 0xc4, 0x8f, 0x79, 0x6b, 0xca, 0x16, 0x7c, + 0x6e, 0xf9, 0xad, 0x0f, 0xf6, 0xfe, 0x27, 0xdb, + 0xc4, 0x13, 0x70, 0xf1, 0x62, 0x1a, 0x4f, 0x79, + 0x40, 0xc9, 0x9b, 0x8b, 0x21, 0xea, 0x84, 0xfa, + 0xf5, 0xf1, 0x89, 0xce, 0xb7, 0x55, 0x0a, 0x80, + 0x39, 0x2f, 0x55, 0x36, 0x16, 0x9c, 0x7b, 0x08, + 0xbd, 0x87, 0x0d, 0xa5, 0x32, 0xf1, 0x52, 0x7c, + 0xe8, 0x55, 0x60, 0x5b, 0xd7, 0x69, 0xe4, 0xfc, + 0xfa, 0x12, 0x85, 0x96, 0xea, 0x50, 0x28, 0xab, + 0x8a, 0xf7, 0xbb, 0x0e, 0x53, 0x74, 0xca, 0xa6, + 0x27, 0x09, 0xc2, 0xb5, 0xde, 0x18, 0x14, 0xd9, + 0xea, 0xe5, 0x29, 0x1c, 0x40, 0x56, 0xcf, 0xd7, + 0xae, 0x05, 0x3f, 0x65, 0xaf, 0x05, 0x73, 0xe2, + 0x35, 0x96, 0x27, 0x07, 0x14, 0xc0, 0xad, 0x33, + 0xf1, 0xdc, 0x44, 0x7a, 0x89, 0x17, 0x77, 0xd2, + 0x9c, 0x58, 0x60, 0xf0, 0x3f, 0x7b, 0x2d, 0x2e, + 0x57, 0x95, 0x54, 0x87, 0xed, 0xf2, 0xc7, 0x4c, + 0xf0, 0xae, 0x56, 0x29, 0x19, 0x7d, 0x66, 0x4b, + 0x9b, 0x83, 0x84, 0x42, 0x3b, 0x01, 0x25, 0x66, + 0x8e, 0x02, 0xde, 0xb9, 0x83, 0x54, 0x19, 0xf6, + 0x9f, 0x79, 0x0d, 0x67, 0xc5, 0x1d, 0x7a, 0x44, + 0x02, 0x98, 0xa7, 0x16, 0x1c, 0x29, 0x0d, 0x74, + 0xff, 0x85, 0x40, 0x06, 0xef, 0x2c, 0xa9, 0xc6, + 0xf5, 0x53, 0x07, 0x06, 0xae, 0xe4, 0xfa, 0x5f, + 0xd8, 0x39, 0x4d, 0xf1, 0x9b, 0x6b, 0xd9, 0x24, + 0x84, 0xfe, 0x03, 0x4c, 0xb2, 0x3f, 0xdf, 0xa1, + 0x05, 0x9e, 0x50, 0x14, 0x5a, 0xd9, 0x1a, 0xa2, + 0xa7, 0xfa, 0xfa, 0x17, 0xf7, 0x78, 0xd6, 0xb5, + 0x92, 0x61, 0x91, 0xac, 0x36, 0xfa, 0x56, 0x0d, + 0x38, 0x32, 0x18, 0x85, 0x08, 0x58, 0x37, 0xf0, + 0x4b, 0xdb, 0x59, 0xe7, 0xa4, 0x34, 0xc0, 0x1b, + 0x01, 0xaf, 0x2d, 0xde, 0xa1, 0xaa, 0x5d, 0xd3, + 0xec, 0xe1, 0xd4, 0xf7, 0xe6, 0x54, 0x68, 0xf0, + 0x51, 0x97, 0xa7, 0x89, 0xea, 0x24, 0xad, 0xd3, + 0x6e, 0x47, 0x93, 0x8b, 0x4b, 0xb4, 0xf7, 0x1c, + 0x42, 0x06, 0x67, 0xe8, 0x99, 0xf6, 0xf5, 0x7b, + 0x85, 0xb5, 0x65, 0xb5, 0xb5, 0xd2, 0x37, 0xf5, + 0xf3, 0x02, 0xa6, 0x4d, 0x11, 0xa7, 0xdc, 0x51, + 0x09, 0x7f, 0xa0, 0xd8, 0x88, 0x1c, 0x13, 0x71, + 0xae, 0x9c, 0xb7, 0x7b, 0x34, 0xd6, 0x4e, 0x68, + 0x26, 0x83, 0x51, 0xaf, 0x1d, 0xee, 0x8b, 0xbb, + 0x69, 0x43, 0x2b, 0x9e, 0x8a, 0xbc, 0x02, 0x0e, + 0xa0, 0x1b, 0xe0, 0xa8, 0x5f, 0x6f, 0xaf, 0x1b, + 0x8f, 0xe7, 0x64, 0x71, 0x74, 0x11, 0x7e, 0xa8, + 0xd8, 0xf9, 0x97, 0x06, 0xc3, 0xb6, 0xfb, 0xfb, + 0xb7, 0x3d, 0x35, 0x9d, 0x3b, 0x52, 0xed, 0x54, + 0xca, 0xf4, 0x81, 0x01, 0x2d, 0x1b, 0xc3, 0xa7, + 0x00, 0x3d, 0x1a, 0x39, 0x54, 0xe1, 0xf6, 0xff, + 0xed, 0x6f, 0x0b, 0x5a, 0x68, 0xda, 0x58, 0xdd, + 0xa9, 0xcf, 0x5c, 0x4a, 0xe5, 0x09, 0x4e, 0xde, + 0x9d, 0xbc, 0x3e, 0xee, 0x5a, 0x00, 0x3b, 0x2c, + 0x87, 0x10, 0x65, 0x60, 0xdd, 0xd7, 0x56, 0xd1, + 0x4c, 0x64, 0x45, 0xe4, 0x21, 0xec, 0x78, 0xf8, + 0x25, 0x7a, 0x3e, 0x16, 0x5d, 0x09, 0x53, 0x14, + 0xbe, 0x4f, 0xae, 0x87, 0xd8, 0xd1, 0xaa, 0x3c, + 0xf6, 0x3e, 0xa4, 0x70, 0x8c, 0x5e, 0x70, 0xa4, + 0xb3, 0x6b, 0x66, 0x73, 0xd3, 0xbf, 0x31, 0x06, + 0x19, 0x62, 0x93, 0x15, 0xf2, 0x86, 0xe4, 0x52, + 0x7e, 0x53, 0x4c, 0x12, 0x38, 0xcc, 0x34, 0x7d, + 0x57, 0xf6, 0x42, 0x93, 0x8a, 0xc4, 0xee, 0x5c, + 0x8a, 0xe1, 0x52, 0x8f, 0x56, 0x64, 0xf6, 0xa6, + 0xd1, 0x91, 0x57, 0x70, 0xcd, 0x11, 0x76, 0xf5, + 0x59, 0x60, 0x60, 0x3c, 0xc1, 0xc3, 0x0b, 0x7f, + 0x58, 0x1a, 0x50, 0x91, 0xf1, 0x68, 0x8f, 0x6e, + 0x74, 0x74, 0xa8, 0x51, 0x0b, 0xf7, 0x7a, 0x98, + 0x37, 0xf2, 0x0a, 0x0e, 0xa4, 0x97, 0x04, 0xb8, + 0x9b, 0xfd, 0xa0, 0xea, 0xf7, 0x0d, 0xe1, 0xdb, + 0x03, 0xf0, 0x31, 0x29, 0xf8, 0xdd, 0x6b, 0x8b, + 0x5d, 0xd8, 0x59, 0xa9, 0x29, 0xcf, 0x9a, 0x79, + 0x89, 0x19, 0x63, 0x46, 0x09, 0x79, 0x6a, 0x11, + 0xda, 0x63, 0x68, 0x48, 0x77, 0x23, 0xfb, 0x7d, + 0x3a, 0x43, 0xcb, 0x02, 0x3b, 0x7a, 0x6d, 0x10, + 0x2a, 0x9e, 0xac, 0xf1, 0xd4, 0x19, 0xf8, 0x23, + 0x64, 0x1d, 0x2c, 0x5f, 0xf2, 0xb0, 0x5c, 0x23, + 0x27, 0xf7, 0x27, 0x30, 0x16, 0x37, 0xb1, 0x90, + 0xab, 0x38, 0xfb, 0x55, 0xcd, 0x78, 0x58, 0xd4, + 0x7d, 0x43, 0xf6, 0x45, 0x5e, 0x55, 0x8d, 0xb1, + 0x02, 0x65, 0x58, 0xb4, 0x13, 0x4b, 0x36, 0xf7, + 0xcc, 0xfe, 0x3d, 0x0b, 0x82, 0xe2, 0x12, 0x11, + 0xbb, 0xe6, 0xb8, 0x3a, 0x48, 0x71, 0xc7, 0x50, + 0x06, 0x16, 0x3a, 0xe6, 0x7c, 0x05, 0xc7, 0xc8, + 0x4d, 0x2f, 0x08, 0x6a, 0x17, 0x9a, 0x95, 0x97, + 0x50, 0x68, 0xdc, 0x28, 0x18, 0xc4, 0x61, 0x38, + 0xb9, 0xe0, 0x3e, 0x78, 0xdb, 0x29, 0xe0, 0x9f, + 0x52, 0xdd, 0xf8, 0x4f, 0x91, 0xc1, 0xd0, 0x33, + 0xa1, 0x7a, 0x8e, 0x30, 0x13, 0x82, 0x07, 0x9f, + 0xd3, 0x31, 0x0f, 0x23, 0xbe, 0x32, 0x5a, 0x75, + 0xcf, 0x96, 0xb2, 0xec, 0xb5, 0x32, 0xac, 0x21, + 0xd1, 0x82, 0x33, 0xd3, 0x15, 0x74, 0xbd, 0x90, + 0xf1, 0x2c, 0xe6, 0x5f, 0x8d, 0xe3, 0x02, 0xe8, + 0xe9, 0xc4, 0xca, 0x96, 0xeb, 0x0e, 0xbc, 0x91, + 0xf4, 0xb9, 0xea, 0xd9, 0x1b, 0x75, 0xbd, 0xe1, + 0xac, 0x2a, 0x05, 0x37, 0x52, 0x9b, 0x1b, 0x3f, + 0x5a, 0xdc, 0x21, 0xc3, 0x98, 0xbb, 0xaf, 0xa3, + 0xf2, 0x00, 0xbf, 0x0d, 0x30, 0x89, 0x05, 0xcc, + 0xa5, 0x76, 0xf5, 0x06, 0xf0, 0xc6, 0x54, 0x8a, + 0x5d, 0xd4, 0x1e, 0xc1, 0xf2, 0xce, 0xb0, 0x62, + 0xc8, 0xfc, 0x59, 0x42, 0x9a, 0x90, 0x60, 0x55, + 0xfe, 0x88, 0xa5, 0x8b, 0xb8, 0x33, 0x0c, 0x23, + 0x24, 0x0d, 0x15, 0x70, 0x37, 0x1e, 0x3d, 0xf6, + 0xd2, 0xea, 0x92, 0x10, 0xb2, 0xc4, 0x51, 0xac, + 0xf2, 0xac, 0xf3, 0x6b, 0x6c, 0xaa, 0xcf, 0x12, + 0xc5, 0x6c, 0x90, 0x50, 0xb5, 0x0c, 0xfc, 0x1a, + 0x15, 0x52, 0xe9, 0x26, 0xc6, 0x52, 0xa4, 0xe7, + 0x81, 0x69, 0xe1, 0xe7, 0x9e, 0x30, 0x01, 0xec, + 0x84, 0x89, 0xb2, 0x0d, 0x66, 0xdd, 0xce, 0x28, + 0x5c, 0xec, 0x98, 0x46, 0x68, 0x21, 0x9f, 0x88, + 0x3f, 0x1f, 0x42, 0x77, 0xce, 0xd0, 0x61, 0xd4, + 0x20, 0xa7, 0xff, 0x53, 0xad, 0x37, 0xd0, 0x17, + 0x35, 0xc9, 0xfc, 0xba, 0x0a, 0x78, 0x3f, 0xf2, + 0xcc, 0x86, 0x89, 0xe8, 0x4b, 0x3c, 0x48, 0x33, + 0x09, 0x7f, 0xc6, 0xc0, 0xdd, 0xb8, 0xfd, 0x7a, + 0x66, 0x66, 0x65, 0xeb, 0x47, 0xa7, 0x04, 0x28, + 0xa3, 0x19, 0x8e, 0xa9, 0xb1, 0x13, 0x67, 0x62, + 0x70, 0xcf, 0xd7 +}; +static const u8 dec_output013[] __initconst = { + 0x74, 0xa6, 0x3e, 0xe4, 0xb1, 0xcb, 0xaf, 0xb0, + 0x40, 0xe5, 0x0f, 0x9e, 0xf1, 0xf2, 0x89, 0xb5, + 0x42, 0x34, 0x8a, 0xa1, 0x03, 0xb7, 0xe9, 0x57, + 0x46, 0xbe, 0x20, 0xe4, 0x6e, 0xb0, 0xeb, 0xff, + 0xea, 0x07, 0x7e, 0xef, 0xe2, 0x55, 0x9f, 0xe5, + 0x78, 0x3a, 0xb7, 0x83, 0xc2, 0x18, 0x40, 0x7b, + 0xeb, 0xcd, 0x81, 0xfb, 0x90, 0x12, 0x9e, 0x46, + 0xa9, 0xd6, 0x4a, 0xba, 0xb0, 0x62, 0xdb, 0x6b, + 0x99, 0xc4, 0xdb, 0x54, 0x4b, 0xb8, 0xa5, 0x71, + 0xcb, 0xcd, 0x63, 0x32, 0x55, 0xfb, 0x31, 0xf0, + 0x38, 0xf5, 0xbe, 0x78, 0xe4, 0x45, 0xce, 0x1b, + 0x6a, 0x5b, 0x0e, 0xf4, 0x16, 0xe4, 0xb1, 0x3d, + 0xf6, 0x63, 0x7b, 0xa7, 0x0c, 0xde, 0x6f, 0x8f, + 0x74, 0xdf, 0xe0, 0x1e, 0x9d, 0xce, 0x8f, 0x24, + 0xef, 0x23, 0x35, 0x33, 0x7b, 0x83, 0x34, 0x23, + 0x58, 0x74, 0x14, 0x77, 0x1f, 0xc2, 0x4f, 0x4e, + 0xc6, 0x89, 0xf9, 0x52, 0x09, 0x37, 0x64, 0x14, + 0xc4, 0x01, 0x6b, 0x9d, 0x77, 0xe8, 0x90, 0x5d, + 0xa8, 0x4a, 0x2a, 0xef, 0x5c, 0x7f, 0xeb, 0xbb, + 0xb2, 0xc6, 0x93, 0x99, 0x66, 0xdc, 0x7f, 0xd4, + 0x9e, 0x2a, 0xca, 0x8d, 0xdb, 0xe7, 0x20, 0xcf, + 0xe4, 0x73, 0xae, 0x49, 0x7d, 0x64, 0x0f, 0x0e, + 0x28, 0x46, 0xa9, 0xa8, 0x32, 0xe4, 0x0e, 0xf6, + 0x51, 0x53, 0xb8, 0x3c, 0xb1, 0xff, 0xa3, 0x33, + 0x41, 0x75, 0xff, 0xf1, 0x6f, 0xf1, 0xfb, 0xbb, + 0x83, 0x7f, 0x06, 0x9b, 0xe7, 0x1b, 0x0a, 0xe0, + 0x5c, 0x33, 0x60, 0x5b, 0xdb, 0x5b, 0xed, 0xfe, + 0xa5, 0x16, 0x19, 0x72, 0xa3, 0x64, 0x23, 0x00, + 0x02, 0xc7, 0xf3, 0x6a, 0x81, 0x3e, 0x44, 0x1d, + 0x79, 0x15, 0x5f, 0x9a, 0xde, 0xe2, 0xfd, 0x1b, + 0x73, 0xc1, 0xbc, 0x23, 0xba, 0x31, 0xd2, 0x50, + 0xd5, 0xad, 0x7f, 0x74, 0xa7, 0xc9, 0xf8, 0x3e, + 0x2b, 0x26, 0x10, 0xf6, 0x03, 0x36, 0x74, 0xe4, + 0x0e, 0x6a, 0x72, 0xb7, 0x73, 0x0a, 0x42, 0x28, + 0xc2, 0xad, 0x5e, 0x03, 0xbe, 0xb8, 0x0b, 0xa8, + 0x5b, 0xd4, 0xb8, 0xba, 0x52, 0x89, 0xb1, 0x9b, + 0xc1, 0xc3, 0x65, 0x87, 0xed, 0xa5, 0xf4, 0x86, + 0xfd, 0x41, 0x80, 0x91, 0x27, 0x59, 0x53, 0x67, + 0x15, 0x78, 0x54, 0x8b, 0x2d, 0x3d, 0xc7, 0xff, + 0x02, 0x92, 0x07, 0x5f, 0x7a, 0x4b, 0x60, 0x59, + 0x3c, 0x6f, 0x5c, 0xd8, 0xec, 0x95, 0xd2, 0xfe, + 0xa0, 0x3b, 0xd8, 0x3f, 0xd1, 0x69, 0xa6, 0xd6, + 0x41, 0xb2, 0xf4, 0x4d, 0x12, 0xf4, 0x58, 0x3e, + 0x66, 0x64, 0x80, 0x31, 0x9b, 0xa8, 0x4c, 0x8b, + 0x07, 0xb2, 0xec, 0x66, 0x94, 0x66, 0x47, 0x50, + 0x50, 0x5f, 0x18, 0x0b, 0x0e, 0xd6, 0xc0, 0x39, + 0x21, 0x13, 0x9e, 0x33, 0xbc, 0x79, 0x36, 0x02, + 0x96, 0x70, 0xf0, 0x48, 0x67, 0x2f, 0x26, 0xe9, + 0x6d, 0x10, 0xbb, 0xd6, 0x3f, 0xd1, 0x64, 0x7a, + 0x2e, 0xbe, 0x0c, 0x61, 0xf0, 0x75, 0x42, 0x38, + 0x23, 0xb1, 0x9e, 0x9f, 0x7c, 0x67, 0x66, 0xd9, + 0x58, 0x9a, 0xf1, 0xbb, 0x41, 0x2a, 0x8d, 0x65, + 0x84, 0x94, 0xfc, 0xdc, 0x6a, 0x50, 0x64, 0xdb, + 0x56, 0x33, 0x76, 0x00, 0x10, 0xed, 0xbe, 0xd2, + 0x12, 0xf6, 0xf6, 0x1b, 0xa2, 0x16, 0xde, 0xae, + 0x31, 0x95, 0xdd, 0xb1, 0x08, 0x7e, 0x4e, 0xee, + 0xe7, 0xf9, 0xa5, 0xfb, 0x5b, 0x61, 0x43, 0x00, + 0x40, 0xf6, 0x7e, 0x02, 0x04, 0x32, 0x4e, 0x0c, + 0xe2, 0x66, 0x0d, 0xd7, 0x07, 0x98, 0x0e, 0xf8, + 0x72, 0x34, 0x6d, 0x95, 0x86, 0xd7, 0xcb, 0x31, + 0x54, 0x47, 0xd0, 0x38, 0x29, 0x9c, 0x5a, 0x68, + 0xd4, 0x87, 0x76, 0xc9, 0xe7, 0x7e, 0xe3, 0xf4, + 0x81, 0x6d, 0x18, 0xcb, 0xc9, 0x05, 0xaf, 0xa0, + 0xfb, 0x66, 0xf7, 0xf1, 0x1c, 0xc6, 0x14, 0x11, + 0x4f, 0x2b, 0x79, 0x42, 0x8b, 0xbc, 0xac, 0xe7, + 0x6c, 0xfe, 0x0f, 0x58, 0xe7, 0x7c, 0x78, 0x39, + 0x30, 0xb0, 0x66, 0x2c, 0x9b, 0x6d, 0x3a, 0xe1, + 0xcf, 0xc9, 0xa4, 0x0e, 0x6d, 0x6d, 0x8a, 0xa1, + 0x3a, 0xe7, 0x28, 0xd4, 0x78, 0x4c, 0xa6, 0xa2, + 0x2a, 0xa6, 0x03, 0x30, 0xd7, 0xa8, 0x25, 0x66, + 0x87, 0x2f, 0x69, 0x5c, 0x4e, 0xdd, 0xa5, 0x49, + 0x5d, 0x37, 0x4a, 0x59, 0xc4, 0xaf, 0x1f, 0xa2, + 0xe4, 0xf8, 0xa6, 0x12, 0x97, 0xd5, 0x79, 0xf5, + 0xe2, 0x4a, 0x2b, 0x5f, 0x61, 0xe4, 0x9e, 0xe3, + 0xee, 0xb8, 0xa7, 0x5b, 0x2f, 0xf4, 0x9e, 0x6c, + 0xfb, 0xd1, 0xc6, 0x56, 0x77, 0xba, 0x75, 0xaa, + 0x3d, 0x1a, 0xa8, 0x0b, 0xb3, 0x68, 0x24, 0x00, + 0x10, 0x7f, 0xfd, 0xd7, 0xa1, 0x8d, 0x83, 0x54, + 0x4f, 0x1f, 0xd8, 0x2a, 0xbe, 0x8a, 0x0c, 0x87, + 0xab, 0xa2, 0xde, 0xc3, 0x39, 0xbf, 0x09, 0x03, + 0xa5, 0xf3, 0x05, 0x28, 0xe1, 0xe1, 0xee, 0x39, + 0x70, 0x9c, 0xd8, 0x81, 0x12, 0x1e, 0x02, 0x40, + 0xd2, 0x6e, 0xf0, 0xeb, 0x1b, 0x3d, 0x22, 0xc6, + 0xe5, 0xe3, 0xb4, 0x5a, 0x98, 0xbb, 0xf0, 0x22, + 0x28, 0x8d, 0xe5, 0xd3, 0x16, 0x48, 0x24, 0xa5, + 0xe6, 0x66, 0x0c, 0xf9, 0x08, 0xf9, 0x7e, 0x1e, + 0xe1, 0x28, 0x26, 0x22, 0xc7, 0xc7, 0x0a, 0x32, + 0x47, 0xfa, 0xa3, 0xbe, 0x3c, 0xc4, 0xc5, 0x53, + 0x0a, 0xd5, 0x94, 0x4a, 0xd7, 0x93, 0xd8, 0x42, + 0x99, 0xb9, 0x0a, 0xdb, 0x56, 0xf7, 0xb9, 0x1c, + 0x53, 0x4f, 0xfa, 0xd3, 0x74, 0xad, 0xd9, 0x68, + 0xf1, 0x1b, 0xdf, 0x61, 0xc6, 0x5e, 0xa8, 0x48, + 0xfc, 0xd4, 0x4a, 0x4c, 0x3c, 0x32, 0xf7, 0x1c, + 0x96, 0x21, 0x9b, 0xf9, 0xa3, 0xcc, 0x5a, 0xce, + 0xd5, 0xd7, 0x08, 0x24, 0xf6, 0x1c, 0xfd, 0xdd, + 0x38, 0xc2, 0x32, 0xe9, 0xb8, 0xe7, 0xb6, 0xfa, + 0x9d, 0x45, 0x13, 0x2c, 0x83, 0xfd, 0x4a, 0x69, + 0x82, 0xcd, 0xdc, 0xb3, 0x76, 0x0c, 0x9e, 0xd8, + 0xf4, 0x1b, 0x45, 0x15, 0xb4, 0x97, 0xe7, 0x58, + 0x34, 0xe2, 0x03, 0x29, 0x5a, 0xbf, 0xb6, 0xe0, + 0x5d, 0x13, 0xd9, 0x2b, 0xb4, 0x80, 0xb2, 0x45, + 0x81, 0x6a, 0x2e, 0x6c, 0x89, 0x7d, 0xee, 0xbb, + 0x52, 0xdd, 0x1f, 0x18, 0xe7, 0x13, 0x6b, 0x33, + 0x0e, 0xea, 0x36, 0x92, 0x77, 0x7b, 0x6d, 0x9c, + 0x5a, 0x5f, 0x45, 0x7b, 0x7b, 0x35, 0x62, 0x23, + 0xd1, 0xbf, 0x0f, 0xd0, 0x08, 0x1b, 0x2b, 0x80, + 0x6b, 0x7e, 0xf1, 0x21, 0x47, 0xb0, 0x57, 0xd1, + 0x98, 0x72, 0x90, 0x34, 0x1c, 0x20, 0x04, 0xff, + 0x3d, 0x5c, 0xee, 0x0e, 0x57, 0x5f, 0x6f, 0x24, + 0x4e, 0x3c, 0xea, 0xfc, 0xa5, 0xa9, 0x83, 0xc9, + 0x61, 0xb4, 0x51, 0x24, 0xf8, 0x27, 0x5e, 0x46, + 0x8c, 0xb1, 0x53, 0x02, 0x96, 0x35, 0xba, 0xb8, + 0x4c, 0x71, 0xd3, 0x15, 0x59, 0x35, 0x22, 0x20, + 0xad, 0x03, 0x9f, 0x66, 0x44, 0x3b, 0x9c, 0x35, + 0x37, 0x1f, 0x9b, 0xbb, 0xf3, 0xdb, 0x35, 0x63, + 0x30, 0x64, 0xaa, 0xa2, 0x06, 0xa8, 0x5d, 0xbb, + 0xe1, 0x9f, 0x70, 0xec, 0x82, 0x11, 0x06, 0x36, + 0xec, 0x8b, 0x69, 0x66, 0x24, 0x44, 0xc9, 0x4a, + 0x57, 0xbb, 0x9b, 0x78, 0x13, 0xce, 0x9c, 0x0c, + 0xba, 0x92, 0x93, 0x63, 0xb8, 0xe2, 0x95, 0x0f, + 0x0f, 0x16, 0x39, 0x52, 0xfd, 0x3a, 0x6d, 0x02, + 0x4b, 0xdf, 0x13, 0xd3, 0x2a, 0x22, 0xb4, 0x03, + 0x7c, 0x54, 0x49, 0x96, 0x68, 0x54, 0x10, 0xfa, + 0xef, 0xaa, 0x6c, 0xe8, 0x22, 0xdc, 0x71, 0x16, + 0x13, 0x1a, 0xf6, 0x28, 0xe5, 0x6d, 0x77, 0x3d, + 0xcd, 0x30, 0x63, 0xb1, 0x70, 0x52, 0xa1, 0xc5, + 0x94, 0x5f, 0xcf, 0xe8, 0xb8, 0x26, 0x98, 0xf7, + 0x06, 0xa0, 0x0a, 0x70, 0xfa, 0x03, 0x80, 0xac, + 0xc1, 0xec, 0xd6, 0x4c, 0x54, 0xd7, 0xfe, 0x47, + 0xb6, 0x88, 0x4a, 0xf7, 0x71, 0x24, 0xee, 0xf3, + 0xd2, 0xc2, 0x4a, 0x7f, 0xfe, 0x61, 0xc7, 0x35, + 0xc9, 0x37, 0x67, 0xcb, 0x24, 0x35, 0xda, 0x7e, + 0xca, 0x5f, 0xf3, 0x8d, 0xd4, 0x13, 0x8e, 0xd6, + 0xcb, 0x4d, 0x53, 0x8f, 0x53, 0x1f, 0xc0, 0x74, + 0xf7, 0x53, 0xb9, 0x5e, 0x23, 0x37, 0xba, 0x6e, + 0xe3, 0x9d, 0x07, 0x55, 0x25, 0x7b, 0xe6, 0x2a, + 0x64, 0xd1, 0x32, 0xdd, 0x54, 0x1b, 0x4b, 0xc0, + 0xe1, 0xd7, 0x69, 0x58, 0xf8, 0x93, 0x29, 0xc4, + 0xdd, 0x23, 0x2f, 0xa5, 0xfc, 0x9d, 0x7e, 0xf8, + 0xd4, 0x90, 0xcd, 0x82, 0x55, 0xdc, 0x16, 0x16, + 0x9f, 0x07, 0x52, 0x9b, 0x9d, 0x25, 0xed, 0x32, + 0xc5, 0x7b, 0xdf, 0xf6, 0x83, 0x46, 0x3d, 0x65, + 0xb7, 0xef, 0x87, 0x7a, 0x12, 0x69, 0x8f, 0x06, + 0x7c, 0x51, 0x15, 0x4a, 0x08, 0xe8, 0xac, 0x9a, + 0x0c, 0x24, 0xa7, 0x27, 0xd8, 0x46, 0x2f, 0xe7, + 0x01, 0x0e, 0x1c, 0xc6, 0x91, 0xb0, 0x6e, 0x85, + 0x65, 0xf0, 0x29, 0x0d, 0x2e, 0x6b, 0x3b, 0xfb, + 0x4b, 0xdf, 0xe4, 0x80, 0x93, 0x03, 0x66, 0x46, + 0x3e, 0x8a, 0x6e, 0xf3, 0x5e, 0x4d, 0x62, 0x0e, + 0x49, 0x05, 0xaf, 0xd4, 0xf8, 0x21, 0x20, 0x61, + 0x1d, 0x39, 0x17, 0xf4, 0x61, 0x47, 0x95, 0xfb, + 0x15, 0x2e, 0xb3, 0x4f, 0xd0, 0x5d, 0xf5, 0x7d, + 0x40, 0xda, 0x90, 0x3c, 0x6b, 0xcb, 0x17, 0x00, + 0x13, 0x3b, 0x64, 0x34, 0x1b, 0xf0, 0xf2, 0xe5, + 0x3b, 0xb2, 0xc7, 0xd3, 0x5f, 0x3a, 0x44, 0xa6, + 0x9b, 0xb7, 0x78, 0x0e, 0x42, 0x5d, 0x4c, 0xc1, + 0xe9, 0xd2, 0xcb, 0xb7, 0x78, 0xd1, 0xfe, 0x9a, + 0xb5, 0x07, 0xe9, 0xe0, 0xbe, 0xe2, 0x8a, 0xa7, + 0x01, 0x83, 0x00, 0x8c, 0x5c, 0x08, 0xe6, 0x63, + 0x12, 0x92, 0xb7, 0xb7, 0xa6, 0x19, 0x7d, 0x38, + 0x13, 0x38, 0x92, 0x87, 0x24, 0xf9, 0x48, 0xb3, + 0x5e, 0x87, 0x6a, 0x40, 0x39, 0x5c, 0x3f, 0xed, + 0x8f, 0xee, 0xdb, 0x15, 0x82, 0x06, 0xda, 0x49, + 0x21, 0x2b, 0xb5, 0xbf, 0x32, 0x7c, 0x9f, 0x42, + 0x28, 0x63, 0xcf, 0xaf, 0x1e, 0xf8, 0xc6, 0xa0, + 0xd1, 0x02, 0x43, 0x57, 0x62, 0xec, 0x9b, 0x0f, + 0x01, 0x9e, 0x71, 0xd8, 0x87, 0x9d, 0x01, 0xc1, + 0x58, 0x77, 0xd9, 0xaf, 0xb1, 0x10, 0x7e, 0xdd, + 0xa6, 0x50, 0x96, 0xe5, 0xf0, 0x72, 0x00, 0x6d, + 0x4b, 0xf8, 0x2a, 0x8f, 0x19, 0xf3, 0x22, 0x88, + 0x11, 0x4a, 0x8b, 0x7c, 0xfd, 0xb7, 0xed, 0xe1, + 0xf6, 0x40, 0x39, 0xe0, 0xe9, 0xf6, 0x3d, 0x25, + 0xe6, 0x74, 0x3c, 0x58, 0x57, 0x7f, 0xe1, 0x22, + 0x96, 0x47, 0x31, 0x91, 0xba, 0x70, 0x85, 0x28, + 0x6b, 0x9f, 0x6e, 0x25, 0xac, 0x23, 0x66, 0x2f, + 0x29, 0x88, 0x28, 0xce, 0x8c, 0x5c, 0x88, 0x53, + 0xd1, 0x3b, 0xcc, 0x6a, 0x51, 0xb2, 0xe1, 0x28, + 0x3f, 0x91, 0xb4, 0x0d, 0x00, 0x3a, 0xe3, 0xf8, + 0xc3, 0x8f, 0xd7, 0x96, 0x62, 0x0e, 0x2e, 0xfc, + 0xc8, 0x6c, 0x77, 0xa6, 0x1d, 0x22, 0xc1, 0xb8, + 0xe6, 0x61, 0xd7, 0x67, 0x36, 0x13, 0x7b, 0xbb, + 0x9b, 0x59, 0x09, 0xa6, 0xdf, 0xf7, 0x6b, 0xa3, + 0x40, 0x1a, 0xf5, 0x4f, 0xb4, 0xda, 0xd3, 0xf3, + 0x81, 0x93, 0xc6, 0x18, 0xd9, 0x26, 0xee, 0xac, + 0xf0, 0xaa, 0xdf, 0xc5, 0x9c, 0xca, 0xc2, 0xa2, + 0xcc, 0x7b, 0x5c, 0x24, 0xb0, 0xbc, 0xd0, 0x6a, + 0x4d, 0x89, 0x09, 0xb8, 0x07, 0xfe, 0x87, 0xad, + 0x0a, 0xea, 0xb8, 0x42, 0xf9, 0x5e, 0xb3, 0x3e, + 0x36, 0x4c, 0xaf, 0x75, 0x9e, 0x1c, 0xeb, 0xbd, + 0xbc, 0xbb, 0x80, 0x40, 0xa7, 0x3a, 0x30, 0xbf, + 0xa8, 0x44, 0xf4, 0xeb, 0x38, 0xad, 0x29, 0xba, + 0x23, 0xed, 0x41, 0x0c, 0xea, 0xd2, 0xbb, 0x41, + 0x18, 0xd6, 0xb9, 0xba, 0x65, 0x2b, 0xa3, 0x91, + 0x6d, 0x1f, 0xa9, 0xf4, 0xd1, 0x25, 0x8d, 0x4d, + 0x38, 0xff, 0x64, 0xa0, 0xec, 0xde, 0xa6, 0xb6, + 0x79, 0xab, 0x8e, 0x33, 0x6c, 0x47, 0xde, 0xaf, + 0x94, 0xa4, 0xa5, 0x86, 0x77, 0x55, 0x09, 0x92, + 0x81, 0x31, 0x76, 0xc7, 0x34, 0x22, 0x89, 0x8e, + 0x3d, 0x26, 0x26, 0xd7, 0xfc, 0x1e, 0x16, 0x72, + 0x13, 0x33, 0x63, 0xd5, 0x22, 0xbe, 0xb8, 0x04, + 0x34, 0x84, 0x41, 0xbb, 0x80, 0xd0, 0x9f, 0x46, + 0x48, 0x07, 0xa7, 0xfc, 0x2b, 0x3a, 0x75, 0x55, + 0x8c, 0xc7, 0x6a, 0xbd, 0x7e, 0x46, 0x08, 0x84, + 0x0f, 0xd5, 0x74, 0xc0, 0x82, 0x8e, 0xaa, 0x61, + 0x05, 0x01, 0xb2, 0x47, 0x6e, 0x20, 0x6a, 0x2d, + 0x58, 0x70, 0x48, 0x32, 0xa7, 0x37, 0xd2, 0xb8, + 0x82, 0x1a, 0x51, 0xb9, 0x61, 0xdd, 0xfd, 0x9d, + 0x6b, 0x0e, 0x18, 0x97, 0xf8, 0x45, 0x5f, 0x87, + 0x10, 0xcf, 0x34, 0x72, 0x45, 0x26, 0x49, 0x70, + 0xe7, 0xa3, 0x78, 0xe0, 0x52, 0x89, 0x84, 0x94, + 0x83, 0x82, 0xc2, 0x69, 0x8f, 0xe3, 0xe1, 0x3f, + 0x60, 0x74, 0x88, 0xc4, 0xf7, 0x75, 0x2c, 0xfb, + 0xbd, 0xb6, 0xc4, 0x7e, 0x10, 0x0a, 0x6c, 0x90, + 0x04, 0x9e, 0xc3, 0x3f, 0x59, 0x7c, 0xce, 0x31, + 0x18, 0x60, 0x57, 0x73, 0x46, 0x94, 0x7d, 0x06, + 0xa0, 0x6d, 0x44, 0xec, 0xa2, 0x0a, 0x9e, 0x05, + 0x15, 0xef, 0xca, 0x5c, 0xbf, 0x00, 0xeb, 0xf7, + 0x3d, 0x32, 0xd4, 0xa5, 0xef, 0x49, 0x89, 0x5e, + 0x46, 0xb0, 0xa6, 0x63, 0x5b, 0x8a, 0x73, 0xae, + 0x6f, 0xd5, 0x9d, 0xf8, 0x4f, 0x40, 0xb5, 0xb2, + 0x6e, 0xd3, 0xb6, 0x01, 0xa9, 0x26, 0xa2, 0x21, + 0xcf, 0x33, 0x7a, 0x3a, 0xa4, 0x23, 0x13, 0xb0, + 0x69, 0x6a, 0xee, 0xce, 0xd8, 0x9d, 0x01, 0x1d, + 0x50, 0xc1, 0x30, 0x6c, 0xb1, 0xcd, 0xa0, 0xf0, + 0xf0, 0xa2, 0x64, 0x6f, 0xbb, 0xbf, 0x5e, 0xe6, + 0xab, 0x87, 0xb4, 0x0f, 0x4f, 0x15, 0xaf, 0xb5, + 0x25, 0xa1, 0xb2, 0xd0, 0x80, 0x2c, 0xfb, 0xf9, + 0xfe, 0xd2, 0x33, 0xbb, 0x76, 0xfe, 0x7c, 0xa8, + 0x66, 0xf7, 0xe7, 0x85, 0x9f, 0x1f, 0x85, 0x57, + 0x88, 0xe1, 0xe9, 0x63, 0xe4, 0xd8, 0x1c, 0xa1, + 0xfb, 0xda, 0x44, 0x05, 0x2e, 0x1d, 0x3a, 0x1c, + 0xff, 0xc8, 0x3b, 0xc0, 0xfe, 0xda, 0x22, 0x0b, + 0x43, 0xd6, 0x88, 0x39, 0x4c, 0x4a, 0xa6, 0x69, + 0x18, 0x93, 0x42, 0x4e, 0xb5, 0xcc, 0x66, 0x0d, + 0x09, 0xf8, 0x1e, 0x7c, 0xd3, 0x3c, 0x99, 0x0d, + 0x50, 0x1d, 0x62, 0xe9, 0x57, 0x06, 0xbf, 0x19, + 0x88, 0xdd, 0xad, 0x7b, 0x4f, 0xf9, 0xc7, 0x82, + 0x6d, 0x8d, 0xc8, 0xc4, 0xc5, 0x78, 0x17, 0x20, + 0x15, 0xc5, 0x52, 0x41, 0xcf, 0x5b, 0xd6, 0x7f, + 0x94, 0x02, 0x41, 0xe0, 0x40, 0x22, 0x03, 0x5e, + 0xd1, 0x53, 0xd4, 0x86, 0xd3, 0x2c, 0x9f, 0x0f, + 0x96, 0xe3, 0x6b, 0x9a, 0x76, 0x32, 0x06, 0x47, + 0x4b, 0x11, 0xb3, 0xdd, 0x03, 0x65, 0xbd, 0x9b, + 0x01, 0xda, 0x9c, 0xb9, 0x7e, 0x3f, 0x6a, 0xc4, + 0x7b, 0xea, 0xd4, 0x3c, 0xb9, 0xfb, 0x5c, 0x6b, + 0x64, 0x33, 0x52, 0xba, 0x64, 0x78, 0x8f, 0xa4, + 0xaf, 0x7a, 0x61, 0x8d, 0xbc, 0xc5, 0x73, 0xe9, + 0x6b, 0x58, 0x97, 0x4b, 0xbf, 0x63, 0x22, 0xd3, + 0x37, 0x02, 0x54, 0xc5, 0xb9, 0x16, 0x4a, 0xf0, + 0x19, 0xd8, 0x94, 0x57, 0xb8, 0x8a, 0xb3, 0x16, + 0x3b, 0xd0, 0x84, 0x8e, 0x67, 0xa6, 0xa3, 0x7d, + 0x78, 0xec, 0x00 +}; +static const u8 dec_assoc013[] __initconst = { + 0xb1, 0x69, 0x83, 0x87, 0x30, 0xaa, 0x5d, 0xb8, + 0x77, 0xe8, 0x21, 0xff, 0x06, 0x59, 0x35, 0xce, + 0x75, 0xfe, 0x38, 0xef, 0xb8, 0x91, 0x43, 0x8c, + 0xcf, 0x70, 0xdd, 0x0a, 0x68, 0xbf, 0xd4, 0xbc, + 0x16, 0x76, 0x99, 0x36, 0x1e, 0x58, 0x79, 0x5e, + 0xd4, 0x29, 0xf7, 0x33, 0x93, 0x48, 0xdb, 0x5f, + 0x01, 0xae, 0x9c, 0xb6, 0xe4, 0x88, 0x6d, 0x2b, + 0x76, 0x75, 0xe0, 0xf3, 0x74, 0xe2, 0xc9 +}; +static const u8 dec_nonce013[] __initconst = { + 0x05, 0xa3, 0x93, 0xed, 0x30, 0xc5, 0xa2, 0x06 +}; +static const u8 dec_key013[] __initconst = { + 0xb3, 0x35, 0x50, 0x03, 0x54, 0x2e, 0x40, 0x5e, + 0x8f, 0x59, 0x8e, 0xc5, 0x90, 0xd5, 0x27, 0x2d, + 0xba, 0x29, 0x2e, 0xcb, 0x1b, 0x70, 0x44, 0x1e, + 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 +}; + +static const struct chacha20poly1305_testvec +chacha20poly1305_dec_vectors[] __initconst = { + { dec_input001, dec_output001, dec_assoc001, dec_nonce001, dec_key001, + sizeof(dec_input001), sizeof(dec_assoc001), sizeof(dec_nonce001) }, + { dec_input002, dec_output002, dec_assoc002, dec_nonce002, dec_key002, + sizeof(dec_input002), sizeof(dec_assoc002), sizeof(dec_nonce002) }, + { dec_input003, dec_output003, dec_assoc003, dec_nonce003, dec_key003, + sizeof(dec_input003), sizeof(dec_assoc003), sizeof(dec_nonce003) }, + { dec_input004, dec_output004, dec_assoc004, dec_nonce004, dec_key004, + sizeof(dec_input004), sizeof(dec_assoc004), sizeof(dec_nonce004) }, + { dec_input005, dec_output005, dec_assoc005, dec_nonce005, dec_key005, + sizeof(dec_input005), sizeof(dec_assoc005), sizeof(dec_nonce005) }, + { dec_input006, dec_output006, dec_assoc006, dec_nonce006, dec_key006, + sizeof(dec_input006), sizeof(dec_assoc006), sizeof(dec_nonce006) }, + { dec_input007, dec_output007, dec_assoc007, dec_nonce007, dec_key007, + sizeof(dec_input007), sizeof(dec_assoc007), sizeof(dec_nonce007) }, + { dec_input008, dec_output008, dec_assoc008, dec_nonce008, dec_key008, + sizeof(dec_input008), sizeof(dec_assoc008), sizeof(dec_nonce008) }, + { dec_input009, dec_output009, dec_assoc009, dec_nonce009, dec_key009, + sizeof(dec_input009), sizeof(dec_assoc009), sizeof(dec_nonce009) }, + { dec_input010, dec_output010, dec_assoc010, dec_nonce010, dec_key010, + sizeof(dec_input010), sizeof(dec_assoc010), sizeof(dec_nonce010) }, + { dec_input011, dec_output011, dec_assoc011, dec_nonce011, dec_key011, + sizeof(dec_input011), sizeof(dec_assoc011), sizeof(dec_nonce011) }, + { dec_input012, dec_output012, dec_assoc012, dec_nonce012, dec_key012, + sizeof(dec_input012), sizeof(dec_assoc012), sizeof(dec_nonce012) }, + { dec_input013, dec_output013, dec_assoc013, dec_nonce013, dec_key013, + sizeof(dec_input013), sizeof(dec_assoc013), sizeof(dec_nonce013), + true } +}; + +static const u8 xenc_input001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 xenc_output001[] __initconst = { + 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, + 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, + 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, + 0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d, + 0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, 0x1f, 0x7e, + 0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86, + 0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2, + 0x60, 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85, + 0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09, + 0xec, 0x5e, 0x11, 0x90, 0xa1, 0xc5, 0x4e, 0x49, + 0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd, + 0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, 0xc8, + 0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f, + 0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79, + 0x50, 0x33, 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8, + 0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0, + 0x85, 0xca, 0x46, 0x6a, 0x10, 0xa7, 0xa3, 0x88, + 0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71, + 0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91, + 0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf, + 0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89, + 0xeb, 0xef, 0x8e, 0xfd, 0xdd, 0xb4, 0x0d, 0x46, + 0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e, + 0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, 0xe6, 0x90, + 0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b, + 0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58, + 0xa5, 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54, + 0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1, + 0xe5, 0x0c, 0xba, 0x31, 0xde, 0x34, 0x64, 0x73, + 0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69, + 0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, 0x05, + 0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83, + 0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13, + 0x5b, 0x5e, 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8, + 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, + 0x9c +}; +static const u8 xenc_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 xenc_nonce001[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 +}; +static const u8 xenc_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const struct chacha20poly1305_testvec +xchacha20poly1305_enc_vectors[] __initconst = { + { xenc_input001, xenc_output001, xenc_assoc001, xenc_nonce001, xenc_key001, + sizeof(xenc_input001), sizeof(xenc_assoc001), sizeof(xenc_nonce001) } +}; + +static const u8 xdec_input001[] __initconst = { + 0x1a, 0x6e, 0x3a, 0xd9, 0xfd, 0x41, 0x3f, 0x77, + 0x54, 0x72, 0x0a, 0x70, 0x9a, 0xa0, 0x29, 0x92, + 0x2e, 0xed, 0x93, 0xcf, 0x0f, 0x71, 0x88, 0x18, + 0x7a, 0x9d, 0x2d, 0x24, 0xe0, 0xf5, 0xea, 0x3d, + 0x55, 0x64, 0xd7, 0xad, 0x2a, 0x1a, 0x1f, 0x7e, + 0x86, 0x6d, 0xb0, 0xce, 0x80, 0x41, 0x72, 0x86, + 0x26, 0xee, 0x84, 0xd7, 0xef, 0x82, 0x9e, 0xe2, + 0x60, 0x9d, 0x5a, 0xfc, 0xf0, 0xe4, 0x19, 0x85, + 0xea, 0x09, 0xc6, 0xfb, 0xb3, 0xa9, 0x50, 0x09, + 0xec, 0x5e, 0x11, 0x90, 0xa1, 0xc5, 0x4e, 0x49, + 0xef, 0x50, 0xd8, 0x8f, 0xe0, 0x78, 0xd7, 0xfd, + 0xb9, 0x3b, 0xc9, 0xf2, 0x91, 0xc8, 0x25, 0xc8, + 0xa7, 0x63, 0x60, 0xce, 0x10, 0xcd, 0xc6, 0x7f, + 0xf8, 0x16, 0xf8, 0xe1, 0x0a, 0xd9, 0xde, 0x79, + 0x50, 0x33, 0xf2, 0x16, 0x0f, 0x17, 0xba, 0xb8, + 0x5d, 0xd8, 0xdf, 0x4e, 0x51, 0xa8, 0x39, 0xd0, + 0x85, 0xca, 0x46, 0x6a, 0x10, 0xa7, 0xa3, 0x88, + 0xef, 0x79, 0xb9, 0xf8, 0x24, 0xf3, 0xe0, 0x71, + 0x7b, 0x76, 0x28, 0x46, 0x3a, 0x3a, 0x1b, 0x91, + 0xb6, 0xd4, 0x3e, 0x23, 0xe5, 0x44, 0x15, 0xbf, + 0x60, 0x43, 0x9d, 0xa4, 0xbb, 0xd5, 0x5f, 0x89, + 0xeb, 0xef, 0x8e, 0xfd, 0xdd, 0xb4, 0x0d, 0x46, + 0xf0, 0x69, 0x23, 0x63, 0xae, 0x94, 0xf5, 0x5e, + 0xa5, 0xad, 0x13, 0x1c, 0x41, 0x76, 0xe6, 0x90, + 0xd6, 0x6d, 0xa2, 0x8f, 0x97, 0x4c, 0xa8, 0x0b, + 0xcf, 0x8d, 0x43, 0x2b, 0x9c, 0x9b, 0xc5, 0x58, + 0xa5, 0xb6, 0x95, 0x9a, 0xbf, 0x81, 0xc6, 0x54, + 0xc9, 0x66, 0x0c, 0xe5, 0x4f, 0x6a, 0x53, 0xa1, + 0xe5, 0x0c, 0xba, 0x31, 0xde, 0x34, 0x64, 0x73, + 0x8a, 0x3b, 0xbd, 0x92, 0x01, 0xdb, 0x71, 0x69, + 0xf3, 0x58, 0x99, 0xbc, 0xd1, 0xcb, 0x4a, 0x05, + 0xe2, 0x58, 0x9c, 0x25, 0x17, 0xcd, 0xdc, 0x83, + 0xb7, 0xff, 0xfb, 0x09, 0x61, 0xad, 0xbf, 0x13, + 0x5b, 0x5e, 0xed, 0x46, 0x82, 0x6f, 0x22, 0xd8, + 0x93, 0xa6, 0x85, 0x5b, 0x40, 0x39, 0x5c, 0xc5, + 0x9c +}; +static const u8 xdec_output001[] __initconst = { + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x65, 0x74, + 0x2d, 0x44, 0x72, 0x61, 0x66, 0x74, 0x73, 0x20, + 0x61, 0x72, 0x65, 0x20, 0x64, 0x72, 0x61, 0x66, + 0x74, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x76, 0x61, 0x6c, 0x69, + 0x64, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x61, 0x20, + 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x20, + 0x6f, 0x66, 0x20, 0x73, 0x69, 0x78, 0x20, 0x6d, + 0x6f, 0x6e, 0x74, 0x68, 0x73, 0x20, 0x61, 0x6e, + 0x64, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x62, 0x65, + 0x20, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x2c, 0x20, 0x72, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6f, + 0x62, 0x73, 0x6f, 0x6c, 0x65, 0x74, 0x65, 0x64, + 0x20, 0x62, 0x79, 0x20, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x20, 0x64, 0x6f, 0x63, 0x75, 0x6d, 0x65, + 0x6e, 0x74, 0x73, 0x20, 0x61, 0x74, 0x20, 0x61, + 0x6e, 0x79, 0x20, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x20, 0x69, + 0x6e, 0x61, 0x70, 0x70, 0x72, 0x6f, 0x70, 0x72, + 0x69, 0x61, 0x74, 0x65, 0x20, 0x74, 0x6f, 0x20, + 0x75, 0x73, 0x65, 0x20, 0x49, 0x6e, 0x74, 0x65, + 0x72, 0x6e, 0x65, 0x74, 0x2d, 0x44, 0x72, 0x61, + 0x66, 0x74, 0x73, 0x20, 0x61, 0x73, 0x20, 0x72, + 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, + 0x20, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, + 0x6c, 0x20, 0x6f, 0x72, 0x20, 0x74, 0x6f, 0x20, + 0x63, 0x69, 0x74, 0x65, 0x20, 0x74, 0x68, 0x65, + 0x6d, 0x20, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x20, + 0x74, 0x68, 0x61, 0x6e, 0x20, 0x61, 0x73, 0x20, + 0x2f, 0xe2, 0x80, 0x9c, 0x77, 0x6f, 0x72, 0x6b, + 0x20, 0x69, 0x6e, 0x20, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x2e, 0x2f, 0xe2, 0x80, + 0x9d +}; +static const u8 xdec_assoc001[] __initconst = { + 0xf3, 0x33, 0x88, 0x86, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x4e, 0x91 +}; +static const u8 xdec_nonce001[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 +}; +static const u8 xdec_key001[] __initconst = { + 0x1c, 0x92, 0x40, 0xa5, 0xeb, 0x55, 0xd3, 0x8a, + 0xf3, 0x33, 0x88, 0x86, 0x04, 0xf6, 0xb5, 0xf0, + 0x47, 0x39, 0x17, 0xc1, 0x40, 0x2b, 0x80, 0x09, + 0x9d, 0xca, 0x5c, 0xbc, 0x20, 0x70, 0x75, 0xc0 +}; + +static const struct chacha20poly1305_testvec +xchacha20poly1305_dec_vectors[] __initconst = { + { xdec_input001, xdec_output001, xdec_assoc001, xdec_nonce001, xdec_key001, + sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } +}; + +static void __init +chacha20poly1305_selftest_encrypt_bignonce(u8 *dst, const u8 *src, + const size_t src_len, const u8 *ad, + const size_t ad_len, + const u8 nonce[12], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + simd_context_t simd_context; + struct poly1305_ctx poly1305_state; + struct chacha20_ctx chacha20_state; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = {{ 0 }}; + + simd_get(&simd_context); + chacha20_init(&chacha20_state, key, 0); + chacha20_state.counter[1] = get_unaligned_le32(nonce + 0); + chacha20_state.counter[2] = get_unaligned_le32(nonce + 4); + chacha20_state.counter[3] = get_unaligned_le32(nonce + 8); + chacha20(&chacha20_state, b.block0, b.block0, sizeof(b.block0), + &simd_context); + poly1305_init(&poly1305_state, b.block0); + poly1305_update(&poly1305_state, ad, ad_len, &simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf, + &simd_context); + chacha20(&chacha20_state, dst, src, src_len, &simd_context); + poly1305_update(&poly1305_state, dst, src_len, &simd_context); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf, + &simd_context); + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens), + &simd_context); + poly1305_final(&poly1305_state, dst + src_len, &simd_context); + simd_put(&simd_context); + memzero_explicit(&chacha20_state, sizeof(chacha20_state)); + memzero_explicit(&b, sizeof(b)); +} + +static void __init +chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 *nonce, const size_t nonce_len, + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + if (nonce_len == 8) + chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, + get_unaligned_le64(nonce), key); + else if (nonce_len == 12) + chacha20poly1305_selftest_encrypt_bignonce(dst, src, src_len, + ad, ad_len, nonce, + key); + else + BUG(); +} + +static bool __init +decryption_success(bool func_ret, bool expect_failure, int memcmp_result) +{ + if (expect_failure) + return !func_ret; + return func_ret && !memcmp_result; +} + +static bool __init chacha20poly1305_selftest(void) +{ + enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; + size_t i, j, k, total_len; + u8 *computed_output = NULL, *input = NULL; + bool success = true, ret; + simd_context_t simd_context; + struct scatterlist sg_src[3]; + + computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + if (!computed_output || !input) { + pr_err("chacha20poly1305 self-test malloc: FAIL\n"); + success = false; + goto out; + } + + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + chacha20poly1305_selftest_encrypt(computed_output, + chacha20poly1305_enc_vectors[i].input, + chacha20poly1305_enc_vectors[i].ilen, + chacha20poly1305_enc_vectors[i].assoc, + chacha20poly1305_enc_vectors[i].alen, + chacha20poly1305_enc_vectors[i].nonce, + chacha20poly1305_enc_vectors[i].nlen, + chacha20poly1305_enc_vectors[i].key); + if (memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("chacha20poly1305 encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { + if (chacha20poly1305_enc_vectors[i].nlen != 8) + continue; + memcpy(computed_output, chacha20poly1305_enc_vectors[i].input, + chacha20poly1305_enc_vectors[i].ilen); + sg_init_one(sg_src, computed_output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE); + ret = chacha20poly1305_encrypt_sg_inplace(sg_src, + chacha20poly1305_enc_vectors[i].ilen, + chacha20poly1305_enc_vectors[i].assoc, + chacha20poly1305_enc_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), + chacha20poly1305_enc_vectors[i].key, + &simd_context); + if (!ret || memcmp(computed_output, + chacha20poly1305_enc_vectors[i].output, + chacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("chacha20poly1305 sg encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_put(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + ret = chacha20poly1305_decrypt(computed_output, + chacha20poly1305_dec_vectors[i].input, + chacha20poly1305_dec_vectors[i].ilen, + chacha20poly1305_dec_vectors[i].assoc, + chacha20poly1305_dec_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), + chacha20poly1305_dec_vectors[i].key); + if (!decryption_success(ret, + chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("chacha20poly1305 decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { + memcpy(computed_output, chacha20poly1305_dec_vectors[i].input, + chacha20poly1305_dec_vectors[i].ilen); + sg_init_one(sg_src, computed_output, + chacha20poly1305_dec_vectors[i].ilen); + ret = chacha20poly1305_decrypt_sg_inplace(sg_src, + chacha20poly1305_dec_vectors[i].ilen, + chacha20poly1305_dec_vectors[i].assoc, + chacha20poly1305_dec_vectors[i].alen, + get_unaligned_le64(chacha20poly1305_dec_vectors[i].nonce), + chacha20poly1305_dec_vectors[i].key, &simd_context); + if (!decryption_success(ret, + chacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, + chacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + simd_put(&simd_context); + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_enc_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + xchacha20poly1305_encrypt(computed_output, + xchacha20poly1305_enc_vectors[i].input, + xchacha20poly1305_enc_vectors[i].ilen, + xchacha20poly1305_enc_vectors[i].assoc, + xchacha20poly1305_enc_vectors[i].alen, + xchacha20poly1305_enc_vectors[i].nonce, + xchacha20poly1305_enc_vectors[i].key); + if (memcmp(computed_output, + xchacha20poly1305_enc_vectors[i].output, + xchacha20poly1305_enc_vectors[i].ilen + + POLY1305_MAC_SIZE)) { + pr_err("xchacha20poly1305 encryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { + memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); + ret = xchacha20poly1305_decrypt(computed_output, + xchacha20poly1305_dec_vectors[i].input, + xchacha20poly1305_dec_vectors[i].ilen, + xchacha20poly1305_dec_vectors[i].assoc, + xchacha20poly1305_dec_vectors[i].alen, + xchacha20poly1305_dec_vectors[i].nonce, + xchacha20poly1305_dec_vectors[i].key); + if (!decryption_success(ret, + xchacha20poly1305_dec_vectors[i].failure, + memcmp(computed_output, + xchacha20poly1305_dec_vectors[i].output, + xchacha20poly1305_dec_vectors[i].ilen - + POLY1305_MAC_SIZE))) { + pr_err("xchacha20poly1305 decryption self-test %zu: FAIL\n", + i + 1); + success = false; + } + } + + simd_get(&simd_context); + for (total_len = POLY1305_MAC_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) + && total_len <= 1 << 10; ++total_len) { + for (i = 0; i <= total_len; ++i) { + for (j = i; j <= total_len; ++j) { + sg_init_table(sg_src, 3); + sg_set_buf(&sg_src[0], input, i); + sg_set_buf(&sg_src[1], input + i, j - i); + sg_set_buf(&sg_src[2], input + j, total_len - j); + memset(computed_output, 0, total_len); + memset(input, 0, total_len); + + if (!chacha20poly1305_encrypt_sg_inplace(sg_src, + total_len - POLY1305_MAC_SIZE, NULL, 0, + 0, enc_key001, &simd_context)) + goto chunkfail; + chacha20poly1305_encrypt(computed_output, + computed_output, + total_len - POLY1305_MAC_SIZE, NULL, 0, 0, + enc_key001); + if (memcmp(computed_output, input, total_len)) + goto chunkfail;; + if (!chacha20poly1305_decrypt(computed_output, + input, total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_MAC_SIZE; ++k) { + if (computed_output[k]) + goto chunkfail; + } + if (!chacha20poly1305_decrypt_sg_inplace(sg_src, + total_len, NULL, 0, 0, enc_key001, + &simd_context)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_MAC_SIZE; ++k) { + if (input[k]) + goto chunkfail; + } + continue; + + chunkfail: + pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", + total_len, i, j); + success = false; + } + + } + } + simd_put(&simd_context); + +out: + kfree(computed_output); + kfree(input); + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/curve25519.c b/net/wireguard/crypto/zinc/selftest/curve25519.c new file mode 100644 index 0000000000000..0e3e3af06ba44 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/curve25519.c @@ -0,0 +1,1315 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct curve25519_test_vector { + u8 private[CURVE25519_KEY_SIZE]; + u8 public[CURVE25519_KEY_SIZE]; + u8 result[CURVE25519_KEY_SIZE]; + bool valid; +}; +static const struct curve25519_test_vector curve25519_test_vectors[] __initconst = { + { + .private = { 0x77, 0x07, 0x6d, 0x0a, 0x73, 0x18, 0xa5, 0x7d, + 0x3c, 0x16, 0xc1, 0x72, 0x51, 0xb2, 0x66, 0x45, + 0xdf, 0x4c, 0x2f, 0x87, 0xeb, 0xc0, 0x99, 0x2a, + 0xb1, 0x77, 0xfb, 0xa5, 0x1d, 0xb9, 0x2c, 0x2a }, + .public = { 0xde, 0x9e, 0xdb, 0x7d, 0x7b, 0x7d, 0xc1, 0xb4, + 0xd3, 0x5b, 0x61, 0xc2, 0xec, 0xe4, 0x35, 0x37, + 0x3f, 0x83, 0x43, 0xc8, 0x5b, 0x78, 0x67, 0x4d, + 0xad, 0xfc, 0x7e, 0x14, 0x6f, 0x88, 0x2b, 0x4f }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 0x5d, 0xab, 0x08, 0x7e, 0x62, 0x4a, 0x8a, 0x4b, + 0x79, 0xe1, 0x7f, 0x8b, 0x83, 0x80, 0x0e, 0xe6, + 0x6f, 0x3b, 0xb1, 0x29, 0x26, 0x18, 0xb6, 0xfd, + 0x1c, 0x2f, 0x8b, 0x27, 0xff, 0x88, 0xe0, 0xeb }, + .public = { 0x85, 0x20, 0xf0, 0x09, 0x89, 0x30, 0xa7, 0x54, + 0x74, 0x8b, 0x7d, 0xdc, 0xb4, 0x3e, 0xf7, 0x5a, + 0x0d, 0xbf, 0x3a, 0x0d, 0x26, 0x38, 0x1a, 0xf4, + 0xeb, 0xa4, 0xa9, 0x8e, 0xaa, 0x9b, 0x4e, 0x6a }, + .result = { 0x4a, 0x5d, 0x9d, 0x5b, 0xa4, 0xce, 0x2d, 0xe1, + 0x72, 0x8e, 0x3b, 0xf4, 0x80, 0x35, 0x0f, 0x25, + 0xe0, 0x7e, 0x21, 0xc9, 0x47, 0xd1, 0x9e, 0x33, + 0x76, 0xf0, 0x9b, 0x3c, 0x1e, 0x16, 0x17, 0x42 }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x3c, 0x77, 0x77, 0xca, 0xf9, 0x97, 0xb2, 0x64, + 0x41, 0x60, 0x77, 0x66, 0x5b, 0x4e, 0x22, 0x9d, + 0x0b, 0x95, 0x48, 0xdc, 0x0c, 0xd8, 0x19, 0x98, + 0xdd, 0xcd, 0xc5, 0xc8, 0x53, 0x3c, 0x79, 0x7f }, + .valid = true + }, + { + .private = { 1 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xb3, 0x2d, 0x13, 0x62, 0xc2, 0x48, 0xd6, 0x2f, + 0xe6, 0x26, 0x19, 0xcf, 0xf0, 0x4d, 0xd4, 0x3d, + 0xb7, 0x3f, 0xfc, 0x1b, 0x63, 0x08, 0xed, 0xe3, + 0x0b, 0x78, 0xd8, 0x73, 0x80, 0xf1, 0xe8, 0x34 }, + .valid = true + }, + { + .private = { 0xa5, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0xc4 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + { + .private = { 1, 2, 3, 4 }, + .public = { 0 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 2, 4, 6, 8 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8 }, + .result = { 0 }, + .valid = false + }, + { + .private = { 0xff, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x0a, 0x00, 0xfb, 0x9f }, + .result = { 0x77, 0x52, 0xb6, 0x18, 0xc1, 0x2d, 0x48, 0xd2, + 0xc6, 0x93, 0x46, 0x83, 0x81, 0x7c, 0xc6, 0x57, + 0xf3, 0x31, 0x03, 0x19, 0x49, 0x48, 0x20, 0x05, + 0x42, 0x2b, 0x4e, 0xae, 0x8d, 0x1d, 0x43, 0x23 }, + .valid = true + }, + { + .private = { 0x8e, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x06 }, + .result = { 0x5a, 0xdf, 0xaa, 0x25, 0x86, 0x8e, 0x32, 0x3d, + 0xae, 0x49, 0x62, 0xc1, 0x01, 0x5c, 0xb3, 0x12, + 0xe1, 0xc5, 0xc7, 0x9e, 0x95, 0x3f, 0x03, 0x99, + 0xb0, 0xba, 0x16, 0x22, 0xf3, 0xb6, 0xf7, 0x0c }, + .valid = true + }, + /* wycheproof - normal case */ + { + .private = { 0x48, 0x52, 0x83, 0x4d, 0x9d, 0x6b, 0x77, 0xda, + 0xde, 0xab, 0xaa, 0xf2, 0xe1, 0x1d, 0xca, 0x66, + 0xd1, 0x9f, 0xe7, 0x49, 0x93, 0xa7, 0xbe, 0xc3, + 0x6c, 0x6e, 0x16, 0xa0, 0x98, 0x3f, 0xea, 0xba }, + .public = { 0x9c, 0x64, 0x7d, 0x9a, 0xe5, 0x89, 0xb9, 0xf5, + 0x8f, 0xdc, 0x3c, 0xa4, 0x94, 0x7e, 0xfb, 0xc9, + 0x15, 0xc4, 0xb2, 0xe0, 0x8e, 0x74, 0x4a, 0x0e, + 0xdf, 0x46, 0x9d, 0xac, 0x59, 0xc8, 0xf8, 0x5a }, + .result = { 0x87, 0xb7, 0xf2, 0x12, 0xb6, 0x27, 0xf7, 0xa5, + 0x4c, 0xa5, 0xe0, 0xbc, 0xda, 0xdd, 0xd5, 0x38, + 0x9d, 0x9d, 0xe6, 0x15, 0x6c, 0xdb, 0xcf, 0x8e, + 0xbe, 0x14, 0xff, 0xbc, 0xfb, 0x43, 0x65, 0x51 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x58, 0x8c, 0x06, 0x1a, 0x50, 0x80, 0x4a, 0xc4, + 0x88, 0xad, 0x77, 0x4a, 0xc7, 0x16, 0xc3, 0xf5, + 0xba, 0x71, 0x4b, 0x27, 0x12, 0xe0, 0x48, 0x49, + 0x13, 0x79, 0xa5, 0x00, 0x21, 0x19, 0x98, 0xa8 }, + .public = { 0x63, 0xaa, 0x40, 0xc6, 0xe3, 0x83, 0x46, 0xc5, + 0xca, 0xf2, 0x3a, 0x6d, 0xf0, 0xa5, 0xe6, 0xc8, + 0x08, 0x89, 0xa0, 0x86, 0x47, 0xe5, 0x51, 0xb3, + 0x56, 0x34, 0x49, 0xbe, 0xfc, 0xfc, 0x97, 0x33 }, + .result = { 0xb1, 0xa7, 0x07, 0x51, 0x94, 0x95, 0xff, 0xff, + 0xb2, 0x98, 0xff, 0x94, 0x17, 0x16, 0xb0, 0x6d, + 0xfa, 0xb8, 0x7c, 0xf8, 0xd9, 0x11, 0x23, 0xfe, + 0x2b, 0xe9, 0xa2, 0x33, 0xdd, 0xa2, 0x22, 0x12 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xb0, 0x5b, 0xfd, 0x32, 0xe5, 0x53, 0x25, 0xd9, + 0xfd, 0x64, 0x8c, 0xb3, 0x02, 0x84, 0x80, 0x39, + 0x00, 0x0b, 0x39, 0x0e, 0x44, 0xd5, 0x21, 0xe5, + 0x8a, 0xab, 0x3b, 0x29, 0xa6, 0x96, 0x0b, 0xa8 }, + .public = { 0x0f, 0x83, 0xc3, 0x6f, 0xde, 0xd9, 0xd3, 0x2f, + 0xad, 0xf4, 0xef, 0xa3, 0xae, 0x93, 0xa9, 0x0b, + 0xb5, 0xcf, 0xa6, 0x68, 0x93, 0xbc, 0x41, 0x2c, + 0x43, 0xfa, 0x72, 0x87, 0xdb, 0xb9, 0x97, 0x79 }, + .result = { 0x67, 0xdd, 0x4a, 0x6e, 0x16, 0x55, 0x33, 0x53, + 0x4c, 0x0e, 0x3f, 0x17, 0x2e, 0x4a, 0xb8, 0x57, + 0x6b, 0xca, 0x92, 0x3a, 0x5f, 0x07, 0xb2, 0xc0, + 0x69, 0xb4, 0xc3, 0x10, 0xff, 0x2e, 0x93, 0x5b }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x70, 0xe3, 0x4b, 0xcb, 0xe1, 0xf4, 0x7f, 0xbc, + 0x0f, 0xdd, 0xfd, 0x7c, 0x1e, 0x1a, 0xa5, 0x3d, + 0x57, 0xbf, 0xe0, 0xf6, 0x6d, 0x24, 0x30, 0x67, + 0xb4, 0x24, 0xbb, 0x62, 0x10, 0xbe, 0xd1, 0x9c }, + .public = { 0x0b, 0x82, 0x11, 0xa2, 0xb6, 0x04, 0x90, 0x97, + 0xf6, 0x87, 0x1c, 0x6c, 0x05, 0x2d, 0x3c, 0x5f, + 0xc1, 0xba, 0x17, 0xda, 0x9e, 0x32, 0xae, 0x45, + 0x84, 0x03, 0xb0, 0x5b, 0xb2, 0x83, 0x09, 0x2a }, + .result = { 0x4a, 0x06, 0x38, 0xcf, 0xaa, 0x9e, 0xf1, 0x93, + 0x3b, 0x47, 0xf8, 0x93, 0x92, 0x96, 0xa6, 0xb2, + 0x5b, 0xe5, 0x41, 0xef, 0x7f, 0x70, 0xe8, 0x44, + 0xc0, 0xbc, 0xc0, 0x0b, 0x13, 0x4d, 0xe6, 0x4a }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0x68, 0xc1, 0xf3, 0xa6, 0x53, 0xa4, 0xcd, 0xb1, + 0xd3, 0x7b, 0xba, 0x94, 0x73, 0x8f, 0x8b, 0x95, + 0x7a, 0x57, 0xbe, 0xb2, 0x4d, 0x64, 0x6e, 0x99, + 0x4d, 0xc2, 0x9a, 0x27, 0x6a, 0xad, 0x45, 0x8d }, + .public = { 0x34, 0x3a, 0xc2, 0x0a, 0x3b, 0x9c, 0x6a, 0x27, + 0xb1, 0x00, 0x81, 0x76, 0x50, 0x9a, 0xd3, 0x07, + 0x35, 0x85, 0x6e, 0xc1, 0xc8, 0xd8, 0xfc, 0xae, + 0x13, 0x91, 0x2d, 0x08, 0xd1, 0x52, 0xf4, 0x6c }, + .result = { 0x39, 0x94, 0x91, 0xfc, 0xe8, 0xdf, 0xab, 0x73, + 0xb4, 0xf9, 0xf6, 0x11, 0xde, 0x8e, 0xa0, 0xb2, + 0x7b, 0x28, 0xf8, 0x59, 0x94, 0x25, 0x0b, 0x0f, + 0x47, 0x5d, 0x58, 0x5d, 0x04, 0x2a, 0xc2, 0x07 }, + .valid = true + }, + /* wycheproof - public key on twist */ + { + .private = { 0xd8, 0x77, 0xb2, 0x6d, 0x06, 0xdf, 0xf9, 0xd9, + 0xf7, 0xfd, 0x4c, 0x5b, 0x37, 0x69, 0xf8, 0xcd, + 0xd5, 0xb3, 0x05, 0x16, 0xa5, 0xab, 0x80, 0x6b, + 0xe3, 0x24, 0xff, 0x3e, 0xb6, 0x9e, 0xa0, 0xb2 }, + .public = { 0xfa, 0x69, 0x5f, 0xc7, 0xbe, 0x8d, 0x1b, 0xe5, + 0xbf, 0x70, 0x48, 0x98, 0xf3, 0x88, 0xc4, 0x52, + 0xba, 0xfd, 0xd3, 0xb8, 0xea, 0xe8, 0x05, 0xf8, + 0x68, 0x1a, 0x8d, 0x15, 0xc2, 0xd4, 0xe1, 0x42 }, + .result = { 0x2c, 0x4f, 0xe1, 0x1d, 0x49, 0x0a, 0x53, 0x86, + 0x17, 0x76, 0xb1, 0x3b, 0x43, 0x54, 0xab, 0xd4, + 0xcf, 0x5a, 0x97, 0x69, 0x9d, 0xb6, 0xe6, 0xc6, + 0x8c, 0x16, 0x26, 0xd0, 0x76, 0x62, 0xf7, 0x58 }, + .valid = true + }, + /* wycheproof - public key = 0 */ + { + .private = { 0x20, 0x74, 0x94, 0x03, 0x8f, 0x2b, 0xb8, 0x11, + 0xd4, 0x78, 0x05, 0xbc, 0xdf, 0x04, 0xa2, 0xac, + 0x58, 0x5a, 0xda, 0x7f, 0x2f, 0x23, 0x38, 0x9b, + 0xfd, 0x46, 0x58, 0xf9, 0xdd, 0xd4, 0xde, 0xbc }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key = 1 */ + { + .private = { 0x20, 0x2e, 0x89, 0x72, 0xb6, 0x1c, 0x7e, 0x61, + 0x93, 0x0e, 0xb9, 0x45, 0x0b, 0x50, 0x70, 0xea, + 0xe1, 0xc6, 0x70, 0x47, 0x56, 0x85, 0x54, 0x1f, + 0x04, 0x76, 0x21, 0x7e, 0x48, 0x18, 0xcf, 0xab }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x38, 0xdd, 0xe9, 0xf3, 0xe7, 0xb7, 0x99, 0x04, + 0x5f, 0x9a, 0xc3, 0x79, 0x3d, 0x4a, 0x92, 0x77, + 0xda, 0xde, 0xad, 0xc4, 0x1b, 0xec, 0x02, 0x90, + 0xf8, 0x1f, 0x74, 0x4f, 0x73, 0x77, 0x5f, 0x84 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x9a, 0x2c, 0xfe, 0x84, 0xff, 0x9c, 0x4a, 0x97, + 0x39, 0x62, 0x5c, 0xae, 0x4a, 0x3b, 0x82, 0xa9, + 0x06, 0x87, 0x7a, 0x44, 0x19, 0x46, 0xf8, 0xd7, + 0xb3, 0xd7, 0x95, 0xfe, 0x8f, 0x5d, 0x16, 0x39 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x98, 0x57, 0xa9, 0x14, 0xe3, 0xc2, 0x90, 0x36, + 0xfd, 0x9a, 0x44, 0x2b, 0xa5, 0x26, 0xb5, 0xcd, + 0xcd, 0xf2, 0x82, 0x16, 0x15, 0x3e, 0x63, 0x6c, + 0x10, 0x67, 0x7a, 0xca, 0xb6, 0xbd, 0x6a, 0xa5 }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x4d, 0xa4, 0xe0, 0xaa, 0x07, 0x2c, 0x23, 0x2e, + 0xe2, 0xf0, 0xfa, 0x4e, 0x51, 0x9a, 0xe5, 0x0b, + 0x52, 0xc1, 0xed, 0xd0, 0x8a, 0x53, 0x4d, 0x4e, + 0xf3, 0x46, 0xc2, 0xe1, 0x06, 0xd2, 0x1d, 0x60 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x48, 0xe2, 0x13, 0x0d, 0x72, 0x33, 0x05, 0xed, + 0x05, 0xe6, 0xe5, 0x89, 0x4d, 0x39, 0x8a, 0x5e, + 0x33, 0x36, 0x7a, 0x8c, 0x6a, 0xac, 0x8f, 0xcd, + 0xf0, 0xa8, 0x8e, 0x4b, 0x42, 0x82, 0x0d, 0xb7 }, + .public = { 0xff, 0xff, 0xff, 0x03, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x1f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0xfe, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xf0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00 }, + .result = { 0x9e, 0xd1, 0x0c, 0x53, 0x74, 0x7f, 0x64, 0x7f, + 0x82, 0xf4, 0x51, 0x25, 0xd3, 0xde, 0x15, 0xa1, + 0xe6, 0xb8, 0x24, 0x49, 0x6a, 0xb4, 0x04, 0x10, + 0xff, 0xcc, 0x3c, 0xfe, 0x95, 0x76, 0x0f, 0x3b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x28, 0xf4, 0x10, 0x11, 0x69, 0x18, 0x51, 0xb3, + 0xa6, 0x2b, 0x64, 0x15, 0x53, 0xb3, 0x0d, 0x0d, + 0xfd, 0xdc, 0xb8, 0xff, 0xfc, 0xf5, 0x37, 0x00, + 0xa7, 0xbe, 0x2f, 0x6a, 0x87, 0x2e, 0x9f, 0xb0 }, + .public = { 0x00, 0x00, 0x00, 0xfc, 0xff, 0xff, 0x07, 0x00, + 0x00, 0xe0, 0xff, 0xff, 0x3f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0x01, 0x00, 0x00, 0xf8, 0xff, + 0xff, 0x0f, 0x00, 0x00, 0xc0, 0xff, 0xff, 0x7f }, + .result = { 0xcf, 0x72, 0xb4, 0xaa, 0x6a, 0xa1, 0xc9, 0xf8, + 0x94, 0xf4, 0x16, 0x5b, 0x86, 0x10, 0x9a, 0xa4, + 0x68, 0x51, 0x76, 0x48, 0xe1, 0xf0, 0xcc, 0x70, + 0xe1, 0xab, 0x08, 0x46, 0x01, 0x76, 0x50, 0x6b }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0x18, 0xa9, 0x3b, 0x64, 0x99, 0xb9, 0xf6, 0xb3, + 0x22, 0x5c, 0xa0, 0x2f, 0xef, 0x41, 0x0e, 0x0a, + 0xde, 0xc2, 0x35, 0x32, 0x32, 0x1d, 0x2d, 0x8e, + 0xf1, 0xa6, 0xd6, 0x02, 0xa8, 0xc6, 0x5b, 0x83 }, + .public = { 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x5d, 0x50, 0xb6, 0x28, 0x36, 0xbb, 0x69, 0x57, + 0x94, 0x10, 0x38, 0x6c, 0xf7, 0xbb, 0x81, 0x1c, + 0x14, 0xbf, 0x85, 0xb1, 0xc7, 0xb1, 0x7e, 0x59, + 0x24, 0xc7, 0xff, 0xea, 0x91, 0xef, 0x9e, 0x12 }, + .valid = true + }, + /* wycheproof - edge case on twist */ + { + .private = { 0xc0, 0x1d, 0x13, 0x05, 0xa1, 0x33, 0x8a, 0x1f, + 0xca, 0xc2, 0xba, 0x7e, 0x2e, 0x03, 0x2b, 0x42, + 0x7e, 0x0b, 0x04, 0x90, 0x31, 0x65, 0xac, 0xa9, + 0x57, 0xd8, 0xd0, 0x55, 0x3d, 0x87, 0x17, 0xb0 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x19, 0x23, 0x0e, 0xb1, 0x48, 0xd5, 0xd6, 0x7c, + 0x3c, 0x22, 0xab, 0x1d, 0xae, 0xff, 0x80, 0xa5, + 0x7e, 0xae, 0x42, 0x65, 0xce, 0x28, 0x72, 0x65, + 0x7b, 0x2c, 0x80, 0x99, 0xfc, 0x69, 0x8e, 0x50 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x38, 0x6f, 0x7f, 0x16, 0xc5, 0x07, 0x31, 0xd6, + 0x4f, 0x82, 0xe6, 0xa1, 0x70, 0xb1, 0x42, 0xa4, + 0xe3, 0x4f, 0x31, 0xfd, 0x77, 0x68, 0xfc, 0xb8, + 0x90, 0x29, 0x25, 0xe7, 0xd1, 0xe2, 0x1a, 0xbe }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x0f, 0xca, 0xb5, 0xd8, 0x42, 0xa0, 0x78, 0xd7, + 0xa7, 0x1f, 0xc5, 0x9b, 0x57, 0xbf, 0xb4, 0xca, + 0x0b, 0xe6, 0x87, 0x3b, 0x49, 0xdc, 0xdb, 0x9f, + 0x44, 0xe1, 0x4a, 0xe8, 0xfb, 0xdf, 0xa5, 0x42 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x23, 0xa2, 0x89, 0xbd, 0x5e, 0x90, 0xfa, + 0x28, 0x04, 0xdd, 0xc0, 0x19, 0xa0, 0x5e, 0xf3, + 0xe7, 0x9d, 0x43, 0x4b, 0xb6, 0xea, 0x2f, 0x52, + 0x2e, 0xcb, 0x64, 0x3a, 0x75, 0x29, 0x6e, 0x95 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 }, + .result = { 0x54, 0xce, 0x8f, 0x22, 0x75, 0xc0, 0x77, 0xe3, + 0xb1, 0x30, 0x6a, 0x39, 0x39, 0xc5, 0xe0, 0x3e, + 0xef, 0x6b, 0xbb, 0x88, 0x06, 0x05, 0x44, 0x75, + 0x8d, 0x9f, 0xef, 0x59, 0xb0, 0xbc, 0x3e, 0x4f }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x68, 0xf0, 0x10, 0xd6, 0x2e, 0xe8, 0xd9, 0x26, + 0x05, 0x3a, 0x36, 0x1c, 0x3a, 0x75, 0xc6, 0xea, + 0x4e, 0xbd, 0xc8, 0x60, 0x6a, 0xb2, 0x85, 0x00, + 0x3a, 0x6f, 0x8f, 0x40, 0x76, 0xb0, 0x1e, 0x83 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .result = { 0xf1, 0x36, 0x77, 0x5c, 0x5b, 0xeb, 0x0a, 0xf8, + 0x11, 0x0a, 0xf1, 0x0b, 0x20, 0x37, 0x23, 0x32, + 0x04, 0x3c, 0xab, 0x75, 0x24, 0x19, 0x67, 0x87, + 0x75, 0xa2, 0x23, 0xdf, 0x57, 0xc9, 0xd3, 0x0d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x58, 0xeb, 0xcb, 0x35, 0xb0, 0xf8, 0x84, 0x5c, + 0xaf, 0x1e, 0xc6, 0x30, 0xf9, 0x65, 0x76, 0xb6, + 0x2c, 0x4b, 0x7b, 0x6c, 0x36, 0xb2, 0x9d, 0xeb, + 0x2c, 0xb0, 0x08, 0x46, 0x51, 0x75, 0x5c, 0x96 }, + .public = { 0xff, 0xff, 0xff, 0xfb, 0xff, 0xff, 0xfb, 0xff, + 0xff, 0xdf, 0xff, 0xff, 0xdf, 0xff, 0xff, 0xff, + 0xfe, 0xff, 0xff, 0xfe, 0xff, 0xff, 0xf7, 0xff, + 0xff, 0xf7, 0xff, 0xff, 0xbf, 0xff, 0xff, 0x3f }, + .result = { 0xbf, 0x9a, 0xff, 0xd0, 0x6b, 0x84, 0x40, 0x85, + 0x58, 0x64, 0x60, 0x96, 0x2e, 0xf2, 0x14, 0x6f, + 0xf3, 0xd4, 0x53, 0x3d, 0x94, 0x44, 0xaa, 0xb0, + 0x06, 0xeb, 0x88, 0xcc, 0x30, 0x54, 0x40, 0x7d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0x18, 0x8c, 0x4b, 0xc5, 0xb9, 0xc4, 0x4b, 0x38, + 0xbb, 0x65, 0x8b, 0x9b, 0x2a, 0xe8, 0x2d, 0x5b, + 0x01, 0x01, 0x5e, 0x09, 0x31, 0x84, 0xb1, 0x7c, + 0xb7, 0x86, 0x35, 0x03, 0xa7, 0x83, 0xe1, 0xbb }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .result = { 0xd4, 0x80, 0xde, 0x04, 0xf6, 0x99, 0xcb, 0x3b, + 0xe0, 0x68, 0x4a, 0x9c, 0xc2, 0xe3, 0x12, 0x81, + 0xea, 0x0b, 0xc5, 0xa9, 0xdc, 0xc1, 0x57, 0xd3, + 0xd2, 0x01, 0x58, 0xd4, 0x6c, 0xa5, 0x24, 0x6d }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xe0, 0x6c, 0x11, 0xbb, 0x2e, 0x13, 0xce, 0x3d, + 0xc7, 0x67, 0x3f, 0x67, 0xf5, 0x48, 0x22, 0x42, + 0x90, 0x94, 0x23, 0xa9, 0xae, 0x95, 0xee, 0x98, + 0x6a, 0x98, 0x8d, 0x98, 0xfa, 0xee, 0x23, 0xa2 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x44, 0x01, 0xcc, 0xe6, 0xb5, 0x1e, 0x4c, + 0xb1, 0x8f, 0x27, 0x90, 0x24, 0x6c, 0x9b, 0xf9, + 0x14, 0xdb, 0x66, 0x77, 0x50, 0xa1, 0xcb, 0x89, + 0x06, 0x90, 0x92, 0xaf, 0x07, 0x29, 0x22, 0x76 }, + .valid = true + }, + /* wycheproof - edge case for public key */ + { + .private = { 0xc0, 0x65, 0x8c, 0x46, 0xdd, 0xe1, 0x81, 0x29, + 0x29, 0x38, 0x77, 0x53, 0x5b, 0x11, 0x62, 0xb6, + 0xf9, 0xf5, 0x41, 0x4a, 0x23, 0xcf, 0x4d, 0x2c, + 0xbc, 0x14, 0x0a, 0x4d, 0x99, 0xda, 0x2b, 0x8f }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x57, 0x8b, 0xa8, 0xcc, 0x2d, 0xbd, 0xc5, 0x75, + 0xaf, 0xcf, 0x9d, 0xf2, 0xb3, 0xee, 0x61, 0x89, + 0xf5, 0x33, 0x7d, 0x68, 0x54, 0xc7, 0x9b, 0x4c, + 0xe1, 0x65, 0xea, 0x12, 0x29, 0x3b, 0x3a, 0x0f }, + .valid = true + }, + /* wycheproof - public key with low order */ + { + .private = { 0x10, 0x25, 0x5c, 0x92, 0x30, 0xa9, 0x7a, 0x30, + 0xa4, 0x58, 0xca, 0x28, 0x4a, 0x62, 0x96, 0x69, + 0x29, 0x3a, 0x31, 0x89, 0x0c, 0xda, 0x9d, 0x14, + 0x7f, 0xeb, 0xc7, 0xd1, 0xe2, 0x2d, 0x6b, 0xb1 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x00 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xf1, 0xe8, 0xed, 0xf1, 0x44, 0x81, 0xb3, + 0x89, 0x44, 0x8d, 0xac, 0x8f, 0x59, 0xc7, 0x0b, + 0x03, 0x8e, 0x7c, 0xf9, 0x2e, 0xf2, 0xc7, 0xef, + 0xf5, 0x7a, 0x72, 0x46, 0x6e, 0x11, 0x52, 0x96 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0x57 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xa0, 0xa0, 0x5a, 0x3e, 0x8f, 0x9f, 0x44, 0x20, + 0x4d, 0x5f, 0x80, 0x59, 0xa9, 0x4a, 0xc7, 0xdf, + 0xc3, 0x9a, 0x49, 0xac, 0x01, 0x6d, 0xd7, 0x43, + 0xdb, 0xfa, 0x43, 0xc5, 0xd6, 0x71, 0xfd, 0x88 }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xd0, 0xdb, 0xb3, 0xed, 0x19, 0x06, 0x66, 0x3f, + 0x15, 0x42, 0x0a, 0xf3, 0x1f, 0x4e, 0xaf, 0x65, + 0x09, 0xd9, 0xa9, 0x94, 0x97, 0x23, 0x50, 0x06, + 0x05, 0xad, 0x7c, 0x1c, 0x6e, 0x74, 0x50, 0xa9 }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0xb1, 0xd0, 0xeb, 0x22, 0xb2, 0x44, 0xfe, + 0x32, 0x91, 0x14, 0x00, 0x72, 0xcd, 0xd9, 0xd9, + 0x89, 0xb5, 0xf0, 0xec, 0xd9, 0x6c, 0x10, 0x0f, + 0xeb, 0x5b, 0xca, 0x24, 0x1c, 0x1d, 0x9f, 0x8f }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x48, 0x0b, 0xf4, 0x5f, 0x59, 0x49, 0x42, 0xa8, + 0xbc, 0x0f, 0x33, 0x53, 0xc6, 0xe8, 0xb8, 0x85, + 0x3d, 0x77, 0xf3, 0x51, 0xf1, 0xc2, 0xca, 0x6c, + 0x2d, 0x1a, 0xbf, 0x8a, 0x00, 0xb4, 0x22, 0x9c }, + .public = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x30, 0xf9, 0x93, 0xfc, 0xf8, 0x51, 0x4f, 0xc8, + 0x9b, 0xd8, 0xdb, 0x14, 0xcd, 0x43, 0xba, 0x0d, + 0x4b, 0x25, 0x30, 0xe7, 0x3c, 0x42, 0x76, 0xa0, + 0x5e, 0x1b, 0x14, 0x5d, 0x42, 0x0c, 0xed, 0xb4 }, + .public = { 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0xc0, 0x49, 0x74, 0xb7, 0x58, 0x38, 0x0e, 0x2a, + 0x5b, 0x5d, 0xf6, 0xeb, 0x09, 0xbb, 0x2f, 0x6b, + 0x34, 0x34, 0xf9, 0x82, 0x72, 0x2a, 0x8e, 0x67, + 0x6d, 0x3d, 0xa2, 0x51, 0xd1, 0xb3, 0xde, 0x83 }, + .public = { 0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, + 0x16, 0x56, 0xe3, 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, + 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, 0xb1, 0xfd, + 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x80 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x50, 0x2a, 0x31, 0x37, 0x3d, 0xb3, 0x24, 0x46, + 0x84, 0x2f, 0xe5, 0xad, 0xd3, 0xe0, 0x24, 0x02, + 0x2e, 0xa5, 0x4f, 0x27, 0x41, 0x82, 0xaf, 0xc3, + 0xd9, 0xf1, 0xbb, 0x3d, 0x39, 0x53, 0x4e, 0xb5 }, + .public = { 0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, + 0xb1, 0xd0, 0xb1, 0x55, 0x9c, 0x83, 0xef, 0x5b, + 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, 0x8e, 0x86, + 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0xd7 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x90, 0xfa, 0x64, 0x17, 0xb0, 0xe3, 0x70, 0x30, + 0xfd, 0x6e, 0x43, 0xef, 0xf2, 0xab, 0xae, 0xf1, + 0x4c, 0x67, 0x93, 0x11, 0x7a, 0x03, 0x9c, 0xf6, + 0x21, 0x31, 0x8b, 0xa9, 0x0f, 0x4e, 0x98, 0xbe }, + .public = { 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x78, 0xad, 0x3f, 0x26, 0x02, 0x7f, 0x1c, 0x9f, + 0xdd, 0x97, 0x5a, 0x16, 0x13, 0xb9, 0x47, 0x77, + 0x9b, 0xad, 0x2c, 0xf2, 0xb7, 0x41, 0xad, 0xe0, + 0x18, 0x40, 0x88, 0x5a, 0x30, 0xbb, 0x97, 0x9c }, + .public = { 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key with low order */ + { + .private = { 0x98, 0xe2, 0x3d, 0xe7, 0xb1, 0xe0, 0x92, 0x6e, + 0xd9, 0xc8, 0x7e, 0x7b, 0x14, 0xba, 0xf5, 0x5f, + 0x49, 0x7a, 0x1d, 0x70, 0x96, 0xf9, 0x39, 0x77, + 0x68, 0x0e, 0x44, 0xdc, 0x1c, 0x7b, 0x7b, 0x8b }, + .public = { 0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = false + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x1e, 0x48, 0xda, 0xfa, 0xc9, 0xd7, 0xbc, + 0xf5, 0x89, 0xcb, 0xc3, 0x82, 0xc8, 0x78, 0xd1, + 0x8b, 0xda, 0x35, 0x50, 0x58, 0x9f, 0xfb, 0x5d, + 0x50, 0xb5, 0x23, 0xbe, 0xbe, 0x32, 0x9d, 0xae }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xbd, 0x36, 0xa0, 0x79, 0x0e, 0xb8, 0x83, 0x09, + 0x8c, 0x98, 0x8b, 0x21, 0x78, 0x67, 0x73, 0xde, + 0x0b, 0x3a, 0x4d, 0xf1, 0x62, 0x28, 0x2c, 0xf1, + 0x10, 0xde, 0x18, 0xdd, 0x48, 0x4c, 0xe7, 0x4b }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x28, 0x87, 0x96, 0xbc, 0x5a, 0xff, 0x4b, 0x81, + 0xa3, 0x75, 0x01, 0x75, 0x7b, 0xc0, 0x75, 0x3a, + 0x3c, 0x21, 0x96, 0x47, 0x90, 0xd3, 0x86, 0x99, + 0x30, 0x8d, 0xeb, 0xc1, 0x7a, 0x6e, 0xaf, 0x8d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0xb4, 0xe0, 0xdd, 0x76, 0xda, 0x7b, 0x07, 0x17, + 0x28, 0xb6, 0x1f, 0x85, 0x67, 0x71, 0xaa, 0x35, + 0x6e, 0x57, 0xed, 0xa7, 0x8a, 0x5b, 0x16, 0x55, + 0xcc, 0x38, 0x20, 0xfb, 0x5f, 0x85, 0x4c, 0x5c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x98, 0xdf, 0x84, 0x5f, 0x66, 0x51, 0xbf, 0x11, + 0x38, 0x22, 0x1f, 0x11, 0x90, 0x41, 0xf7, 0x2b, + 0x6d, 0xbc, 0x3c, 0x4a, 0xce, 0x71, 0x43, 0xd9, + 0x9f, 0xd5, 0x5a, 0xd8, 0x67, 0x48, 0x0d, 0xa8 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x6f, 0xdf, 0x6c, 0x37, 0x61, 0x1d, 0xbd, 0x53, + 0x04, 0xdc, 0x0f, 0x2e, 0xb7, 0xc9, 0x51, 0x7e, + 0xb3, 0xc5, 0x0e, 0x12, 0xfd, 0x05, 0x0a, 0xc6, + 0xde, 0xc2, 0x70, 0x71, 0xd4, 0xbf, 0xc0, 0x34 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf0, 0x94, 0x98, 0xe4, 0x6f, 0x02, 0xf8, 0x78, + 0x82, 0x9e, 0x78, 0xb8, 0x03, 0xd3, 0x16, 0xa2, + 0xed, 0x69, 0x5d, 0x04, 0x98, 0xa0, 0x8a, 0xbd, + 0xf8, 0x27, 0x69, 0x30, 0xe2, 0x4e, 0xdc, 0xb0 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .result = { 0x4c, 0x8f, 0xc4, 0xb1, 0xc6, 0xab, 0x88, 0xfb, + 0x21, 0xf1, 0x8f, 0x6d, 0x4c, 0x81, 0x02, 0x40, + 0xd4, 0xe9, 0x46, 0x51, 0xba, 0x44, 0xf7, 0xa2, + 0xc8, 0x63, 0xce, 0xc7, 0xdc, 0x56, 0x60, 0x2d }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x18, 0x13, 0xc1, 0x0a, 0x5c, 0x7f, 0x21, 0xf9, + 0x6e, 0x17, 0xf2, 0x88, 0xc0, 0xcc, 0x37, 0x60, + 0x7c, 0x04, 0xc5, 0xf5, 0xae, 0xa2, 0xdb, 0x13, + 0x4f, 0x9e, 0x2f, 0xfc, 0x66, 0xbd, 0x9d, 0xb8 }, + .public = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x1c, 0xd0, 0xb2, 0x82, 0x67, 0xdc, 0x54, 0x1c, + 0x64, 0x2d, 0x6d, 0x7d, 0xca, 0x44, 0xa8, 0xb3, + 0x8a, 0x63, 0x73, 0x6e, 0xef, 0x5c, 0x4e, 0x65, + 0x01, 0xff, 0xbb, 0xb1, 0x78, 0x0c, 0x03, 0x3c }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0x57, 0xfb, 0x80, 0x86, 0x53, 0x64, 0x5a, + 0x0b, 0xeb, 0x13, 0x8a, 0x64, 0xf5, 0xf4, 0xd7, + 0x33, 0xa4, 0x5e, 0xa8, 0x4c, 0x3c, 0xda, 0x11, + 0xa9, 0xc0, 0x6f, 0x7e, 0x71, 0x39, 0x14, 0x9e }, + .public = { 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x87, 0x55, 0xbe, 0x01, 0xc6, 0x0a, 0x7e, 0x82, + 0x5c, 0xff, 0x3e, 0x0e, 0x78, 0xcb, 0x3a, 0xa4, + 0x33, 0x38, 0x61, 0x51, 0x6a, 0xa5, 0x9b, 0x1c, + 0x51, 0xa8, 0xb2, 0xa5, 0x43, 0xdf, 0xa8, 0x22 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xe0, 0x3a, 0xa8, 0x42, 0xe2, 0xab, 0xc5, 0x6e, + 0x81, 0xe8, 0x7b, 0x8b, 0x9f, 0x41, 0x7b, 0x2a, + 0x1e, 0x59, 0x13, 0xc7, 0x23, 0xee, 0xd2, 0x8d, + 0x75, 0x2f, 0x8d, 0x47, 0xa5, 0x9f, 0x49, 0x8f }, + .public = { 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 }, + .result = { 0x54, 0xc9, 0xa1, 0xed, 0x95, 0xe5, 0x46, 0xd2, + 0x78, 0x22, 0xa3, 0x60, 0x93, 0x1d, 0xda, 0x60, + 0xa1, 0xdf, 0x04, 0x9d, 0xa6, 0xf9, 0x04, 0x25, + 0x3c, 0x06, 0x12, 0xbb, 0xdc, 0x08, 0x74, 0x76 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xf8, 0xf7, 0x07, 0xb7, 0x99, 0x9b, 0x18, 0xcb, + 0x0d, 0x6b, 0x96, 0x12, 0x4f, 0x20, 0x45, 0x97, + 0x2c, 0xa2, 0x74, 0xbf, 0xc1, 0x54, 0xad, 0x0c, + 0x87, 0x03, 0x8c, 0x24, 0xc6, 0xd0, 0xd4, 0xb2 }, + .public = { 0xda, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xcc, 0x1f, 0x40, 0xd7, 0x43, 0xcd, 0xc2, 0x23, + 0x0e, 0x10, 0x43, 0xda, 0xba, 0x8b, 0x75, 0xe8, + 0x10, 0xf1, 0xfb, 0xab, 0x7f, 0x25, 0x52, 0x69, + 0xbd, 0x9e, 0xbb, 0x29, 0xe6, 0xbf, 0x49, 0x4f }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xa0, 0x34, 0xf6, 0x84, 0xfa, 0x63, 0x1e, 0x1a, + 0x34, 0x81, 0x18, 0xc1, 0xce, 0x4c, 0x98, 0x23, + 0x1f, 0x2d, 0x9e, 0xec, 0x9b, 0xa5, 0x36, 0x5b, + 0x4a, 0x05, 0xd6, 0x9a, 0x78, 0x5b, 0x07, 0x96 }, + .public = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x54, 0x99, 0x8e, 0xe4, 0x3a, 0x5b, 0x00, 0x7b, + 0xf4, 0x99, 0xf0, 0x78, 0xe7, 0x36, 0x52, 0x44, + 0x00, 0xa8, 0xb5, 0xc7, 0xe9, 0xb9, 0xb4, 0x37, + 0x71, 0x74, 0x8c, 0x7c, 0xdf, 0x88, 0x04, 0x12 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x30, 0xb6, 0xc6, 0xa0, 0xf2, 0xff, 0xa6, 0x80, + 0x76, 0x8f, 0x99, 0x2b, 0xa8, 0x9e, 0x15, 0x2d, + 0x5b, 0xc9, 0x89, 0x3d, 0x38, 0xc9, 0x11, 0x9b, + 0xe4, 0xf7, 0x67, 0xbf, 0xab, 0x6e, 0x0c, 0xa5 }, + .public = { 0xdc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xea, 0xd9, 0xb3, 0x8e, 0xfd, 0xd7, 0x23, 0x63, + 0x79, 0x34, 0xe5, 0x5a, 0xb7, 0x17, 0xa7, 0xae, + 0x09, 0xeb, 0x86, 0xa2, 0x1d, 0xc3, 0x6a, 0x3f, + 0xee, 0xb8, 0x8b, 0x75, 0x9e, 0x39, 0x1e, 0x09 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x90, 0x1b, 0x9d, 0xcf, 0x88, 0x1e, 0x01, 0xe0, + 0x27, 0x57, 0x50, 0x35, 0xd4, 0x0b, 0x43, 0xbd, + 0xc1, 0xc5, 0x24, 0x2e, 0x03, 0x08, 0x47, 0x49, + 0x5b, 0x0c, 0x72, 0x86, 0x46, 0x9b, 0x65, 0x91 }, + .public = { 0xea, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x60, 0x2f, 0xf4, 0x07, 0x89, 0xb5, 0x4b, 0x41, + 0x80, 0x59, 0x15, 0xfe, 0x2a, 0x62, 0x21, 0xf0, + 0x7a, 0x50, 0xff, 0xc2, 0xc3, 0xfc, 0x94, 0xcf, + 0x61, 0xf1, 0x3d, 0x79, 0x04, 0xe8, 0x8e, 0x0e }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x80, 0x46, 0x67, 0x7c, 0x28, 0xfd, 0x82, 0xc9, + 0xa1, 0xbd, 0xb7, 0x1a, 0x1a, 0x1a, 0x34, 0xfa, + 0xba, 0x12, 0x25, 0xe2, 0x50, 0x7f, 0xe3, 0xf5, + 0x4d, 0x10, 0xbd, 0x5b, 0x0d, 0x86, 0x5f, 0x8e }, + .public = { 0xeb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0xe0, 0x0a, 0xe8, 0xb1, 0x43, 0x47, 0x12, 0x47, + 0xba, 0x24, 0xf1, 0x2c, 0x88, 0x55, 0x36, 0xc3, + 0xcb, 0x98, 0x1b, 0x58, 0xe1, 0xe5, 0x6b, 0x2b, + 0xaf, 0x35, 0xc1, 0x2a, 0xe1, 0xf7, 0x9c, 0x26 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x2f, 0x7e, 0x2f, 0x68, 0xa8, 0x46, 0xb8, + 0x2c, 0xc2, 0x69, 0xb1, 0xd4, 0x8e, 0x93, 0x98, + 0x86, 0xae, 0x54, 0xfd, 0x63, 0x6c, 0x1f, 0xe0, + 0x74, 0xd7, 0x10, 0x12, 0x7d, 0x47, 0x24, 0x91 }, + .public = { 0xef, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x98, 0xcb, 0x9b, 0x50, 0xdd, 0x3f, 0xc2, 0xb0, + 0xd4, 0xf2, 0xd2, 0xbf, 0x7c, 0x5c, 0xfd, 0xd1, + 0x0c, 0x8f, 0xcd, 0x31, 0xfc, 0x40, 0xaf, 0x1a, + 0xd4, 0x4f, 0x47, 0xc1, 0x31, 0x37, 0x63, 0x62 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x60, 0x88, 0x7b, 0x3d, 0xc7, 0x24, 0x43, 0x02, + 0x6e, 0xbe, 0xdb, 0xbb, 0xb7, 0x06, 0x65, 0xf4, + 0x2b, 0x87, 0xad, 0xd1, 0x44, 0x0e, 0x77, 0x68, + 0xfb, 0xd7, 0xe8, 0xe2, 0xce, 0x5f, 0x63, 0x9d }, + .public = { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x38, 0xd6, 0x30, 0x4c, 0x4a, 0x7e, 0x6d, 0x9f, + 0x79, 0x59, 0x33, 0x4f, 0xb5, 0x24, 0x5b, 0xd2, + 0xc7, 0x54, 0x52, 0x5d, 0x4c, 0x91, 0xdb, 0x95, + 0x02, 0x06, 0x92, 0x62, 0x34, 0xc1, 0xf6, 0x33 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0x78, 0xd3, 0x1d, 0xfa, 0x85, 0x44, 0x97, 0xd7, + 0x2d, 0x8d, 0xef, 0x8a, 0x1b, 0x7f, 0xb0, 0x06, + 0xce, 0xc2, 0xd8, 0xc4, 0x92, 0x46, 0x47, 0xc9, + 0x38, 0x14, 0xae, 0x56, 0xfa, 0xed, 0xa4, 0x95 }, + .public = { 0xf1, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x78, 0x6c, 0xd5, 0x49, 0x96, 0xf0, 0x14, 0xa5, + 0xa0, 0x31, 0xec, 0x14, 0xdb, 0x81, 0x2e, 0xd0, + 0x83, 0x55, 0x06, 0x1f, 0xdb, 0x5d, 0xe6, 0x80, + 0xa8, 0x00, 0xac, 0x52, 0x1f, 0x31, 0x8e, 0x23 }, + .valid = true + }, + /* wycheproof - public key >= p */ + { + .private = { 0xc0, 0x4c, 0x5b, 0xae, 0xfa, 0x83, 0x02, 0xdd, + 0xde, 0xd6, 0xa4, 0xbb, 0x95, 0x77, 0x61, 0xb4, + 0xeb, 0x97, 0xae, 0xfa, 0x4f, 0xc3, 0xb8, 0x04, + 0x30, 0x85, 0xf9, 0x6a, 0x56, 0x59, 0xb3, 0xa5 }, + .public = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, + .result = { 0x29, 0xae, 0x8b, 0xc7, 0x3e, 0x9b, 0x10, 0xa0, + 0x8b, 0x4f, 0x68, 0x1c, 0x43, 0xc3, 0xe0, 0xac, + 0x1a, 0x17, 0x1d, 0x31, 0xb3, 0x8f, 0x1a, 0x48, + 0xef, 0xba, 0x29, 0xae, 0x63, 0x9e, 0xa1, 0x34 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0xa0, 0x46, 0xe3, 0x6b, 0xf0, 0x52, 0x7c, 0x9d, + 0x3b, 0x16, 0x15, 0x4b, 0x82, 0x46, 0x5e, 0xdd, + 0x62, 0x14, 0x4c, 0x0a, 0xc1, 0xfc, 0x5a, 0x18, + 0x50, 0x6a, 0x22, 0x44, 0xba, 0x44, 0x9a, 0x44 }, + .public = { 0xe6, 0xdb, 0x68, 0x67, 0x58, 0x30, 0x30, 0xdb, + 0x35, 0x94, 0xc1, 0xa4, 0x24, 0xb1, 0x5f, 0x7c, + 0x72, 0x66, 0x24, 0xec, 0x26, 0xb3, 0x35, 0x3b, + 0x10, 0xa9, 0x03, 0xa6, 0xd0, 0xab, 0x1c, 0x4c }, + .result = { 0xc3, 0xda, 0x55, 0x37, 0x9d, 0xe9, 0xc6, 0x90, + 0x8e, 0x94, 0xea, 0x4d, 0xf2, 0x8d, 0x08, 0x4f, + 0x32, 0xec, 0xcf, 0x03, 0x49, 0x1c, 0x71, 0xf7, + 0x54, 0xb4, 0x07, 0x55, 0x77, 0xa2, 0x85, 0x52 }, + .valid = true + }, + /* wycheproof - RFC 7748 */ + { + .private = { 0x48, 0x66, 0xe9, 0xd4, 0xd1, 0xb4, 0x67, 0x3c, + 0x5a, 0xd2, 0x26, 0x91, 0x95, 0x7d, 0x6a, 0xf5, + 0xc1, 0x1b, 0x64, 0x21, 0xe0, 0xea, 0x01, 0xd4, + 0x2c, 0xa4, 0x16, 0x9e, 0x79, 0x18, 0xba, 0x4d }, + .public = { 0xe5, 0x21, 0x0f, 0x12, 0x78, 0x68, 0x11, 0xd3, + 0xf4, 0xb7, 0x95, 0x9d, 0x05, 0x38, 0xae, 0x2c, + 0x31, 0xdb, 0xe7, 0x10, 0x6f, 0xc0, 0x3c, 0x3e, + 0xfc, 0x4c, 0xd5, 0x49, 0xc7, 0x15, 0xa4, 0x13 }, + .result = { 0x95, 0xcb, 0xde, 0x94, 0x76, 0xe8, 0x90, 0x7d, + 0x7a, 0xad, 0xe4, 0x5c, 0xb4, 0xb8, 0x73, 0xf8, + 0x8b, 0x59, 0x5a, 0x68, 0x79, 0x9f, 0xa1, 0x52, + 0xe6, 0xf8, 0xf7, 0x64, 0x7a, 0xac, 0x79, 0x57 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x0a, 0xb4, 0xe7, 0x63, 0x80, 0xd8, 0x4d, 0xde, + 0x4f, 0x68, 0x33, 0xc5, 0x8f, 0x2a, 0x9f, 0xb8, + 0xf8, 0x3b, 0xb0, 0x16, 0x9b, 0x17, 0x2b, 0xe4, + 0xb6, 0xe0, 0x59, 0x28, 0x87, 0x74, 0x1a, 0x36 }, + .result = { 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x89, 0xe1, 0x0d, 0x57, 0x01, 0xb4, 0x33, 0x7d, + 0x2d, 0x03, 0x21, 0x81, 0x53, 0x8b, 0x10, 0x64, + 0xbd, 0x40, 0x84, 0x40, 0x1c, 0xec, 0xa1, 0xfd, + 0x12, 0x66, 0x3a, 0x19, 0x59, 0x38, 0x80, 0x00 }, + .result = { 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x2b, 0x55, 0xd3, 0xaa, 0x4a, 0x8f, 0x80, 0xc8, + 0xc0, 0xb2, 0xae, 0x5f, 0x93, 0x3e, 0x85, 0xaf, + 0x49, 0xbe, 0xac, 0x36, 0xc2, 0xfa, 0x73, 0x94, + 0xba, 0xb7, 0x6c, 0x89, 0x33, 0xf8, 0xf8, 0x1d }, + .result = { 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x63, 0xe5, 0xb1, 0xfe, 0x96, 0x01, 0xfe, 0x84, + 0x38, 0x5d, 0x88, 0x66, 0xb0, 0x42, 0x12, 0x62, + 0xf7, 0x8f, 0xbf, 0xa5, 0xaf, 0xf9, 0x58, 0x5e, + 0x62, 0x66, 0x79, 0xb1, 0x85, 0x47, 0xd9, 0x59 }, + .result = { 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xe4, 0x28, 0xf3, 0xda, 0xc1, 0x78, 0x09, 0xf8, + 0x27, 0xa5, 0x22, 0xce, 0x32, 0x35, 0x50, 0x58, + 0xd0, 0x73, 0x69, 0x36, 0x4a, 0xa7, 0x89, 0x02, + 0xee, 0x10, 0x13, 0x9b, 0x9f, 0x9d, 0xd6, 0x53 }, + .result = { 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xb3, 0xb5, 0x0e, 0x3e, 0xd3, 0xa4, 0x07, 0xb9, + 0x5d, 0xe9, 0x42, 0xef, 0x74, 0x57, 0x5b, 0x5a, + 0xb8, 0xa1, 0x0c, 0x09, 0xee, 0x10, 0x35, 0x44, + 0xd6, 0x0b, 0xdf, 0xed, 0x81, 0x38, 0xab, 0x2b }, + .result = { 0xf9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x21, 0x3f, 0xff, 0xe9, 0x3d, 0x5e, 0xa8, 0xcd, + 0x24, 0x2e, 0x46, 0x28, 0x44, 0x02, 0x99, 0x22, + 0xc4, 0x3c, 0x77, 0xc9, 0xe3, 0xe4, 0x2f, 0x56, + 0x2f, 0x48, 0x5d, 0x24, 0xc5, 0x01, 0xa2, 0x0b }, + .result = { 0xf3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x91, 0xb2, 0x32, 0xa1, 0x78, 0xb3, 0xcd, 0x53, + 0x09, 0x32, 0x44, 0x1e, 0x61, 0x39, 0x41, 0x8f, + 0x72, 0x17, 0x22, 0x92, 0xf1, 0xda, 0x4c, 0x18, + 0x34, 0xfc, 0x5e, 0xbf, 0xef, 0xb5, 0x1e, 0x3f }, + .result = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x04, 0x5c, 0x6e, 0x11, 0xc5, 0xd3, 0x32, 0x55, + 0x6c, 0x78, 0x22, 0xfe, 0x94, 0xeb, 0xf8, 0x9b, + 0x56, 0xa3, 0x87, 0x8d, 0xc2, 0x7c, 0xa0, 0x79, + 0x10, 0x30, 0x58, 0x84, 0x9f, 0xab, 0xcb, 0x4f }, + .result = { 0xe5, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x1c, 0xa2, 0x19, 0x0b, 0x71, 0x16, 0x35, 0x39, + 0x06, 0x3c, 0x35, 0x77, 0x3b, 0xda, 0x0c, 0x9c, + 0x92, 0x8e, 0x91, 0x36, 0xf0, 0x62, 0x0a, 0xeb, + 0x09, 0x3f, 0x09, 0x91, 0x97, 0xb7, 0xf7, 0x4e }, + .result = { 0xe3, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xf7, 0x6e, 0x90, 0x10, 0xac, 0x33, 0xc5, 0x04, + 0x3b, 0x2d, 0x3b, 0x76, 0xa8, 0x42, 0x17, 0x10, + 0x00, 0xc4, 0x91, 0x62, 0x22, 0xe9, 0xe8, 0x58, + 0x97, 0xa0, 0xae, 0xc7, 0xf6, 0x35, 0x0b, 0x3c }, + .result = { 0xdd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0xbb, 0x72, 0x68, 0x8d, 0x8f, 0x8a, 0xa7, 0xa3, + 0x9c, 0xd6, 0x06, 0x0c, 0xd5, 0xc8, 0x09, 0x3c, + 0xde, 0xc6, 0xfe, 0x34, 0x19, 0x37, 0xc3, 0x88, + 0x6a, 0x99, 0x34, 0x6c, 0xd0, 0x7f, 0xaa, 0x55 }, + .result = { 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x88, 0xfd, 0xde, 0xa1, 0x93, 0x39, 0x1c, 0x6a, + 0x59, 0x33, 0xef, 0x9b, 0x71, 0x90, 0x15, 0x49, + 0x44, 0x72, 0x05, 0xaa, 0xe9, 0xda, 0x92, 0x8a, + 0x6b, 0x91, 0xa3, 0x52, 0xba, 0x10, 0xf4, 0x1f }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 }, + .valid = true + }, + /* wycheproof - edge case for shared secret */ + { + .private = { 0xa0, 0xa4, 0xf1, 0x30, 0xb9, 0x8a, 0x5b, 0xe4, + 0xb1, 0xce, 0xdb, 0x7c, 0xb8, 0x55, 0x84, 0xa3, + 0x52, 0x0e, 0x14, 0x2d, 0x47, 0x4d, 0xc9, 0xcc, + 0xb9, 0x09, 0xa0, 0x73, 0xa9, 0x76, 0xbf, 0x63 }, + .public = { 0x30, 0x3b, 0x39, 0x2f, 0x15, 0x31, 0x16, 0xca, + 0xd9, 0xcc, 0x68, 0x2a, 0x00, 0xcc, 0xc4, 0x4c, + 0x95, 0xff, 0x0d, 0x3b, 0xbe, 0x56, 0x8b, 0xeb, + 0x6c, 0x4e, 0x73, 0x9b, 0xaf, 0xdc, 0x2c, 0x68 }, + .result = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xfd, 0x30, 0x0a, 0xeb, 0x40, 0xe1, 0xfa, 0x58, + 0x25, 0x18, 0x41, 0x2b, 0x49, 0xb2, 0x08, 0xa7, + 0x84, 0x2b, 0x1e, 0x1f, 0x05, 0x6a, 0x04, 0x01, + 0x78, 0xea, 0x41, 0x41, 0x53, 0x4f, 0x65, 0x2d }, + .result = { 0xb7, 0x34, 0x10, 0x5d, 0xc2, 0x57, 0x58, 0x5d, + 0x73, 0xb5, 0x66, 0xcc, 0xb7, 0x6f, 0x06, 0x27, + 0x95, 0xcc, 0xbe, 0xc8, 0x91, 0x28, 0xe5, 0x2b, + 0x02, 0xf3, 0xe5, 0x96, 0x39, 0xf1, 0x3c, 0x46 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xc8, 0xef, 0x79, 0xb5, 0x14, 0xd7, 0x68, 0x26, + 0x77, 0xbc, 0x79, 0x31, 0xe0, 0x6e, 0xe5, 0xc2, + 0x7c, 0x9b, 0x39, 0x2b, 0x4a, 0xe9, 0x48, 0x44, + 0x73, 0xf5, 0x54, 0xe6, 0x67, 0x8e, 0xcc, 0x2e }, + .result = { 0x64, 0x7a, 0x46, 0xb6, 0xfc, 0x3f, 0x40, 0xd6, + 0x21, 0x41, 0xee, 0x3c, 0xee, 0x70, 0x6b, 0x4d, + 0x7a, 0x92, 0x71, 0x59, 0x3a, 0x7b, 0x14, 0x3e, + 0x8e, 0x2e, 0x22, 0x79, 0x88, 0x3e, 0x45, 0x50 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x64, 0xae, 0xac, 0x25, 0x04, 0x14, 0x48, 0x61, + 0x53, 0x2b, 0x7b, 0xbc, 0xb6, 0xc8, 0x7d, 0x67, + 0xdd, 0x4c, 0x1f, 0x07, 0xeb, 0xc2, 0xe0, 0x6e, + 0xff, 0xb9, 0x5a, 0xec, 0xc6, 0x17, 0x0b, 0x2c }, + .result = { 0x4f, 0xf0, 0x3d, 0x5f, 0xb4, 0x3c, 0xd8, 0x65, + 0x7a, 0x3c, 0xf3, 0x7c, 0x13, 0x8c, 0xad, 0xce, + 0xcc, 0xe5, 0x09, 0xe4, 0xeb, 0xa0, 0x89, 0xd0, + 0xef, 0x40, 0xb4, 0xe4, 0xfb, 0x94, 0x61, 0x55 }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0xbf, 0x68, 0xe3, 0x5e, 0x9b, 0xdb, 0x7e, 0xee, + 0x1b, 0x50, 0x57, 0x02, 0x21, 0x86, 0x0f, 0x5d, + 0xcd, 0xad, 0x8a, 0xcb, 0xab, 0x03, 0x1b, 0x14, + 0x97, 0x4c, 0xc4, 0x90, 0x13, 0xc4, 0x98, 0x31 }, + .result = { 0x21, 0xce, 0xe5, 0x2e, 0xfd, 0xbc, 0x81, 0x2e, + 0x1d, 0x02, 0x1a, 0x4a, 0xf1, 0xe1, 0xd8, 0xbc, + 0x4d, 0xb3, 0xc4, 0x00, 0xe4, 0xd2, 0xa2, 0xc5, + 0x6a, 0x39, 0x26, 0xdb, 0x4d, 0x99, 0xc6, 0x5b }, + .valid = true + }, + /* wycheproof - checking for overflow */ + { + .private = { 0xc8, 0x17, 0x24, 0x70, 0x40, 0x00, 0xb2, 0x6d, + 0x31, 0x70, 0x3c, 0xc9, 0x7e, 0x3a, 0x37, 0x8d, + 0x56, 0xfa, 0xd8, 0x21, 0x93, 0x61, 0xc8, 0x8c, + 0xca, 0x8b, 0xd7, 0xc5, 0x71, 0x9b, 0x12, 0xb2 }, + .public = { 0x53, 0x47, 0xc4, 0x91, 0x33, 0x1a, 0x64, 0xb4, + 0x3d, 0xdc, 0x68, 0x30, 0x34, 0xe6, 0x77, 0xf5, + 0x3d, 0xc3, 0x2b, 0x52, 0xa5, 0x2a, 0x57, 0x7c, + 0x15, 0xa8, 0x3b, 0xf2, 0x98, 0xe9, 0x9f, 0x19 }, + .result = { 0x18, 0xcb, 0x89, 0xe4, 0xe2, 0x0c, 0x0c, 0x2b, + 0xd3, 0x24, 0x30, 0x52, 0x45, 0x26, 0x6c, 0x93, + 0x27, 0x69, 0x0b, 0xbe, 0x79, 0xac, 0xb8, 0x8f, + 0x5b, 0x8f, 0xb3, 0xf7, 0x4e, 0xca, 0x3e, 0x52 }, + .valid = true + }, + /* wycheproof - private key == -1 (mod order) */ + { + .private = { 0xa0, 0x23, 0xcd, 0xd0, 0x83, 0xef, 0x5b, 0xb8, + 0x2f, 0x10, 0xd6, 0x2e, 0x59, 0xe1, 0x5a, 0x68, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50 }, + .public = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .result = { 0x25, 0x8e, 0x04, 0x52, 0x3b, 0x8d, 0x25, 0x3e, + 0xe6, 0x57, 0x19, 0xfc, 0x69, 0x06, 0xc6, 0x57, + 0x19, 0x2d, 0x80, 0x71, 0x7e, 0xdc, 0x82, 0x8f, + 0xa0, 0xaf, 0x21, 0x68, 0x6e, 0x2f, 0xaa, 0x75 }, + .valid = true + }, + /* wycheproof - private key == 1 (mod order) on twist */ + { + .private = { 0x58, 0x08, 0x3d, 0xd2, 0x61, 0xad, 0x91, 0xef, + 0xf9, 0x52, 0x32, 0x2e, 0xc8, 0x24, 0xc6, 0x82, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x5f }, + .public = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .result = { 0x2e, 0xae, 0x5e, 0xc3, 0xdd, 0x49, 0x4e, 0x9f, + 0x2d, 0x37, 0xd2, 0x58, 0xf8, 0x73, 0xa8, 0xe6, + 0xe9, 0xd0, 0xdb, 0xd1, 0xe3, 0x83, 0xef, 0x64, + 0xd9, 0x8b, 0xb9, 0x1b, 0x3e, 0x0b, 0xe0, 0x35 }, + .valid = true + } +}; + +static bool __init curve25519_selftest(void) +{ + bool success = true, ret, ret2; + size_t i = 0, j; + u8 in[CURVE25519_KEY_SIZE]; + u8 out[CURVE25519_KEY_SIZE], out2[CURVE25519_KEY_SIZE]; + + for (i = 0; i < ARRAY_SIZE(curve25519_test_vectors); ++i) { + memset(out, 0, CURVE25519_KEY_SIZE); + ret = curve25519(out, curve25519_test_vectors[i].private, + curve25519_test_vectors[i].public); + if (ret != curve25519_test_vectors[i].valid || + memcmp(out, curve25519_test_vectors[i].result, + CURVE25519_KEY_SIZE)) { + pr_err("curve25519 self-test %zu: FAIL\n", i + 1); + success = false; + } + } + + for (i = 0; i < 5; ++i) { + get_random_bytes(in, sizeof(in)); + ret = curve25519_generate_public(out, in); + ret2 = curve25519(out2, in, (u8[CURVE25519_KEY_SIZE]){ 9 }); + if (ret != ret2 || memcmp(out, out2, CURVE25519_KEY_SIZE)) { + pr_err("curve25519 basepoint self-test %zu: FAIL: input - 0x", + i + 1); + for (j = CURVE25519_KEY_SIZE; j-- > 0;) + printk(KERN_CONT "%02x", in[j]); + printk(KERN_CONT "\n"); + success = false; + } + } + + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/poly1305.c b/net/wireguard/crypto/zinc/selftest/poly1305.c new file mode 100644 index 0000000000000..b4d7a9c2f6ec7 --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/poly1305.c @@ -0,0 +1,1107 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +struct poly1305_testvec { + const u8 *input, *output, *key; + size_t ilen; +}; + +/* RFC7539 */ +static const u8 input01[] __initconst = { + 0x43, 0x72, 0x79, 0x70, 0x74, 0x6f, 0x67, 0x72, + 0x61, 0x70, 0x68, 0x69, 0x63, 0x20, 0x46, 0x6f, + 0x72, 0x75, 0x6d, 0x20, 0x52, 0x65, 0x73, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x20, 0x47, 0x72, 0x6f, + 0x75, 0x70 +}; +static const u8 output01[] __initconst = { + 0xa8, 0x06, 0x1d, 0xc1, 0x30, 0x51, 0x36, 0xc6, + 0xc2, 0x2b, 0x8b, 0xaf, 0x0c, 0x01, 0x27, 0xa9 +}; +static const u8 key01[] __initconst = { + 0x85, 0xd6, 0xbe, 0x78, 0x57, 0x55, 0x6d, 0x33, + 0x7f, 0x44, 0x52, 0xfe, 0x42, 0xd5, 0x06, 0xa8, + 0x01, 0x03, 0x80, 0x8a, 0xfb, 0x0d, 0xb2, 0xfd, + 0x4a, 0xbf, 0xf6, 0xaf, 0x41, 0x49, 0xf5, 0x1b +}; + +/* "The Poly1305-AES message-authentication code" */ +static const u8 input02[] __initconst = { + 0xf3, 0xf6 +}; +static const u8 output02[] __initconst = { + 0xf4, 0xc6, 0x33, 0xc3, 0x04, 0x4f, 0xc1, 0x45, + 0xf8, 0x4f, 0x33, 0x5c, 0xb8, 0x19, 0x53, 0xde +}; +static const u8 key02[] __initconst = { + 0x85, 0x1f, 0xc4, 0x0c, 0x34, 0x67, 0xac, 0x0b, + 0xe0, 0x5c, 0xc2, 0x04, 0x04, 0xf3, 0xf7, 0x00, + 0x58, 0x0b, 0x3b, 0x0f, 0x94, 0x47, 0xbb, 0x1e, + 0x69, 0xd0, 0x95, 0xb5, 0x92, 0x8b, 0x6d, 0xbc +}; + +static const u8 input03[] __initconst = { }; +static const u8 output03[] __initconst = { + 0xdd, 0x3f, 0xab, 0x22, 0x51, 0xf1, 0x1a, 0xc7, + 0x59, 0xf0, 0x88, 0x71, 0x29, 0xcc, 0x2e, 0xe7 +}; +static const u8 key03[] __initconst = { + 0xa0, 0xf3, 0x08, 0x00, 0x00, 0xf4, 0x64, 0x00, + 0xd0, 0xc7, 0xe9, 0x07, 0x6c, 0x83, 0x44, 0x03, + 0xdd, 0x3f, 0xab, 0x22, 0x51, 0xf1, 0x1a, 0xc7, + 0x59, 0xf0, 0x88, 0x71, 0x29, 0xcc, 0x2e, 0xe7 +}; + +static const u8 input04[] __initconst = { + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output04[] __initconst = { + 0x0e, 0xe1, 0xc1, 0x6b, 0xb7, 0x3f, 0x0f, 0x4f, + 0xd1, 0x98, 0x81, 0x75, 0x3c, 0x01, 0xcd, 0xbe +}; +static const u8 key04[] __initconst = { + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef +}; + +static const u8 input05[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9 +}; +static const u8 output05[] __initconst = { + 0x51, 0x54, 0xad, 0x0d, 0x2c, 0xb2, 0x6e, 0x01, + 0x27, 0x4f, 0xc5, 0x11, 0x48, 0x49, 0x1f, 0x1b +}; +static const u8 key05[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +/* self-generated vectors exercise "significant" lengths, such that they + * are handled by different code paths */ +static const u8 input06[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf +}; +static const u8 output06[] __initconst = { + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66 +}; +static const u8 key06[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input07[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67 +}; +static const u8 output07[] __initconst = { + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 key07[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input08[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output08[] __initconst = { + 0xbb, 0xb6, 0x13, 0xb2, 0xb6, 0xd7, 0x53, 0xba, + 0x07, 0x39, 0x5b, 0x91, 0x6a, 0xae, 0xce, 0x15 +}; +static const u8 key08[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input09[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24 +}; +static const u8 output09[] __initconst = { + 0xc7, 0x94, 0xd7, 0x05, 0x7d, 0x17, 0x78, 0xc4, + 0xbb, 0xee, 0x0a, 0x39, 0xb3, 0xd9, 0x73, 0x42 +}; +static const u8 key09[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input10[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output10[] __initconst = { + 0xff, 0xbc, 0xb9, 0xb3, 0x71, 0x42, 0x31, 0x52, + 0xd7, 0xfc, 0xa5, 0xad, 0x04, 0x2f, 0xba, 0xa9 +}; +static const u8 key10[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input11[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66 +}; +static const u8 output11[] __initconst = { + 0x06, 0x9e, 0xd6, 0xb8, 0xef, 0x0f, 0x20, 0x7b, + 0x3e, 0x24, 0x3b, 0xb1, 0x01, 0x9f, 0xe6, 0x32 +}; +static const u8 key11[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input12[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 output12[] __initconst = { + 0xcc, 0xa3, 0x39, 0xd9, 0xa4, 0x5f, 0xa2, 0x36, + 0x8c, 0x2c, 0x68, 0xb3, 0xa4, 0x17, 0x91, 0x33 +}; +static const u8 key12[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input13[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61, + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36 +}; +static const u8 output13[] __initconst = { + 0x53, 0xf6, 0xe8, 0x28, 0xa2, 0xf0, 0xfe, 0x0e, + 0xe8, 0x15, 0xbf, 0x0b, 0xd5, 0x84, 0x1a, 0x34 +}; +static const u8 key13[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +static const u8 input14[] __initconst = { + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61, + 0xab, 0x08, 0x12, 0x72, 0x4a, 0x7f, 0x1e, 0x34, + 0x27, 0x42, 0xcb, 0xed, 0x37, 0x4d, 0x94, 0xd1, + 0x36, 0xc6, 0xb8, 0x79, 0x5d, 0x45, 0xb3, 0x81, + 0x98, 0x30, 0xf2, 0xc0, 0x44, 0x91, 0xfa, 0xf0, + 0x99, 0x0c, 0x62, 0xe4, 0x8b, 0x80, 0x18, 0xb2, + 0xc3, 0xe4, 0xa0, 0xfa, 0x31, 0x34, 0xcb, 0x67, + 0xfa, 0x83, 0xe1, 0x58, 0xc9, 0x94, 0xd9, 0x61, + 0xc4, 0xcb, 0x21, 0x09, 0x5c, 0x1b, 0xf9, 0xaf, + 0x48, 0x44, 0x3d, 0x0b, 0xb0, 0xd2, 0x11, 0x09, + 0xc8, 0x9a, 0x10, 0x0b, 0x5c, 0xe2, 0xc2, 0x08, + 0x83, 0x14, 0x9c, 0x69, 0xb5, 0x61, 0xdd, 0x88, + 0x29, 0x8a, 0x17, 0x98, 0xb1, 0x07, 0x16, 0xef, + 0x66, 0x3c, 0xea, 0x19, 0x0f, 0xfb, 0x83, 0xd8, + 0x95, 0x93, 0xf3, 0xf4, 0x76, 0xb6, 0xbc, 0x24, + 0xd7, 0xe6, 0x79, 0x10, 0x7e, 0xa2, 0x6a, 0xdb, + 0x8c, 0xaf, 0x66, 0x52, 0xd0, 0x65, 0x61, 0x36, + 0x81, 0x20, 0x59, 0xa5, 0xda, 0x19, 0x86, 0x37, + 0xca, 0xc7, 0xc4, 0xa6, 0x31, 0xbe, 0xe4, 0x66, + 0x5b, 0x88, 0xd7, 0xf6, 0x22, 0x8b, 0x11, 0xe2, + 0xe2, 0x85, 0x79, 0xa5, 0xc0, 0xc1, 0xf7, 0x61 +}; +static const u8 output14[] __initconst = { + 0xb8, 0x46, 0xd4, 0x4e, 0x9b, 0xbd, 0x53, 0xce, + 0xdf, 0xfb, 0xfb, 0xb6, 0xb7, 0xfa, 0x49, 0x33 +}; +static const u8 key14[] __initconst = { + 0x12, 0x97, 0x6a, 0x08, 0xc4, 0x42, 0x6d, 0x0c, + 0xe8, 0xa8, 0x24, 0x07, 0xc4, 0xf4, 0x82, 0x07, + 0x80, 0xf8, 0xc2, 0x0a, 0xa7, 0x12, 0x02, 0xd1, + 0xe2, 0x91, 0x79, 0xcb, 0xcb, 0x55, 0x5a, 0x57 +}; + +/* 4th power of the key spills to 131th bit in SIMD key setup */ +static const u8 input15[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output15[] __initconst = { + 0x07, 0x14, 0x5a, 0x4c, 0x02, 0xfe, 0x5f, 0xa3, + 0x20, 0x36, 0xde, 0x68, 0xfa, 0xbe, 0x90, 0x66 +}; +static const u8 key15[] __initconst = { + 0xad, 0x62, 0x81, 0x07, 0xe8, 0x35, 0x1d, 0x0f, + 0x2c, 0x23, 0x1a, 0x05, 0xdc, 0x4a, 0x41, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* OpenSSL's poly1305_ieee754.c failed this in final stage */ +static const u8 input16[] __initconst = { + 0x84, 0x23, 0x64, 0xe1, 0x56, 0x33, 0x6c, 0x09, + 0x98, 0xb9, 0x33, 0xa6, 0x23, 0x77, 0x26, 0x18, + 0x0d, 0x9e, 0x3f, 0xdc, 0xbd, 0xe4, 0xcd, 0x5d, + 0x17, 0x08, 0x0f, 0xc3, 0xbe, 0xb4, 0x96, 0x14, + 0xd7, 0x12, 0x2c, 0x03, 0x74, 0x63, 0xff, 0x10, + 0x4d, 0x73, 0xf1, 0x9c, 0x12, 0x70, 0x46, 0x28, + 0xd4, 0x17, 0xc4, 0xc5, 0x4a, 0x3f, 0xe3, 0x0d, + 0x3c, 0x3d, 0x77, 0x14, 0x38, 0x2d, 0x43, 0xb0, + 0x38, 0x2a, 0x50, 0xa5, 0xde, 0xe5, 0x4b, 0xe8, + 0x44, 0xb0, 0x76, 0xe8, 0xdf, 0x88, 0x20, 0x1a, + 0x1c, 0xd4, 0x3b, 0x90, 0xeb, 0x21, 0x64, 0x3f, + 0xa9, 0x6f, 0x39, 0xb5, 0x18, 0xaa, 0x83, 0x40, + 0xc9, 0x42, 0xff, 0x3c, 0x31, 0xba, 0xf7, 0xc9, + 0xbd, 0xbf, 0x0f, 0x31, 0xae, 0x3f, 0xa0, 0x96, + 0xbf, 0x8c, 0x63, 0x03, 0x06, 0x09, 0x82, 0x9f, + 0xe7, 0x2e, 0x17, 0x98, 0x24, 0x89, 0x0b, 0xc8, + 0xe0, 0x8c, 0x31, 0x5c, 0x1c, 0xce, 0x2a, 0x83, + 0x14, 0x4d, 0xbb, 0xff, 0x09, 0xf7, 0x4e, 0x3e, + 0xfc, 0x77, 0x0b, 0x54, 0xd0, 0x98, 0x4a, 0x8f, + 0x19, 0xb1, 0x47, 0x19, 0xe6, 0x36, 0x35, 0x64, + 0x1d, 0x6b, 0x1e, 0xed, 0xf6, 0x3e, 0xfb, 0xf0, + 0x80, 0xe1, 0x78, 0x3d, 0x32, 0x44, 0x54, 0x12, + 0x11, 0x4c, 0x20, 0xde, 0x0b, 0x83, 0x7a, 0x0d, + 0xfa, 0x33, 0xd6, 0xb8, 0x28, 0x25, 0xff, 0xf4, + 0x4c, 0x9a, 0x70, 0xea, 0x54, 0xce, 0x47, 0xf0, + 0x7d, 0xf6, 0x98, 0xe6, 0xb0, 0x33, 0x23, 0xb5, + 0x30, 0x79, 0x36, 0x4a, 0x5f, 0xc3, 0xe9, 0xdd, + 0x03, 0x43, 0x92, 0xbd, 0xde, 0x86, 0xdc, 0xcd, + 0xda, 0x94, 0x32, 0x1c, 0x5e, 0x44, 0x06, 0x04, + 0x89, 0x33, 0x6c, 0xb6, 0x5b, 0xf3, 0x98, 0x9c, + 0x36, 0xf7, 0x28, 0x2c, 0x2f, 0x5d, 0x2b, 0x88, + 0x2c, 0x17, 0x1e, 0x74 +}; +static const u8 output16[] __initconst = { + 0xf2, 0x48, 0x31, 0x2e, 0x57, 0x8d, 0x9d, 0x58, + 0xf8, 0xb7, 0xbb, 0x4d, 0x19, 0x10, 0x54, 0x31 +}; +static const u8 key16[] __initconst = { + 0x95, 0xd5, 0xc0, 0x05, 0x50, 0x3e, 0x51, 0x0d, + 0x8c, 0xd0, 0xaa, 0x07, 0x2c, 0x4a, 0x4d, 0x06, + 0x6e, 0xab, 0xc5, 0x2d, 0x11, 0x65, 0x3d, 0xf4, + 0x7f, 0xbf, 0x63, 0xab, 0x19, 0x8b, 0xcc, 0x26 +}; + +/* AVX2 in OpenSSL's poly1305-x86.pl failed this with 176+32 split */ +static const u8 input17[] __initconst = { + 0x24, 0x8a, 0xc3, 0x10, 0x85, 0xb6, 0xc2, 0xad, + 0xaa, 0xa3, 0x82, 0x59, 0xa0, 0xd7, 0x19, 0x2c, + 0x5c, 0x35, 0xd1, 0xbb, 0x4e, 0xf3, 0x9a, 0xd9, + 0x4c, 0x38, 0xd1, 0xc8, 0x24, 0x79, 0xe2, 0xdd, + 0x21, 0x59, 0xa0, 0x77, 0x02, 0x4b, 0x05, 0x89, + 0xbc, 0x8a, 0x20, 0x10, 0x1b, 0x50, 0x6f, 0x0a, + 0x1a, 0xd0, 0xbb, 0xab, 0x76, 0xe8, 0x3a, 0x83, + 0xf1, 0xb9, 0x4b, 0xe6, 0xbe, 0xae, 0x74, 0xe8, + 0x74, 0xca, 0xb6, 0x92, 0xc5, 0x96, 0x3a, 0x75, + 0x43, 0x6b, 0x77, 0x61, 0x21, 0xec, 0x9f, 0x62, + 0x39, 0x9a, 0x3e, 0x66, 0xb2, 0xd2, 0x27, 0x07, + 0xda, 0xe8, 0x19, 0x33, 0xb6, 0x27, 0x7f, 0x3c, + 0x85, 0x16, 0xbc, 0xbe, 0x26, 0xdb, 0xbd, 0x86, + 0xf3, 0x73, 0x10, 0x3d, 0x7c, 0xf4, 0xca, 0xd1, + 0x88, 0x8c, 0x95, 0x21, 0x18, 0xfb, 0xfb, 0xd0, + 0xd7, 0xb4, 0xbe, 0xdc, 0x4a, 0xe4, 0x93, 0x6a, + 0xff, 0x91, 0x15, 0x7e, 0x7a, 0xa4, 0x7c, 0x54, + 0x44, 0x2e, 0xa7, 0x8d, 0x6a, 0xc2, 0x51, 0xd3, + 0x24, 0xa0, 0xfb, 0xe4, 0x9d, 0x89, 0xcc, 0x35, + 0x21, 0xb6, 0x6d, 0x16, 0xe9, 0xc6, 0x6a, 0x37, + 0x09, 0x89, 0x4e, 0x4e, 0xb0, 0xa4, 0xee, 0xdc, + 0x4a, 0xe1, 0x94, 0x68, 0xe6, 0x6b, 0x81, 0xf2, + 0x71, 0x35, 0x1b, 0x1d, 0x92, 0x1e, 0xa5, 0x51, + 0x04, 0x7a, 0xbc, 0xc6, 0xb8, 0x7a, 0x90, 0x1f, + 0xde, 0x7d, 0xb7, 0x9f, 0xa1, 0x81, 0x8c, 0x11, + 0x33, 0x6d, 0xbc, 0x07, 0x24, 0x4a, 0x40, 0xeb +}; +static const u8 output17[] __initconst = { + 0xbc, 0x93, 0x9b, 0xc5, 0x28, 0x14, 0x80, 0xfa, + 0x99, 0xc6, 0xd6, 0x8c, 0x25, 0x8e, 0xc4, 0x2f +}; +static const u8 key17[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* test vectors from Google */ +static const u8 input18[] __initconst = { }; +static const u8 output18[] __initconst = { + 0x47, 0x10, 0x13, 0x0e, 0x9f, 0x6f, 0xea, 0x8d, + 0x72, 0x29, 0x38, 0x50, 0xa6, 0x67, 0xd8, 0x6c +}; +static const u8 key18[] __initconst = { + 0xc8, 0xaf, 0xaa, 0xc3, 0x31, 0xee, 0x37, 0x2c, + 0xd6, 0x08, 0x2d, 0xe1, 0x34, 0x94, 0x3b, 0x17, + 0x47, 0x10, 0x13, 0x0e, 0x9f, 0x6f, 0xea, 0x8d, + 0x72, 0x29, 0x38, 0x50, 0xa6, 0x67, 0xd8, 0x6c +}; + +static const u8 input19[] __initconst = { + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, + 0x72, 0x6c, 0x64, 0x21 +}; +static const u8 output19[] __initconst = { + 0xa6, 0xf7, 0x45, 0x00, 0x8f, 0x81, 0xc9, 0x16, + 0xa2, 0x0d, 0xcc, 0x74, 0xee, 0xf2, 0xb2, 0xf0 +}; +static const u8 key19[] __initconst = { + 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, + 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, + 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35 +}; + +static const u8 input20[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output20[] __initconst = { + 0x49, 0xec, 0x78, 0x09, 0x0e, 0x48, 0x1e, 0xc6, + 0xc2, 0x6b, 0x33, 0xb9, 0x1c, 0xcc, 0x03, 0x07 +}; +static const u8 key20[] __initconst = { + 0x74, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, + 0x33, 0x32, 0x2d, 0x62, 0x79, 0x74, 0x65, 0x20, + 0x6b, 0x65, 0x79, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x50, 0x6f, 0x6c, 0x79, 0x31, 0x33, 0x30, 0x35 +}; + +static const u8 input21[] __initconst = { + 0x89, 0xda, 0xb8, 0x0b, 0x77, 0x17, 0xc1, 0xdb, + 0x5d, 0xb4, 0x37, 0x86, 0x0a, 0x3f, 0x70, 0x21, + 0x8e, 0x93, 0xe1, 0xb8, 0xf4, 0x61, 0xfb, 0x67, + 0x7f, 0x16, 0xf3, 0x5f, 0x6f, 0x87, 0xe2, 0xa9, + 0x1c, 0x99, 0xbc, 0x3a, 0x47, 0xac, 0xe4, 0x76, + 0x40, 0xcc, 0x95, 0xc3, 0x45, 0xbe, 0x5e, 0xcc, + 0xa5, 0xa3, 0x52, 0x3c, 0x35, 0xcc, 0x01, 0x89, + 0x3a, 0xf0, 0xb6, 0x4a, 0x62, 0x03, 0x34, 0x27, + 0x03, 0x72, 0xec, 0x12, 0x48, 0x2d, 0x1b, 0x1e, + 0x36, 0x35, 0x61, 0x69, 0x8a, 0x57, 0x8b, 0x35, + 0x98, 0x03, 0x49, 0x5b, 0xb4, 0xe2, 0xef, 0x19, + 0x30, 0xb1, 0x7a, 0x51, 0x90, 0xb5, 0x80, 0xf1, + 0x41, 0x30, 0x0d, 0xf3, 0x0a, 0xdb, 0xec, 0xa2, + 0x8f, 0x64, 0x27, 0xa8, 0xbc, 0x1a, 0x99, 0x9f, + 0xd5, 0x1c, 0x55, 0x4a, 0x01, 0x7d, 0x09, 0x5d, + 0x8c, 0x3e, 0x31, 0x27, 0xda, 0xf9, 0xf5, 0x95 +}; +static const u8 output21[] __initconst = { + 0xc8, 0x5d, 0x15, 0xed, 0x44, 0xc3, 0x78, 0xd6, + 0xb0, 0x0e, 0x23, 0x06, 0x4c, 0x7b, 0xcd, 0x51 +}; +static const u8 key21[] __initconst = { + 0x2d, 0x77, 0x3b, 0xe3, 0x7a, 0xdb, 0x1e, 0x4d, + 0x68, 0x3b, 0xf0, 0x07, 0x5e, 0x79, 0xc4, 0xee, + 0x03, 0x79, 0x18, 0x53, 0x5a, 0x7f, 0x99, 0xcc, + 0xb7, 0x04, 0x0f, 0xb5, 0xf5, 0xf4, 0x3a, 0xea +}; + +static const u8 input22[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, + 0x17, 0x03, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x06, 0xdb, 0x1f, 0x1f, 0x36, 0x8d, 0x69, 0x6a, + 0x81, 0x0a, 0x34, 0x9c, 0x0c, 0x71, 0x4c, 0x9a, + 0x5e, 0x78, 0x50, 0xc2, 0x40, 0x7d, 0x72, 0x1a, + 0xcd, 0xed, 0x95, 0xe0, 0x18, 0xd7, 0xa8, 0x52, + 0x66, 0xa6, 0xe1, 0x28, 0x9c, 0xdb, 0x4a, 0xeb, + 0x18, 0xda, 0x5a, 0xc8, 0xa2, 0xb0, 0x02, 0x6d, + 0x24, 0xa5, 0x9a, 0xd4, 0x85, 0x22, 0x7f, 0x3e, + 0xae, 0xdb, 0xb2, 0xe7, 0xe3, 0x5e, 0x1c, 0x66, + 0xcd, 0x60, 0xf9, 0xab, 0xf7, 0x16, 0xdc, 0xc9, + 0xac, 0x42, 0x68, 0x2d, 0xd7, 0xda, 0xb2, 0x87, + 0xa7, 0x02, 0x4c, 0x4e, 0xef, 0xc3, 0x21, 0xcc, + 0x05, 0x74, 0xe1, 0x67, 0x93, 0xe3, 0x7c, 0xec, + 0x03, 0xc5, 0xbd, 0xa4, 0x2b, 0x54, 0xc1, 0x14, + 0xa8, 0x0b, 0x57, 0xaf, 0x26, 0x41, 0x6c, 0x7b, + 0xe7, 0x42, 0x00, 0x5e, 0x20, 0x85, 0x5c, 0x73, + 0xe2, 0x1d, 0xc8, 0xe2, 0xed, 0xc9, 0xd4, 0x35, + 0xcb, 0x6f, 0x60, 0x59, 0x28, 0x00, 0x11, 0xc2, + 0x70, 0xb7, 0x15, 0x70, 0x05, 0x1c, 0x1c, 0x9b, + 0x30, 0x52, 0x12, 0x66, 0x20, 0xbc, 0x1e, 0x27, + 0x30, 0xfa, 0x06, 0x6c, 0x7a, 0x50, 0x9d, 0x53, + 0xc6, 0x0e, 0x5a, 0xe1, 0xb4, 0x0a, 0xa6, 0xe3, + 0x9e, 0x49, 0x66, 0x92, 0x28, 0xc9, 0x0e, 0xec, + 0xb4, 0xa5, 0x0d, 0xb3, 0x2a, 0x50, 0xbc, 0x49, + 0xe9, 0x0b, 0x4f, 0x4b, 0x35, 0x9a, 0x1d, 0xfd, + 0x11, 0x74, 0x9c, 0xd3, 0x86, 0x7f, 0xcf, 0x2f, + 0xb7, 0xbb, 0x6c, 0xd4, 0x73, 0x8f, 0x6a, 0x4a, + 0xd6, 0xf7, 0xca, 0x50, 0x58, 0xf7, 0x61, 0x88, + 0x45, 0xaf, 0x9f, 0x02, 0x0f, 0x6c, 0x3b, 0x96, + 0x7b, 0x8f, 0x4c, 0xd4, 0xa9, 0x1e, 0x28, 0x13, + 0xb5, 0x07, 0xae, 0x66, 0xf2, 0xd3, 0x5c, 0x18, + 0x28, 0x4f, 0x72, 0x92, 0x18, 0x60, 0x62, 0xe1, + 0x0f, 0xd5, 0x51, 0x0d, 0x18, 0x77, 0x53, 0x51, + 0xef, 0x33, 0x4e, 0x76, 0x34, 0xab, 0x47, 0x43, + 0xf5, 0xb6, 0x8f, 0x49, 0xad, 0xca, 0xb3, 0x84, + 0xd3, 0xfd, 0x75, 0xf7, 0x39, 0x0f, 0x40, 0x06, + 0xef, 0x2a, 0x29, 0x5c, 0x8c, 0x7a, 0x07, 0x6a, + 0xd5, 0x45, 0x46, 0xcd, 0x25, 0xd2, 0x10, 0x7f, + 0xbe, 0x14, 0x36, 0xc8, 0x40, 0x92, 0x4a, 0xae, + 0xbe, 0x5b, 0x37, 0x08, 0x93, 0xcd, 0x63, 0xd1, + 0x32, 0x5b, 0x86, 0x16, 0xfc, 0x48, 0x10, 0x88, + 0x6b, 0xc1, 0x52, 0xc5, 0x32, 0x21, 0xb6, 0xdf, + 0x37, 0x31, 0x19, 0x39, 0x32, 0x55, 0xee, 0x72, + 0xbc, 0xaa, 0x88, 0x01, 0x74, 0xf1, 0x71, 0x7f, + 0x91, 0x84, 0xfa, 0x91, 0x64, 0x6f, 0x17, 0xa2, + 0x4a, 0xc5, 0x5d, 0x16, 0xbf, 0xdd, 0xca, 0x95, + 0x81, 0xa9, 0x2e, 0xda, 0x47, 0x92, 0x01, 0xf0, + 0xed, 0xbf, 0x63, 0x36, 0x00, 0xd6, 0x06, 0x6d, + 0x1a, 0xb3, 0x6d, 0x5d, 0x24, 0x15, 0xd7, 0x13, + 0x51, 0xbb, 0xcd, 0x60, 0x8a, 0x25, 0x10, 0x8d, + 0x25, 0x64, 0x19, 0x92, 0xc1, 0xf2, 0x6c, 0x53, + 0x1c, 0xf9, 0xf9, 0x02, 0x03, 0xbc, 0x4c, 0xc1, + 0x9f, 0x59, 0x27, 0xd8, 0x34, 0xb0, 0xa4, 0x71, + 0x16, 0xd3, 0x88, 0x4b, 0xbb, 0x16, 0x4b, 0x8e, + 0xc8, 0x83, 0xd1, 0xac, 0x83, 0x2e, 0x56, 0xb3, + 0x91, 0x8a, 0x98, 0x60, 0x1a, 0x08, 0xd1, 0x71, + 0x88, 0x15, 0x41, 0xd5, 0x94, 0xdb, 0x39, 0x9c, + 0x6a, 0xe6, 0x15, 0x12, 0x21, 0x74, 0x5a, 0xec, + 0x81, 0x4c, 0x45, 0xb0, 0xb0, 0x5b, 0x56, 0x54, + 0x36, 0xfd, 0x6f, 0x13, 0x7a, 0xa1, 0x0a, 0x0c, + 0x0b, 0x64, 0x37, 0x61, 0xdb, 0xd6, 0xf9, 0xa9, + 0xdc, 0xb9, 0x9b, 0x1a, 0x6e, 0x69, 0x08, 0x54, + 0xce, 0x07, 0x69, 0xcd, 0xe3, 0x97, 0x61, 0xd8, + 0x2f, 0xcd, 0xec, 0x15, 0xf0, 0xd9, 0x2d, 0x7d, + 0x8e, 0x94, 0xad, 0xe8, 0xeb, 0x83, 0xfb, 0xe0 +}; +static const u8 output22[] __initconst = { + 0x26, 0x37, 0x40, 0x8f, 0xe1, 0x30, 0x86, 0xea, + 0x73, 0xf9, 0x71, 0xe3, 0x42, 0x5e, 0x28, 0x20 +}; +static const u8 key22[] __initconst = { + 0x99, 0xe5, 0x82, 0x2d, 0xd4, 0x17, 0x3c, 0x99, + 0x5e, 0x3d, 0xae, 0x0d, 0xde, 0xfb, 0x97, 0x74, + 0x3f, 0xde, 0x3b, 0x08, 0x01, 0x34, 0xb3, 0x9f, + 0x76, 0xe9, 0xbf, 0x8d, 0x0e, 0x88, 0xd5, 0x46 +}; + +/* test vectors from Hanno Böck */ +static const u8 input23[] __initconst = { + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0x80, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xce, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xc5, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xe3, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xac, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xe6, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x00, 0x00, 0x00, + 0xaf, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, + 0xcc, 0xcc, 0xff, 0xff, 0xff, 0xf5, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xff, 0xe7, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x71, 0x92, 0x05, 0xa8, 0x52, 0x1d, + 0xfc +}; +static const u8 output23[] __initconst = { + 0x85, 0x59, 0xb8, 0x76, 0xec, 0xee, 0xd6, 0x6e, + 0xb3, 0x77, 0x98, 0xc0, 0x45, 0x7b, 0xaf, 0xf9 +}; +static const u8 key23[] __initconst = { + 0x7f, 0x1b, 0x02, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc +}; + +static const u8 input24[] __initconst = { + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x02, 0x64 +}; +static const u8 output24[] __initconst = { + 0x00, 0xbd, 0x12, 0x58, 0x97, 0x8e, 0x20, 0x54, + 0x44, 0xc9, 0xaa, 0xaa, 0x82, 0x00, 0x6f, 0xed +}; +static const u8 key24[] __initconst = { + 0xe0, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, + 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa +}; + +static const u8 input25[] __initconst = { + 0x02, 0xfc +}; +static const u8 output25[] __initconst = { + 0x06, 0x12, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c +}; +static const u8 key25[] __initconst = { + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c +}; + +static const u8 input26[] __initconst = { + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7a, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x5c, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x6e, 0x7b, 0x00, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7a, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x5c, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, 0x7b, + 0x7b, 0x6e, 0x7b, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x00, 0xef, 0xff, 0x00, + 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x64, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, + 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x00, 0xef, 0xff, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x7a, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x09, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc +}; +static const u8 output26[] __initconst = { + 0x33, 0x20, 0x5b, 0xbf, 0x9e, 0x9f, 0x8f, 0x72, + 0x12, 0xab, 0x9e, 0x2a, 0xb9, 0xb7, 0xe4, 0xa5 +}; +static const u8 key26[] __initconst = { + 0x00, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0x7b +}; + +static const u8 input27[] __initconst = { + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0xff, 0xff, 0xff, 0xe9, + 0xe9, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, + 0xac, 0xac, 0xac, 0xac, 0x00, 0x00, 0xac, 0xac, + 0xec, 0x01, 0x00, 0xac, 0xac, 0xac, 0x2c, 0xac, + 0xa2, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, 0xac, + 0xac, 0xac, 0xac, 0xac, 0x64, 0xf2 +}; +static const u8 output27[] __initconst = { + 0x02, 0xee, 0x7c, 0x8c, 0x54, 0x6d, 0xde, 0xb1, + 0xa4, 0x67, 0xe4, 0xc3, 0x98, 0x11, 0x58, 0xb9 +}; +static const u8 key27[] __initconst = { + 0x00, 0x00, 0x00, 0x7f, 0x00, 0x00, 0x00, 0x7f, + 0x01, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xcf, 0x77, 0x77, 0x77, 0x77, 0x77, + 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77, 0x77 +}; + +/* nacl */ +static const u8 input28[] __initconst = { + 0x8e, 0x99, 0x3b, 0x9f, 0x48, 0x68, 0x12, 0x73, + 0xc2, 0x96, 0x50, 0xba, 0x32, 0xfc, 0x76, 0xce, + 0x48, 0x33, 0x2e, 0xa7, 0x16, 0x4d, 0x96, 0xa4, + 0x47, 0x6f, 0xb8, 0xc5, 0x31, 0xa1, 0x18, 0x6a, + 0xc0, 0xdf, 0xc1, 0x7c, 0x98, 0xdc, 0xe8, 0x7b, + 0x4d, 0xa7, 0xf0, 0x11, 0xec, 0x48, 0xc9, 0x72, + 0x71, 0xd2, 0xc2, 0x0f, 0x9b, 0x92, 0x8f, 0xe2, + 0x27, 0x0d, 0x6f, 0xb8, 0x63, 0xd5, 0x17, 0x38, + 0xb4, 0x8e, 0xee, 0xe3, 0x14, 0xa7, 0xcc, 0x8a, + 0xb9, 0x32, 0x16, 0x45, 0x48, 0xe5, 0x26, 0xae, + 0x90, 0x22, 0x43, 0x68, 0x51, 0x7a, 0xcf, 0xea, + 0xbd, 0x6b, 0xb3, 0x73, 0x2b, 0xc0, 0xe9, 0xda, + 0x99, 0x83, 0x2b, 0x61, 0xca, 0x01, 0xb6, 0xde, + 0x56, 0x24, 0x4a, 0x9e, 0x88, 0xd5, 0xf9, 0xb3, + 0x79, 0x73, 0xf6, 0x22, 0xa4, 0x3d, 0x14, 0xa6, + 0x59, 0x9b, 0x1f, 0x65, 0x4c, 0xb4, 0x5a, 0x74, + 0xe3, 0x55, 0xa5 +}; +static const u8 output28[] __initconst = { + 0xf3, 0xff, 0xc7, 0x70, 0x3f, 0x94, 0x00, 0xe5, + 0x2a, 0x7d, 0xfb, 0x4b, 0x3d, 0x33, 0x05, 0xd9 +}; +static const u8 key28[] __initconst = { + 0xee, 0xa6, 0xa7, 0x25, 0x1c, 0x1e, 0x72, 0x91, + 0x6d, 0x11, 0xc2, 0xcb, 0x21, 0x4d, 0x3c, 0x25, + 0x25, 0x39, 0x12, 0x1d, 0x8e, 0x23, 0x4e, 0x65, + 0x2d, 0x65, 0x1f, 0xa4, 0xc8, 0xcf, 0xf8, 0x80 +}; + +/* wrap 2^130-5 */ +static const u8 input29[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output29[] __initconst = { + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key29[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* wrap 2^128 */ +static const u8 input30[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output30[] __initconst = { + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key30[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; + +/* limb carry */ +static const u8 input31[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output31[] __initconst = { + 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key31[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 2^130-5 */ +static const u8 input32[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfb, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 +}; +static const u8 output32[] __initconst = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key32[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 2^130-6 */ +static const u8 input33[] __initconst = { + 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 output33[] __initconst = { + 0xfa, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 key33[] __initconst = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 5*H+L reduction intermediate */ +static const u8 input34[] __initconst = { + 0xe3, 0x35, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0xb9, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0x79, 0xcd, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output34[] __initconst = { + 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key34[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +/* 5*H+L reduction final */ +static const u8 input35[] __initconst = { + 0xe3, 0x35, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0xb9, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x94, 0xd7, 0x50, 0x5e, 0x43, 0x79, 0xcd, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 output35[] __initconst = { + 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 key35[] __initconst = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +static const struct poly1305_testvec poly1305_testvecs[] __initconst = { + { input01, output01, key01, sizeof(input01) }, + { input02, output02, key02, sizeof(input02) }, + { input03, output03, key03, sizeof(input03) }, + { input04, output04, key04, sizeof(input04) }, + { input05, output05, key05, sizeof(input05) }, + { input06, output06, key06, sizeof(input06) }, + { input07, output07, key07, sizeof(input07) }, + { input08, output08, key08, sizeof(input08) }, + { input09, output09, key09, sizeof(input09) }, + { input10, output10, key10, sizeof(input10) }, + { input11, output11, key11, sizeof(input11) }, + { input12, output12, key12, sizeof(input12) }, + { input13, output13, key13, sizeof(input13) }, + { input14, output14, key14, sizeof(input14) }, + { input15, output15, key15, sizeof(input15) }, + { input16, output16, key16, sizeof(input16) }, + { input17, output17, key17, sizeof(input17) }, + { input18, output18, key18, sizeof(input18) }, + { input19, output19, key19, sizeof(input19) }, + { input20, output20, key20, sizeof(input20) }, + { input21, output21, key21, sizeof(input21) }, + { input22, output22, key22, sizeof(input22) }, + { input23, output23, key23, sizeof(input23) }, + { input24, output24, key24, sizeof(input24) }, + { input25, output25, key25, sizeof(input25) }, + { input26, output26, key26, sizeof(input26) }, + { input27, output27, key27, sizeof(input27) }, + { input28, output28, key28, sizeof(input28) }, + { input29, output29, key29, sizeof(input29) }, + { input30, output30, key30, sizeof(input30) }, + { input31, output31, key31, sizeof(input31) }, + { input32, output32, key32, sizeof(input32) }, + { input33, output33, key33, sizeof(input33) }, + { input34, output34, key34, sizeof(input34) }, + { input35, output35, key35, sizeof(input35) } +}; + +static bool __init poly1305_selftest(void) +{ + simd_context_t simd_context; + bool success = true; + size_t i, j; + + simd_get(&simd_context); + for (i = 0; i < ARRAY_SIZE(poly1305_testvecs); ++i) { + struct poly1305_ctx poly1305; + u8 out[POLY1305_MAC_SIZE]; + + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + poly1305_testvecs[i].ilen, &simd_context); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu: FAIL\n", i + 1); + success = false; + } + simd_relax(&simd_context); + + if (poly1305_testvecs[i].ilen <= 1) + continue; + + for (j = 1; j < poly1305_testvecs[i].ilen - 1; ++j) { + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + j, &simd_context); + poly1305_update(&poly1305, + poly1305_testvecs[i].input + j, + poly1305_testvecs[i].ilen - j, + &simd_context); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu (split %zu): FAIL\n", + i + 1, j); + success = false; + } + + memset(out, 0, sizeof(out)); + memset(&poly1305, 0, sizeof(poly1305)); + poly1305_init(&poly1305, poly1305_testvecs[i].key); + poly1305_update(&poly1305, poly1305_testvecs[i].input, + j, &simd_context); + poly1305_update(&poly1305, + poly1305_testvecs[i].input + j, + poly1305_testvecs[i].ilen - j, + DONT_USE_SIMD); + poly1305_final(&poly1305, out, &simd_context); + if (memcmp(out, poly1305_testvecs[i].output, + POLY1305_MAC_SIZE)) { + pr_err("poly1305 self-test %zu (split %zu, mixed simd): FAIL\n", + i + 1, j); + success = false; + } + simd_relax(&simd_context); + } + } + simd_put(&simd_context); + + return success; +} diff --git a/net/wireguard/crypto/zinc/selftest/run.h b/net/wireguard/crypto/zinc/selftest/run.h new file mode 100644 index 0000000000000..4ffaf6089eeaf --- /dev/null +++ b/net/wireguard/crypto/zinc/selftest/run.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _ZINC_SELFTEST_RUN_H +#define _ZINC_SELFTEST_RUN_H + +#include +#include +#include + +static inline bool selftest_run(const char *name, bool (*selftest)(void), + bool *const nobs[], unsigned int nobs_len) +{ + unsigned long set = 0, subset = 0, largest_subset = 0; + unsigned int i; + + BUILD_BUG_ON(!__builtin_constant_p(nobs_len) || + nobs_len >= BITS_PER_LONG); + + if (!IS_ENABLED(CONFIG_ZINC_SELFTEST)) + return true; + + for (i = 0; i < nobs_len; ++i) + set |= ((unsigned long)*nobs[i]) << i; + + do { + for (i = 0; i < nobs_len; ++i) + *nobs[i] = BIT(i) & subset; + if (selftest()) + largest_subset = max(subset, largest_subset); + else + pr_err("%s self-test combination 0x%lx: FAIL\n", name, + subset); + subset = (subset - set) & set; + } while (subset); + + for (i = 0; i < nobs_len; ++i) + *nobs[i] = BIT(i) & largest_subset; + + if (largest_subset == set) + pr_info("%s self-tests: pass\n", name); + + return !WARN_ON(largest_subset != set); +} + +#endif diff --git a/net/wireguard/device.c b/net/wireguard/device.c new file mode 100644 index 0000000000000..e888ac37f9a89 --- /dev/null +++ b/net/wireguard/device.c @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "socket.h" +#include "timers.h" +#include "device.h" +#include "ratelimiter.h" +#include "peer.h" +#include "messages.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static LIST_HEAD(device_list); + +static int wg_open(struct net_device *dev) +{ + struct in_device *dev_v4 = __in_dev_get_rtnl(dev); +#ifndef COMPAT_CANNOT_USE_IN6_DEV_GET + struct inet6_dev *dev_v6 = __in6_dev_get(dev); +#endif + struct wg_device *wg = netdev_priv(dev); + struct wg_peer *peer; + int ret; + + if (dev_v4) { + /* At some point we might put this check near the ip_rt_send_ + * redirect call of ip_forward in net/ipv4/ip_forward.c, similar + * to the current secpath check. + */ + IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); + IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; + } +#ifndef COMPAT_CANNOT_USE_IN6_DEV_GET + if (dev_v6) +#ifndef COMPAT_CANNOT_USE_DEV_CNF + dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; +#else + dev_v6->addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; +#endif +#endif + + ret = wg_socket_init(wg, wg->incoming_port); + if (ret < 0) + return ret; + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_packet_send_staged_packets(peer); + if (peer->persistent_keepalive_interval) + wg_packet_send_keepalive(peer); + } + mutex_unlock(&wg->device_update_lock); + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int wg_pm_notification(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct wg_device *wg; + struct wg_peer *peer; + + /* If the machine is constantly suspending and resuming, as part of + * its normal operation rather than as a somewhat rare event, then we + * don't actually want to clear keys. + */ + if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID)) + return 0; + + if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) + return 0; + + rtnl_lock(); + list_for_each_entry(wg, &device_list, device_list) { + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + del_timer(&peer->timer_zero_key_material); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + } + mutex_unlock(&wg->device_update_lock); + } + rtnl_unlock(); + rcu_barrier(); + return 0; +} + +static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification }; +#endif + +static int wg_stop(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + struct wg_peer *peer; + + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_packet_purge_staged_packets(peer); + wg_timers_stop(peer); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + } + mutex_unlock(&wg->device_update_lock); + skb_queue_purge(&wg->incoming_handshakes); + wg_socket_reinit(wg, NULL, NULL); + return 0; +} + +static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + struct sk_buff_head packets; + struct wg_peer *peer; + struct sk_buff *next; + sa_family_t family; + u32 mtu; + int ret; + + if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol)) { + ret = -EPROTONOSUPPORT; + net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); + goto err; + } + + peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb); + if (unlikely(!peer)) { + ret = -ENOKEY; + if (skb->protocol == htons(ETH_P_IP)) + net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n", + dev->name, &ip_hdr(skb)->daddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", + dev->name, &ipv6_hdr(skb)->daddr); + goto err; + } + + family = READ_ONCE(peer->endpoint.addr.sa_family); + if (unlikely(family != AF_INET && family != AF_INET6)) { + ret = -EDESTADDRREQ; + net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n", + dev->name, peer->internal_id); + goto err_peer; + } + + mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu; + + __skb_queue_head_init(&packets); + if (!skb_is_gso(skb)) { + skb_mark_not_on_list(skb); + } else { + struct sk_buff *segs = skb_gso_segment(skb, 0); + + if (unlikely(IS_ERR(segs))) { + ret = PTR_ERR(segs); + goto err_peer; + } + dev_kfree_skb(skb); + skb = segs; + } + + skb_list_walk_safe(skb, skb, next) { + skb_mark_not_on_list(skb); + + skb = skb_share_check(skb, GFP_ATOMIC); + if (unlikely(!skb)) + continue; + + /* We only need to keep the original dst around for icmp, + * so at this point we're in a position to drop it. + */ + skb_dst_drop(skb); + + PACKET_CB(skb)->mtu = mtu; + + __skb_queue_tail(&packets, skb); + } + + spin_lock_bh(&peer->staged_packet_queue.lock); + /* If the queue is getting too big, we start removing the oldest packets + * until it's small again. We do this before adding the new packet, so + * we don't remove GSO segments that are in excess. + */ + while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) { + dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue)); + ++dev->stats.tx_dropped; + } + skb_queue_splice_tail(&packets, &peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); + + wg_packet_send_staged_packets(peer); + + wg_peer_put(peer); + return NETDEV_TX_OK; + +err_peer: + wg_peer_put(peer); +err: + ++dev->stats.tx_errors; + if (skb->protocol == htons(ETH_P_IP)) + icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); + else if (skb->protocol == htons(ETH_P_IPV6)) + icmpv6_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); + kfree_skb(skb); + return ret; +} + +static const struct net_device_ops netdev_ops = { + .ndo_open = wg_open, + .ndo_stop = wg_stop, + .ndo_start_xmit = wg_xmit, + .ndo_get_stats64 = ip_tunnel_get_stats64 +}; + +static void wg_destruct(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + + rtnl_lock(); + list_del(&wg->device_list); + rtnl_unlock(); + mutex_lock(&wg->device_update_lock); + wg->incoming_port = 0; + wg_socket_reinit(wg, NULL, NULL); + /* The final references are cleared in the below calls to destroy_workqueue. */ + wg_peer_remove_all(wg); + destroy_workqueue(wg->handshake_receive_wq); + destroy_workqueue(wg->handshake_send_wq); + destroy_workqueue(wg->packet_crypt_wq); + wg_packet_queue_free(&wg->decrypt_queue, true); + wg_packet_queue_free(&wg->encrypt_queue, true); + rcu_barrier(); /* Wait for all the peers to be actually freed. */ + wg_ratelimiter_uninit(); + memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); + skb_queue_purge(&wg->incoming_handshakes); + free_percpu(dev->tstats); + free_percpu(wg->incoming_handshakes_worker); + if (wg->have_creating_net_ref) + put_net(wg->creating_net); + kvfree(wg->index_hashtable); + kvfree(wg->peer_hashtable); + mutex_unlock(&wg->device_update_lock); + + pr_debug("%s: Interface deleted\n", dev->name); + free_netdev(dev); +} + +static const struct device_type device_type = { .name = KBUILD_MODNAME }; + +static void wg_setup(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM | + NETIF_F_SG | NETIF_F_GSO | + NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; + + dev->netdev_ops = &netdev_ops; + dev->hard_header_len = 0; + dev->addr_len = 0; + dev->needed_headroom = DATA_PACKET_HEAD_ROOM; + dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); + dev->type = ARPHRD_NONE; + dev->flags = IFF_POINTOPOINT | IFF_NOARP; +#ifndef COMPAT_CANNOT_USE_IFF_NO_QUEUE + dev->priv_flags |= IFF_NO_QUEUE; +#else + dev->tx_queue_len = 0; +#endif + dev->features |= NETIF_F_LLTX; + dev->features |= WG_NETDEV_FEATURES; + dev->hw_features |= WG_NETDEV_FEATURES; + dev->hw_enc_features |= WG_NETDEV_FEATURES; + dev->mtu = ETH_DATA_LEN - MESSAGE_MINIMUM_LENGTH - + sizeof(struct udphdr) - + max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); + + SET_NETDEV_DEVTYPE(dev, &device_type); + + /* We need to keep the dst around in case of icmp replies. */ + netif_keep_dst(dev); + + memset(wg, 0, sizeof(*wg)); + wg->dev = dev; +} + +static int wg_newlink(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + struct wg_device *wg = netdev_priv(dev); + int ret = -ENOMEM; + + wg->creating_net = src_net; + init_rwsem(&wg->static_identity.lock); + mutex_init(&wg->socket_update_lock); + mutex_init(&wg->device_update_lock); + skb_queue_head_init(&wg->incoming_handshakes); + wg_allowedips_init(&wg->peer_allowedips); + wg_cookie_checker_init(&wg->cookie_checker, wg); + INIT_LIST_HEAD(&wg->peer_list); + wg->device_update_gen = 1; + + wg->peer_hashtable = wg_pubkey_hashtable_alloc(); + if (!wg->peer_hashtable) + return ret; + + wg->index_hashtable = wg_index_hashtable_alloc(); + if (!wg->index_hashtable) + goto err_free_peer_hashtable; + + dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); + if (!dev->tstats) + goto err_free_index_hashtable; + + wg->incoming_handshakes_worker = + wg_packet_percpu_multicore_worker_alloc( + wg_packet_handshake_receive_worker, wg); + if (!wg->incoming_handshakes_worker) + goto err_free_tstats; + + wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s", + WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name); + if (!wg->handshake_receive_wq) + goto err_free_incoming_handshakes; + + wg->handshake_send_wq = alloc_workqueue("wg-kex-%s", + WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name); + if (!wg->handshake_send_wq) + goto err_destroy_handshake_receive; + + wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s", + WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name); + if (!wg->packet_crypt_wq) + goto err_destroy_handshake_send; + + ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, + true, MAX_QUEUED_PACKETS); + if (ret < 0) + goto err_destroy_packet_crypt; + + ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, + true, MAX_QUEUED_PACKETS); + if (ret < 0) + goto err_free_encrypt_queue; + + ret = wg_ratelimiter_init(); + if (ret < 0) + goto err_free_decrypt_queue; + + ret = register_netdevice(dev); + if (ret < 0) + goto err_uninit_ratelimiter; + + list_add(&wg->device_list, &device_list); + + /* We wait until the end to assign priv_destructor, so that + * register_netdevice doesn't call it for us if it fails. + */ + dev->priv_destructor = wg_destruct; + + pr_debug("%s: Interface created\n", dev->name); + return ret; + +err_uninit_ratelimiter: + wg_ratelimiter_uninit(); +err_free_decrypt_queue: + wg_packet_queue_free(&wg->decrypt_queue, true); +err_free_encrypt_queue: + wg_packet_queue_free(&wg->encrypt_queue, true); +err_destroy_packet_crypt: + destroy_workqueue(wg->packet_crypt_wq); +err_destroy_handshake_send: + destroy_workqueue(wg->handshake_send_wq); +err_destroy_handshake_receive: + destroy_workqueue(wg->handshake_receive_wq); +err_free_incoming_handshakes: + free_percpu(wg->incoming_handshakes_worker); +err_free_tstats: + free_percpu(dev->tstats); +err_free_index_hashtable: + kvfree(wg->index_hashtable); +err_free_peer_hashtable: + kvfree(wg->peer_hashtable); + return ret; +} + +static struct rtnl_link_ops link_ops __read_mostly = { + .kind = KBUILD_MODNAME, + .priv_size = sizeof(struct wg_device), + .setup = wg_setup, + .newlink = wg_newlink, +}; + +static int wg_netdevice_notification(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct net_device *dev = ((struct netdev_notifier_info *)data)->dev; + struct wg_device *wg = netdev_priv(dev); + + ASSERT_RTNL(); + + if (action != NETDEV_REGISTER || dev->netdev_ops != &netdev_ops) + return 0; + + if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) { + put_net(wg->creating_net); + wg->have_creating_net_ref = false; + } else if (dev_net(dev) != wg->creating_net && + !wg->have_creating_net_ref) { + wg->have_creating_net_ref = true; + get_net(wg->creating_net); + } + return 0; +} + +static struct notifier_block netdevice_notifier = { + .notifier_call = wg_netdevice_notification +}; + +int __init wg_device_init(void) +{ + int ret; + +#ifdef CONFIG_PM_SLEEP + ret = register_pm_notifier(&pm_notifier); + if (ret) + return ret; +#endif + + ret = register_netdevice_notifier(&netdevice_notifier); + if (ret) + goto error_pm; + + ret = rtnl_link_register(&link_ops); + if (ret) + goto error_netdevice; + + return 0; + +error_netdevice: + unregister_netdevice_notifier(&netdevice_notifier); +error_pm: +#ifdef CONFIG_PM_SLEEP + unregister_pm_notifier(&pm_notifier); +#endif + return ret; +} + +void wg_device_uninit(void) +{ + rtnl_link_unregister(&link_ops); + unregister_netdevice_notifier(&netdevice_notifier); +#ifdef CONFIG_PM_SLEEP + unregister_pm_notifier(&pm_notifier); +#endif + rcu_barrier(); +} diff --git a/net/wireguard/device.h b/net/wireguard/device.h new file mode 100644 index 0000000000000..c91f3051c5c78 --- /dev/null +++ b/net/wireguard/device.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_DEVICE_H +#define _WG_DEVICE_H + +#include "noise.h" +#include "allowedips.h" +#include "peerlookup.h" +#include "cookie.h" + +#include +#include +#include +#include +#include +#include + +struct wg_device; + +struct multicore_worker { + void *ptr; + struct work_struct work; +}; + +struct crypt_queue { + struct ptr_ring ring; + union { + struct { + struct multicore_worker __percpu *worker; + int last_cpu; + }; + struct work_struct work; + }; +}; + +struct wg_device { + struct net_device *dev; + struct crypt_queue encrypt_queue, decrypt_queue; + struct sock __rcu *sock4, *sock6; + struct net *creating_net; + struct noise_static_identity static_identity; + struct workqueue_struct *handshake_receive_wq, *handshake_send_wq; + struct workqueue_struct *packet_crypt_wq; + struct sk_buff_head incoming_handshakes; + int incoming_handshake_cpu; + struct multicore_worker __percpu *incoming_handshakes_worker; + struct cookie_checker cookie_checker; + struct pubkey_hashtable *peer_hashtable; + struct index_hashtable *index_hashtable; + struct allowedips peer_allowedips; + struct mutex device_update_lock, socket_update_lock; + struct list_head device_list, peer_list; + unsigned int num_peers, device_update_gen; + u32 fwmark; + u16 incoming_port; + bool have_creating_net_ref; +}; + +int wg_device_init(void); +void wg_device_uninit(void); + +/* Later after the dust settles, this can be moved into include/linux/skbuff.h, + * where virtually all code that deals with GSO segs can benefit, around ~30 + * drivers as of writing. + */ +#define skb_list_walk_safe(first, skb, next) \ + for (skb = first, next = skb->next; skb; \ + skb = next, next = skb ? skb->next : NULL) + +#endif /* _WG_DEVICE_H */ diff --git a/net/wireguard/main.c b/net/wireguard/main.c new file mode 100644 index 0000000000000..9ca4195bc9850 --- /dev/null +++ b/net/wireguard/main.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "version.h" +#include "device.h" +#include "noise.h" +#include "queueing.h" +#include "ratelimiter.h" +#include "netlink.h" +#include "uapi/wireguard.h" +#include "crypto/zinc.h" + +#include +#include +#include +#include + +static int __init mod_init(void) +{ + int ret; + + if ((ret = chacha20_mod_init()) || (ret = poly1305_mod_init()) || + (ret = chacha20poly1305_mod_init()) || (ret = blake2s_mod_init()) || + (ret = curve25519_mod_init())) + return ret; + +#ifdef DEBUG + if (!wg_allowedips_selftest() || !wg_packet_counter_selftest() || + !wg_ratelimiter_selftest()) + return -ENOTRECOVERABLE; +#endif + wg_noise_init(); + + ret = wg_device_init(); + if (ret < 0) + goto err_device; + + ret = wg_genetlink_init(); + if (ret < 0) + goto err_netlink; + + pr_info("WireGuard " WIREGUARD_VERSION " loaded. See www.wireguard.com for information.\n"); + pr_info("Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved.\n"); + + return 0; + +err_netlink: + wg_device_uninit(); +err_device: + return ret; +} + +static void __exit mod_exit(void) +{ + wg_genetlink_uninit(); + wg_device_uninit(); +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("WireGuard secure network tunnel"); +MODULE_AUTHOR("Jason A. Donenfeld "); +MODULE_VERSION(WIREGUARD_VERSION); +MODULE_ALIAS_RTNL_LINK(KBUILD_MODNAME); +MODULE_ALIAS_GENL_FAMILY(WG_GENL_NAME); diff --git a/net/wireguard/messages.h b/net/wireguard/messages.h new file mode 100644 index 0000000000000..f415cdd2e90a9 --- /dev/null +++ b/net/wireguard/messages.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_MESSAGES_H +#define _WG_MESSAGES_H + +#include +#include +#include + +#include +#include +#include + +enum noise_lengths { + NOISE_PUBLIC_KEY_LEN = CURVE25519_KEY_SIZE, + NOISE_SYMMETRIC_KEY_LEN = CHACHA20POLY1305_KEY_SIZE, + NOISE_TIMESTAMP_LEN = sizeof(u64) + sizeof(u32), + NOISE_AUTHTAG_LEN = CHACHA20POLY1305_AUTHTAG_SIZE, + NOISE_HASH_LEN = BLAKE2S_HASH_SIZE +}; + +#define noise_encrypted_len(plain_len) ((plain_len) + NOISE_AUTHTAG_LEN) + +enum cookie_values { + COOKIE_SECRET_MAX_AGE = 2 * 60, + COOKIE_SECRET_LATENCY = 5, + COOKIE_NONCE_LEN = XCHACHA20POLY1305_NONCE_SIZE, + COOKIE_LEN = 16 +}; + +enum counter_values { + COUNTER_BITS_TOTAL = 2048, + COUNTER_REDUNDANT_BITS = BITS_PER_LONG, + COUNTER_WINDOW_SIZE = COUNTER_BITS_TOTAL - COUNTER_REDUNDANT_BITS +}; + +enum limits { + REKEY_AFTER_MESSAGES = 1ULL << 60, + REJECT_AFTER_MESSAGES = U64_MAX - COUNTER_WINDOW_SIZE - 1, + REKEY_TIMEOUT = 5, + REKEY_TIMEOUT_JITTER_MAX_JIFFIES = HZ / 3, + REKEY_AFTER_TIME = 120, + REJECT_AFTER_TIME = 180, + INITIATIONS_PER_SECOND = 50, + MAX_PEERS_PER_DEVICE = 1U << 20, + KEEPALIVE_TIMEOUT = 10, + MAX_TIMER_HANDSHAKES = 90 / REKEY_TIMEOUT, + MAX_QUEUED_INCOMING_HANDSHAKES = 4096, /* TODO: replace this with DQL */ + MAX_STAGED_PACKETS = 128, + MAX_QUEUED_PACKETS = 1024 /* TODO: replace this with DQL */ +}; + +enum message_type { + MESSAGE_INVALID = 0, + MESSAGE_HANDSHAKE_INITIATION = 1, + MESSAGE_HANDSHAKE_RESPONSE = 2, + MESSAGE_HANDSHAKE_COOKIE = 3, + MESSAGE_DATA = 4 +}; + +struct message_header { + /* The actual layout of this that we want is: + * u8 type + * u8 reserved_zero[3] + * + * But it turns out that by encoding this as little endian, + * we achieve the same thing, and it makes checking faster. + */ + __le32 type; +}; + +struct message_macs { + u8 mac1[COOKIE_LEN]; + u8 mac2[COOKIE_LEN]; +}; + +struct message_handshake_initiation { + struct message_header header; + __le32 sender_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_static[noise_encrypted_len(NOISE_PUBLIC_KEY_LEN)]; + u8 encrypted_timestamp[noise_encrypted_len(NOISE_TIMESTAMP_LEN)]; + struct message_macs macs; +}; + +struct message_handshake_response { + struct message_header header; + __le32 sender_index; + __le32 receiver_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_nothing[noise_encrypted_len(0)]; + struct message_macs macs; +}; + +struct message_handshake_cookie { + struct message_header header; + __le32 receiver_index; + u8 nonce[COOKIE_NONCE_LEN]; + u8 encrypted_cookie[noise_encrypted_len(COOKIE_LEN)]; +}; + +struct message_data { + struct message_header header; + __le32 key_idx; + __le64 counter; + u8 encrypted_data[]; +}; + +#define message_data_len(plain_len) \ + (noise_encrypted_len(plain_len) + sizeof(struct message_data)) + +enum message_alignments { + MESSAGE_PADDING_MULTIPLE = 16, + MESSAGE_MINIMUM_LENGTH = message_data_len(0) +}; + +#define SKB_HEADER_LEN \ + (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ + sizeof(struct udphdr) + NET_SKB_PAD) +#define DATA_PACKET_HEAD_ROOM \ + ALIGN(sizeof(struct message_data) + SKB_HEADER_LEN, 4) + +enum { HANDSHAKE_DSCP = 0x88 /* AF41, plus 00 ECN */ }; + +#endif /* _WG_MESSAGES_H */ diff --git a/net/wireguard/netlink.c b/net/wireguard/netlink.c new file mode 100644 index 0000000000000..9bf2f84edda81 --- /dev/null +++ b/net/wireguard/netlink.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "netlink.h" +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "queueing.h" +#include "messages.h" +#include "uapi/wireguard.h" +#include +#include +#include +#include + +static struct genl_family genl_family; + +static const struct nla_policy device_policy[WGDEVICE_A_MAX + 1] = { + [WGDEVICE_A_IFINDEX] = { .type = NLA_U32 }, + [WGDEVICE_A_IFNAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, + [WGDEVICE_A_PRIVATE_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGDEVICE_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGDEVICE_A_FLAGS] = { .type = NLA_U32 }, + [WGDEVICE_A_LISTEN_PORT] = { .type = NLA_U16 }, + [WGDEVICE_A_FWMARK] = { .type = NLA_U32 }, + [WGDEVICE_A_PEERS] = { .type = NLA_NESTED } +}; + +static const struct nla_policy peer_policy[WGPEER_A_MAX + 1] = { + [WGPEER_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGPEER_A_PRESHARED_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_SYMMETRIC_KEY_LEN }, + [WGPEER_A_FLAGS] = { .type = NLA_U32 }, + [WGPEER_A_ENDPOINT] = { .type = NLA_MIN_LEN, .len = sizeof(struct sockaddr) }, + [WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL] = { .type = NLA_U16 }, + [WGPEER_A_LAST_HANDSHAKE_TIME] = { .type = NLA_EXACT_LEN, .len = sizeof(struct __kernel_timespec) }, + [WGPEER_A_RX_BYTES] = { .type = NLA_U64 }, + [WGPEER_A_TX_BYTES] = { .type = NLA_U64 }, + [WGPEER_A_ALLOWEDIPS] = { .type = NLA_NESTED }, + [WGPEER_A_PROTOCOL_VERSION] = { .type = NLA_U32 } +}; + +static const struct nla_policy allowedip_policy[WGALLOWEDIP_A_MAX + 1] = { + [WGALLOWEDIP_A_FAMILY] = { .type = NLA_U16 }, + [WGALLOWEDIP_A_IPADDR] = { .type = NLA_MIN_LEN, .len = sizeof(struct in_addr) }, + [WGALLOWEDIP_A_CIDR_MASK] = { .type = NLA_U8 } +}; + +static struct wg_device *lookup_interface(struct nlattr **attrs, + struct sk_buff *skb) +{ + struct net_device *dev = NULL; + + if (!attrs[WGDEVICE_A_IFINDEX] == !attrs[WGDEVICE_A_IFNAME]) + return ERR_PTR(-EBADR); + if (attrs[WGDEVICE_A_IFINDEX]) + dev = dev_get_by_index(sock_net(skb->sk), + nla_get_u32(attrs[WGDEVICE_A_IFINDEX])); + else if (attrs[WGDEVICE_A_IFNAME]) + dev = dev_get_by_name(sock_net(skb->sk), + nla_data(attrs[WGDEVICE_A_IFNAME])); + if (!dev) + return ERR_PTR(-ENODEV); + if (!dev->rtnl_link_ops || !dev->rtnl_link_ops->kind || + strcmp(dev->rtnl_link_ops->kind, KBUILD_MODNAME)) { + dev_put(dev); + return ERR_PTR(-EOPNOTSUPP); + } + return netdev_priv(dev); +} + +static int get_allowedips(struct sk_buff *skb, const u8 *ip, u8 cidr, + int family) +{ + struct nlattr *allowedip_nest; + + allowedip_nest = nla_nest_start(skb, 0); + if (!allowedip_nest) + return -EMSGSIZE; + + if (nla_put_u8(skb, WGALLOWEDIP_A_CIDR_MASK, cidr) || + nla_put_u16(skb, WGALLOWEDIP_A_FAMILY, family) || + nla_put(skb, WGALLOWEDIP_A_IPADDR, family == AF_INET6 ? + sizeof(struct in6_addr) : sizeof(struct in_addr), ip)) { + nla_nest_cancel(skb, allowedip_nest); + return -EMSGSIZE; + } + + nla_nest_end(skb, allowedip_nest); + return 0; +} + +struct dump_ctx { + struct wg_device *wg; + struct wg_peer *next_peer; + u64 allowedips_seq; + struct allowedips_node *next_allowedip; +}; + +#define DUMP_CTX(cb) ((struct dump_ctx *)(cb)->args) + +static int +get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) +{ + + struct nlattr *allowedips_nest, *peer_nest = nla_nest_start(skb, 0); + struct allowedips_node *allowedips_node = ctx->next_allowedip; + bool fail; + + if (!peer_nest) + return -EMSGSIZE; + + down_read(&peer->handshake.lock); + fail = nla_put(skb, WGPEER_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN, + peer->handshake.remote_static); + up_read(&peer->handshake.lock); + if (fail) + goto err; + + if (!allowedips_node) { + const struct __kernel_timespec last_handshake = { + .tv_sec = peer->walltime_last_handshake.tv_sec, + .tv_nsec = peer->walltime_last_handshake.tv_nsec + }; + + down_read(&peer->handshake.lock); + fail = nla_put(skb, WGPEER_A_PRESHARED_KEY, + NOISE_SYMMETRIC_KEY_LEN, + peer->handshake.preshared_key); + up_read(&peer->handshake.lock); + if (fail) + goto err; + + if (nla_put(skb, WGPEER_A_LAST_HANDSHAKE_TIME, + sizeof(last_handshake), &last_handshake) || + nla_put_u16(skb, WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, + peer->persistent_keepalive_interval) || + nla_put_u64_64bit(skb, WGPEER_A_TX_BYTES, peer->tx_bytes, + WGPEER_A_UNSPEC) || + nla_put_u64_64bit(skb, WGPEER_A_RX_BYTES, peer->rx_bytes, + WGPEER_A_UNSPEC) || + nla_put_u32(skb, WGPEER_A_PROTOCOL_VERSION, 1)) + goto err; + + read_lock_bh(&peer->endpoint_lock); + if (peer->endpoint.addr.sa_family == AF_INET) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->endpoint.addr4), + &peer->endpoint.addr4); + else if (peer->endpoint.addr.sa_family == AF_INET6) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->endpoint.addr6), + &peer->endpoint.addr6); + read_unlock_bh(&peer->endpoint_lock); + if (fail) + goto err; + allowedips_node = + list_first_entry_or_null(&peer->allowedips_list, + struct allowedips_node, peer_list); + } + if (!allowedips_node) + goto no_allowedips; + if (!ctx->allowedips_seq) + ctx->allowedips_seq = peer->device->peer_allowedips.seq; + else if (ctx->allowedips_seq != peer->device->peer_allowedips.seq) + goto no_allowedips; + + allowedips_nest = nla_nest_start(skb, WGPEER_A_ALLOWEDIPS); + if (!allowedips_nest) + goto err; + + list_for_each_entry_from(allowedips_node, &peer->allowedips_list, + peer_list) { + u8 cidr, ip[16] __aligned(__alignof(u64)); + int family; + + family = wg_allowedips_read_node(allowedips_node, ip, &cidr); + if (get_allowedips(skb, ip, cidr, family)) { + nla_nest_end(skb, allowedips_nest); + nla_nest_end(skb, peer_nest); + ctx->next_allowedip = allowedips_node; + return -EMSGSIZE; + } + } + nla_nest_end(skb, allowedips_nest); +no_allowedips: + nla_nest_end(skb, peer_nest); + ctx->next_allowedip = NULL; + ctx->allowedips_seq = 0; + return 0; +err: + nla_nest_cancel(skb, peer_nest); + return -EMSGSIZE; +} + +static int wg_get_device_start(struct netlink_callback *cb) +{ + struct wg_device *wg; + + wg = lookup_interface(genl_dumpit_info(cb)->attrs, cb->skb); + if (IS_ERR(wg)) + return PTR_ERR(wg); + DUMP_CTX(cb)->wg = wg; + return 0; +} + +static int wg_get_device_dump(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct wg_peer *peer, *next_peer_cursor; + struct dump_ctx *ctx = DUMP_CTX(cb); + struct wg_device *wg = ctx->wg; + struct nlattr *peers_nest; + int ret = -EMSGSIZE; + bool done = true; + void *hdr; + + rtnl_lock(); + mutex_lock(&wg->device_update_lock); + cb->seq = wg->device_update_gen; + next_peer_cursor = ctx->next_peer; + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, + &genl_family, NLM_F_MULTI, WG_CMD_GET_DEVICE); + if (!hdr) + goto out; + genl_dump_check_consistent(cb, hdr); + + if (!ctx->next_peer) { + if (nla_put_u16(skb, WGDEVICE_A_LISTEN_PORT, + wg->incoming_port) || + nla_put_u32(skb, WGDEVICE_A_FWMARK, wg->fwmark) || + nla_put_u32(skb, WGDEVICE_A_IFINDEX, wg->dev->ifindex) || + nla_put_string(skb, WGDEVICE_A_IFNAME, wg->dev->name)) + goto out; + + down_read(&wg->static_identity.lock); + if (wg->static_identity.has_identity) { + if (nla_put(skb, WGDEVICE_A_PRIVATE_KEY, + NOISE_PUBLIC_KEY_LEN, + wg->static_identity.static_private) || + nla_put(skb, WGDEVICE_A_PUBLIC_KEY, + NOISE_PUBLIC_KEY_LEN, + wg->static_identity.static_public)) { + up_read(&wg->static_identity.lock); + goto out; + } + } + up_read(&wg->static_identity.lock); + } + + peers_nest = nla_nest_start(skb, WGDEVICE_A_PEERS); + if (!peers_nest) + goto out; + ret = 0; + /* If the last cursor was removed via list_del_init in peer_remove, then + * we just treat this the same as there being no more peers left. The + * reason is that seq_nr should indicate to userspace that this isn't a + * coherent dump anyway, so they'll try again. + */ + if (list_empty(&wg->peer_list) || + (ctx->next_peer && list_empty(&ctx->next_peer->peer_list))) { + nla_nest_cancel(skb, peers_nest); + goto out; + } + lockdep_assert_held(&wg->device_update_lock); + peer = list_prepare_entry(ctx->next_peer, &wg->peer_list, peer_list); + list_for_each_entry_continue(peer, &wg->peer_list, peer_list) { + if (get_peer(peer, skb, ctx)) { + done = false; + break; + } + next_peer_cursor = peer; + } + nla_nest_end(skb, peers_nest); + +out: + if (!ret && !done && next_peer_cursor) + wg_peer_get(next_peer_cursor); + wg_peer_put(ctx->next_peer); + mutex_unlock(&wg->device_update_lock); + rtnl_unlock(); + + if (ret) { + genlmsg_cancel(skb, hdr); + return ret; + } + genlmsg_end(skb, hdr); + if (done) { + ctx->next_peer = NULL; + return 0; + } + ctx->next_peer = next_peer_cursor; + return skb->len; + + /* At this point, we can't really deal ourselves with safely zeroing out + * the private key material after usage. This will need an additional API + * in the kernel for marking skbs as zero_on_free. + */ +} + +static int wg_get_device_done(struct netlink_callback *cb) +{ + struct dump_ctx *ctx = DUMP_CTX(cb); + + if (ctx->wg) + dev_put(ctx->wg->dev); + wg_peer_put(ctx->next_peer); + return 0; +} + +static int set_port(struct wg_device *wg, u16 port) +{ + struct wg_peer *peer; + + if (wg->incoming_port == port) + return 0; + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_socket_clear_peer_endpoint_src(peer); + if (!netif_running(wg->dev)) { + wg->incoming_port = port; + return 0; + } + return wg_socket_init(wg, port); +} + +static int set_allowedip(struct wg_peer *peer, struct nlattr **attrs) +{ + int ret = -EINVAL; + u16 family; + u8 cidr; + + if (!attrs[WGALLOWEDIP_A_FAMILY] || !attrs[WGALLOWEDIP_A_IPADDR] || + !attrs[WGALLOWEDIP_A_CIDR_MASK]) + return ret; + family = nla_get_u16(attrs[WGALLOWEDIP_A_FAMILY]); + cidr = nla_get_u8(attrs[WGALLOWEDIP_A_CIDR_MASK]); + + if (family == AF_INET && cidr <= 32 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in_addr)) + ret = wg_allowedips_insert_v4( + &peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + else if (family == AF_INET6 && cidr <= 128 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in6_addr)) + ret = wg_allowedips_insert_v6( + &peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + + return ret; +} + +static int set_peer(struct wg_device *wg, struct nlattr **attrs) +{ + u8 *public_key = NULL, *preshared_key = NULL; + struct wg_peer *peer = NULL; + u32 flags = 0; + int ret; + + ret = -EINVAL; + if (attrs[WGPEER_A_PUBLIC_KEY] && + nla_len(attrs[WGPEER_A_PUBLIC_KEY]) == NOISE_PUBLIC_KEY_LEN) + public_key = nla_data(attrs[WGPEER_A_PUBLIC_KEY]); + else + goto out; + if (attrs[WGPEER_A_PRESHARED_KEY] && + nla_len(attrs[WGPEER_A_PRESHARED_KEY]) == NOISE_SYMMETRIC_KEY_LEN) + preshared_key = nla_data(attrs[WGPEER_A_PRESHARED_KEY]); + + if (attrs[WGPEER_A_FLAGS]) + flags = nla_get_u32(attrs[WGPEER_A_FLAGS]); + ret = -EOPNOTSUPP; + if (flags & ~__WGPEER_F_ALL) + goto out; + + ret = -EPFNOSUPPORT; + if (attrs[WGPEER_A_PROTOCOL_VERSION]) { + if (nla_get_u32(attrs[WGPEER_A_PROTOCOL_VERSION]) != 1) + goto out; + } + + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, + nla_data(attrs[WGPEER_A_PUBLIC_KEY])); + ret = 0; + if (!peer) { /* Peer doesn't exist yet. Add a new one. */ + if (flags & (WGPEER_F_REMOVE_ME | WGPEER_F_UPDATE_ONLY)) + goto out; + + /* The peer is new, so there aren't allowed IPs to remove. */ + flags &= ~WGPEER_F_REPLACE_ALLOWEDIPS; + + down_read(&wg->static_identity.lock); + if (wg->static_identity.has_identity && + !memcmp(nla_data(attrs[WGPEER_A_PUBLIC_KEY]), + wg->static_identity.static_public, + NOISE_PUBLIC_KEY_LEN)) { + /* We silently ignore peers that have the same public + * key as the device. The reason we do it silently is + * that we'd like for people to be able to reuse the + * same set of API calls across peers. + */ + up_read(&wg->static_identity.lock); + ret = 0; + goto out; + } + up_read(&wg->static_identity.lock); + + peer = wg_peer_create(wg, public_key, preshared_key); + if (IS_ERR(peer)) { + /* Similar to the above, if the key is invalid, we skip + * it without fanfare, so that services don't need to + * worry about doing key validation themselves. + */ + ret = PTR_ERR(peer) == -EKEYREJECTED ? 0 : PTR_ERR(peer); + peer = NULL; + goto out; + } + /* Take additional reference, as though we've just been + * looked up. + */ + wg_peer_get(peer); + } + + if (flags & WGPEER_F_REMOVE_ME) { + wg_peer_remove(peer); + goto out; + } + + if (preshared_key) { + down_write(&peer->handshake.lock); + memcpy(&peer->handshake.preshared_key, preshared_key, + NOISE_SYMMETRIC_KEY_LEN); + up_write(&peer->handshake.lock); + } + + if (attrs[WGPEER_A_ENDPOINT]) { + struct sockaddr *addr = nla_data(attrs[WGPEER_A_ENDPOINT]); + size_t len = nla_len(attrs[WGPEER_A_ENDPOINT]); + + if ((len == sizeof(struct sockaddr_in) && + addr->sa_family == AF_INET) || + (len == sizeof(struct sockaddr_in6) && + addr->sa_family == AF_INET6)) { + struct endpoint endpoint = { { { 0 } } }; + + memcpy(&endpoint.addr, addr, len); + wg_socket_set_peer_endpoint(peer, &endpoint); + } + } + + if (flags & WGPEER_F_REPLACE_ALLOWEDIPS) + wg_allowedips_remove_by_peer(&wg->peer_allowedips, peer, + &wg->device_update_lock); + + if (attrs[WGPEER_A_ALLOWEDIPS]) { + struct nlattr *attr, *allowedip[WGALLOWEDIP_A_MAX + 1]; + int rem; + + nla_for_each_nested(attr, attrs[WGPEER_A_ALLOWEDIPS], rem) { + ret = nla_parse_nested(allowedip, WGALLOWEDIP_A_MAX, + attr, allowedip_policy, NULL); + if (ret < 0) + goto out; + ret = set_allowedip(peer, allowedip); + if (ret < 0) + goto out; + } + } + + if (attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]) { + const u16 persistent_keepalive_interval = nla_get_u16( + attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]); + const bool send_keepalive = + !peer->persistent_keepalive_interval && + persistent_keepalive_interval && + netif_running(wg->dev); + + peer->persistent_keepalive_interval = persistent_keepalive_interval; + if (send_keepalive) + wg_packet_send_keepalive(peer); + } + + if (netif_running(wg->dev)) + wg_packet_send_staged_packets(peer); + +out: + wg_peer_put(peer); + if (attrs[WGPEER_A_PRESHARED_KEY]) + memzero_explicit(nla_data(attrs[WGPEER_A_PRESHARED_KEY]), + nla_len(attrs[WGPEER_A_PRESHARED_KEY])); + return ret; +} + +static int wg_set_device(struct sk_buff *skb, struct genl_info *info) +{ + struct wg_device *wg = lookup_interface(info->attrs, skb); + u32 flags = 0; + int ret; + + if (IS_ERR(wg)) { + ret = PTR_ERR(wg); + goto out_nodev; + } + + rtnl_lock(); + mutex_lock(&wg->device_update_lock); + + if (info->attrs[WGDEVICE_A_FLAGS]) + flags = nla_get_u32(info->attrs[WGDEVICE_A_FLAGS]); + ret = -EOPNOTSUPP; + if (flags & ~__WGDEVICE_F_ALL) + goto out; + + ret = -EPERM; + if ((info->attrs[WGDEVICE_A_LISTEN_PORT] || + info->attrs[WGDEVICE_A_FWMARK]) && + !ns_capable(wg->creating_net->user_ns, CAP_NET_ADMIN)) + goto out; + + ++wg->device_update_gen; + + if (info->attrs[WGDEVICE_A_FWMARK]) { + struct wg_peer *peer; + + wg->fwmark = nla_get_u32(info->attrs[WGDEVICE_A_FWMARK]); + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_socket_clear_peer_endpoint_src(peer); + } + + if (info->attrs[WGDEVICE_A_LISTEN_PORT]) { + ret = set_port(wg, + nla_get_u16(info->attrs[WGDEVICE_A_LISTEN_PORT])); + if (ret) + goto out; + } + + if (flags & WGDEVICE_F_REPLACE_PEERS) + wg_peer_remove_all(wg); + + if (info->attrs[WGDEVICE_A_PRIVATE_KEY] && + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]) == + NOISE_PUBLIC_KEY_LEN) { + u8 *private_key = nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]); + u8 public_key[NOISE_PUBLIC_KEY_LEN]; + struct wg_peer *peer, *temp; + + if (!crypto_memneq(wg->static_identity.static_private, + private_key, NOISE_PUBLIC_KEY_LEN)) + goto skip_set_private_key; + + /* We remove before setting, to prevent race, which means doing + * two 25519-genpub ops. + */ + if (curve25519_generate_public(public_key, private_key)) { + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, + public_key); + if (peer) { + wg_peer_put(peer); + wg_peer_remove(peer); + } + } + + down_write(&wg->static_identity.lock); + wg_noise_set_static_identity_private_key(&wg->static_identity, + private_key); + list_for_each_entry_safe(peer, temp, &wg->peer_list, + peer_list) { + if (wg_noise_precompute_static_static(peer)) + wg_noise_expire_current_peer_keypairs(peer); + else + wg_peer_remove(peer); + } + wg_cookie_checker_precompute_device_keys(&wg->cookie_checker); + up_write(&wg->static_identity.lock); + } +skip_set_private_key: + + if (info->attrs[WGDEVICE_A_PEERS]) { + struct nlattr *attr, *peer[WGPEER_A_MAX + 1]; + int rem; + + nla_for_each_nested(attr, info->attrs[WGDEVICE_A_PEERS], rem) { + ret = nla_parse_nested(peer, WGPEER_A_MAX, attr, + peer_policy, NULL); + if (ret < 0) + goto out; + ret = set_peer(wg, peer); + if (ret < 0) + goto out; + } + } + ret = 0; + +out: + mutex_unlock(&wg->device_update_lock); + rtnl_unlock(); + dev_put(wg->dev); +out_nodev: + if (info->attrs[WGDEVICE_A_PRIVATE_KEY]) + memzero_explicit(nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]), + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY])); + return ret; +} + +#ifndef COMPAT_CANNOT_USE_CONST_GENL_OPS +static const +#else +static +#endif +struct genl_ops genl_ops[] = { + { + .cmd = WG_CMD_GET_DEVICE, +#ifndef COMPAT_CANNOT_USE_NETLINK_START + .start = wg_get_device_start, +#endif + .dumpit = wg_get_device_dump, + .done = wg_get_device_done, +#ifdef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .flags = GENL_UNS_ADMIN_PERM + }, { + .cmd = WG_CMD_SET_DEVICE, + .doit = wg_set_device, +#ifdef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .flags = GENL_UNS_ADMIN_PERM + } +}; + +static struct genl_family genl_family +#ifndef COMPAT_CANNOT_USE_GENL_NOPS +__ro_after_init = { + .ops = genl_ops, + .n_ops = ARRAY_SIZE(genl_ops), +#else += { +#endif + .name = WG_GENL_NAME, + .version = WG_GENL_VERSION, + .maxattr = WGDEVICE_A_MAX, + .module = THIS_MODULE, +#ifndef COMPAT_CANNOT_INDIVIDUAL_NETLINK_OPS_POLICY + .policy = device_policy, +#endif + .netnsok = true +}; + +int __init wg_genetlink_init(void) +{ + return genl_register_family(&genl_family); +} + +void __exit wg_genetlink_uninit(void) +{ + genl_unregister_family(&genl_family); +} diff --git a/net/wireguard/netlink.h b/net/wireguard/netlink.h new file mode 100644 index 0000000000000..15100d92e2e32 --- /dev/null +++ b/net/wireguard/netlink.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_NETLINK_H +#define _WG_NETLINK_H + +int wg_genetlink_init(void); +void wg_genetlink_uninit(void); + +#endif /* _WG_NETLINK_H */ diff --git a/net/wireguard/noise.c b/net/wireguard/noise.c new file mode 100644 index 0000000000000..269b69faf11a4 --- /dev/null +++ b/net/wireguard/noise.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "noise.h" +#include "device.h" +#include "peer.h" +#include "messages.h" +#include "queueing.h" +#include "peerlookup.h" + +#include +#include +#include +#include +#include +#include + +/* This implements Noise_IKpsk2: + * + * <- s + * ****** + * -> e, es, s, ss, {t} + * <- e, ee, se, psk, {} + */ + +static const u8 handshake_name[37] = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; +static const u8 identifier_name[34] = "WireGuard v1 zx2c4 Jason@zx2c4.com"; +static u8 handshake_init_hash[NOISE_HASH_LEN] __ro_after_init; +static u8 handshake_init_chaining_key[NOISE_HASH_LEN] __ro_after_init; +static atomic64_t keypair_counter = ATOMIC64_INIT(0); + +void __init wg_noise_init(void) +{ + struct blake2s_state blake; + + blake2s(handshake_init_chaining_key, handshake_name, NULL, + NOISE_HASH_LEN, sizeof(handshake_name), 0); + blake2s_init(&blake, NOISE_HASH_LEN); + blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN); + blake2s_update(&blake, identifier_name, sizeof(identifier_name)); + blake2s_final(&blake, handshake_init_hash); +} + +/* Must hold peer->handshake.static_identity->lock */ +bool wg_noise_precompute_static_static(struct wg_peer *peer) +{ + bool ret = true; + + down_write(&peer->handshake.lock); + if (peer->handshake.static_identity->has_identity) + ret = curve25519( + peer->handshake.precomputed_static_static, + peer->handshake.static_identity->static_private, + peer->handshake.remote_static); + else + memset(peer->handshake.precomputed_static_static, 0, + NOISE_PUBLIC_KEY_LEN); + up_write(&peer->handshake.lock); + return ret; +} + +bool wg_noise_handshake_init(struct noise_handshake *handshake, + struct noise_static_identity *static_identity, + const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], + const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], + struct wg_peer *peer) +{ + memset(handshake, 0, sizeof(*handshake)); + init_rwsem(&handshake->lock); + handshake->entry.type = INDEX_HASHTABLE_HANDSHAKE; + handshake->entry.peer = peer; + memcpy(handshake->remote_static, peer_public_key, NOISE_PUBLIC_KEY_LEN); + if (peer_preshared_key) + memcpy(handshake->preshared_key, peer_preshared_key, + NOISE_SYMMETRIC_KEY_LEN); + handshake->static_identity = static_identity; + handshake->state = HANDSHAKE_ZEROED; + return wg_noise_precompute_static_static(peer); +} + +static void handshake_zero(struct noise_handshake *handshake) +{ + memset(&handshake->ephemeral_private, 0, NOISE_PUBLIC_KEY_LEN); + memset(&handshake->remote_ephemeral, 0, NOISE_PUBLIC_KEY_LEN); + memset(&handshake->hash, 0, NOISE_HASH_LEN); + memset(&handshake->chaining_key, 0, NOISE_HASH_LEN); + handshake->remote_index = 0; + handshake->state = HANDSHAKE_ZEROED; +} + +void wg_noise_handshake_clear(struct noise_handshake *handshake) +{ + wg_index_hashtable_remove( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + down_write(&handshake->lock); + handshake_zero(handshake); + up_write(&handshake->lock); + wg_index_hashtable_remove( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); +} + +static struct noise_keypair *keypair_create(struct wg_peer *peer) +{ + struct noise_keypair *keypair = kzalloc(sizeof(*keypair), GFP_KERNEL); + + if (unlikely(!keypair)) + return NULL; + keypair->internal_id = atomic64_inc_return(&keypair_counter); + keypair->entry.type = INDEX_HASHTABLE_KEYPAIR; + keypair->entry.peer = peer; + kref_init(&keypair->refcount); + return keypair; +} + +static void keypair_free_rcu(struct rcu_head *rcu) +{ + kzfree(container_of(rcu, struct noise_keypair, rcu)); +} + +static void keypair_free_kref(struct kref *kref) +{ + struct noise_keypair *keypair = + container_of(kref, struct noise_keypair, refcount); + + net_dbg_ratelimited("%s: Keypair %llu destroyed for peer %llu\n", + keypair->entry.peer->device->dev->name, + keypair->internal_id, + keypair->entry.peer->internal_id); + wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable, + &keypair->entry); + call_rcu(&keypair->rcu, keypair_free_rcu); +} + +void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now) +{ + if (unlikely(!keypair)) + return; + if (unlikely(unreference_now)) + wg_index_hashtable_remove( + keypair->entry.peer->device->index_hashtable, + &keypair->entry); + kref_put(&keypair->refcount, keypair_free_kref); +} + +struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), + "Taking noise keypair reference without holding the RCU BH read lock"); + if (unlikely(!keypair || !kref_get_unless_zero(&keypair->refcount))) + return NULL; + return keypair; +} + +void wg_noise_keypairs_clear(struct noise_keypairs *keypairs) +{ + struct noise_keypair *old; + + spin_lock_bh(&keypairs->keypair_update_lock); + + /* We zero the next_keypair before zeroing the others, so that + * wg_noise_received_with_keypair returns early before subsequent ones + * are zeroed. + */ + old = rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + wg_noise_keypair_put(old, true); + + old = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->previous_keypair, NULL); + wg_noise_keypair_put(old, true); + + old = rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->current_keypair, NULL); + wg_noise_keypair_put(old, true); + + spin_unlock_bh(&keypairs->keypair_update_lock); +} + +void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + + wg_noise_handshake_clear(&peer->handshake); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + + spin_lock_bh(&peer->keypairs.keypair_update_lock); + keypair = rcu_dereference_protected(peer->keypairs.next_keypair, + lockdep_is_held(&peer->keypairs.keypair_update_lock)); + if (keypair) + keypair->sending.is_valid = false; + keypair = rcu_dereference_protected(peer->keypairs.current_keypair, + lockdep_is_held(&peer->keypairs.keypair_update_lock)); + if (keypair) + keypair->sending.is_valid = false; + spin_unlock_bh(&peer->keypairs.keypair_update_lock); +} + +static void add_new_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *new_keypair) +{ + struct noise_keypair *previous_keypair, *next_keypair, *current_keypair; + + spin_lock_bh(&keypairs->keypair_update_lock); + previous_keypair = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + next_keypair = rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + current_keypair = rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + if (new_keypair->i_am_the_initiator) { + /* If we're the initiator, it means we've sent a handshake, and + * received a confirmation response, which means this new + * keypair can now be used. + */ + if (next_keypair) { + /* If there already was a next keypair pending, we + * demote it to be the previous keypair, and free the + * existing current. Note that this means KCI can result + * in this transition. It would perhaps be more sound to + * always just get rid of the unused next keypair + * instead of putting it in the previous slot, but this + * might be a bit less robust. Something to think about + * for the future. + */ + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + rcu_assign_pointer(keypairs->previous_keypair, + next_keypair); + wg_noise_keypair_put(current_keypair, true); + } else /* If there wasn't an existing next keypair, we replace + * the previous with the current one. + */ + rcu_assign_pointer(keypairs->previous_keypair, + current_keypair); + /* At this point we can get rid of the old previous keypair, and + * set up the new keypair. + */ + wg_noise_keypair_put(previous_keypair, true); + rcu_assign_pointer(keypairs->current_keypair, new_keypair); + } else { + /* If we're the responder, it means we can't use the new keypair + * until we receive confirmation via the first data packet, so + * we get rid of the existing previous one, the possibly + * existing next one, and slide in the new next one. + */ + rcu_assign_pointer(keypairs->next_keypair, new_keypair); + wg_noise_keypair_put(next_keypair, true); + RCU_INIT_POINTER(keypairs->previous_keypair, NULL); + wg_noise_keypair_put(previous_keypair, true); + } + spin_unlock_bh(&keypairs->keypair_update_lock); +} + +bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *received_keypair) +{ + struct noise_keypair *old_keypair; + bool key_is_new; + + /* We first check without taking the spinlock. */ + key_is_new = received_keypair == + rcu_access_pointer(keypairs->next_keypair); + if (likely(!key_is_new)) + return false; + + spin_lock_bh(&keypairs->keypair_update_lock); + /* After locking, we double check that things didn't change from + * beneath us. + */ + if (unlikely(received_keypair != + rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)))) { + spin_unlock_bh(&keypairs->keypair_update_lock); + return false; + } + + /* When we've finally received the confirmation, we slide the next + * into the current, the current into the previous, and get rid of + * the old previous. + */ + old_keypair = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + rcu_assign_pointer(keypairs->previous_keypair, + rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock))); + wg_noise_keypair_put(old_keypair, true); + rcu_assign_pointer(keypairs->current_keypair, received_keypair); + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + + spin_unlock_bh(&keypairs->keypair_update_lock); + return true; +} + +/* Must hold static_identity->lock */ +void wg_noise_set_static_identity_private_key( + struct noise_static_identity *static_identity, + const u8 private_key[NOISE_PUBLIC_KEY_LEN]) +{ + memcpy(static_identity->static_private, private_key, + NOISE_PUBLIC_KEY_LEN); + curve25519_clamp_secret(static_identity->static_private); + static_identity->has_identity = curve25519_generate_public( + static_identity->static_public, private_key); +} + +/* This is Hugo Krawczyk's HKDF: + * - https://eprint.iacr.org/2010/264.pdf + * - https://tools.ietf.org/html/rfc5869 + */ +static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data, + size_t first_len, size_t second_len, size_t third_len, + size_t data_len, const u8 chaining_key[NOISE_HASH_LEN]) +{ + u8 output[BLAKE2S_HASH_SIZE + 1]; + u8 secret[BLAKE2S_HASH_SIZE]; + + WARN_ON(IS_ENABLED(DEBUG) && + (first_len > BLAKE2S_HASH_SIZE || + second_len > BLAKE2S_HASH_SIZE || + third_len > BLAKE2S_HASH_SIZE || + ((second_len || second_dst || third_len || third_dst) && + (!first_len || !first_dst)) || + ((third_len || third_dst) && (!second_len || !second_dst)))); + + /* Extract entropy from data into secret */ + blake2s_hmac(secret, data, chaining_key, BLAKE2S_HASH_SIZE, data_len, + NOISE_HASH_LEN); + + if (!first_dst || !first_len) + goto out; + + /* Expand first key: key = secret, data = 0x1 */ + output[0] = 1; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, 1, + BLAKE2S_HASH_SIZE); + memcpy(first_dst, output, first_len); + + if (!second_dst || !second_len) + goto out; + + /* Expand second key: key = secret, data = first-key || 0x2 */ + output[BLAKE2S_HASH_SIZE] = 2; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, + BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); + memcpy(second_dst, output, second_len); + + if (!third_dst || !third_len) + goto out; + + /* Expand third key: key = secret, data = second-key || 0x3 */ + output[BLAKE2S_HASH_SIZE] = 3; + blake2s_hmac(output, output, secret, BLAKE2S_HASH_SIZE, + BLAKE2S_HASH_SIZE + 1, BLAKE2S_HASH_SIZE); + memcpy(third_dst, output, third_len); + +out: + /* Clear sensitive data from stack */ + memzero_explicit(secret, BLAKE2S_HASH_SIZE); + memzero_explicit(output, BLAKE2S_HASH_SIZE + 1); +} + +static void symmetric_key_init(struct noise_symmetric_key *key) +{ + spin_lock_init(&key->counter.receive.lock); + atomic64_set(&key->counter.counter, 0); + memset(key->counter.receive.backtrack, 0, + sizeof(key->counter.receive.backtrack)); + key->birthdate = ktime_get_coarse_boottime_ns(); + key->is_valid = true; +} + +static void derive_keys(struct noise_symmetric_key *first_dst, + struct noise_symmetric_key *second_dst, + const u8 chaining_key[NOISE_HASH_LEN]) +{ + kdf(first_dst->key, second_dst->key, NULL, NULL, + NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0, + chaining_key); + symmetric_key_init(first_dst); + symmetric_key_init(second_dst); +} + +static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 private[NOISE_PUBLIC_KEY_LEN], + const u8 public[NOISE_PUBLIC_KEY_LEN]) +{ + u8 dh_calculation[NOISE_PUBLIC_KEY_LEN]; + + if (unlikely(!curve25519(dh_calculation, private, public))) + return false; + kdf(chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, chaining_key); + memzero_explicit(dh_calculation, NOISE_PUBLIC_KEY_LEN); + return true; +} + +static void mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t src_len) +{ + struct blake2s_state blake; + + blake2s_init(&blake, NOISE_HASH_LEN); + blake2s_update(&blake, hash, NOISE_HASH_LEN); + blake2s_update(&blake, src, src_len); + blake2s_final(&blake, hash); +} + +static void mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 psk[NOISE_SYMMETRIC_KEY_LEN]) +{ + u8 temp_hash[NOISE_HASH_LEN]; + + kdf(chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, chaining_key); + mix_hash(hash, temp_hash, NOISE_HASH_LEN); + memzero_explicit(temp_hash, NOISE_HASH_LEN); +} + +static void handshake_init(u8 chaining_key[NOISE_HASH_LEN], + u8 hash[NOISE_HASH_LEN], + const u8 remote_static[NOISE_PUBLIC_KEY_LEN]) +{ + memcpy(hash, handshake_init_hash, NOISE_HASH_LEN); + memcpy(chaining_key, handshake_init_chaining_key, NOISE_HASH_LEN); + mix_hash(hash, remote_static, NOISE_PUBLIC_KEY_LEN); +} + +static void message_encrypt(u8 *dst_ciphertext, const u8 *src_plaintext, + size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash, + NOISE_HASH_LEN, + 0 /* Always zero for Noise_IK */, key); + mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len)); +} + +static bool message_decrypt(u8 *dst_plaintext, const u8 *src_ciphertext, + size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len, + hash, NOISE_HASH_LEN, + 0 /* Always zero for Noise_IK */, key)) + return false; + mix_hash(hash, src_ciphertext, src_len); + return true; +} + +static void message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], + const u8 ephemeral_src[NOISE_PUBLIC_KEY_LEN], + u8 chaining_key[NOISE_HASH_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (ephemeral_dst != ephemeral_src) + memcpy(ephemeral_dst, ephemeral_src, NOISE_PUBLIC_KEY_LEN); + mix_hash(hash, ephemeral_src, NOISE_PUBLIC_KEY_LEN); + kdf(chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0, + NOISE_PUBLIC_KEY_LEN, chaining_key); +} + +static void tai64n_now(u8 output[NOISE_TIMESTAMP_LEN]) +{ + struct timespec64 now; + + ktime_get_real_ts64(&now); + + /* In order to prevent some sort of infoleak from precise timers, we + * round down the nanoseconds part to the closest rounded-down power of + * two to the maximum initiations per second allowed anyway by the + * implementation. + */ + now.tv_nsec = ALIGN_DOWN(now.tv_nsec, + rounddown_pow_of_two(NSEC_PER_SEC / INITIATIONS_PER_SECOND)); + + /* https://cr.yp.to/libtai/tai64.html */ + *(__be64 *)output = cpu_to_be64(0x400000000000000aULL + now.tv_sec); + *(__be32 *)(output + sizeof(__be64)) = cpu_to_be32(now.tv_nsec); +} + +bool +wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + struct noise_handshake *handshake) +{ + u8 timestamp[NOISE_TIMESTAMP_LEN]; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + bool ret = false; + + /* We need to wait for crng _before_ taking any locks, since + * curve25519_generate_secret uses get_random_bytes_wait. + */ + wait_for_random_bytes(); + + down_read(&handshake->static_identity->lock); + down_write(&handshake->lock); + + if (unlikely(!handshake->static_identity->has_identity)) + goto out; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION); + + handshake_init(handshake->chaining_key, handshake->hash, + handshake->remote_static); + + /* e */ + curve25519_generate_secret(handshake->ephemeral_private); + if (!curve25519_generate_public(dst->unencrypted_ephemeral, + handshake->ephemeral_private)) + goto out; + message_ephemeral(dst->unencrypted_ephemeral, + dst->unencrypted_ephemeral, handshake->chaining_key, + handshake->hash); + + /* es */ + if (!mix_dh(handshake->chaining_key, key, handshake->ephemeral_private, + handshake->remote_static)) + goto out; + + /* s */ + message_encrypt(dst->encrypted_static, + handshake->static_identity->static_public, + NOISE_PUBLIC_KEY_LEN, key, handshake->hash); + + /* ss */ + kdf(handshake->chaining_key, key, NULL, + handshake->precomputed_static_static, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, + handshake->chaining_key); + + /* {t} */ + tai64n_now(timestamp); + message_encrypt(dst->encrypted_timestamp, timestamp, + NOISE_TIMESTAMP_LEN, key, handshake->hash); + + dst->sender_index = wg_index_hashtable_insert( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + + handshake->state = HANDSHAKE_CREATED_INITIATION; + ret = true; + +out: + up_write(&handshake->lock); + up_read(&handshake->static_identity->lock); + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + return ret; +} + +struct wg_peer * +wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, + struct wg_device *wg) +{ + struct wg_peer *peer = NULL, *ret_peer = NULL; + struct noise_handshake *handshake; + bool replay_attack, flood_attack; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + u8 hash[NOISE_HASH_LEN]; + u8 s[NOISE_PUBLIC_KEY_LEN]; + u8 e[NOISE_PUBLIC_KEY_LEN]; + u8 t[NOISE_TIMESTAMP_LEN]; + u64 initiation_consumption; + + down_read(&wg->static_identity.lock); + if (unlikely(!wg->static_identity.has_identity)) + goto out; + + handshake_init(chaining_key, hash, wg->static_identity.static_public); + + /* e */ + message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); + + /* es */ + if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e)) + goto out; + + /* s */ + if (!message_decrypt(s, src->encrypted_static, + sizeof(src->encrypted_static), key, hash)) + goto out; + + /* Lookup which peer we're actually talking to */ + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, s); + if (!peer) + goto out; + handshake = &peer->handshake; + + /* ss */ + kdf(chaining_key, key, NULL, handshake->precomputed_static_static, + NOISE_HASH_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, + chaining_key); + + /* {t} */ + if (!message_decrypt(t, src->encrypted_timestamp, + sizeof(src->encrypted_timestamp), key, hash)) + goto out; + + down_read(&handshake->lock); + replay_attack = memcmp(t, handshake->latest_timestamp, + NOISE_TIMESTAMP_LEN) <= 0; + flood_attack = (s64)handshake->last_initiation_consumption + + NSEC_PER_SEC / INITIATIONS_PER_SECOND > + (s64)ktime_get_coarse_boottime_ns(); + up_read(&handshake->lock); + if (replay_attack || flood_attack) + goto out; + + /* Success! Copy everything to peer */ + down_write(&handshake->lock); + memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); + if (memcmp(t, handshake->latest_timestamp, NOISE_TIMESTAMP_LEN) > 0) + memcpy(handshake->latest_timestamp, t, NOISE_TIMESTAMP_LEN); + memcpy(handshake->hash, hash, NOISE_HASH_LEN); + memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); + handshake->remote_index = src->sender_index; + if ((s64)(handshake->last_initiation_consumption - + (initiation_consumption = ktime_get_coarse_boottime_ns())) < 0) + handshake->last_initiation_consumption = initiation_consumption; + handshake->state = HANDSHAKE_CONSUMED_INITIATION; + up_write(&handshake->lock); + ret_peer = peer; + +out: + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(hash, NOISE_HASH_LEN); + memzero_explicit(chaining_key, NOISE_HASH_LEN); + up_read(&wg->static_identity.lock); + if (!ret_peer) + wg_peer_put(peer); + return ret_peer; +} + +bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + struct noise_handshake *handshake) +{ + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + bool ret = false; + + /* We need to wait for crng _before_ taking any locks, since + * curve25519_generate_secret uses get_random_bytes_wait. + */ + wait_for_random_bytes(); + + down_read(&handshake->static_identity->lock); + down_write(&handshake->lock); + + if (handshake->state != HANDSHAKE_CONSUMED_INITIATION) + goto out; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE); + dst->receiver_index = handshake->remote_index; + + /* e */ + curve25519_generate_secret(handshake->ephemeral_private); + if (!curve25519_generate_public(dst->unencrypted_ephemeral, + handshake->ephemeral_private)) + goto out; + message_ephemeral(dst->unencrypted_ephemeral, + dst->unencrypted_ephemeral, handshake->chaining_key, + handshake->hash); + + /* ee */ + if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, + handshake->remote_ephemeral)) + goto out; + + /* se */ + if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, + handshake->remote_static)) + goto out; + + /* psk */ + mix_psk(handshake->chaining_key, handshake->hash, key, + handshake->preshared_key); + + /* {} */ + message_encrypt(dst->encrypted_nothing, NULL, 0, key, handshake->hash); + + dst->sender_index = wg_index_hashtable_insert( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + + handshake->state = HANDSHAKE_CREATED_RESPONSE; + ret = true; + +out: + up_write(&handshake->lock); + up_read(&handshake->static_identity->lock); + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + return ret; +} + +struct wg_peer * +wg_noise_handshake_consume_response(struct message_handshake_response *src, + struct wg_device *wg) +{ + enum noise_handshake_state state = HANDSHAKE_ZEROED; + struct wg_peer *peer = NULL, *ret_peer = NULL; + struct noise_handshake *handshake; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 hash[NOISE_HASH_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + u8 e[NOISE_PUBLIC_KEY_LEN]; + u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; + u8 static_private[NOISE_PUBLIC_KEY_LEN]; + + down_read(&wg->static_identity.lock); + + if (unlikely(!wg->static_identity.has_identity)) + goto out; + + handshake = (struct noise_handshake *)wg_index_hashtable_lookup( + wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE, + src->receiver_index, &peer); + if (unlikely(!handshake)) + goto out; + + down_read(&handshake->lock); + state = handshake->state; + memcpy(hash, handshake->hash, NOISE_HASH_LEN); + memcpy(chaining_key, handshake->chaining_key, NOISE_HASH_LEN); + memcpy(ephemeral_private, handshake->ephemeral_private, + NOISE_PUBLIC_KEY_LEN); + up_read(&handshake->lock); + + if (state != HANDSHAKE_CREATED_INITIATION) + goto fail; + + /* e */ + message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); + + /* ee */ + if (!mix_dh(chaining_key, NULL, ephemeral_private, e)) + goto fail; + + /* se */ + if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e)) + goto fail; + + /* psk */ + mix_psk(chaining_key, hash, key, handshake->preshared_key); + + /* {} */ + if (!message_decrypt(NULL, src->encrypted_nothing, + sizeof(src->encrypted_nothing), key, hash)) + goto fail; + + /* Success! Copy everything to peer */ + down_write(&handshake->lock); + /* It's important to check that the state is still the same, while we + * have an exclusive lock. + */ + if (handshake->state != state) { + up_write(&handshake->lock); + goto fail; + } + memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); + memcpy(handshake->hash, hash, NOISE_HASH_LEN); + memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); + handshake->remote_index = src->sender_index; + handshake->state = HANDSHAKE_CONSUMED_RESPONSE; + up_write(&handshake->lock); + ret_peer = peer; + goto out; + +fail: + wg_peer_put(peer); +out: + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(hash, NOISE_HASH_LEN); + memzero_explicit(chaining_key, NOISE_HASH_LEN); + memzero_explicit(ephemeral_private, NOISE_PUBLIC_KEY_LEN); + memzero_explicit(static_private, NOISE_PUBLIC_KEY_LEN); + up_read(&wg->static_identity.lock); + return ret_peer; +} + +bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, + struct noise_keypairs *keypairs) +{ + struct noise_keypair *new_keypair; + bool ret = false; + + down_write(&handshake->lock); + if (handshake->state != HANDSHAKE_CREATED_RESPONSE && + handshake->state != HANDSHAKE_CONSUMED_RESPONSE) + goto out; + + new_keypair = keypair_create(handshake->entry.peer); + if (!new_keypair) + goto out; + new_keypair->i_am_the_initiator = handshake->state == + HANDSHAKE_CONSUMED_RESPONSE; + new_keypair->remote_index = handshake->remote_index; + + if (new_keypair->i_am_the_initiator) + derive_keys(&new_keypair->sending, &new_keypair->receiving, + handshake->chaining_key); + else + derive_keys(&new_keypair->receiving, &new_keypair->sending, + handshake->chaining_key); + + handshake_zero(handshake); + rcu_read_lock_bh(); + if (likely(!READ_ONCE(container_of(handshake, struct wg_peer, + handshake)->is_dead))) { + add_new_keypair(keypairs, new_keypair); + net_dbg_ratelimited("%s: Keypair %llu created for peer %llu\n", + handshake->entry.peer->device->dev->name, + new_keypair->internal_id, + handshake->entry.peer->internal_id); + ret = wg_index_hashtable_replace( + handshake->entry.peer->device->index_hashtable, + &handshake->entry, &new_keypair->entry); + } else { + kzfree(new_keypair); + } + rcu_read_unlock_bh(); + +out: + up_write(&handshake->lock); + return ret; +} diff --git a/net/wireguard/noise.h b/net/wireguard/noise.h new file mode 100644 index 0000000000000..138a07bb817ce --- /dev/null +++ b/net/wireguard/noise.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ +#ifndef _WG_NOISE_H +#define _WG_NOISE_H + +#include "messages.h" +#include "peerlookup.h" + +#include +#include +#include +#include +#include +#include + +union noise_counter { + struct { + u64 counter; + unsigned long backtrack[COUNTER_BITS_TOTAL / BITS_PER_LONG]; + spinlock_t lock; + } receive; + atomic64_t counter; +}; + +struct noise_symmetric_key { + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + union noise_counter counter; + u64 birthdate; + bool is_valid; +}; + +struct noise_keypair { + struct index_hashtable_entry entry; + struct noise_symmetric_key sending; + struct noise_symmetric_key receiving; + __le32 remote_index; + bool i_am_the_initiator; + struct kref refcount; + struct rcu_head rcu; + u64 internal_id; +}; + +struct noise_keypairs { + struct noise_keypair __rcu *current_keypair; + struct noise_keypair __rcu *previous_keypair; + struct noise_keypair __rcu *next_keypair; + spinlock_t keypair_update_lock; +}; + +struct noise_static_identity { + u8 static_public[NOISE_PUBLIC_KEY_LEN]; + u8 static_private[NOISE_PUBLIC_KEY_LEN]; + struct rw_semaphore lock; + bool has_identity; +}; + +enum noise_handshake_state { + HANDSHAKE_ZEROED, + HANDSHAKE_CREATED_INITIATION, + HANDSHAKE_CONSUMED_INITIATION, + HANDSHAKE_CREATED_RESPONSE, + HANDSHAKE_CONSUMED_RESPONSE +}; + +struct noise_handshake { + struct index_hashtable_entry entry; + + enum noise_handshake_state state; + u64 last_initiation_consumption; + + struct noise_static_identity *static_identity; + + u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; + u8 remote_static[NOISE_PUBLIC_KEY_LEN]; + u8 remote_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 precomputed_static_static[NOISE_PUBLIC_KEY_LEN]; + + u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]; + + u8 hash[NOISE_HASH_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + + u8 latest_timestamp[NOISE_TIMESTAMP_LEN]; + __le32 remote_index; + + /* Protects all members except the immutable (after noise_handshake_ + * init): remote_static, precomputed_static_static, static_identity. + */ + struct rw_semaphore lock; +}; + +struct wg_device; + +void wg_noise_init(void); +bool wg_noise_handshake_init(struct noise_handshake *handshake, + struct noise_static_identity *static_identity, + const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], + const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], + struct wg_peer *peer); +void wg_noise_handshake_clear(struct noise_handshake *handshake); +static inline void wg_noise_reset_last_sent_handshake(atomic64_t *handshake_ns) +{ + atomic64_set(handshake_ns, ktime_get_coarse_boottime_ns() - + (u64)(REKEY_TIMEOUT + 1) * NSEC_PER_SEC); +} + +void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now); +struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair); +void wg_noise_keypairs_clear(struct noise_keypairs *keypairs); +bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *received_keypair); +void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer); + +void wg_noise_set_static_identity_private_key( + struct noise_static_identity *static_identity, + const u8 private_key[NOISE_PUBLIC_KEY_LEN]); +bool wg_noise_precompute_static_static(struct wg_peer *peer); + +bool +wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + struct noise_handshake *handshake); +struct wg_peer * +wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, + struct wg_device *wg); + +bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + struct noise_handshake *handshake); +struct wg_peer * +wg_noise_handshake_consume_response(struct message_handshake_response *src, + struct wg_device *wg); + +bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, + struct noise_keypairs *keypairs); + +#endif /* _WG_NOISE_H */ diff --git a/net/wireguard/peer.c b/net/wireguard/peer.c new file mode 100644 index 0000000000000..071eedf33f5aa --- /dev/null +++ b/net/wireguard/peer.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "peer.h" +#include "device.h" +#include "queueing.h" +#include "timers.h" +#include "peerlookup.h" +#include "noise.h" + +#include +#include +#include +#include + +static atomic64_t peer_counter = ATOMIC64_INIT(0); + +struct wg_peer *wg_peer_create(struct wg_device *wg, + const u8 public_key[NOISE_PUBLIC_KEY_LEN], + const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]) +{ + struct wg_peer *peer; + int ret = -ENOMEM; + + lockdep_assert_held(&wg->device_update_lock); + + if (wg->num_peers >= MAX_PEERS_PER_DEVICE) + return ERR_PTR(ret); + + peer = kzalloc(sizeof(*peer), GFP_KERNEL); + if (unlikely(!peer)) + return ERR_PTR(ret); + peer->device = wg; + + if (!wg_noise_handshake_init(&peer->handshake, &wg->static_identity, + public_key, preshared_key, peer)) { + ret = -EKEYREJECTED; + goto err_1; + } + if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) + goto err_1; + if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false, + MAX_QUEUED_PACKETS)) + goto err_2; + if (wg_packet_queue_init(&peer->rx_queue, NULL, false, + MAX_QUEUED_PACKETS)) + goto err_3; + + peer->internal_id = atomic64_inc_return(&peer_counter); + peer->serial_work_cpu = nr_cpumask_bits; + wg_cookie_init(&peer->latest_cookie); + wg_timers_init(peer); + wg_cookie_checker_precompute_peer_keys(peer); + spin_lock_init(&peer->keypairs.keypair_update_lock); + INIT_WORK(&peer->transmit_handshake_work, + wg_packet_handshake_send_worker); + rwlock_init(&peer->endpoint_lock); + kref_init(&peer->refcount); + skb_queue_head_init(&peer->staged_packet_queue); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); + netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, + NAPI_POLL_WEIGHT); + napi_enable(&peer->napi); + list_add_tail(&peer->peer_list, &wg->peer_list); + INIT_LIST_HEAD(&peer->allowedips_list); + wg_pubkey_hashtable_add(wg->peer_hashtable, peer); + ++wg->num_peers; + pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); + return peer; + +err_3: + wg_packet_queue_free(&peer->tx_queue, false); +err_2: + dst_cache_destroy(&peer->endpoint_cache); +err_1: + kfree(peer); + return ERR_PTR(ret); +} + +struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), + "Taking peer reference without holding the RCU read lock"); + if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) + return NULL; + return peer; +} + +static void peer_make_dead(struct wg_peer *peer) +{ + /* Remove from configuration-time lookup structures. */ + list_del_init(&peer->peer_list); + wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, + &peer->device->device_update_lock); + wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer); + + /* Mark as dead, so that we don't allow jumping contexts after. */ + WRITE_ONCE(peer->is_dead, true); + + /* The caller must now synchronize_rcu() for this to take effect. */ +} + +static void peer_remove_after_dead(struct wg_peer *peer) +{ + WARN_ON(!peer->is_dead); + + /* No more keypairs can be created for this peer, since is_dead protects + * add_new_keypair, so we can now destroy existing ones. + */ + wg_noise_keypairs_clear(&peer->keypairs); + + /* Destroy all ongoing timers that were in-flight at the beginning of + * this function. + */ + wg_timers_stop(peer); + + /* The transition between packet encryption/decryption queues isn't + * guarded by is_dead, but each reference's life is strictly bounded by + * two generations: once for parallel crypto and once for serial + * ingestion, so we can simply flush twice, and be sure that we no + * longer have references inside these queues. + */ + + /* a) For encrypt/decrypt. */ + flush_workqueue(peer->device->packet_crypt_wq); + /* b.1) For send (but not receive, since that's napi). */ + flush_workqueue(peer->device->packet_crypt_wq); + /* b.2.1) For receive (but not send, since that's wq). */ + napi_disable(&peer->napi); + /* b.2.1) It's now safe to remove the napi struct, which must be done + * here from process context. + */ + netif_napi_del(&peer->napi); + + /* Ensure any workstructs we own (like transmit_handshake_work or + * clear_peer_work) no longer are in use. + */ + flush_workqueue(peer->device->handshake_send_wq); + + /* After the above flushes, a peer might still be active in a few + * different contexts: 1) from xmit(), before hitting is_dead and + * returning, 2) from wg_packet_consume_data(), before hitting is_dead + * and returning, 3) from wg_receive_handshake_packet() after a point + * where it has processed an incoming handshake packet, but where + * all calls to pass it off to timers fails because of is_dead. We won't + * have new references in (1) eventually, because we're removed from + * allowedips; we won't have new references in (2) eventually, because + * wg_index_hashtable_lookup will always return NULL, since we removed + * all existing keypairs and no more can be created; we won't have new + * references in (3) eventually, because we're removed from the pubkey + * hash table, which allows for a maximum of one handshake response, + * via the still-uncleared index hashtable entry, but not more than one, + * and in wg_cookie_message_consume, the lookup eventually gets a peer + * with a refcount of zero, so no new reference is taken. + */ + + --peer->device->num_peers; + wg_peer_put(peer); +} + +/* We have a separate "remove" function make sure that all active places where + * a peer is currently operating will eventually come to an end and not pass + * their reference onto another context. + */ +void wg_peer_remove(struct wg_peer *peer) +{ + if (unlikely(!peer)) + return; + lockdep_assert_held(&peer->device->device_update_lock); + + peer_make_dead(peer); + synchronize_rcu(); + peer_remove_after_dead(peer); +} + +void wg_peer_remove_all(struct wg_device *wg) +{ + struct wg_peer *peer, *temp; + LIST_HEAD(dead_peers); + + lockdep_assert_held(&wg->device_update_lock); + + /* Avoid having to traverse individually for each one. */ + wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock); + + list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) { + peer_make_dead(peer); + list_add_tail(&peer->peer_list, &dead_peers); + } + synchronize_rcu(); + list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) + peer_remove_after_dead(peer); +} + +static void rcu_release(struct rcu_head *rcu) +{ + struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); + + dst_cache_destroy(&peer->endpoint_cache); + wg_packet_queue_free(&peer->rx_queue, false); + wg_packet_queue_free(&peer->tx_queue, false); + + /* The final zeroing takes care of clearing any remaining handshake key + * material and other potentially sensitive information. + */ + kzfree(peer); +} + +static void kref_release(struct kref *refcount) +{ + struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount); + + pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + /* Remove ourself from dynamic runtime lookup structures, now that the + * last reference is gone. + */ + wg_index_hashtable_remove(peer->device->index_hashtable, + &peer->handshake.entry); + + /* Remove any lingering packets that didn't have a chance to be + * transmitted. + */ + wg_packet_purge_staged_packets(peer); + + /* Free the memory used. */ + call_rcu(&peer->rcu, rcu_release); +} + +void wg_peer_put(struct wg_peer *peer) +{ + if (unlikely(!peer)) + return; + kref_put(&peer->refcount, kref_release); +} diff --git a/net/wireguard/peer.h b/net/wireguard/peer.h new file mode 100644 index 0000000000000..23af409229972 --- /dev/null +++ b/net/wireguard/peer.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_PEER_H +#define _WG_PEER_H + +#include "device.h" +#include "noise.h" +#include "cookie.h" + +#include +#include +#include +#include +#include + +struct wg_device; + +struct endpoint { + union { + struct sockaddr addr; + struct sockaddr_in addr4; + struct sockaddr_in6 addr6; + }; + union { + struct { + struct in_addr src4; + /* Essentially the same as addr6->scope_id */ + int src_if4; + }; + struct in6_addr src6; + }; +}; + +struct wg_peer { + struct wg_device *device; + struct crypt_queue tx_queue, rx_queue; + struct sk_buff_head staged_packet_queue; + int serial_work_cpu; + struct noise_keypairs keypairs; + struct endpoint endpoint; + struct dst_cache endpoint_cache; + rwlock_t endpoint_lock; + struct noise_handshake handshake; + atomic64_t last_sent_handshake; + struct work_struct transmit_handshake_work, clear_peer_work; + struct cookie latest_cookie; + struct hlist_node pubkey_hash; + u64 rx_bytes, tx_bytes; + struct timer_list timer_retransmit_handshake, timer_send_keepalive; + struct timer_list timer_new_handshake, timer_zero_key_material; + struct timer_list timer_persistent_keepalive; + unsigned int timer_handshake_attempts; + u16 persistent_keepalive_interval; + bool timer_need_another_keepalive; + bool sent_lastminute_handshake; + struct timespec64 walltime_last_handshake; + struct kref refcount; + struct rcu_head rcu; + struct list_head peer_list; + struct list_head allowedips_list; + u64 internal_id; + struct napi_struct napi; + bool is_dead; +}; + +struct wg_peer *wg_peer_create(struct wg_device *wg, + const u8 public_key[NOISE_PUBLIC_KEY_LEN], + const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]); + +struct wg_peer *__must_check wg_peer_get_maybe_zero(struct wg_peer *peer); +static inline struct wg_peer *wg_peer_get(struct wg_peer *peer) +{ + kref_get(&peer->refcount); + return peer; +} +void wg_peer_put(struct wg_peer *peer); +void wg_peer_remove(struct wg_peer *peer); +void wg_peer_remove_all(struct wg_device *wg); + +#endif /* _WG_PEER_H */ diff --git a/net/wireguard/peerlookup.c b/net/wireguard/peerlookup.c new file mode 100644 index 0000000000000..e4deb331476b3 --- /dev/null +++ b/net/wireguard/peerlookup.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "peerlookup.h" +#include "peer.h" +#include "noise.h" + +static struct hlist_head *pubkey_bucket(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + /* siphash gives us a secure 64bit number based on a random key. Since + * the bits are uniformly distributed, we can then mask off to get the + * bits we need. + */ + const u64 hash = siphash(pubkey, NOISE_PUBLIC_KEY_LEN, &table->key); + + return &table->hashtable[hash & (HASH_SIZE(table->hashtable) - 1)]; +} + +struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void) +{ + struct pubkey_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); + + if (!table) + return NULL; + + get_random_bytes(&table->key, sizeof(table->key)); + hash_init(table->hashtable); + mutex_init(&table->lock); + return table; +} + +void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, + struct wg_peer *peer) +{ + mutex_lock(&table->lock); + hlist_add_head_rcu(&peer->pubkey_hash, + pubkey_bucket(table, peer->handshake.remote_static)); + mutex_unlock(&table->lock); +} + +void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, + struct wg_peer *peer) +{ + mutex_lock(&table->lock); + hlist_del_init_rcu(&peer->pubkey_hash); + mutex_unlock(&table->lock); +} + +/* Returns a strong reference to a peer */ +struct wg_peer * +wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + struct wg_peer *iter_peer, *peer = NULL; + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu_bh(iter_peer, pubkey_bucket(table, pubkey), + pubkey_hash) { + if (!memcmp(pubkey, iter_peer->handshake.remote_static, + NOISE_PUBLIC_KEY_LEN)) { + peer = iter_peer; + break; + } + } + peer = wg_peer_get_maybe_zero(peer); + rcu_read_unlock_bh(); + return peer; +} + +static struct hlist_head *index_bucket(struct index_hashtable *table, + const __le32 index) +{ + /* Since the indices are random and thus all bits are uniformly + * distributed, we can find its bucket simply by masking. + */ + return &table->hashtable[(__force u32)index & + (HASH_SIZE(table->hashtable) - 1)]; +} + +struct index_hashtable *wg_index_hashtable_alloc(void) +{ + struct index_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); + + if (!table) + return NULL; + + hash_init(table->hashtable); + spin_lock_init(&table->lock); + return table; +} + +/* At the moment, we limit ourselves to 2^20 total peers, which generally might + * amount to 2^20*3 items in this hashtable. The algorithm below works by + * picking a random number and testing it. We can see that these limits mean we + * usually succeed pretty quickly: + * + * >>> def calculation(tries, size): + * ... return (size / 2**32)**(tries - 1) * (1 - (size / 2**32)) + * ... + * >>> calculation(1, 2**20 * 3) + * 0.999267578125 + * >>> calculation(2, 2**20 * 3) + * 0.0007318854331970215 + * >>> calculation(3, 2**20 * 3) + * 5.360489012673497e-07 + * >>> calculation(4, 2**20 * 3) + * 3.9261394135792216e-10 + * + * At the moment, we don't do any masking, so this algorithm isn't exactly + * constant time in either the random guessing or in the hash list lookup. We + * could require a minimum of 3 tries, which would successfully mask the + * guessing. this would not, however, help with the growing hash lengths, which + * is another thing to consider moving forward. + */ + +__le32 wg_index_hashtable_insert(struct index_hashtable *table, + struct index_hashtable_entry *entry) +{ + struct index_hashtable_entry *existing_entry; + + spin_lock_bh(&table->lock); + hlist_del_init_rcu(&entry->index_hash); + spin_unlock_bh(&table->lock); + + rcu_read_lock_bh(); + +search_unused_slot: + /* First we try to find an unused slot, randomly, while unlocked. */ + entry->index = (__force __le32)get_random_u32(); + hlist_for_each_entry_rcu_bh(existing_entry, + index_bucket(table, entry->index), + index_hash) { + if (existing_entry->index == entry->index) + /* If it's already in use, we continue searching. */ + goto search_unused_slot; + } + + /* Once we've found an unused slot, we lock it, and then double-check + * that nobody else stole it from us. + */ + spin_lock_bh(&table->lock); + hlist_for_each_entry_rcu_bh(existing_entry, + index_bucket(table, entry->index), + index_hash) { + if (existing_entry->index == entry->index) { + spin_unlock_bh(&table->lock); + /* If it was stolen, we start over. */ + goto search_unused_slot; + } + } + /* Otherwise, we know we have it exclusively (since we're locked), + * so we insert. + */ + hlist_add_head_rcu(&entry->index_hash, + index_bucket(table, entry->index)); + spin_unlock_bh(&table->lock); + + rcu_read_unlock_bh(); + + return entry->index; +} + +bool wg_index_hashtable_replace(struct index_hashtable *table, + struct index_hashtable_entry *old, + struct index_hashtable_entry *new) +{ + if (unlikely(hlist_unhashed(&old->index_hash))) + return false; + spin_lock_bh(&table->lock); + new->index = old->index; + hlist_replace_rcu(&old->index_hash, &new->index_hash); + + /* Calling init here NULLs out index_hash, and in fact after this + * function returns, it's theoretically possible for this to get + * reinserted elsewhere. That means the RCU lookup below might either + * terminate early or jump between buckets, in which case the packet + * simply gets dropped, which isn't terrible. + */ + INIT_HLIST_NODE(&old->index_hash); + spin_unlock_bh(&table->lock); + return true; +} + +void wg_index_hashtable_remove(struct index_hashtable *table, + struct index_hashtable_entry *entry) +{ + spin_lock_bh(&table->lock); + hlist_del_init_rcu(&entry->index_hash); + spin_unlock_bh(&table->lock); +} + +/* Returns a strong reference to a entry->peer */ +struct index_hashtable_entry * +wg_index_hashtable_lookup(struct index_hashtable *table, + const enum index_hashtable_type type_mask, + const __le32 index, struct wg_peer **peer) +{ + struct index_hashtable_entry *iter_entry, *entry = NULL; + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu_bh(iter_entry, index_bucket(table, index), + index_hash) { + if (iter_entry->index == index) { + if (likely(iter_entry->type & type_mask)) + entry = iter_entry; + break; + } + } + if (likely(entry)) { + entry->peer = wg_peer_get_maybe_zero(entry->peer); + if (likely(entry->peer)) + *peer = entry->peer; + else + entry = NULL; + } + rcu_read_unlock_bh(); + return entry; +} diff --git a/net/wireguard/peerlookup.h b/net/wireguard/peerlookup.h new file mode 100644 index 0000000000000..ced8117976800 --- /dev/null +++ b/net/wireguard/peerlookup.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_PEERLOOKUP_H +#define _WG_PEERLOOKUP_H + +#include "messages.h" + +#include +#include +#include + +struct wg_peer; + +struct pubkey_hashtable { + /* TODO: move to rhashtable */ + DECLARE_HASHTABLE(hashtable, 11); + siphash_key_t key; + struct mutex lock; +}; + +struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void); +void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, + struct wg_peer *peer); +void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, + struct wg_peer *peer); +struct wg_peer * +wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]); + +struct index_hashtable { + /* TODO: move to rhashtable */ + DECLARE_HASHTABLE(hashtable, 13); + spinlock_t lock; +}; + +enum index_hashtable_type { + INDEX_HASHTABLE_HANDSHAKE = 1U << 0, + INDEX_HASHTABLE_KEYPAIR = 1U << 1 +}; + +struct index_hashtable_entry { + struct wg_peer *peer; + struct hlist_node index_hash; + enum index_hashtable_type type; + __le32 index; +}; + +struct index_hashtable *wg_index_hashtable_alloc(void); +__le32 wg_index_hashtable_insert(struct index_hashtable *table, + struct index_hashtable_entry *entry); +bool wg_index_hashtable_replace(struct index_hashtable *table, + struct index_hashtable_entry *old, + struct index_hashtable_entry *new); +void wg_index_hashtable_remove(struct index_hashtable *table, + struct index_hashtable_entry *entry); +struct index_hashtable_entry * +wg_index_hashtable_lookup(struct index_hashtable *table, + const enum index_hashtable_type type_mask, + const __le32 index, struct wg_peer **peer); + +#endif /* _WG_PEERLOOKUP_H */ diff --git a/net/wireguard/queueing.c b/net/wireguard/queueing.c new file mode 100644 index 0000000000000..5c964fcb994ec --- /dev/null +++ b/net/wireguard/queueing.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" + +struct multicore_worker __percpu * +wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr) +{ + int cpu; + struct multicore_worker __percpu *worker = + alloc_percpu(struct multicore_worker); + + if (!worker) + return NULL; + + for_each_possible_cpu(cpu) { + per_cpu_ptr(worker, cpu)->ptr = ptr; + INIT_WORK(&per_cpu_ptr(worker, cpu)->work, function); + } + return worker; +} + +int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, + bool multicore, unsigned int len) +{ + int ret; + + memset(queue, 0, sizeof(*queue)); + ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL); + if (ret) + return ret; + if (function) { + if (multicore) { + queue->worker = wg_packet_percpu_multicore_worker_alloc( + function, queue); + if (!queue->worker) + return -ENOMEM; + } else { + INIT_WORK(&queue->work, function); + } + } + return 0; +} + +void wg_packet_queue_free(struct crypt_queue *queue, bool multicore) +{ + if (multicore) + free_percpu(queue->worker); + WARN_ON(!__ptr_ring_empty(&queue->ring)); + ptr_ring_cleanup(&queue->ring, NULL); +} diff --git a/net/wireguard/queueing.h b/net/wireguard/queueing.h new file mode 100644 index 0000000000000..e49a464238fde --- /dev/null +++ b/net/wireguard/queueing.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_QUEUEING_H +#define _WG_QUEUEING_H + +#include "peer.h" +#include +#include +#include +#include + +struct wg_device; +struct wg_peer; +struct multicore_worker; +struct crypt_queue; +struct sk_buff; + +/* queueing.c APIs: */ +int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, + bool multicore, unsigned int len); +void wg_packet_queue_free(struct crypt_queue *queue, bool multicore); +struct multicore_worker __percpu * +wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr); + +/* receive.c APIs: */ +void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb); +void wg_packet_handshake_receive_worker(struct work_struct *work); +/* NAPI poll function: */ +int wg_packet_rx_poll(struct napi_struct *napi, int budget); +/* Workqueue worker: */ +void wg_packet_decrypt_worker(struct work_struct *work); + +/* send.c APIs: */ +void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, + bool is_retry); +void wg_packet_send_handshake_response(struct wg_peer *peer); +void wg_packet_send_handshake_cookie(struct wg_device *wg, + struct sk_buff *initiating_skb, + __le32 sender_index); +void wg_packet_send_keepalive(struct wg_peer *peer); +void wg_packet_purge_staged_packets(struct wg_peer *peer); +void wg_packet_send_staged_packets(struct wg_peer *peer); +/* Workqueue workers: */ +void wg_packet_handshake_send_worker(struct work_struct *work); +void wg_packet_tx_worker(struct work_struct *work); +void wg_packet_encrypt_worker(struct work_struct *work); + +enum packet_state { + PACKET_STATE_UNCRYPTED, + PACKET_STATE_CRYPTED, + PACKET_STATE_DEAD +}; + +struct packet_cb { + u64 nonce; + struct noise_keypair *keypair; + atomic_t state; + u32 mtu; + u8 ds; +}; + +#define PACKET_CB(skb) ((struct packet_cb *)((skb)->cb)) +#define PACKET_PEER(skb) (PACKET_CB(skb)->keypair->entry.peer) + +/* Returns either the correct skb->protocol value, or 0 if invalid. */ +static inline __be16 wg_skb_examine_untrusted_ip_hdr(struct sk_buff *skb) +{ + if (skb_network_header(skb) >= skb->head && + (skb_network_header(skb) + sizeof(struct iphdr)) <= + skb_tail_pointer(skb) && + ip_hdr(skb)->version == 4) + return htons(ETH_P_IP); + if (skb_network_header(skb) >= skb->head && + (skb_network_header(skb) + sizeof(struct ipv6hdr)) <= + skb_tail_pointer(skb) && + ipv6_hdr(skb)->version == 6) + return htons(ETH_P_IPV6); + return 0; +} + +static inline void wg_reset_packet(struct sk_buff *skb) +{ + const int pfmemalloc = skb->pfmemalloc; + + skb_scrub_packet(skb, true); + memset(&skb->headers_start, 0, + offsetof(struct sk_buff, headers_end) - + offsetof(struct sk_buff, headers_start)); + skb->pfmemalloc = pfmemalloc; + skb->queue_mapping = 0; + skb->nohdr = 0; + skb->peeked = 0; + skb->mac_len = 0; + skb->dev = NULL; +#ifdef CONFIG_NET_SCHED + skb->tc_index = 0; + skb_reset_tc(skb); +#endif + skb->hdr_len = skb_headroom(skb); + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + skb_probe_transport_header(skb); + skb_reset_inner_headers(skb); +} + +static inline int wg_cpumask_choose_online(int *stored_cpu, unsigned int id) +{ + unsigned int cpu = *stored_cpu, cpu_index, i; + + if (unlikely(cpu == nr_cpumask_bits || + !cpumask_test_cpu(cpu, cpu_online_mask))) { + cpu_index = id % cpumask_weight(cpu_online_mask); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < cpu_index; ++i) + cpu = cpumask_next(cpu, cpu_online_mask); + *stored_cpu = cpu; + } + return cpu; +} + +/* This function is racy, in the sense that next is unlocked, so it could return + * the same CPU twice. A race-free version of this would be to instead store an + * atomic sequence number, do an increment-and-return, and then iterate through + * every possible CPU until we get to that index -- choose_cpu. However that's + * a bit slower, and it doesn't seem like this potential race actually + * introduces any performance loss, so we live with it. + */ +static inline int wg_cpumask_next_online(int *next) +{ + int cpu = *next; + + while (unlikely(!cpumask_test_cpu(cpu, cpu_online_mask))) + cpu = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; + *next = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; + return cpu; +} + +static inline int wg_queue_enqueue_per_device_and_peer( + struct crypt_queue *device_queue, struct crypt_queue *peer_queue, + struct sk_buff *skb, struct workqueue_struct *wq, int *next_cpu) +{ + int cpu; + + atomic_set_release(&PACKET_CB(skb)->state, PACKET_STATE_UNCRYPTED); + /* We first queue this up for the peer ingestion, but the consumer + * will wait for the state to change to CRYPTED or DEAD before. + */ + if (unlikely(ptr_ring_produce_bh(&peer_queue->ring, skb))) + return -ENOSPC; + /* Then we queue it up in the device queue, which consumes the + * packet as soon as it can. + */ + cpu = wg_cpumask_next_online(next_cpu); + if (unlikely(ptr_ring_produce_bh(&device_queue->ring, skb))) + return -EPIPE; + queue_work_on(cpu, wq, &per_cpu_ptr(device_queue->worker, cpu)->work); + return 0; +} + +static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue, + struct sk_buff *skb, + enum packet_state state) +{ + /* We take a reference, because as soon as we call atomic_set, the + * peer can be freed from below us. + */ + struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); + + atomic_set_release(&PACKET_CB(skb)->state, state); + queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, + peer->internal_id), + peer->device->packet_crypt_wq, &queue->work); + wg_peer_put(peer); +} + +static inline void wg_queue_enqueue_per_peer_napi(struct sk_buff *skb, + enum packet_state state) +{ + /* We take a reference, because as soon as we call atomic_set, the + * peer can be freed from below us. + */ + struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); + + atomic_set_release(&PACKET_CB(skb)->state, state); + napi_schedule(&peer->napi); + wg_peer_put(peer); +} + +#ifdef DEBUG +bool wg_packet_counter_selftest(void); +#endif + +#endif /* _WG_QUEUEING_H */ diff --git a/net/wireguard/ratelimiter.c b/net/wireguard/ratelimiter.c new file mode 100644 index 0000000000000..e33ec72a96421 --- /dev/null +++ b/net/wireguard/ratelimiter.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef COMPAT_CANNOT_DEPRECIATE_BH_RCU +/* We normally alias all non-_bh functions to the _bh ones in the compat layer, + * but that's not appropriate here, where we actually do want non-_bh ones. + */ +#undef synchronize_rcu +#define synchronize_rcu old_synchronize_rcu +#undef call_rcu +#define call_rcu old_call_rcu +#undef rcu_barrier +#define rcu_barrier old_rcu_barrier +#endif + +#include "ratelimiter.h" +#include +#include +#include +#include + +static struct kmem_cache *entry_cache; +static hsiphash_key_t key; +static spinlock_t table_lock = __SPIN_LOCK_UNLOCKED("ratelimiter_table_lock"); +static DEFINE_MUTEX(init_lock); +static u64 init_refcnt; /* Protected by init_lock, hence not atomic. */ +static atomic_t total_entries = ATOMIC_INIT(0); +static unsigned int max_entries, table_size; +static void wg_ratelimiter_gc_entries(struct work_struct *); +static DECLARE_DEFERRABLE_WORK(gc_work, wg_ratelimiter_gc_entries); +static struct hlist_head *table_v4; +#if IS_ENABLED(CONFIG_IPV6) +static struct hlist_head *table_v6; +#endif + +struct ratelimiter_entry { + u64 last_time_ns, tokens, ip; + void *net; + spinlock_t lock; + struct hlist_node hash; + struct rcu_head rcu; +}; + +enum { + PACKETS_PER_SECOND = 20, + PACKETS_BURSTABLE = 5, + PACKET_COST = NSEC_PER_SEC / PACKETS_PER_SECOND, + TOKEN_MAX = PACKET_COST * PACKETS_BURSTABLE +}; + +static void entry_free(struct rcu_head *rcu) +{ + kmem_cache_free(entry_cache, + container_of(rcu, struct ratelimiter_entry, rcu)); + atomic_dec(&total_entries); +} + +static void entry_uninit(struct ratelimiter_entry *entry) +{ + hlist_del_rcu(&entry->hash); + call_rcu(&entry->rcu, entry_free); +} + +/* Calling this function with a NULL work uninits all entries. */ +static void wg_ratelimiter_gc_entries(struct work_struct *work) +{ + const u64 now = ktime_get_coarse_boottime_ns(); + struct ratelimiter_entry *entry; + struct hlist_node *temp; + unsigned int i; + + for (i = 0; i < table_size; ++i) { + spin_lock(&table_lock); + hlist_for_each_entry_safe(entry, temp, &table_v4[i], hash) { + if (unlikely(!work) || + now - entry->last_time_ns > NSEC_PER_SEC) + entry_uninit(entry); + } +#if IS_ENABLED(CONFIG_IPV6) + hlist_for_each_entry_safe(entry, temp, &table_v6[i], hash) { + if (unlikely(!work) || + now - entry->last_time_ns > NSEC_PER_SEC) + entry_uninit(entry); + } +#endif + spin_unlock(&table_lock); + if (likely(work)) + cond_resched(); + } + if (likely(work)) + queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); +} + +bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net) +{ + /* We only take the bottom half of the net pointer, so that we can hash + * 3 words in the end. This way, siphash's len param fits into the final + * u32, and we don't incur an extra round. + */ + const u32 net_word = (unsigned long)net; + struct ratelimiter_entry *entry; + struct hlist_head *bucket; + u64 ip; + + if (skb->protocol == htons(ETH_P_IP)) { + ip = (u64 __force)ip_hdr(skb)->saddr; + bucket = &table_v4[hsiphash_2u32(net_word, ip, &key) & + (table_size - 1)]; + } +#if IS_ENABLED(CONFIG_IPV6) + else if (skb->protocol == htons(ETH_P_IPV6)) { + /* Only use 64 bits, so as to ratelimit the whole /64. */ + memcpy(&ip, &ipv6_hdr(skb)->saddr, sizeof(ip)); + bucket = &table_v6[hsiphash_3u32(net_word, ip >> 32, ip, &key) & + (table_size - 1)]; + } +#endif + else + return false; + rcu_read_lock(); + hlist_for_each_entry_rcu(entry, bucket, hash) { + if (entry->net == net && entry->ip == ip) { + u64 now, tokens; + bool ret; + /* Quasi-inspired by nft_limit.c, but this is actually a + * slightly different algorithm. Namely, we incorporate + * the burst as part of the maximum tokens, rather than + * as part of the rate. + */ + spin_lock(&entry->lock); + now = ktime_get_coarse_boottime_ns(); + tokens = min_t(u64, TOKEN_MAX, + entry->tokens + now - + entry->last_time_ns); + entry->last_time_ns = now; + ret = tokens >= PACKET_COST; + entry->tokens = ret ? tokens - PACKET_COST : tokens; + spin_unlock(&entry->lock); + rcu_read_unlock(); + return ret; + } + } + rcu_read_unlock(); + + if (atomic_inc_return(&total_entries) > max_entries) + goto err_oom; + + entry = kmem_cache_alloc(entry_cache, GFP_KERNEL); + if (unlikely(!entry)) + goto err_oom; + + entry->net = net; + entry->ip = ip; + INIT_HLIST_NODE(&entry->hash); + spin_lock_init(&entry->lock); + entry->last_time_ns = ktime_get_coarse_boottime_ns(); + entry->tokens = TOKEN_MAX - PACKET_COST; + spin_lock(&table_lock); + hlist_add_head_rcu(&entry->hash, bucket); + spin_unlock(&table_lock); + return true; + +err_oom: + atomic_dec(&total_entries); + return false; +} + +int wg_ratelimiter_init(void) +{ + mutex_lock(&init_lock); + if (++init_refcnt != 1) + goto out; + + entry_cache = KMEM_CACHE(ratelimiter_entry, 0); + if (!entry_cache) + goto err; + + /* xt_hashlimit.c uses a slightly different algorithm for ratelimiting, + * but what it shares in common is that it uses a massive hashtable. So, + * we borrow their wisdom about good table sizes on different systems + * dependent on RAM. This calculation here comes from there. + */ + table_size = (totalram_pages() > (1U << 30) / PAGE_SIZE) ? 8192 : + max_t(unsigned long, 16, roundup_pow_of_two( + (totalram_pages() << PAGE_SHIFT) / + (1U << 14) / sizeof(struct hlist_head))); + max_entries = table_size * 8; + + table_v4 = kvzalloc(table_size * sizeof(*table_v4), GFP_KERNEL); + if (unlikely(!table_v4)) + goto err_kmemcache; + +#if IS_ENABLED(CONFIG_IPV6) + table_v6 = kvzalloc(table_size * sizeof(*table_v6), GFP_KERNEL); + if (unlikely(!table_v6)) { + kvfree(table_v4); + goto err_kmemcache; + } +#endif + + queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); + get_random_bytes(&key, sizeof(key)); +out: + mutex_unlock(&init_lock); + return 0; + +err_kmemcache: + kmem_cache_destroy(entry_cache); +err: + --init_refcnt; + mutex_unlock(&init_lock); + return -ENOMEM; +} + +void wg_ratelimiter_uninit(void) +{ + mutex_lock(&init_lock); + if (!init_refcnt || --init_refcnt) + goto out; + + cancel_delayed_work_sync(&gc_work); + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + kvfree(table_v4); +#if IS_ENABLED(CONFIG_IPV6) + kvfree(table_v6); +#endif + kmem_cache_destroy(entry_cache); +out: + mutex_unlock(&init_lock); +} + +#include "selftest/ratelimiter.c" diff --git a/net/wireguard/ratelimiter.h b/net/wireguard/ratelimiter.h new file mode 100644 index 0000000000000..83067f71ea99e --- /dev/null +++ b/net/wireguard/ratelimiter.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_RATELIMITER_H +#define _WG_RATELIMITER_H + +#include + +int wg_ratelimiter_init(void); +void wg_ratelimiter_uninit(void); +bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net); + +#ifdef DEBUG +bool wg_ratelimiter_selftest(void); +#endif + +#endif /* _WG_RATELIMITER_H */ diff --git a/net/wireguard/receive.c b/net/wireguard/receive.c new file mode 100644 index 0000000000000..e00f0f4b09827 --- /dev/null +++ b/net/wireguard/receive.c @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "device.h" +#include "peer.h" +#include "timers.h" +#include "messages.h" +#include "cookie.h" +#include "socket.h" + +#include +#include +#include +#include +#include + +/* Must be called with bh disabled. */ +static void update_rx_stats(struct wg_peer *peer, size_t len) +{ + struct pcpu_sw_netstats *tstats = + get_cpu_ptr(peer->device->dev->tstats); + + u64_stats_update_begin(&tstats->syncp); + ++tstats->rx_packets; + tstats->rx_bytes += len; + peer->rx_bytes += len; + u64_stats_update_end(&tstats->syncp); + put_cpu_ptr(tstats); +} + +#define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) + +static size_t validate_header_len(struct sk_buff *skb) +{ + if (unlikely(skb->len < sizeof(struct message_header))) + return 0; + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && + skb->len >= MESSAGE_MINIMUM_LENGTH) + return sizeof(struct message_data); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && + skb->len == sizeof(struct message_handshake_initiation)) + return sizeof(struct message_handshake_initiation); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && + skb->len == sizeof(struct message_handshake_response)) + return sizeof(struct message_handshake_response); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && + skb->len == sizeof(struct message_handshake_cookie)) + return sizeof(struct message_handshake_cookie); + return 0; +} + +static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) +{ + size_t data_offset, data_len, header_len; + struct udphdr *udp; + + if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol || + skb_transport_header(skb) < skb->head || + (skb_transport_header(skb) + sizeof(struct udphdr)) > + skb_tail_pointer(skb))) + return -EINVAL; /* Bogus IP header */ + udp = udp_hdr(skb); + data_offset = (u8 *)udp - skb->data; + if (unlikely(data_offset > U16_MAX || + data_offset + sizeof(struct udphdr) > skb->len)) + /* Packet has offset at impossible location or isn't big enough + * to have UDP fields. + */ + return -EINVAL; + data_len = ntohs(udp->len); + if (unlikely(data_len < sizeof(struct udphdr) || + data_len > skb->len - data_offset)) + /* UDP packet is reporting too small of a size or lying about + * its size. + */ + return -EINVAL; + data_len -= sizeof(struct udphdr); + data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; + if (unlikely(!pskb_may_pull(skb, + data_offset + sizeof(struct message_header)) || + pskb_trim(skb, data_len + data_offset) < 0)) + return -EINVAL; + skb_pull(skb, data_offset); + if (unlikely(skb->len != data_len)) + /* Final len does not agree with calculated len */ + return -EINVAL; + header_len = validate_header_len(skb); + if (unlikely(!header_len)) + return -EINVAL; + __skb_push(skb, data_offset); + if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) + return -EINVAL; + __skb_pull(skb, data_offset); + return 0; +} + +static void wg_receive_handshake_packet(struct wg_device *wg, + struct sk_buff *skb) +{ + enum cookie_mac_state mac_state; + struct wg_peer *peer = NULL; + /* This is global, so that our load calculation applies to the whole + * system. We don't care about races with it at all. + */ + static u64 last_under_load; + bool packet_needs_cookie; + bool under_load; + + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { + net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", + wg->dev->name, skb); + wg_cookie_message_consume( + (struct message_handshake_cookie *)skb->data, wg); + return; + } + + under_load = skb_queue_len(&wg->incoming_handshakes) >= + MAX_QUEUED_INCOMING_HANDSHAKES / 8; + if (under_load) + last_under_load = ktime_get_coarse_boottime_ns(); + else if (last_under_load) + under_load = !wg_birthdate_has_expired(last_under_load, 1); + mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, + under_load); + if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || + (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { + packet_needs_cookie = false; + } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { + packet_needs_cookie = true; + } else { + net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", + wg->dev->name, skb); + return; + } + + switch (SKB_TYPE_LE32(skb)) { + case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { + struct message_handshake_initiation *message = + (struct message_handshake_initiation *)skb->data; + + if (packet_needs_cookie) { + wg_packet_send_handshake_cookie(wg, skb, + message->sender_index); + return; + } + peer = wg_noise_handshake_consume_initiation(message, wg); + if (unlikely(!peer)) { + net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", + wg->dev->name, skb); + return; + } + wg_socket_set_peer_endpoint_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", + wg->dev->name, peer->internal_id, + &peer->endpoint.addr); + wg_packet_send_handshake_response(peer); + break; + } + case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { + struct message_handshake_response *message = + (struct message_handshake_response *)skb->data; + + if (packet_needs_cookie) { + wg_packet_send_handshake_cookie(wg, skb, + message->sender_index); + return; + } + peer = wg_noise_handshake_consume_response(message, wg); + if (unlikely(!peer)) { + net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", + wg->dev->name, skb); + return; + } + wg_socket_set_peer_endpoint_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", + wg->dev->name, peer->internal_id, + &peer->endpoint.addr); + if (wg_noise_handshake_begin_session(&peer->handshake, + &peer->keypairs)) { + wg_timers_session_derived(peer); + wg_timers_handshake_complete(peer); + /* Calling this function will either send any existing + * packets in the queue and not send a keepalive, which + * is the best case, Or, if there's nothing in the + * queue, it will send a keepalive, in order to give + * immediate confirmation of the session. + */ + wg_packet_send_keepalive(peer); + } + break; + } + } + + if (unlikely(!peer)) { + WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n"); + return; + } + + local_bh_disable(); + update_rx_stats(peer, skb->len); + local_bh_enable(); + + wg_timers_any_authenticated_packet_received(peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_peer_put(peer); +} + +void wg_packet_handshake_receive_worker(struct work_struct *work) +{ + struct wg_device *wg = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *skb; + + while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) { + wg_receive_handshake_packet(wg, skb); + dev_kfree_skb(skb); + cond_resched(); + } +} + +static void keep_key_fresh(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + bool send = false; + + if (peer->sent_lastminute_handshake) + return; + + rcu_read_lock_bh(); + keypair = rcu_dereference_bh(peer->keypairs.current_keypair); + if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && + keypair->i_am_the_initiator && + unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, + REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT))) + send = true; + rcu_read_unlock_bh(); + + if (send) { + peer->sent_lastminute_handshake = true; + wg_packet_send_queued_handshake_initiation(peer, false); + } +} + +static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key, + simd_context_t *simd_context) +{ + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct sk_buff *trailer; + unsigned int offset; + int num_frags; + + if (unlikely(!key)) + return false; + + if (unlikely(!READ_ONCE(key->is_valid) || + wg_birthdate_has_expired(key->birthdate, REJECT_AFTER_TIME) || + key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) { + WRITE_ONCE(key->is_valid, false); + return false; + } + + PACKET_CB(skb)->nonce = + le64_to_cpu(((struct message_data *)skb->data)->counter); + + /* We ensure that the network header is part of the packet before we + * call skb_cow_data, so that there's no chance that data is removed + * from the skb, so that later we can extract the original endpoint. + */ + offset = skb->data - skb_network_header(skb); + skb_push(skb, offset); + num_frags = skb_cow_data(skb, 0, &trailer); + offset += sizeof(struct message_data); + skb_pull(skb, offset); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + return false; + + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) + return false; + + if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0, + PACKET_CB(skb)->nonce, key->key, + simd_context)) + return false; + + /* Another ugly situation of pushing and pulling the header so as to + * keep endpoint information intact. + */ + skb_push(skb, offset); + if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) + return false; + skb_pull(skb, offset); + + return true; +} + +/* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ +static bool counter_validate(union noise_counter *counter, u64 their_counter) +{ + unsigned long index, index_current, top, i; + bool ret = false; + + spin_lock_bh(&counter->receive.lock); + + if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 || + their_counter >= REJECT_AFTER_MESSAGES)) + goto out; + + ++their_counter; + + if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < + counter->receive.counter)) + goto out; + + index = their_counter >> ilog2(BITS_PER_LONG); + + if (likely(their_counter > counter->receive.counter)) { + index_current = counter->receive.counter >> ilog2(BITS_PER_LONG); + top = min_t(unsigned long, index - index_current, + COUNTER_BITS_TOTAL / BITS_PER_LONG); + for (i = 1; i <= top; ++i) + counter->receive.backtrack[(i + index_current) & + ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; + counter->receive.counter = their_counter; + } + + index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; + ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1), + &counter->receive.backtrack[index]); + +out: + spin_unlock_bh(&counter->receive.lock); + return ret; +} + +#include "selftest/counter.c" + +static void wg_packet_consume_data_done(struct wg_peer *peer, + struct sk_buff *skb, + struct endpoint *endpoint) +{ + struct net_device *dev = peer->device->dev; + unsigned int len, len_before_trim; + struct wg_peer *routed_peer; + + wg_socket_set_peer_endpoint(peer, endpoint); + + if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, + PACKET_CB(skb)->keypair))) { + wg_timers_handshake_complete(peer); + wg_packet_send_staged_packets(peer); + } + + keep_key_fresh(peer); + + wg_timers_any_authenticated_packet_received(peer); + wg_timers_any_authenticated_packet_traversal(peer); + + /* A packet with length 0 is a keepalive packet */ + if (unlikely(!skb->len)) { + update_rx_stats(peer, message_data_len(0)); + net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, + &peer->endpoint.addr); + goto packet_processed; + } + + wg_timers_data_received(peer); + + if (unlikely(skb_network_header(skb) < skb->head)) + goto dishonest_packet_size; + if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && + (ip_hdr(skb)->version == 4 || + (ip_hdr(skb)->version == 6 && + pskb_network_may_pull(skb, sizeof(struct ipv6hdr))))))) + goto dishonest_packet_type; + + skb->dev = dev; + /* We've already verified the Poly1305 auth tag, which means this packet + * was not modified in transit. We can therefore tell the networking + * stack that all checksums of every layer of encapsulation have already + * been checked "by the hardware" and therefore is unnecessary to check + * again in software. + */ + skb->ip_summed = CHECKSUM_UNNECESSARY; +#ifndef COMPAT_CANNOT_USE_CSUM_LEVEL + skb->csum_level = ~0; /* All levels */ +#endif + skb->protocol = wg_skb_examine_untrusted_ip_hdr(skb); + if (skb->protocol == htons(ETH_P_IP)) { + len = ntohs(ip_hdr(skb)->tot_len); + if (unlikely(len < sizeof(struct iphdr))) + goto dishonest_packet_size; + if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) + IP_ECN_set_ce(ip_hdr(skb)); + } else if (skb->protocol == htons(ETH_P_IPV6)) { + len = ntohs(ipv6_hdr(skb)->payload_len) + + sizeof(struct ipv6hdr); + if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) + IP6_ECN_set_ce(skb, ipv6_hdr(skb)); + } else { + goto dishonest_packet_type; + } + + if (unlikely(len > skb->len)) + goto dishonest_packet_size; + len_before_trim = skb->len; + if (unlikely(pskb_trim(skb, len))) + goto packet_processed; + + routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips, + skb); + wg_peer_put(routed_peer); /* We don't need the extra reference. */ + + if (unlikely(routed_peer != peer)) + goto dishonest_packet_peer; + + if (unlikely(napi_gro_receive(&peer->napi, skb) == GRO_DROP)) { + ++dev->stats.rx_dropped; + net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, + &peer->endpoint.addr); + } else { + update_rx_stats(peer, message_data_len(len_before_trim)); + } + return; + +dishonest_packet_peer: + net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n", + dev->name, skb, peer->internal_id, + &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_frame_errors; + goto packet_processed; +dishonest_packet_type: + net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_frame_errors; + goto packet_processed; +dishonest_packet_size: + net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_length_errors; + goto packet_processed; +packet_processed: + dev_kfree_skb(skb); +} + +int wg_packet_rx_poll(struct napi_struct *napi, int budget) +{ + struct wg_peer *peer = container_of(napi, struct wg_peer, napi); + struct crypt_queue *queue = &peer->rx_queue; + struct noise_keypair *keypair; + struct endpoint endpoint; + enum packet_state state; + struct sk_buff *skb; + int work_done = 0; + bool free; + + if (unlikely(budget <= 0)) + return 0; + + while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && + (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != + PACKET_STATE_UNCRYPTED) { + __ptr_ring_discard_one(&queue->ring); + peer = PACKET_PEER(skb); + keypair = PACKET_CB(skb)->keypair; + free = true; + + if (unlikely(state != PACKET_STATE_CRYPTED)) + goto next; + + if (unlikely(!counter_validate(&keypair->receiving.counter, + PACKET_CB(skb)->nonce))) { + net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", + peer->device->dev->name, + PACKET_CB(skb)->nonce, + keypair->receiving.counter.receive.counter); + goto next; + } + + if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) + goto next; + + wg_reset_packet(skb); + wg_packet_consume_data_done(peer, skb, &endpoint); + free = false; + +next: + wg_noise_keypair_put(keypair, false); + wg_peer_put(peer); + if (unlikely(free)) + dev_kfree_skb(skb); + + if (++work_done >= budget) + break; + } + + if (work_done < budget) + napi_complete_done(napi, work_done); + + return work_done; +} + +void wg_packet_decrypt_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct multicore_worker, + work)->ptr; + simd_context_t simd_context; + struct sk_buff *skb; + + simd_get(&simd_context); + while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { + enum packet_state state = likely(decrypt_packet(skb, + &PACKET_CB(skb)->keypair->receiving, + &simd_context)) ? + PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; + wg_queue_enqueue_per_peer_napi(skb, state); + simd_relax(&simd_context); + } + + simd_put(&simd_context); +} + +static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) +{ + __le32 idx = ((struct message_data *)skb->data)->key_idx; + struct wg_peer *peer = NULL; + int ret; + + rcu_read_lock_bh(); + PACKET_CB(skb)->keypair = + (struct noise_keypair *)wg_index_hashtable_lookup( + wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx, + &peer); + if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair))) + goto err_keypair; + + if (unlikely(READ_ONCE(peer->is_dead))) + goto err; + + ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, + &peer->rx_queue, skb, + wg->packet_crypt_wq, + &wg->decrypt_queue.last_cpu); + if (unlikely(ret == -EPIPE)) + wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); + if (likely(!ret || ret == -EPIPE)) { + rcu_read_unlock_bh(); + return; + } +err: + wg_noise_keypair_put(PACKET_CB(skb)->keypair, false); +err_keypair: + rcu_read_unlock_bh(); + wg_peer_put(peer); + dev_kfree_skb(skb); +} + +void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) +{ + if (unlikely(prepare_skb_header(skb, wg) < 0)) + goto err; + switch (SKB_TYPE_LE32(skb)) { + case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): + case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): + case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { + int cpu; + + if (skb_queue_len(&wg->incoming_handshakes) > + MAX_QUEUED_INCOMING_HANDSHAKES || + unlikely(!rng_is_initialized())) { + net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", + wg->dev->name, skb); + goto err; + } + skb_queue_tail(&wg->incoming_handshakes, skb); + /* Queues up a call to packet_process_queued_handshake_ + * packets(skb): + */ + cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu); + queue_work_on(cpu, wg->handshake_receive_wq, + &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work); + break; + } + case cpu_to_le32(MESSAGE_DATA): + PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb); + wg_packet_consume_data(wg, skb); + break; + default: + net_dbg_skb_ratelimited("%s: Invalid packet from %pISpfsc\n", + wg->dev->name, skb); + goto err; + } + return; + +err: + dev_kfree_skb(skb); +} diff --git a/net/wireguard/selftest/allowedips.c b/net/wireguard/selftest/allowedips.c new file mode 100644 index 0000000000000..846db14cb046b --- /dev/null +++ b/net/wireguard/selftest/allowedips.c @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This contains some basic static unit tests for the allowedips data structure. + * It also has two additional modes that are disabled and meant to be used by + * folks directly playing with this file. If you define the macro + * DEBUG_PRINT_TRIE_GRAPHVIZ to be 1, then every time there's a full tree in + * memory, it will be printed out as KERN_DEBUG in a format that can be passed + * to graphviz (the dot command) to visualize it. If you define the macro + * DEBUG_RANDOM_TRIE to be 1, then there will be an extremely costly set of + * randomized tests done against a trivial implementation, which may take + * upwards of a half-hour to complete. There's no set of users who should be + * enabling these, and the only developers that should go anywhere near these + * nobs are the ones who are reading this comment. + */ + +#ifdef DEBUG + +#include + +static __init void swap_endian_and_apply_cidr(u8 *dst, const u8 *src, u8 bits, + u8 cidr) +{ + swap_endian(dst, src, bits); + memset(dst + (cidr + 7) / 8, 0, bits / 8 - (cidr + 7) / 8); + if (cidr) + dst[(cidr + 7) / 8 - 1] &= ~0U << ((8 - (cidr % 8)) % 8); +} + +static __init void print_node(struct allowedips_node *node, u8 bits) +{ + char *fmt_connection = KERN_DEBUG "\t\"%p/%d\" -> \"%p/%d\";\n"; + char *fmt_declaration = KERN_DEBUG + "\t\"%p/%d\"[style=%s, color=\"#%06x\"];\n"; + char *style = "dotted"; + u8 ip1[16], ip2[16]; + u32 color = 0; + + if (bits == 32) { + fmt_connection = KERN_DEBUG "\t\"%pI4/%d\" -> \"%pI4/%d\";\n"; + fmt_declaration = KERN_DEBUG + "\t\"%pI4/%d\"[style=%s, color=\"#%06x\"];\n"; + } else if (bits == 128) { + fmt_connection = KERN_DEBUG "\t\"%pI6/%d\" -> \"%pI6/%d\";\n"; + fmt_declaration = KERN_DEBUG + "\t\"%pI6/%d\"[style=%s, color=\"#%06x\"];\n"; + } + if (node->peer) { + hsiphash_key_t key = { { 0 } }; + + memcpy(&key, &node->peer, sizeof(node->peer)); + color = hsiphash_1u32(0xdeadbeef, &key) % 200 << 16 | + hsiphash_1u32(0xbabecafe, &key) % 200 << 8 | + hsiphash_1u32(0xabad1dea, &key) % 200; + style = "bold"; + } + swap_endian_and_apply_cidr(ip1, node->bits, bits, node->cidr); + printk(fmt_declaration, ip1, node->cidr, style, color); + if (node->bit[0]) { + swap_endian_and_apply_cidr(ip2, + rcu_dereference_raw(node->bit[0])->bits, bits, + node->cidr); + printk(fmt_connection, ip1, node->cidr, ip2, + rcu_dereference_raw(node->bit[0])->cidr); + print_node(rcu_dereference_raw(node->bit[0]), bits); + } + if (node->bit[1]) { + swap_endian_and_apply_cidr(ip2, + rcu_dereference_raw(node->bit[1])->bits, + bits, node->cidr); + printk(fmt_connection, ip1, node->cidr, ip2, + rcu_dereference_raw(node->bit[1])->cidr); + print_node(rcu_dereference_raw(node->bit[1]), bits); + } +} + +static __init void print_tree(struct allowedips_node __rcu *top, u8 bits) +{ + printk(KERN_DEBUG "digraph trie {\n"); + print_node(rcu_dereference_raw(top), bits); + printk(KERN_DEBUG "}\n"); +} + +enum { + NUM_PEERS = 2000, + NUM_RAND_ROUTES = 400, + NUM_MUTATED_ROUTES = 100, + NUM_QUERIES = NUM_RAND_ROUTES * NUM_MUTATED_ROUTES * 30 +}; + +struct horrible_allowedips { + struct hlist_head head; +}; + +struct horrible_allowedips_node { + struct hlist_node table; + union nf_inet_addr ip; + union nf_inet_addr mask; + u8 ip_version; + void *value; +}; + +static __init void horrible_allowedips_init(struct horrible_allowedips *table) +{ + INIT_HLIST_HEAD(&table->head); +} + +static __init void horrible_allowedips_free(struct horrible_allowedips *table) +{ + struct horrible_allowedips_node *node; + struct hlist_node *h; + + hlist_for_each_entry_safe(node, h, &table->head, table) { + hlist_del(&node->table); + kfree(node); + } +} + +static __init inline union nf_inet_addr horrible_cidr_to_mask(u8 cidr) +{ + union nf_inet_addr mask; + + memset(&mask, 0x00, 128 / 8); + memset(&mask, 0xff, cidr / 8); + if (cidr % 32) + mask.all[cidr / 32] = (__force u32)htonl( + (0xFFFFFFFFUL << (32 - (cidr % 32))) & 0xFFFFFFFFUL); + return mask; +} + +static __init inline u8 horrible_mask_to_cidr(union nf_inet_addr subnet) +{ + return hweight32(subnet.all[0]) + hweight32(subnet.all[1]) + + hweight32(subnet.all[2]) + hweight32(subnet.all[3]); +} + +static __init inline void +horrible_mask_self(struct horrible_allowedips_node *node) +{ + if (node->ip_version == 4) { + node->ip.ip &= node->mask.ip; + } else if (node->ip_version == 6) { + node->ip.ip6[0] &= node->mask.ip6[0]; + node->ip.ip6[1] &= node->mask.ip6[1]; + node->ip.ip6[2] &= node->mask.ip6[2]; + node->ip.ip6[3] &= node->mask.ip6[3]; + } +} + +static __init inline bool +horrible_match_v4(const struct horrible_allowedips_node *node, + struct in_addr *ip) +{ + return (ip->s_addr & node->mask.ip) == node->ip.ip; +} + +static __init inline bool +horrible_match_v6(const struct horrible_allowedips_node *node, + struct in6_addr *ip) +{ + return (ip->in6_u.u6_addr32[0] & node->mask.ip6[0]) == + node->ip.ip6[0] && + (ip->in6_u.u6_addr32[1] & node->mask.ip6[1]) == + node->ip.ip6[1] && + (ip->in6_u.u6_addr32[2] & node->mask.ip6[2]) == + node->ip.ip6[2] && + (ip->in6_u.u6_addr32[3] & node->mask.ip6[3]) == node->ip.ip6[3]; +} + +static __init void +horrible_insert_ordered(struct horrible_allowedips *table, + struct horrible_allowedips_node *node) +{ + struct horrible_allowedips_node *other = NULL, *where = NULL; + u8 my_cidr = horrible_mask_to_cidr(node->mask); + + hlist_for_each_entry(other, &table->head, table) { + if (!memcmp(&other->mask, &node->mask, + sizeof(union nf_inet_addr)) && + !memcmp(&other->ip, &node->ip, + sizeof(union nf_inet_addr)) && + other->ip_version == node->ip_version) { + other->value = node->value; + kfree(node); + return; + } + where = other; + if (horrible_mask_to_cidr(other->mask) <= my_cidr) + break; + } + if (!other && !where) + hlist_add_head(&node->table, &table->head); + else if (!other) + hlist_add_behind(&node->table, &where->table); + else + hlist_add_before(&node->table, &where->table); +} + +static __init int +horrible_allowedips_insert_v4(struct horrible_allowedips *table, + struct in_addr *ip, u8 cidr, void *value) +{ + struct horrible_allowedips_node *node = kzalloc(sizeof(*node), + GFP_KERNEL); + + if (unlikely(!node)) + return -ENOMEM; + node->ip.in = *ip; + node->mask = horrible_cidr_to_mask(cidr); + node->ip_version = 4; + node->value = value; + horrible_mask_self(node); + horrible_insert_ordered(table, node); + return 0; +} + +static __init int +horrible_allowedips_insert_v6(struct horrible_allowedips *table, + struct in6_addr *ip, u8 cidr, void *value) +{ + struct horrible_allowedips_node *node = kzalloc(sizeof(*node), + GFP_KERNEL); + + if (unlikely(!node)) + return -ENOMEM; + node->ip.in6 = *ip; + node->mask = horrible_cidr_to_mask(cidr); + node->ip_version = 6; + node->value = value; + horrible_mask_self(node); + horrible_insert_ordered(table, node); + return 0; +} + +static __init void * +horrible_allowedips_lookup_v4(struct horrible_allowedips *table, + struct in_addr *ip) +{ + struct horrible_allowedips_node *node; + void *ret = NULL; + + hlist_for_each_entry(node, &table->head, table) { + if (node->ip_version != 4) + continue; + if (horrible_match_v4(node, ip)) { + ret = node->value; + break; + } + } + return ret; +} + +static __init void * +horrible_allowedips_lookup_v6(struct horrible_allowedips *table, + struct in6_addr *ip) +{ + struct horrible_allowedips_node *node; + void *ret = NULL; + + hlist_for_each_entry(node, &table->head, table) { + if (node->ip_version != 6) + continue; + if (horrible_match_v6(node, ip)) { + ret = node->value; + break; + } + } + return ret; +} + +static __init bool randomized_test(void) +{ + unsigned int i, j, k, mutate_amount, cidr; + u8 ip[16], mutate_mask[16], mutated[16]; + struct wg_peer **peers, *peer; + struct horrible_allowedips h; + DEFINE_MUTEX(mutex); + struct allowedips t; + bool ret = false; + + mutex_init(&mutex); + + wg_allowedips_init(&t); + horrible_allowedips_init(&h); + + peers = kcalloc(NUM_PEERS, sizeof(*peers), GFP_KERNEL); + if (unlikely(!peers)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free; + } + for (i = 0; i < NUM_PEERS; ++i) { + peers[i] = kzalloc(sizeof(*peers[i]), GFP_KERNEL); + if (unlikely(!peers[i])) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free; + } + kref_init(&peers[i]->refcount); + } + + mutex_lock(&mutex); + + for (i = 0; i < NUM_RAND_ROUTES; ++i) { + prandom_bytes(ip, 4); + cidr = prandom_u32_max(32) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v4(&t, (struct in_addr *)ip, cidr, + peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v4(&h, (struct in_addr *)ip, + cidr, peer) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { + memcpy(mutated, ip, 4); + prandom_bytes(mutate_mask, 4); + mutate_amount = prandom_u32_max(32); + for (k = 0; k < mutate_amount / 8; ++k) + mutate_mask[k] = 0xff; + mutate_mask[k] = 0xff + << ((8 - (mutate_amount % 8)) % 8); + for (; k < 4; ++k) + mutate_mask[k] = 0; + for (k = 0; k < 4; ++k) + mutated[k] = (mutated[k] & mutate_mask[k]) | + (~mutate_mask[k] & + prandom_u32_max(256)); + cidr = prandom_u32_max(32) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v4(&t, + (struct in_addr *)mutated, + cidr, peer, &mutex) < 0) { + pr_err("allowedips random malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v4(&h, + (struct in_addr *)mutated, cidr, peer)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + } + } + + for (i = 0; i < NUM_RAND_ROUTES; ++i) { + prandom_bytes(ip, 16); + cidr = prandom_u32_max(128) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v6(&t, (struct in6_addr *)ip, cidr, + peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v6(&h, (struct in6_addr *)ip, + cidr, peer) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { + memcpy(mutated, ip, 16); + prandom_bytes(mutate_mask, 16); + mutate_amount = prandom_u32_max(128); + for (k = 0; k < mutate_amount / 8; ++k) + mutate_mask[k] = 0xff; + mutate_mask[k] = 0xff + << ((8 - (mutate_amount % 8)) % 8); + for (; k < 4; ++k) + mutate_mask[k] = 0; + for (k = 0; k < 4; ++k) + mutated[k] = (mutated[k] & mutate_mask[k]) | + (~mutate_mask[k] & + prandom_u32_max(256)); + cidr = prandom_u32_max(128) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v6(&t, + (struct in6_addr *)mutated, + cidr, peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v6( + &h, (struct in6_addr *)mutated, cidr, + peer)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + } + } + + mutex_unlock(&mutex); + + if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { + print_tree(t.root4, 32); + print_tree(t.root6, 128); + } + + for (i = 0; i < NUM_QUERIES; ++i) { + prandom_bytes(ip, 4); + if (lookup(t.root4, 32, ip) != + horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip)) { + pr_err("allowedips random self-test: FAIL\n"); + goto free; + } + } + + for (i = 0; i < NUM_QUERIES; ++i) { + prandom_bytes(ip, 16); + if (lookup(t.root6, 128, ip) != + horrible_allowedips_lookup_v6(&h, (struct in6_addr *)ip)) { + pr_err("allowedips random self-test: FAIL\n"); + goto free; + } + } + ret = true; + +free: + mutex_lock(&mutex); +free_locked: + wg_allowedips_free(&t, &mutex); + mutex_unlock(&mutex); + horrible_allowedips_free(&h); + if (peers) { + for (i = 0; i < NUM_PEERS; ++i) + kfree(peers[i]); + } + kfree(peers); + return ret; +} + +static __init inline struct in_addr *ip4(u8 a, u8 b, u8 c, u8 d) +{ + static struct in_addr ip; + u8 *split = (u8 *)&ip; + + split[0] = a; + split[1] = b; + split[2] = c; + split[3] = d; + return &ip; +} + +static __init inline struct in6_addr *ip6(u32 a, u32 b, u32 c, u32 d) +{ + static struct in6_addr ip; + __be32 *split = (__be32 *)&ip; + + split[0] = cpu_to_be32(a); + split[1] = cpu_to_be32(b); + split[2] = cpu_to_be32(c); + split[3] = cpu_to_be32(d); + return &ip; +} + +static __init struct wg_peer *init_peer(void) +{ + struct wg_peer *peer = kzalloc(sizeof(*peer), GFP_KERNEL); + + if (!peer) + return NULL; + kref_init(&peer->refcount); + INIT_LIST_HEAD(&peer->allowedips_list); + return peer; +} + +#define insert(version, mem, ipa, ipb, ipc, ipd, cidr) \ + wg_allowedips_insert_v##version(&t, ip##version(ipa, ipb, ipc, ipd), \ + cidr, mem, &mutex) + +#define maybe_fail() do { \ + ++i; \ + if (!_s) { \ + pr_info("allowedips self-test %zu: FAIL\n", i); \ + success = false; \ + } \ + } while (0) + +#define test(version, mem, ipa, ipb, ipc, ipd) do { \ + bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ + ip##version(ipa, ipb, ipc, ipd)) == (mem); \ + maybe_fail(); \ + } while (0) + +#define test_negative(version, mem, ipa, ipb, ipc, ipd) do { \ + bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ + ip##version(ipa, ipb, ipc, ipd)) != (mem); \ + maybe_fail(); \ + } while (0) + +#define test_boolean(cond) do { \ + bool _s = (cond); \ + maybe_fail(); \ + } while (0) + +bool __init wg_allowedips_selftest(void) +{ + bool found_a = false, found_b = false, found_c = false, found_d = false, + found_e = false, found_other = false; + struct wg_peer *a = init_peer(), *b = init_peer(), *c = init_peer(), + *d = init_peer(), *e = init_peer(), *f = init_peer(), + *g = init_peer(), *h = init_peer(); + struct allowedips_node *iter_node; + bool success = false; + struct allowedips t; + DEFINE_MUTEX(mutex); + struct in6_addr ip; + size_t i = 0, count = 0; + __be64 part; + + mutex_init(&mutex); + mutex_lock(&mutex); + wg_allowedips_init(&t); + + if (!a || !b || !c || !d || !e || !f || !g || !h) { + pr_err("allowedips self-test malloc: FAIL\n"); + goto free; + } + + insert(4, a, 192, 168, 4, 0, 24); + insert(4, b, 192, 168, 4, 4, 32); + insert(4, c, 192, 168, 0, 0, 16); + insert(4, d, 192, 95, 5, 64, 27); + /* replaces previous entry, and maskself is required */ + insert(4, c, 192, 95, 5, 65, 27); + insert(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + insert(6, c, 0x26075300, 0x60006b00, 0, 0, 64); + insert(4, e, 0, 0, 0, 0, 0); + insert(6, e, 0, 0, 0, 0, 0); + /* replaces previous entry */ + insert(6, f, 0, 0, 0, 0, 0); + insert(6, g, 0x24046800, 0, 0, 0, 32); + /* maskself is required */ + insert(6, h, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 64); + insert(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 128); + insert(6, c, 0x24446800, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128); + insert(6, b, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); + insert(4, g, 64, 15, 112, 0, 20); + /* maskself is required */ + insert(4, h, 64, 15, 123, 211, 25); + insert(4, a, 10, 0, 0, 0, 25); + insert(4, b, 10, 0, 0, 128, 25); + insert(4, a, 10, 1, 0, 0, 30); + insert(4, b, 10, 1, 0, 4, 30); + insert(4, c, 10, 1, 0, 8, 29); + insert(4, d, 10, 1, 0, 16, 29); + + if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { + print_tree(t.root4, 32); + print_tree(t.root6, 128); + } + + success = true; + + test(4, a, 192, 168, 4, 20); + test(4, a, 192, 168, 4, 0); + test(4, b, 192, 168, 4, 4); + test(4, c, 192, 168, 200, 182); + test(4, c, 192, 95, 5, 68); + test(4, e, 192, 95, 5, 96); + test(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543); + test(6, c, 0x26075300, 0x60006b00, 0, 0xc02e01ee); + test(6, f, 0x26075300, 0x60006b01, 0, 0); + test(6, g, 0x24046800, 0x40040806, 0, 0x1006); + test(6, g, 0x24046800, 0x40040806, 0x1234, 0x5678); + test(6, f, 0x240467ff, 0x40040806, 0x1234, 0x5678); + test(6, f, 0x24046801, 0x40040806, 0x1234, 0x5678); + test(6, h, 0x24046800, 0x40040800, 0x1234, 0x5678); + test(6, h, 0x24046800, 0x40040800, 0, 0); + test(6, h, 0x24046800, 0x40040800, 0x10101010, 0x10101010); + test(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef); + test(4, g, 64, 15, 116, 26); + test(4, g, 64, 15, 127, 3); + test(4, g, 64, 15, 123, 1); + test(4, h, 64, 15, 123, 128); + test(4, h, 64, 15, 123, 129); + test(4, a, 10, 0, 0, 52); + test(4, b, 10, 0, 0, 220); + test(4, a, 10, 1, 0, 2); + test(4, b, 10, 1, 0, 6); + test(4, c, 10, 1, 0, 10); + test(4, d, 10, 1, 0, 20); + + insert(4, a, 1, 0, 0, 0, 32); + insert(4, a, 64, 0, 0, 0, 32); + insert(4, a, 128, 0, 0, 0, 32); + insert(4, a, 192, 0, 0, 0, 32); + insert(4, a, 255, 0, 0, 0, 32); + wg_allowedips_remove_by_peer(&t, a, &mutex); + test_negative(4, a, 1, 0, 0, 0); + test_negative(4, a, 64, 0, 0, 0); + test_negative(4, a, 128, 0, 0, 0); + test_negative(4, a, 192, 0, 0, 0); + test_negative(4, a, 255, 0, 0, 0); + + wg_allowedips_free(&t, &mutex); + wg_allowedips_init(&t); + insert(4, a, 192, 168, 0, 0, 16); + insert(4, a, 192, 168, 0, 0, 24); + wg_allowedips_remove_by_peer(&t, a, &mutex); + test_negative(4, a, 192, 168, 0, 1); + + /* These will hit the WARN_ON(len >= 128) in free_node if something + * goes wrong. + */ + for (i = 0; i < 128; ++i) { + part = cpu_to_be64(~(1LLU << (i % 64))); + memset(&ip, 0xff, 16); + memcpy((u8 *)&ip + (i < 64) * 8, &part, 8); + wg_allowedips_insert_v6(&t, &ip, 128, a, &mutex); + } + + wg_allowedips_free(&t, &mutex); + + wg_allowedips_init(&t); + insert(4, a, 192, 95, 5, 93, 27); + insert(6, a, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + insert(4, a, 10, 1, 0, 20, 29); + insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 83); + insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 21); + list_for_each_entry(iter_node, &a->allowedips_list, peer_list) { + u8 cidr, ip[16] __aligned(__alignof(u64)); + int family = wg_allowedips_read_node(iter_node, ip, &cidr); + + count++; + + if (cidr == 27 && family == AF_INET && + !memcmp(ip, ip4(192, 95, 5, 64), sizeof(struct in_addr))) + found_a = true; + else if (cidr == 128 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075300, 0x60006b00, 0, 0xc05f0543), + sizeof(struct in6_addr))) + found_b = true; + else if (cidr == 29 && family == AF_INET && + !memcmp(ip, ip4(10, 1, 0, 16), sizeof(struct in_addr))) + found_c = true; + else if (cidr == 83 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075300, 0x6d8a6bf8, 0xdab1e000, 0), + sizeof(struct in6_addr))) + found_d = true; + else if (cidr == 21 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075000, 0, 0, 0), + sizeof(struct in6_addr))) + found_e = true; + else + found_other = true; + } + test_boolean(count == 5); + test_boolean(found_a); + test_boolean(found_b); + test_boolean(found_c); + test_boolean(found_d); + test_boolean(found_e); + test_boolean(!found_other); + + if (IS_ENABLED(DEBUG_RANDOM_TRIE) && success) + success = randomized_test(); + + if (success) + pr_info("allowedips self-tests: pass\n"); + +free: + wg_allowedips_free(&t, &mutex); + kfree(a); + kfree(b); + kfree(c); + kfree(d); + kfree(e); + kfree(f); + kfree(g); + kfree(h); + mutex_unlock(&mutex); + + return success; +} + +#undef test_negative +#undef test +#undef remove +#undef insert +#undef init_peer + +#endif diff --git a/net/wireguard/selftest/counter.c b/net/wireguard/selftest/counter.c new file mode 100644 index 0000000000000..f4fbb9072ed73 --- /dev/null +++ b/net/wireguard/selftest/counter.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef DEBUG +bool __init wg_packet_counter_selftest(void) +{ + unsigned int test_num = 0, i; + union noise_counter counter; + bool success = true; + +#define T_INIT do { \ + memset(&counter, 0, sizeof(union noise_counter)); \ + spin_lock_init(&counter.receive.lock); \ + } while (0) +#define T_LIM (COUNTER_WINDOW_SIZE + 1) +#define T(n, v) do { \ + ++test_num; \ + if (counter_validate(&counter, n) != (v)) { \ + pr_err("nonce counter self-test %u: FAIL\n", \ + test_num); \ + success = false; \ + } \ + } while (0) + + T_INIT; + /* 1 */ T(0, true); + /* 2 */ T(1, true); + /* 3 */ T(1, false); + /* 4 */ T(9, true); + /* 5 */ T(8, true); + /* 6 */ T(7, true); + /* 7 */ T(7, false); + /* 8 */ T(T_LIM, true); + /* 9 */ T(T_LIM - 1, true); + /* 10 */ T(T_LIM - 1, false); + /* 11 */ T(T_LIM - 2, true); + /* 12 */ T(2, true); + /* 13 */ T(2, false); + /* 14 */ T(T_LIM + 16, true); + /* 15 */ T(3, false); + /* 16 */ T(T_LIM + 16, false); + /* 17 */ T(T_LIM * 4, true); + /* 18 */ T(T_LIM * 4 - (T_LIM - 1), true); + /* 19 */ T(10, false); + /* 20 */ T(T_LIM * 4 - T_LIM, false); + /* 21 */ T(T_LIM * 4 - (T_LIM + 1), false); + /* 22 */ T(T_LIM * 4 - (T_LIM - 2), true); + /* 23 */ T(T_LIM * 4 + 1 - T_LIM, false); + /* 24 */ T(0, false); + /* 25 */ T(REJECT_AFTER_MESSAGES, false); + /* 26 */ T(REJECT_AFTER_MESSAGES - 1, true); + /* 27 */ T(REJECT_AFTER_MESSAGES, false); + /* 28 */ T(REJECT_AFTER_MESSAGES - 1, false); + /* 29 */ T(REJECT_AFTER_MESSAGES - 2, true); + /* 30 */ T(REJECT_AFTER_MESSAGES + 1, false); + /* 31 */ T(REJECT_AFTER_MESSAGES + 2, false); + /* 32 */ T(REJECT_AFTER_MESSAGES - 2, false); + /* 33 */ T(REJECT_AFTER_MESSAGES - 3, true); + /* 34 */ T(0, false); + + T_INIT; + for (i = 1; i <= COUNTER_WINDOW_SIZE; ++i) + T(i, true); + T(0, true); + T(0, false); + + T_INIT; + for (i = 2; i <= COUNTER_WINDOW_SIZE + 1; ++i) + T(i, true); + T(1, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 0;) + T(i, true); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 2; i-- > 1;) + T(i, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, true); + T(COUNTER_WINDOW_SIZE + 1, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, true); + T(0, true); + T(COUNTER_WINDOW_SIZE + 1, true); + +#undef T +#undef T_LIM +#undef T_INIT + + if (success) + pr_info("nonce counter self-tests: pass\n"); + return success; +} +#endif diff --git a/net/wireguard/selftest/ratelimiter.c b/net/wireguard/selftest/ratelimiter.c new file mode 100644 index 0000000000000..bcd6462e45401 --- /dev/null +++ b/net/wireguard/selftest/ratelimiter.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef DEBUG + +#include + +static const struct { + bool result; + unsigned int msec_to_sleep_before; +} expected_results[] __initconst = { + [0 ... PACKETS_BURSTABLE - 1] = { true, 0 }, + [PACKETS_BURSTABLE] = { false, 0 }, + [PACKETS_BURSTABLE + 1] = { true, MSEC_PER_SEC / PACKETS_PER_SECOND }, + [PACKETS_BURSTABLE + 2] = { false, 0 }, + [PACKETS_BURSTABLE + 3] = { true, (MSEC_PER_SEC / PACKETS_PER_SECOND) * 2 }, + [PACKETS_BURSTABLE + 4] = { true, 0 }, + [PACKETS_BURSTABLE + 5] = { false, 0 } +}; + +static __init unsigned int maximum_jiffies_at_index(int index) +{ + unsigned int total_msecs = 2 * MSEC_PER_SEC / PACKETS_PER_SECOND / 3; + int i; + + for (i = 0; i <= index; ++i) + total_msecs += expected_results[i].msec_to_sleep_before; + return msecs_to_jiffies(total_msecs); +} + +static __init int timings_test(struct sk_buff *skb4, struct iphdr *hdr4, + struct sk_buff *skb6, struct ipv6hdr *hdr6, + int *test) +{ + unsigned long loop_start_time; + int i; + + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + loop_start_time = jiffies; + + for (i = 0; i < ARRAY_SIZE(expected_results); ++i) { + if (expected_results[i].msec_to_sleep_before) + msleep(expected_results[i].msec_to_sleep_before); + + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (wg_ratelimiter_allow(skb4, &init_net) != + expected_results[i].result) + return -EXFULL; + ++(*test); + + hdr4->saddr = htonl(ntohl(hdr4->saddr) + i + 1); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (!wg_ratelimiter_allow(skb4, &init_net)) + return -EXFULL; + ++(*test); + + hdr4->saddr = htonl(ntohl(hdr4->saddr) - i - 1); + +#if IS_ENABLED(CONFIG_IPV6) + hdr6->saddr.in6_u.u6_addr32[2] = htonl(i); + hdr6->saddr.in6_u.u6_addr32[3] = htonl(i); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (wg_ratelimiter_allow(skb6, &init_net) != + expected_results[i].result) + return -EXFULL; + ++(*test); + + hdr6->saddr.in6_u.u6_addr32[0] = + htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) + i + 1); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (!wg_ratelimiter_allow(skb6, &init_net)) + return -EXFULL; + ++(*test); + + hdr6->saddr.in6_u.u6_addr32[0] = + htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) - i - 1); + + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; +#endif + } + return 0; +} + +static __init int capacity_test(struct sk_buff *skb4, struct iphdr *hdr4, + int *test) +{ + int i; + + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + + if (atomic_read(&total_entries)) + return -EXFULL; + ++(*test); + + for (i = 0; i <= max_entries; ++i) { + hdr4->saddr = htonl(i); + if (wg_ratelimiter_allow(skb4, &init_net) != (i != max_entries)) + return -EXFULL; + ++(*test); + } + return 0; +} + +bool __init wg_ratelimiter_selftest(void) +{ + enum { TRIALS_BEFORE_GIVING_UP = 5000 }; + bool success = false; + int test = 0, trials; + struct sk_buff *skb4, *skb6; + struct iphdr *hdr4; + struct ipv6hdr *hdr6; + + if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_UBSAN)) + return true; + + BUILD_BUG_ON(MSEC_PER_SEC % PACKETS_PER_SECOND != 0); + + if (wg_ratelimiter_init()) + goto out; + ++test; + if (wg_ratelimiter_init()) { + wg_ratelimiter_uninit(); + goto out; + } + ++test; + if (wg_ratelimiter_init()) { + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + goto out; + } + ++test; + + skb4 = alloc_skb(sizeof(struct iphdr), GFP_KERNEL); + if (unlikely(!skb4)) + goto err_nofree; + skb4->protocol = htons(ETH_P_IP); + hdr4 = (struct iphdr *)skb_put(skb4, sizeof(*hdr4)); + hdr4->saddr = htonl(8182); + skb_reset_network_header(skb4); + ++test; + +#if IS_ENABLED(CONFIG_IPV6) + skb6 = alloc_skb(sizeof(struct ipv6hdr), GFP_KERNEL); + if (unlikely(!skb6)) { + kfree_skb(skb4); + goto err_nofree; + } + skb6->protocol = htons(ETH_P_IPV6); + hdr6 = (struct ipv6hdr *)skb_put(skb6, sizeof(*hdr6)); + hdr6->saddr.in6_u.u6_addr32[0] = htonl(1212); + hdr6->saddr.in6_u.u6_addr32[1] = htonl(289188); + skb_reset_network_header(skb6); + ++test; +#endif + + for (trials = TRIALS_BEFORE_GIVING_UP;;) { + int test_count = 0, ret; + + ret = timings_test(skb4, hdr4, skb6, hdr6, &test_count); + if (ret == -ETIMEDOUT) { + if (!trials--) { + test += test_count; + goto err; + } + msleep(500); + continue; + } else if (ret < 0) { + test += test_count; + goto err; + } else { + test += test_count; + break; + } + } + + for (trials = TRIALS_BEFORE_GIVING_UP;;) { + int test_count = 0; + + if (capacity_test(skb4, hdr4, &test_count) < 0) { + if (!trials--) { + test += test_count; + goto err; + } + msleep(50); + continue; + } + test += test_count; + break; + } + + success = true; + +err: + kfree_skb(skb4); +#if IS_ENABLED(CONFIG_IPV6) + kfree_skb(skb6); +#endif +err_nofree: + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + /* Uninit one extra time to check underflow detection. */ + wg_ratelimiter_uninit(); +out: + if (success) + pr_info("ratelimiter self-tests: pass\n"); + else + pr_err("ratelimiter self-test %d: FAIL\n", test); + + return success; +} +#endif diff --git a/net/wireguard/send.c b/net/wireguard/send.c new file mode 100644 index 0000000000000..85b83fafbe4f6 --- /dev/null +++ b/net/wireguard/send.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "messages.h" +#include "cookie.h" + +#include +#include +#include +#include +#include +#include +#include + +static void wg_packet_send_handshake_initiation(struct wg_peer *peer) +{ + struct message_handshake_initiation packet; + + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), + REKEY_TIMEOUT)) + return; /* This function is rate limited. */ + + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { + wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + atomic64_set(&peer->last_sent_handshake, + ktime_get_coarse_boottime_ns()); + wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), + HANDSHAKE_DSCP); + wg_timers_handshake_initiated(peer); + } +} + +void wg_packet_handshake_send_worker(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, + transmit_handshake_work); + + wg_packet_send_handshake_initiation(peer); + wg_peer_put(peer); +} + +void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, + bool is_retry) +{ + if (!is_retry) + peer->timer_handshake_attempts = 0; + + rcu_read_lock_bh(); + /* We check last_sent_handshake here in addition to the actual function + * we're queueing up, so that we don't queue things if not strictly + * necessary: + */ + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), + REKEY_TIMEOUT) || + unlikely(READ_ONCE(peer->is_dead))) + goto out; + + wg_peer_get(peer); + /* Queues up calling packet_send_queued_handshakes(peer), where we do a + * peer_put(peer) after: + */ + if (!queue_work(peer->device->handshake_send_wq, + &peer->transmit_handshake_work)) + /* If the work was already queued, we want to drop the + * extra reference: + */ + wg_peer_put(peer); +out: + rcu_read_unlock_bh(); +} + +void wg_packet_send_handshake_response(struct wg_peer *peer) +{ + struct message_handshake_response packet; + + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { + wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + if (wg_noise_handshake_begin_session(&peer->handshake, + &peer->keypairs)) { + wg_timers_session_derived(peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + atomic64_set(&peer->last_sent_handshake, + ktime_get_coarse_boottime_ns()); + wg_socket_send_buffer_to_peer(peer, &packet, + sizeof(packet), + HANDSHAKE_DSCP); + } + } +} + +void wg_packet_send_handshake_cookie(struct wg_device *wg, + struct sk_buff *initiating_skb, + __le32 sender_index) +{ + struct message_handshake_cookie packet; + + net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n", + wg->dev->name, initiating_skb); + wg_cookie_message_create(&packet, initiating_skb, sender_index, + &wg->cookie_checker); + wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet, + sizeof(packet)); +} + +static void keep_key_fresh(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + bool send = false; + + rcu_read_lock_bh(); + keypair = rcu_dereference_bh(peer->keypairs.current_keypair); + if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && + (unlikely(atomic64_read(&keypair->sending.counter.counter) > + REKEY_AFTER_MESSAGES) || + (keypair->i_am_the_initiator && + unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, + REKEY_AFTER_TIME))))) + send = true; + rcu_read_unlock_bh(); + + if (send) + wg_packet_send_queued_handshake_initiation(peer, false); +} + +static unsigned int calculate_skb_padding(struct sk_buff *skb) +{ + /* We do this modulo business with the MTU, just in case the networking + * layer gives us a packet that's bigger than the MTU. In that case, we + * wouldn't want the final subtraction to overflow in the case of the + * padded_size being clamped. + */ + unsigned int last_unit = skb->len % PACKET_CB(skb)->mtu; + unsigned int padded_size = ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE); + + if (padded_size > PACKET_CB(skb)->mtu) + padded_size = PACKET_CB(skb)->mtu; + return padded_size - last_unit; +} + +static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair, + simd_context_t *simd_context) +{ + unsigned int padding_len, plaintext_len, trailer_len; + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct message_data *header; + struct sk_buff *trailer; + int num_frags; + + /* Calculate lengths. */ + padding_len = calculate_skb_padding(skb); + trailer_len = padding_len + noise_encrypted_len(0); + plaintext_len = skb->len + padding_len; + + /* Expand data section to have room for padding and auth tag. */ + num_frags = skb_cow_data(skb, trailer_len, &trailer); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + return false; + + /* Set the padding to zeros, and make sure it and the auth tag are part + * of the skb. + */ + memset(skb_tail_pointer(trailer), 0, padding_len); + + /* Expand head section to have room for our header and the network + * stack's headers. + */ + if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) + return false; + + /* Finalize checksum calculation for the inner packet, if required. */ + if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && + skb_checksum_help(skb))) + return false; + + /* Only after checksumming can we safely add on the padding at the end + * and the header. + */ + skb_set_inner_network_header(skb, 0); + header = (struct message_data *)skb_push(skb, sizeof(*header)); + header->header.type = cpu_to_le32(MESSAGE_DATA); + header->key_idx = keypair->remote_index; + header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); + pskb_put(skb, trailer, trailer_len); + + /* Now we can encrypt the scattergather segments */ + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, sizeof(struct message_data), + noise_encrypted_len(plaintext_len)) <= 0) + return false; + return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, + PACKET_CB(skb)->nonce, + keypair->sending.key, + simd_context); +} + +void wg_packet_send_keepalive(struct wg_peer *peer) +{ + struct sk_buff *skb; + + if (skb_queue_empty(&peer->staged_packet_queue)) { + skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, + GFP_ATOMIC); + if (unlikely(!skb)) + return; + skb_reserve(skb, DATA_PACKET_HEAD_ROOM); + skb->dev = peer->device->dev; + PACKET_CB(skb)->mtu = skb->dev->mtu; + skb_queue_tail(&peer->staged_packet_queue, skb); + net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + } + + wg_packet_send_staged_packets(peer); +} + +static void wg_packet_create_data_done(struct sk_buff *first, + struct wg_peer *peer) +{ + struct sk_buff *skb, *next; + bool is_keepalive, data_sent = false; + + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + skb_list_walk_safe(first, skb, next) { + is_keepalive = skb->len == message_data_len(0); + if (likely(!wg_socket_send_skb_to_peer(peer, skb, + PACKET_CB(skb)->ds) && !is_keepalive)) + data_sent = true; + } + + if (likely(data_sent)) + wg_timers_data_sent(peer); + + keep_key_fresh(peer); +} + +void wg_packet_tx_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct crypt_queue, + work); + struct noise_keypair *keypair; + enum packet_state state; + struct sk_buff *first; + struct wg_peer *peer; + + while ((first = __ptr_ring_peek(&queue->ring)) != NULL && + (state = atomic_read_acquire(&PACKET_CB(first)->state)) != + PACKET_STATE_UNCRYPTED) { + __ptr_ring_discard_one(&queue->ring); + peer = PACKET_PEER(first); + keypair = PACKET_CB(first)->keypair; + + if (likely(state == PACKET_STATE_CRYPTED)) + wg_packet_create_data_done(first, peer); + else + kfree_skb_list(first); + + wg_noise_keypair_put(keypair, false); + wg_peer_put(peer); + } +} + +void wg_packet_encrypt_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *first, *skb, *next; + simd_context_t simd_context; + + simd_get(&simd_context); + while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) { + enum packet_state state = PACKET_STATE_CRYPTED; + + skb_list_walk_safe(first, skb, next) { + if (likely(encrypt_packet(skb, + PACKET_CB(first)->keypair, + &simd_context))) { + wg_reset_packet(skb); + } else { + state = PACKET_STATE_DEAD; + break; + } + } + wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, + state); + + simd_relax(&simd_context); + } + simd_put(&simd_context); +} + +static void wg_packet_create_data(struct sk_buff *first) +{ + struct wg_peer *peer = PACKET_PEER(first); + struct wg_device *wg = peer->device; + int ret = -EINVAL; + + rcu_read_lock_bh(); + if (unlikely(READ_ONCE(peer->is_dead))) + goto err; + + ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, + &peer->tx_queue, first, + wg->packet_crypt_wq, + &wg->encrypt_queue.last_cpu); + if (unlikely(ret == -EPIPE)) + wg_queue_enqueue_per_peer(&peer->tx_queue, first, + PACKET_STATE_DEAD); +err: + rcu_read_unlock_bh(); + if (likely(!ret || ret == -EPIPE)) + return; + wg_noise_keypair_put(PACKET_CB(first)->keypair, false); + wg_peer_put(peer); + kfree_skb_list(first); +} + +void wg_packet_purge_staged_packets(struct wg_peer *peer) +{ + spin_lock_bh(&peer->staged_packet_queue.lock); + peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen; + __skb_queue_purge(&peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); +} + +void wg_packet_send_staged_packets(struct wg_peer *peer) +{ + struct noise_symmetric_key *key; + struct noise_keypair *keypair; + struct sk_buff_head packets; + struct sk_buff *skb; + + /* Steal the current queue into our local one. */ + __skb_queue_head_init(&packets); + spin_lock_bh(&peer->staged_packet_queue.lock); + skb_queue_splice_init(&peer->staged_packet_queue, &packets); + spin_unlock_bh(&peer->staged_packet_queue.lock); + if (unlikely(skb_queue_empty(&packets))) + return; + + /* First we make sure we have a valid reference to a valid key. */ + rcu_read_lock_bh(); + keypair = wg_noise_keypair_get( + rcu_dereference_bh(peer->keypairs.current_keypair)); + rcu_read_unlock_bh(); + if (unlikely(!keypair)) + goto out_nokey; + key = &keypair->sending; + if (unlikely(!READ_ONCE(key->is_valid))) + goto out_nokey; + if (unlikely(wg_birthdate_has_expired(key->birthdate, + REJECT_AFTER_TIME))) + goto out_invalid; + + /* After we know we have a somewhat valid key, we now try to assign + * nonces to all of the packets in the queue. If we can't assign nonces + * for all of them, we just consider it a failure and wait for the next + * handshake. + */ + skb_queue_walk(&packets, skb) { + /* 0 for no outer TOS: no leak. TODO: at some later point, we + * might consider using flowi->tos as outer instead. + */ + PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb); + PACKET_CB(skb)->nonce = + atomic64_inc_return(&key->counter.counter) - 1; + if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES)) + goto out_invalid; + } + + packets.prev->next = NULL; + wg_peer_get(keypair->entry.peer); + PACKET_CB(packets.next)->keypair = keypair; + wg_packet_create_data(packets.next); + return; + +out_invalid: + WRITE_ONCE(key->is_valid, false); +out_nokey: + wg_noise_keypair_put(keypair, false); + + /* We orphan the packets if we're waiting on a handshake, so that they + * don't block a socket's pool. + */ + skb_queue_walk(&packets, skb) + skb_orphan(skb); + /* Then we put them back on the top of the queue. We're not too + * concerned about accidentally getting things a little out of order if + * packets are being added really fast, because this queue is for before + * packets can even be sent and it's small anyway. + */ + spin_lock_bh(&peer->staged_packet_queue.lock); + skb_queue_splice(&packets, &peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); + + /* If we're exiting because there's something wrong with the key, it + * means we should initiate a new handshake. + */ + wg_packet_send_queued_handshake_initiation(peer, false); +} diff --git a/net/wireguard/socket.c b/net/wireguard/socket.c new file mode 100644 index 0000000000000..c46256d0d81c1 --- /dev/null +++ b/net/wireguard/socket.c @@ -0,0 +1,437 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "queueing.h" +#include "messages.h" + +#include +#include +#include +#include +#include +#include +#include + +static int send4(struct wg_device *wg, struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, struct dst_cache *cache) +{ + struct flowi4 fl = { + .saddr = endpoint->src4.s_addr, + .daddr = endpoint->addr4.sin_addr.s_addr, + .fl4_dport = endpoint->addr4.sin_port, + .flowi4_mark = wg->fwmark, + .flowi4_proto = IPPROTO_UDP + }; + struct rtable *rt = NULL; + struct sock *sock; + int ret = 0; + + skb_mark_not_on_list(skb); + skb->dev = wg->dev; + skb->mark = wg->fwmark; + + rcu_read_lock_bh(); + sock = rcu_dereference_bh(wg->sock4); + + if (unlikely(!sock)) { + ret = -ENONET; + goto err; + } + + fl.fl4_sport = inet_sk(sock)->inet_sport; + + if (cache) + rt = dst_cache_get_ip4(cache, &fl.saddr); + + if (!rt) { + security_sk_classify_flow(sock, flowi4_to_flowi(&fl)); + if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0, + fl.saddr, RT_SCOPE_HOST))) { + endpoint->src4.s_addr = 0; + *(__force __be32 *)&endpoint->src_if4 = 0; + fl.saddr = 0; + if (cache) + dst_cache_reset(cache); + } + rt = ip_route_output_flow(sock_net(sock), &fl, sock); + if (unlikely(endpoint->src_if4 && ((IS_ERR(rt) && + PTR_ERR(rt) == -EINVAL) || (!IS_ERR(rt) && + rt->dst.dev->ifindex != endpoint->src_if4)))) { + endpoint->src4.s_addr = 0; + *(__force __be32 *)&endpoint->src_if4 = 0; + fl.saddr = 0; + if (cache) + dst_cache_reset(cache); + if (!IS_ERR(rt)) + ip_rt_put(rt); + rt = ip_route_output_flow(sock_net(sock), &fl, sock); + } + if (unlikely(IS_ERR(rt))) { + ret = PTR_ERR(rt); + net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", + wg->dev->name, &endpoint->addr, ret); + goto err; + } else if (unlikely(rt->dst.dev == skb->dev)) { + ip_rt_put(rt); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; + } + if (cache) + dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + } + + skb->ignore_df = 1; + udp_tunnel_xmit_skb(rt, sock, skb, fl.saddr, fl.daddr, ds, + ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, + fl.fl4_dport, false, false); + goto out; + +err: + kfree_skb(skb); +out: + rcu_read_unlock_bh(); + return ret; +} + +static int send6(struct wg_device *wg, struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, struct dst_cache *cache) +{ +#if IS_ENABLED(CONFIG_IPV6) + struct flowi6 fl = { + .saddr = endpoint->src6, + .daddr = endpoint->addr6.sin6_addr, + .fl6_dport = endpoint->addr6.sin6_port, + .flowi6_mark = wg->fwmark, + .flowi6_oif = endpoint->addr6.sin6_scope_id, + .flowi6_proto = IPPROTO_UDP + /* TODO: addr->sin6_flowinfo */ + }; + struct dst_entry *dst = NULL; + struct sock *sock; + int ret = 0; + + skb_mark_not_on_list(skb); + skb->dev = wg->dev; + skb->mark = wg->fwmark; + + rcu_read_lock_bh(); + sock = rcu_dereference_bh(wg->sock6); + + if (unlikely(!sock)) { + ret = -ENONET; + goto err; + } + + fl.fl6_sport = inet_sk(sock)->inet_sport; + + if (cache) + dst = dst_cache_get_ip6(cache, &fl.saddr); + + if (!dst) { + security_sk_classify_flow(sock, flowi6_to_flowi(&fl)); + if (unlikely(!ipv6_addr_any(&fl.saddr) && + !ipv6_chk_addr(sock_net(sock), &fl.saddr, NULL, 0))) { + endpoint->src6 = fl.saddr = in6addr_any; + if (cache) + dst_cache_reset(cache); + } + dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl, + NULL); + if (unlikely(IS_ERR(dst))) { + ret = PTR_ERR(dst); + net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", + wg->dev->name, &endpoint->addr, ret); + goto err; + } else if (unlikely(dst->dev == skb->dev)) { + dst_release(dst); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; + } + if (cache) + dst_cache_set_ip6(cache, dst, &fl.saddr); + } + + skb->ignore_df = 1; + udp_tunnel6_xmit_skb(dst, sock, skb, skb->dev, &fl.saddr, &fl.daddr, ds, + ip6_dst_hoplimit(dst), 0, fl.fl6_sport, + fl.fl6_dport, false); + goto out; + +err: + kfree_skb(skb); +out: + rcu_read_unlock_bh(); + return ret; +#else + return -EAFNOSUPPORT; +#endif +} + +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds) +{ + size_t skb_len = skb->len; + int ret = -EAFNOSUPPORT; + + read_lock_bh(&peer->endpoint_lock); + if (peer->endpoint.addr.sa_family == AF_INET) + ret = send4(peer->device, skb, &peer->endpoint, ds, + &peer->endpoint_cache); + else if (peer->endpoint.addr.sa_family == AF_INET6) + ret = send6(peer->device, skb, &peer->endpoint, ds, + &peer->endpoint_cache); + else + dev_kfree_skb(skb); + if (likely(!ret)) + peer->tx_bytes += skb_len; + read_unlock_bh(&peer->endpoint_lock); + + return ret; +} + +int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *buffer, + size_t len, u8 ds) +{ + struct sk_buff *skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + + if (unlikely(!skb)) + return -ENOMEM; + + skb_reserve(skb, SKB_HEADER_LEN); + skb_set_inner_network_header(skb, 0); + skb_put_data(skb, buffer, len); + return wg_socket_send_skb_to_peer(peer, skb, ds); +} + +int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, + struct sk_buff *in_skb, void *buffer, + size_t len) +{ + int ret = 0; + struct sk_buff *skb; + struct endpoint endpoint; + + if (unlikely(!in_skb)) + return -EINVAL; + ret = wg_socket_endpoint_from_skb(&endpoint, in_skb); + if (unlikely(ret < 0)) + return ret; + + skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + if (unlikely(!skb)) + return -ENOMEM; + skb_reserve(skb, SKB_HEADER_LEN); + skb_set_inner_network_header(skb, 0); + skb_put_data(skb, buffer, len); + + if (endpoint.addr.sa_family == AF_INET) + ret = send4(wg, skb, &endpoint, 0, NULL); + else if (endpoint.addr.sa_family == AF_INET6) + ret = send6(wg, skb, &endpoint, 0, NULL); + /* No other possibilities if the endpoint is valid, which it is, + * as we checked above. + */ + + return ret; +} + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, + const struct sk_buff *skb) +{ + memset(endpoint, 0, sizeof(*endpoint)); + if (skb->protocol == htons(ETH_P_IP)) { + endpoint->addr4.sin_family = AF_INET; + endpoint->addr4.sin_port = udp_hdr(skb)->source; + endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr; + endpoint->src4.s_addr = ip_hdr(skb)->daddr; + endpoint->src_if4 = skb->skb_iif; + } else if (skb->protocol == htons(ETH_P_IPV6)) { + endpoint->addr6.sin6_family = AF_INET6; + endpoint->addr6.sin6_port = udp_hdr(skb)->source; + endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr; + endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id( + &ipv6_hdr(skb)->saddr, skb->skb_iif); + endpoint->src6 = ipv6_hdr(skb)->daddr; + } else { + return -EINVAL; + } + return 0; +} + +static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) +{ + return (a->addr.sa_family == AF_INET && b->addr.sa_family == AF_INET && + a->addr4.sin_port == b->addr4.sin_port && + a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr && + a->src4.s_addr == b->src4.s_addr && a->src_if4 == b->src_if4) || + (a->addr.sa_family == AF_INET6 && + b->addr.sa_family == AF_INET6 && + a->addr6.sin6_port == b->addr6.sin6_port && + ipv6_addr_equal(&a->addr6.sin6_addr, &b->addr6.sin6_addr) && + a->addr6.sin6_scope_id == b->addr6.sin6_scope_id && + ipv6_addr_equal(&a->src6, &b->src6)) || + unlikely(!a->addr.sa_family && !b->addr.sa_family); +} + +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint) +{ + /* First we check unlocked, in order to optimize, since it's pretty rare + * that an endpoint will change. If we happen to be mid-write, and two + * CPUs wind up writing the same thing or something slightly different, + * it doesn't really matter much either. + */ + if (endpoint_eq(endpoint, &peer->endpoint)) + return; + write_lock_bh(&peer->endpoint_lock); + if (endpoint->addr.sa_family == AF_INET) { + peer->endpoint.addr4 = endpoint->addr4; + peer->endpoint.src4 = endpoint->src4; + peer->endpoint.src_if4 = endpoint->src_if4; + } else if (endpoint->addr.sa_family == AF_INET6) { + peer->endpoint.addr6 = endpoint->addr6; + peer->endpoint.src6 = endpoint->src6; + } else { + goto out; + } + dst_cache_reset(&peer->endpoint_cache); +out: + write_unlock_bh(&peer->endpoint_lock); +} + +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb) +{ + struct endpoint endpoint; + + if (!wg_socket_endpoint_from_skb(&endpoint, skb)) + wg_socket_set_peer_endpoint(peer, &endpoint); +} + +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) +{ + write_lock_bh(&peer->endpoint_lock); + memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6)); + dst_cache_reset(&peer->endpoint_cache); + write_unlock_bh(&peer->endpoint_lock); +} + +static int wg_receive(struct sock *sk, struct sk_buff *skb) +{ + struct wg_device *wg; + + if (unlikely(!sk)) + goto err; + wg = sk->sk_user_data; + if (unlikely(!wg)) + goto err; + wg_packet_receive(wg, skb); + return 0; + +err: + kfree_skb(skb); + return 0; +} + +static void sock_free(struct sock *sock) +{ + if (unlikely(!sock)) + return; + sk_clear_memalloc(sock); + udp_tunnel_sock_release(sock->sk_socket); +} + +static void set_sock_opts(struct socket *sock) +{ + sock->sk->sk_allocation = GFP_ATOMIC; + sock->sk->sk_sndbuf = INT_MAX; + sk_set_memalloc(sock->sk); +} + +int wg_socket_init(struct wg_device *wg, u16 port) +{ + int ret; + struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = wg, + .encap_type = 1, + .encap_rcv = wg_receive + }; + struct socket *new4 = NULL, *new6 = NULL; + struct udp_port_cfg port4 = { + .family = AF_INET, + .local_ip.s_addr = htonl(INADDR_ANY), + .local_udp_port = htons(port), + .use_udp_checksums = true + }; +#if IS_ENABLED(CONFIG_IPV6) + int retries = 0; + struct udp_port_cfg port6 = { + .family = AF_INET6, + .local_ip6 = IN6ADDR_ANY_INIT, + .use_udp6_tx_checksums = true, + .use_udp6_rx_checksums = true, + .ipv6_v6only = true + }; +#endif + +#if IS_ENABLED(CONFIG_IPV6) +retry: +#endif + + ret = udp_sock_create(wg->creating_net, &port4, &new4); + if (ret < 0) { + pr_err("%s: Could not create IPv4 socket\n", wg->dev->name); + return ret; + } + set_sock_opts(new4); + setup_udp_tunnel_sock(wg->creating_net, new4, &cfg); + +#if IS_ENABLED(CONFIG_IPV6) + if (ipv6_mod_enabled()) { + port6.local_udp_port = inet_sk(new4->sk)->inet_sport; + ret = udp_sock_create(wg->creating_net, &port6, &new6); + if (ret < 0) { + udp_tunnel_sock_release(new4); + if (ret == -EADDRINUSE && !port && retries++ < 100) + goto retry; + pr_err("%s: Could not create IPv6 socket\n", + wg->dev->name); + return ret; + } + set_sock_opts(new6); + setup_udp_tunnel_sock(wg->creating_net, new6, &cfg); + } +#endif + + wg_socket_reinit(wg, new4->sk, new6 ? new6->sk : NULL); + return 0; +} + +void wg_socket_reinit(struct wg_device *wg, struct sock *new4, + struct sock *new6) +{ + struct sock *old4, *old6; + + mutex_lock(&wg->socket_update_lock); + old4 = rcu_dereference_protected(wg->sock4, + lockdep_is_held(&wg->socket_update_lock)); + old6 = rcu_dereference_protected(wg->sock6, + lockdep_is_held(&wg->socket_update_lock)); + rcu_assign_pointer(wg->sock4, new4); + rcu_assign_pointer(wg->sock6, new6); + if (new4) + wg->incoming_port = ntohs(inet_sk(new4)->inet_sport); + mutex_unlock(&wg->socket_update_lock); + synchronize_rcu(); + synchronize_net(); + sock_free(old4); + sock_free(old6); +} diff --git a/net/wireguard/socket.h b/net/wireguard/socket.h new file mode 100644 index 0000000000000..bab5848efbcdf --- /dev/null +++ b/net/wireguard/socket.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_SOCKET_H +#define _WG_SOCKET_H + +#include +#include +#include +#include + +int wg_socket_init(struct wg_device *wg, u16 port); +void wg_socket_reinit(struct wg_device *wg, struct sock *new4, + struct sock *new6); +int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *data, + size_t len, u8 ds); +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, + u8 ds); +int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, + struct sk_buff *in_skb, + void *out_buffer, size_t len); + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, + const struct sk_buff *skb); +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint); +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb); +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer); + +#if defined(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG) +#define net_dbg_skb_ratelimited(fmt, dev, skb, ...) do { \ + struct endpoint __endpoint; \ + wg_socket_endpoint_from_skb(&__endpoint, skb); \ + net_dbg_ratelimited(fmt, dev, &__endpoint.addr, \ + ##__VA_ARGS__); \ + } while (0) +#else +#define net_dbg_skb_ratelimited(fmt, skb, ...) +#endif + +#endif /* _WG_SOCKET_H */ diff --git a/net/wireguard/timers.c b/net/wireguard/timers.c new file mode 100644 index 0000000000000..d54d32ac9bc41 --- /dev/null +++ b/net/wireguard/timers.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "queueing.h" +#include "socket.h" + +/* + * - Timer for retransmitting the handshake if we don't hear back after + * `REKEY_TIMEOUT + jitter` ms. + * + * - Timer for sending empty packet if we have received a packet but after have + * not sent one for `KEEPALIVE_TIMEOUT` ms. + * + * - Timer for initiating new handshake if we have sent a packet but after have + * not received one (even empty) for `(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) + + * jitter` ms. + * + * - Timer for zeroing out all ephemeral keys after `(REJECT_AFTER_TIME * 3)` ms + * if no new keys have been received. + * + * - Timer for, if enabled, sending an empty authenticated packet every user- + * specified seconds. + */ + +static inline void mod_peer_timer(struct wg_peer *peer, + struct timer_list *timer, + unsigned long expires) +{ + rcu_read_lock_bh(); + if (likely(netif_running(peer->device->dev) && + !READ_ONCE(peer->is_dead))) + mod_timer(timer, expires); + rcu_read_unlock_bh(); +} + +static void wg_expired_retransmit_handshake(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, + timer_retransmit_handshake); + + if (peer->timer_handshake_attempts > MAX_TIMER_HANDSHAKES) { + pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d attempts, giving up\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, MAX_TIMER_HANDSHAKES + 2); + + del_timer(&peer->timer_send_keepalive); + /* We drop all packets without a keypair and don't try again, + * if we try unsuccessfully for too long to make a handshake. + */ + wg_packet_purge_staged_packets(peer); + + /* We set a timer for destroying any residue that might be left + * of a partial exchange. + */ + if (!timer_pending(&peer->timer_zero_key_material)) + mod_peer_timer(peer, &peer->timer_zero_key_material, + jiffies + REJECT_AFTER_TIME * 3 * HZ); + } else { + ++peer->timer_handshake_attempts; + pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d seconds, retrying (try %d)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, REKEY_TIMEOUT, + peer->timer_handshake_attempts + 1); + + /* We clear the endpoint address src address, in case this is + * the cause of trouble. + */ + wg_socket_clear_peer_endpoint_src(peer); + + wg_packet_send_queued_handshake_initiation(peer, true); + } +} + +static void wg_expired_send_keepalive(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_send_keepalive); + + wg_packet_send_keepalive(peer); + if (peer->timer_need_another_keepalive) { + peer->timer_need_another_keepalive = false; + mod_peer_timer(peer, &peer->timer_send_keepalive, + jiffies + KEEPALIVE_TIMEOUT * HZ); + } +} + +static void wg_expired_new_handshake(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_new_handshake); + + pr_debug("%s: Retrying handshake with peer %llu (%pISpfsc) because we stopped hearing back after %d seconds\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, KEEPALIVE_TIMEOUT + REKEY_TIMEOUT); + /* We clear the endpoint address src address, in case this is the cause + * of trouble. + */ + wg_socket_clear_peer_endpoint_src(peer); + wg_packet_send_queued_handshake_initiation(peer, false); +} + +static void wg_expired_zero_key_material(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_zero_key_material); + + rcu_read_lock_bh(); + if (!READ_ONCE(peer->is_dead)) { + wg_peer_get(peer); + if (!queue_work(peer->device->handshake_send_wq, + &peer->clear_peer_work)) + /* If the work was already on the queue, we want to drop + * the extra reference. + */ + wg_peer_put(peer); + } + rcu_read_unlock_bh(); +} + +static void wg_queued_expired_zero_key_material(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, + clear_peer_work); + + pr_debug("%s: Zeroing out all keys for peer %llu (%pISpfsc), since we haven't received a new one in %d seconds\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, REJECT_AFTER_TIME * 3); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + wg_peer_put(peer); +} + +static void wg_expired_send_persistent_keepalive(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, + timer_persistent_keepalive); + + if (likely(peer->persistent_keepalive_interval)) + wg_packet_send_keepalive(peer); +} + +/* Should be called after an authenticated data packet is sent. */ +void wg_timers_data_sent(struct wg_peer *peer) +{ + if (!timer_pending(&peer->timer_new_handshake)) + mod_peer_timer(peer, &peer->timer_new_handshake, + jiffies + (KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * HZ + + prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); +} + +/* Should be called after an authenticated data packet is received. */ +void wg_timers_data_received(struct wg_peer *peer) +{ + if (likely(netif_running(peer->device->dev))) { + if (!timer_pending(&peer->timer_send_keepalive)) + mod_peer_timer(peer, &peer->timer_send_keepalive, + jiffies + KEEPALIVE_TIMEOUT * HZ); + else + peer->timer_need_another_keepalive = true; + } +} + +/* Should be called after any type of authenticated packet is sent, whether + * keepalive, data, or handshake. + */ +void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer) +{ + del_timer(&peer->timer_send_keepalive); +} + +/* Should be called after any type of authenticated packet is received, whether + * keepalive, data, or handshake. + */ +void wg_timers_any_authenticated_packet_received(struct wg_peer *peer) +{ + del_timer(&peer->timer_new_handshake); +} + +/* Should be called after a handshake initiation message is sent. */ +void wg_timers_handshake_initiated(struct wg_peer *peer) +{ + mod_peer_timer(peer, &peer->timer_retransmit_handshake, + jiffies + REKEY_TIMEOUT * HZ + + prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); +} + +/* Should be called after a handshake response message is received and processed + * or when getting key confirmation via the first data message. + */ +void wg_timers_handshake_complete(struct wg_peer *peer) +{ + del_timer(&peer->timer_retransmit_handshake); + peer->timer_handshake_attempts = 0; + peer->sent_lastminute_handshake = false; + ktime_get_real_ts64(&peer->walltime_last_handshake); +} + +/* Should be called after an ephemeral key is created, which is before sending a + * handshake response or after receiving a handshake response. + */ +void wg_timers_session_derived(struct wg_peer *peer) +{ + mod_peer_timer(peer, &peer->timer_zero_key_material, + jiffies + REJECT_AFTER_TIME * 3 * HZ); +} + +/* Should be called before a packet with authentication, whether + * keepalive, data, or handshakem is sent, or after one is received. + */ +void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer) +{ + if (peer->persistent_keepalive_interval) + mod_peer_timer(peer, &peer->timer_persistent_keepalive, + jiffies + peer->persistent_keepalive_interval * HZ); +} + +void wg_timers_init(struct wg_peer *peer) +{ + timer_setup(&peer->timer_retransmit_handshake, + wg_expired_retransmit_handshake, 0); + timer_setup(&peer->timer_send_keepalive, wg_expired_send_keepalive, 0); + timer_setup(&peer->timer_new_handshake, wg_expired_new_handshake, 0); + timer_setup(&peer->timer_zero_key_material, + wg_expired_zero_key_material, 0); + timer_setup(&peer->timer_persistent_keepalive, + wg_expired_send_persistent_keepalive, 0); + INIT_WORK(&peer->clear_peer_work, wg_queued_expired_zero_key_material); + peer->timer_handshake_attempts = 0; + peer->sent_lastminute_handshake = false; + peer->timer_need_another_keepalive = false; +} + +void wg_timers_stop(struct wg_peer *peer) +{ + del_timer_sync(&peer->timer_retransmit_handshake); + del_timer_sync(&peer->timer_send_keepalive); + del_timer_sync(&peer->timer_new_handshake); + del_timer_sync(&peer->timer_zero_key_material); + del_timer_sync(&peer->timer_persistent_keepalive); + flush_work(&peer->clear_peer_work); +} diff --git a/net/wireguard/timers.h b/net/wireguard/timers.h new file mode 100644 index 0000000000000..f0653dcb13266 --- /dev/null +++ b/net/wireguard/timers.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_TIMERS_H +#define _WG_TIMERS_H + +#include + +struct wg_peer; + +void wg_timers_init(struct wg_peer *peer); +void wg_timers_stop(struct wg_peer *peer); +void wg_timers_data_sent(struct wg_peer *peer); +void wg_timers_data_received(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_received(struct wg_peer *peer); +void wg_timers_handshake_initiated(struct wg_peer *peer); +void wg_timers_handshake_complete(struct wg_peer *peer); +void wg_timers_session_derived(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer); + +static inline bool wg_birthdate_has_expired(u64 birthday_nanoseconds, + u64 expiration_seconds) +{ + return (s64)(birthday_nanoseconds + expiration_seconds * NSEC_PER_SEC) + <= (s64)ktime_get_coarse_boottime_ns(); +} + +#endif /* _WG_TIMERS_H */ diff --git a/net/wireguard/uapi/wireguard.h b/net/wireguard/uapi/wireguard.h new file mode 100644 index 0000000000000..ae88be14c9478 --- /dev/null +++ b/net/wireguard/uapi/wireguard.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Documentation + * ============= + * + * The below enums and macros are for interfacing with WireGuard, using generic + * netlink, with family WG_GENL_NAME and version WG_GENL_VERSION. It defines two + * methods: get and set. Note that while they share many common attributes, + * these two functions actually accept a slightly different set of inputs and + * outputs. + * + * WG_CMD_GET_DEVICE + * ----------------- + * + * May only be called via NLM_F_REQUEST | NLM_F_DUMP. The command should contain + * one but not both of: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * + * The kernel will then return several messages (NLM_F_MULTI) containing the + * following tree of nested items: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * WGDEVICE_A_PRIVATE_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGDEVICE_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGDEVICE_A_LISTEN_PORT: NLA_U16 + * WGDEVICE_A_FWMARK: NLA_U32 + * WGDEVICE_A_PEERS: NLA_NESTED + * 0: NLA_NESTED + * WGPEER_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGPEER_A_PRESHARED_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGPEER_A_ENDPOINT: NLA_MIN_LEN(struct sockaddr), struct sockaddr_in or struct sockaddr_in6 + * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16 + * WGPEER_A_LAST_HANDSHAKE_TIME: NLA_EXACT_LEN, struct __kernel_timespec + * WGPEER_A_RX_BYTES: NLA_U64 + * WGPEER_A_TX_BYTES: NLA_U64 + * WGPEER_A_ALLOWEDIPS: NLA_NESTED + * 0: NLA_NESTED + * WGALLOWEDIP_A_FAMILY: NLA_U16 + * WGALLOWEDIP_A_IPADDR: NLA_MIN_LEN(struct in_addr), struct in_addr or struct in6_addr + * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * 0: NLA_NESTED + * ... + * 0: NLA_NESTED + * ... + * ... + * WGPEER_A_PROTOCOL_VERSION: NLA_U32 + * 0: NLA_NESTED + * ... + * ... + * + * It is possible that all of the allowed IPs of a single peer will not + * fit within a single netlink message. In that case, the same peer will + * be written in the following message, except it will only contain + * WGPEER_A_PUBLIC_KEY and WGPEER_A_ALLOWEDIPS. This may occur several + * times in a row for the same peer. It is then up to the receiver to + * coalesce adjacent peers. Likewise, it is possible that all peers will + * not fit within a single message. So, subsequent peers will be sent + * in following messages, except those will only contain WGDEVICE_A_IFNAME + * and WGDEVICE_A_PEERS. It is then up to the receiver to coalesce these + * messages to form the complete list of peers. + * + * Since this is an NLA_F_DUMP command, the final message will always be + * NLMSG_DONE, even if an error occurs. However, this NLMSG_DONE message + * contains an integer error code. It is either zero or a negative error + * code corresponding to the errno. + * + * WG_CMD_SET_DEVICE + * ----------------- + * + * May only be called via NLM_F_REQUEST. The command should contain the + * following tree of nested items, containing one but not both of + * WGDEVICE_A_IFINDEX and WGDEVICE_A_IFNAME: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * WGDEVICE_A_FLAGS: NLA_U32, 0 or WGDEVICE_F_REPLACE_PEERS if all current + * peers should be removed prior to adding the list below. + * WGDEVICE_A_PRIVATE_KEY: len WG_KEY_LEN, all zeros to remove + * WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly + * WGDEVICE_A_FWMARK: NLA_U32, 0 to disable + * WGDEVICE_A_PEERS: NLA_NESTED + * 0: NLA_NESTED + * WGPEER_A_PUBLIC_KEY: len WG_KEY_LEN + * WGPEER_A_FLAGS: NLA_U32, 0 and/or WGPEER_F_REMOVE_ME if the + * specified peer should not exist at the end of the + * operation, rather than added/updated and/or + * WGPEER_F_REPLACE_ALLOWEDIPS if all current allowed + * IPs of this peer should be removed prior to adding + * the list below and/or WGPEER_F_UPDATE_ONLY if the + * peer should only be set if it already exists. + * WGPEER_A_PRESHARED_KEY: len WG_KEY_LEN, all zeros to remove + * WGPEER_A_ENDPOINT: struct sockaddr_in or struct sockaddr_in6 + * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16, 0 to disable + * WGPEER_A_ALLOWEDIPS: NLA_NESTED + * 0: NLA_NESTED + * WGALLOWEDIP_A_FAMILY: NLA_U16 + * WGALLOWEDIP_A_IPADDR: struct in_addr or struct in6_addr + * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * 0: NLA_NESTED + * ... + * 0: NLA_NESTED + * ... + * ... + * WGPEER_A_PROTOCOL_VERSION: NLA_U32, should not be set or used at + * all by most users of this API, as the + * most recent protocol will be used when + * this is unset. Otherwise, must be set + * to 1. + * 0: NLA_NESTED + * ... + * ... + * + * It is possible that the amount of configuration data exceeds that of + * the maximum message length accepted by the kernel. In that case, several + * messages should be sent one after another, with each successive one + * filling in information not contained in the prior. Note that if + * WGDEVICE_F_REPLACE_PEERS is specified in the first message, it probably + * should not be specified in fragments that come after, so that the list + * of peers is only cleared the first time but appended after. Likewise for + * peers, if WGPEER_F_REPLACE_ALLOWEDIPS is specified in the first message + * of a peer, it likely should not be specified in subsequent fragments. + * + * If an error occurs, NLMSG_ERROR will reply containing an errno. + */ + +#ifndef _WG_UAPI_WIREGUARD_H +#define _WG_UAPI_WIREGUARD_H + +#define WG_GENL_NAME "wireguard" +#define WG_GENL_VERSION 1 + +#define WG_KEY_LEN 32 + +enum wg_cmd { + WG_CMD_GET_DEVICE, + WG_CMD_SET_DEVICE, + __WG_CMD_MAX +}; +#define WG_CMD_MAX (__WG_CMD_MAX - 1) + +enum wgdevice_flag { + WGDEVICE_F_REPLACE_PEERS = 1U << 0, + __WGDEVICE_F_ALL = WGDEVICE_F_REPLACE_PEERS +}; +enum wgdevice_attribute { + WGDEVICE_A_UNSPEC, + WGDEVICE_A_IFINDEX, + WGDEVICE_A_IFNAME, + WGDEVICE_A_PRIVATE_KEY, + WGDEVICE_A_PUBLIC_KEY, + WGDEVICE_A_FLAGS, + WGDEVICE_A_LISTEN_PORT, + WGDEVICE_A_FWMARK, + WGDEVICE_A_PEERS, + __WGDEVICE_A_LAST +}; +#define WGDEVICE_A_MAX (__WGDEVICE_A_LAST - 1) + +enum wgpeer_flag { + WGPEER_F_REMOVE_ME = 1U << 0, + WGPEER_F_REPLACE_ALLOWEDIPS = 1U << 1, + WGPEER_F_UPDATE_ONLY = 1U << 2, + __WGPEER_F_ALL = WGPEER_F_REMOVE_ME | WGPEER_F_REPLACE_ALLOWEDIPS | + WGPEER_F_UPDATE_ONLY +}; +enum wgpeer_attribute { + WGPEER_A_UNSPEC, + WGPEER_A_PUBLIC_KEY, + WGPEER_A_PRESHARED_KEY, + WGPEER_A_FLAGS, + WGPEER_A_ENDPOINT, + WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, + WGPEER_A_LAST_HANDSHAKE_TIME, + WGPEER_A_RX_BYTES, + WGPEER_A_TX_BYTES, + WGPEER_A_ALLOWEDIPS, + WGPEER_A_PROTOCOL_VERSION, + __WGPEER_A_LAST +}; +#define WGPEER_A_MAX (__WGPEER_A_LAST - 1) + +enum wgallowedip_attribute { + WGALLOWEDIP_A_UNSPEC, + WGALLOWEDIP_A_FAMILY, + WGALLOWEDIP_A_IPADDR, + WGALLOWEDIP_A_CIDR_MASK, + __WGALLOWEDIP_A_LAST +}; +#define WGALLOWEDIP_A_MAX (__WGALLOWEDIP_A_LAST - 1) + +#endif /* _WG_UAPI_WIREGUARD_H */ diff --git a/net/wireguard/version.h b/net/wireguard/version.h new file mode 100644 index 0000000000000..ddba10344448b --- /dev/null +++ b/net/wireguard/version.h @@ -0,0 +1 @@ +#define WIREGUARD_VERSION "0.0.20191219" From a0f0bc2a02868b66c4a4c63adaff6d6f64adea40 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Wed, 9 Oct 2019 14:53:51 -0500 Subject: [PATCH 009/213] Add AM335x CM3 Power Managment Firmware http://git.ti.com/gitweb/?p=processor-firmware/ti-amx3-cm3-pm-firmware.git;a=commit;h=6a849767df85ce9399494f53fb5c753665396653 Signed-off-by: Robert Nelson --- firmware/am335x-bone-scale-data.bin | Bin 0 -> 73 bytes firmware/am335x-evm-scale-data.bin | Bin 0 -> 17 bytes firmware/am335x-pm-firmware.bin | Bin 0 -> 11320 bytes firmware/am335x-pm-firmware.elf | Bin 0 -> 217168 bytes firmware/am43x-evm-scale-data.bin | Bin 0 -> 41 bytes 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 firmware/am335x-bone-scale-data.bin create mode 100644 firmware/am335x-evm-scale-data.bin create mode 100755 firmware/am335x-pm-firmware.bin create mode 100755 firmware/am335x-pm-firmware.elf create mode 100644 firmware/am43x-evm-scale-data.bin diff --git a/firmware/am335x-bone-scale-data.bin b/firmware/am335x-bone-scale-data.bin new file mode 100644 index 0000000000000000000000000000000000000000..1ce3c1c596d7a7f400b0cc89bda5a41eed2780c5 GIT binary patch literal 73 pcmd-HXHZUIU{c}EWl|AfLZWk+R0P|Ad@#)bSHb~R0-{lr003gr3L5|b literal 0 HcmV?d00001 diff --git a/firmware/am335x-evm-scale-data.bin b/firmware/am335x-evm-scale-data.bin new file mode 100644 index 0000000000000000000000000000000000000000..a222389d233fa8f1c76ef35470b57284999c8df5 GIT binary patch literal 17 Ucmd-HXJAiZVA55UX8=>$024d{B>(^b literal 0 HcmV?d00001 diff --git a/firmware/am335x-pm-firmware.bin b/firmware/am335x-pm-firmware.bin new file mode 100755 index 0000000000000000000000000000000000000000..1fc92e2bc40e9aaec0b3f60e0c2083ff40f22c3e GIT binary patch literal 11320 zcmbt)3w#t+miMi$>ZFtIBps4K9z9S=hc(b4HV-v|k0xmzme+IY+BqTHTJzzGl2v`9;05ky40j~kwfFBSl)8rgej!-)I zVO^=h@oDbbGJDz?CsS_f(z$qN3(J&i%b52!&f-*^%Sd@}#PWP)j@)vXWC|hkyw1Q3 zt{i#g)RdVCpTt-4{o8pR&@3~1rk4{TNAMCKV+Lle%y*^AIkM7Wt>TsV_`J%i-Fdeh zA2_DZsLfrRSAKjzKP9*F_<(2nb#73teC_nwspaYN)JnZGy_Cxo@KrTsSdmwm|<&Fq8B zrJ;NWE4<`C({}^fPz)fev%`dr2D&k&7%)m4ws&14l|^fmwZVytD^Fmr4gEglw>=Rc3A!GBX3Qd2^X3r&;aDtM&3LrgSxixN||jr8;}?@{>TwtRS{Qw{z~ zwR=ZFqx2rt;Wcze(mRmOgY4rY-RBGrN>v{vo~WY|yTTz#n}w>>L6EyUT_%pt!)sx1q38W+b&xpHDvMZ3%Lvl6S@;5E3CYYkqj zn~9-r{gU%J#gZEQy?VX$|Iq$)8Yd!h6LoFf z=j;x?t+wpz0rQia-b!(qE3@QVL!!9Yl3Z?aV=fQJ{VYpzTJ+6a6ZuVWCc;NQQJ)+z zH;U3_dFH*1Gq6n9{TTXkH^8|f^PT9kzEqa5e7&?ox=Py9c1T*6BlNI>hhXU@^)2aP zX>y1&Cim-w=>@sc1wXs`f`5UuL=g@1r58dR<9^Z_u$neojt1^XVHeIdoIYn!RMyK0 zTyONxz)eOfhZFIf&J|tp7vuY%ysO7+nJ9fb_uDA_bENJ@dJXUZ_ZYX8dj_c{?w8yu z*x1Vj@f>2JlnP%fyAnly_vdleK~D_}eA@77yd&^sgE0`#Ta|eJ;}C1*0|sN(pl4&8 z@0|ckIqIz#WLtg{V2KHXp1YZKlk{}RVt+fBeq|vUShvnov{SgObpc zsMQwl<68_irhMJNx0tne)*!3Xqu&=b=vuNAPXZs{cpdUP8fIaHsKKT&p=DV+SsIfK zM`gd)&;=?}sv7L|)$MBVpU2S0NmxM&LVhZ_d2r+q*TQT!4~7ou_8;umB@lIMFmNa# zpkB+}rp;rwq9pmPm?9J4(z?z0r zA;ydW!N>}@-N!B*X{fC)l6oTCYl@^(lu|s<@H>shK=b#e4YKtPWes?jD0V}U^y}Uq zhq&FX0lVq7fSn~2G<$h)unS$DBFWUh)0EJ^*o3zH>+9nj1Ai!&$Oqg3lXsnTkHkIq z3M__&%lO@fzNk%c+&ncnJo=`zRkOkjE1G(n(f@NBtcs2&%Q(<})cbiH3;f0N2}(*E z@+3%`znJp=m&{7-THR})@zcxgXEPm2TmyN5d;_jl8dJn`TGLOu*7SnD8#GS0C(;}L z&`^%v=J%?>*GKvNLQ9rjcXD^hUWxleCvB6seK~P2_Um+c=l}FD3mo^v1=IyMwHxuI z-Z(J&Za}>`=wB+8Blqv4^PGb7k0C?b6s!FeTu_sZCjcpp&+W6kR zt2Yy6jUKCF;7b*OFYJ4{KbfnA&gN0>D#7{n*|>5w_{1o*_m)dV5_x|K*2H62Qxt&4 z;X?nBfIVfhk89c9uG`;|&1_tg@sE5!y&j|Ida_m;?b58}S}aaI#+gI;*dw&qw6Uu5 zVP+(}()-=gS51(%*Rz6~2VZ`JWcK$)d`*6~^$Obf-F1li{^r3QhaT@Gt2+F(y)VS| zpV%M&<%zrF`c7oWKYH?A^bg6TQjgdvWpO`xHc_O0Y(q+LJhn-nd?H-qapz;mMIm3+HLQaNAk*e_g2e;d&amuV^E;UD|r-{a^^eXj2W9im$F^byxN(bxQ-X{EPSx7KuNnDc}{B(qG zL!8tvV9#RSF^r4k$`sfBlBECgmu5MW%5xQ-6K3<0!3mk1?Ik66hW&=}gbHZYHI^Ll zIzobN=oI$IcbS)Bu@A66*=Hh!z3Ui2I{!NuI+GDA*oRmbxI{xY7kFynix&kvW_P4|aFlR6njGxnY6lOQEBW{#-2n0G zDjlgMA<>#dPULBtH;aL8Mh|?iE zUcS{UV%!#vmp|gASS}Yl|DsY9my7zl&o%(7RI8YlYsYq1Vuv4_dWG%Li?{Mk@eK(r5*sZP92mK)XSs z%>ZqqMwQKJ=s zcB4kS8Z@TSt_E$cMze!&?rfIY}p#4ds%>gY#qs;~F6OA?( zG^0kF2ikvVw0WR$8tod;J{Uur!)nkOZbTn{9k!!yZwphsRQlq3)VI{vk^H{*sBeEg zUcTo&=H+o-%a+sRQ~Hq;G#4$HI}>OpV>AJL%Mel0sU&hrjPPEj$4-wCy6cj25}gH` zN(?@nGbS~aFkfw%9w(S=jA-5^XzxhBQ=L0!F!sbhFvju(SGIeld%ycrclPR)tEqkw z<)qJ~=`&5xm*7e$Ppn9+Y+n6`)!Fxuep(|E#`6*VB)wSar)zbXOJ}#= zS@VklmT|x#p66IWY3+2+VA z#W|Smlu6U8jt^xs@+?g}FR&es%yB(>7wdO$+3WfK2L9HNUA$AKRV3m^^cU#ISgD*T zGlgd{dLrme=i*p`JV~x}?2YGqbYA~d-X|Mnv%K7425ftp`8b1(dNAIWk9u-Nry9Iv)ZntnTp{zByE&yQ%#;II1#rH^ z+%Y>+Fs`>?ZFwGkDDJ$@bsbK+dMS3+ZFvl}*;e=qP~$x#J&nf(o@QA0hD5o@#XEA| zJT|ZddH)ctDn+VZqW5x@<*LCHmCG$a&Zu&Vit}PB>iyLyhuTVT-d&|y>rcfSPEK{@ z)Z5-1>ys{RM#&pPpY}23$3rycGPN-`Ejs4T`)LN!FHsw5&l*!X?QY>H@*>Xn9a`J-TzOZt zeaE}h^0UKaZNbS{`?sL|nP~sV?UD8`)!Kh(EYki@#;E!FXvt-23u)|GZyT$U^BTBh0M z8jgAEPhn9`hef}QTJ%nsM#+?@MGG~HLZcCH4Qig6J(d8QczCL~J2Em_!+(Z7|7XbG z_iY;tou%?8(Q)~Pw!+dFGrw2diBT+Wz+B57r9R}x?b)x@|NGHMZ*f}vM@9=I4!zbf zYTbhQ&+|5&PY*pbM~WS!u@8GZ&Dq+{IB6_cKE?{1zPu{yVw-j^B;fAr6vo+nUvrrq z<)4k+FXv#)a3`2j9F|f4%R}Rw!okOk?MzuaL^myJ@P*;X&V6V&wsS{rTJ(M@-Gx#$ z*g8BGbYtF}2vO|t;4tkn-$!gv>oCH$EJR!}zZ}I_COac@$c)>Hrjp#~EXt+3jZs0O zx%Ap7&6`(7Y5xCklqvgw1tZvRQOj(s4`abvjE+`#)U36Cb9bCJlm6%fpVhjh~ zI?WC>IO!;hJL9DOC4W2E87JL?@{!eWkj;(kHMIiWVbTg|{T3~9-yFNwB)%ns>_LhM zs71!oh`biZB9^5%7#ixdI2ifj&I%VFXnZKsy_U0Lq!;?v;CZ#*x}MqQ_^ma}c8z}x zp2hxycrNfCMCncbFYsLG{{j@Jc87R<)lA%amiZ1sYOUXTKeOHC{{rbMt(3~DeZ}1r z<*H%Q|DQgJIKyG8<^TF9LK_HET_5@=O8hEJHU82^(P3Yh>OJnGsPIgfYCqW$v=%N_;r{M#A6hFTb{xS9#C;Ei?DCS3&SKOKB$`!NSm~&xj%`0Jwm=A`@y4Em7 z#RtM<-TpAO=_g@o&9h-@$DT0N{7jf4*d1ZAx-m>Xcrr}&Z3|QFTQ$3kRfY2Ni1doP z6DrM-zDLig!P>A*K3Fm^Zn4}Iv(Jn^rMYni`?3u<9;SW3h-emjgt8YiKn;F0Bw+Oa zU`U7OBSSPN9>P@|Qy$bJTbdag-l2PXLnoy@TG`_oUrdJN9|A=UwhS?4NjMp62yaWr z9rA-=+6_y)m@*c^T9p<%tr)ryzl;s^QS6NMr;BQR4R^@&ZGh?`-xhZ+xun)Lm!$gy ziE8kU=(YEABf>?p@8)4aN@^FS;%+K^9{13=?ZsM)r&kU3cy-$DlF>`+B7reNjiBqtvQ0t=;LiPa&W?TZ&cVlsMlq1XS?C8>G;ZG*XqUXNSfOj zx?$DAki$fNr243CtzFSJkv*}x-JsDPM7u^)N!*091LhKXVJl)z>PzbTDV0h`s=N26C+TGbETMQ&m3B>AZly?=kr@|u(ivt}iEoV>_v zJ^q*#$ucW)_~ycn8+~s{x05Bp?V)+1zW)Q3Dj%<*{fS$@ZM~+!M3&TDsYo3H@v}P-LE>C^f2Mrwa6}J{r`S1sKThIg56z{>IXrQ~T*5gV zr|=b+t+izdm3ri`4!POPfG>W+IiKV+uVTyq(0Jc4ov~Gbxzp%+qAe+4PEPr4@|P*h zuvqx16>aM99v@g(cYL5>8GS`w7BZWJHZ%MZ!aV`S9b^m6TbS3z3APxbcL3YZ#Ulb0 zkTc*$Ad-vBK@aM77cf=~*ntzkt3WGYaB&4WGS(orER;(-7@}~Km+m$qCq42R$rpE; zqy)LrV)W_p^`^Ls&X{LZjHqcYOVD53jkQS}#+;{h2lpndxSeU|?bd*fwGDX`M*{8x zp+_GnkJfbg4&|?vMQ;Bh^>Z#xTp_>PM|_-1hjiz7dY}ArEluC+(2|5WNs#2q8rOe? znCEn;?aZ4YcI@7&`^xXDcrV0y3KTAWR^t!bSYtxt6!)~Dofi(`dxL)Ow4rA%2z!5e zxUN1uqIyaAu6nI^yN8VOOVn7f77-4fAuu0|R{j z$D3|7FP+8$PkYoEjsZM#mke&()RG;)fCU;ns?9O*q(`-4WglcE*hL>HDc5^SsyxLu zPiv^re3NToMYbp2lj1pC&t^J(&r9VO`1?J(R`w z+!>#8f}1b+jE(qD28B<#DiT2WOvscA*+M*i*9%W?Xb$n2e?w+ z7V`fi)}Qs3RTj>dA)W8z`kxe%rs;Zw8(uC+_2~+R{CP6u>h1{TnGGbnAmB@sCk;Ao z4*QAcE3{RMBSaeWJ2B4wr}K?uk7CZg1ndMv%y|SJsKHym zL1W@R(8|9-(`mGu#%U!RH$Jp(t@!Y|O?5RJzc1SBr`L-c?-fg{=8Dx-qExf#fk#$v zT35uLM>|`ASAjO*2yo|gdvX0;)9dDo@CO4H-FhotOuS>^t+&v-;uZ;aT1<2(uNybLU-i?RCZ1ufAu^+I9EdC)cdMf5QXc-&p(Lrn-kVKm17jmLHIO!S^?B*noRY zeD7k3$R_~sAd3g$02M#Hm>$pp_}3zhwwS|MV?9nEU=T3&fIlPM1AGRo1w`<;@fPAR zCh>--_9H!9G)?+5ahJtBAc%k&AOi#-%4pJ~JeuB#dTN2^0Zs0|5!B?Og3aJB2W}ge z`)0Wy%D(&MxM-_@JMG)%bsob>m&2 zwfpbwtDae>oqq=Y0{lIFB^I4_<^zR5MDB0vt5($W67V{p8T>as+~8jeZ1_9nMDXVW zg#Zm%@(=Zi08oF>{NVxWA0rT(SH#C=z()8UWX78C-V2})*bV^w!O&k?89mTXY&Sst zLqGIqtUl5mtB2~OX+m{}LM$!0GCSPd3X!*EYRL^(ur{iAA0v& z6M!!9dl8L>oXA^ydhn$6>$~)!h|9!-F%O%fy&Y^a-rvPfZAb^m{<2$cXLSj$eO&YT z#q_t<-m(AUb=DgvZ;rnQMOXZe;2SgjmZmbcIU};>{WaR_;A;im0N<7<-<33uJqw;j z;I&DNeHrDulE$&ez(c=9X^iEfVQ}|fp%FFm`-mEc-(^Z9AA8nD_5Bh0mPYkmN#oe^ zDBl+7+d~eWpm6qtbWCs71lroD>}tq1L}khE#Ny$YNHUb(9!>Y3Ao_^5H=Rd%J2!Hg zcqskkgfw_A(+!u?o@kopOYBGb39%nc(dHBVKOX+8N&9Wsc;7$Wv-#c_%N6hc4_oH7 AF8}}l literal 0 HcmV?d00001 diff --git a/firmware/am335x-pm-firmware.elf b/firmware/am335x-pm-firmware.elf new file mode 100755 index 0000000000000000000000000000000000000000..843fc22ce3484251fc9700f013e13fac20896d67 GIT binary patch literal 217168 zcmeFZ3wTw<)jvFQ_TK00b4hY?A$Q>90)!-BxEjDq4Dk>M$W=u}8U*x!h!GIMS`BC` zDk_LtglMU^YU>3M6cvP4Yps_ysc5ZQg;s%zwn5M&-17Z?v-dfm*7xcA{-5vtf8X;= zVCJk@v(}n5zq#z0*>lbfXOE6bDMkFxBT7U(H4w~-F@_MPum*~Vu#lG_Qr-Nx6l6-B zU@0Tg&G#T>t2`Uz4g~MX`C%}^wiC#Qxo_mBB0m-R4E#$ukw5;^1v4DxH)HCDzsb@m zjetf#BcKt`2xtT}0vZ90fJQ(gpb^jrXaqC@8Uc-fMnEH=5zq)|1T+E~0gZr0KqH_L z&P&-3VVH_>+a`iqH??Y=jFCu0U9Va1X-c2-^@^5X@2OvAj^8IsDM?=MQg{ zMtXh2h>G;xHKO&5=RI{s&03KiYZxK6zg|ZuDd4T*T#7?l@WyOBEO&}s76!mi7U zUDz$9)c>Aul(GLt!-KFoCMvSG%Lw!swo^_BTNI5du1k&O#acHNjWJq%zQR$b)EA!f z^1&TemxhATg|#mqw6Y3Dy?n5-$Eo$;78wma8oJeH#JY{LV!=>-{=_AzOVVSZ=Dgbc zk&XK*#JKv^!l;obac!(iZR^(T5wr4JTQW!bV@7Ygd9_Fz`AW9jf0H>QmcLI8;5@;s>09`+leK4MXjAi_Hc(+Mg2Mp zc*jxAzV2i!znM4r8+qZ)qV>;5MX_l8@Mt6|2a49mjt(sEXsP>pd3x%owAh;J+~&@< zHJze$Ub}4F(;-{G?C2g5txJv$LVW$v&b9|T1*oj{sxMcic}E(cx3!B%cv~bYT9+Q( zwO4H43M?iS1C`C4ZNKZR*nSK=(knZww%gkr`Zmyo(EZWTZTsvhQscH#=7vtv^Ag;% zw~FoXxtP``x*ku5$6qUtlagW{v$UuQJ-%_UFza(SM@;8SEz!gj8EpZZ@N0 zQRRjn$p4@-qg|$ml8%=8yh>oxzS)PoMH^;qNRFNs{ZY2u^+dHFE%G~8uB~NPUi&sX z+P)Pe?0Hml)CV!oY;i+NMv+= za&3M+#_|#GuX3U_`POQ=iny6I0bumg&L1BPFN;J^=!@->q%E@H_jPE?6$rA<8Smux z`A6$mQEWwTi}s4HT>n&bW}dlAlq`U!^E#i6E{tZj%V6?;%j{875dE@6obqMMu;|#< zh&?p=K)V$3r|S+Bg|5!udf@yN@#VfHAMeX=?G$^YDfhVlY?%ovPeu^;kbS@NFD~~# z`5c!hq4;v;X- zp~hL_!23(y_f;ME$CBUypHbB6Gv00&>x=_-Fy~O?mEPtT4v3Vk+eaP}i(fw=0{%md zQ$^9d=_p5`so+i&Klr;aUsJSYjO7k@vYwa@8`mZI% z+OYb}IV9#=X!o<1c-H2$Hu{YN((oXC+mb%$A)Rfjj+<*IZlI@e-40i`>yqce6|G~I zR4f{|p|kD!<7nd~%pfK0EzCLd(9x&lT2Xf9q4uXdYaZI~@soQ)+kvP22RaAKGpf+r zXB?lO-_bI(~DfMXcAfJvRH#9=S%8R<(MU5CvibsdemAl=)cMr>*IGz+VBL^sIDR zqvw)Zw6<|qXWJ9Uj5X%k9Lw|0it4+f@;y&woxZAi{Pt*g`>5p=ble zWuG=I0zc9>O8$~hYop^)pTIG+#t^gSj1BP`zH`*6;Xu_2lRicJ&u`zea1nd%qpIg_ zh~Dt^BTrSJNA`_g46DP@&-z9uf9hE)2Y6P6_juNN@EL-)&+lCAdv522-pxC^`Np;e zMCm&(cEmkJ3`sbzRhO;T+^t0B#PM%1o> zPTq~x)rdP<>{|awSoJKc-m<-_-7Gu&0iKaBitSvBy@KacMJXseyLKSZ=6@CJ&B<6R zR2&x1Ln>l#lLu}+>*rtXPPMrLJ8)~&y03PJ1y;VY=&Wkl+Fgp)?w-lpM^%fGE!(?S zRHZgzD^Xvys!Dt{t;>cjb)3o`aP2jH*hlZjV$njA+~~cKt_ke-pbrSpRWcaUB`{ zBA)gn()Q*sp5U)^h9jc2ZtIM%*ZgX|W;C;e&Nk}>+KWM|vu!afkay&HXZ>hzE`r?) zmE9uPy&ZP#&Bfr2Q@mpEZdAMy@S=)W0^VZ9>kZy%iq{*wD;2L4y#9(;3f@e`D+8}Y z@yfuvNb$!x_+;Eh+jKHz02ULWvAC|+Oi0*cocywer$6!3)NodVtf#j61C>*K_y z0=(Xe*AKkU6|Wz7-4(Atcz;v8{@`^{yaC|7r+5Rv3o70~@cy891HqGuHwe5}j-$>U zoozK8%h1OE>Zm~5PU>K}%>DUGY+JUqlfLgIw(V<){FN_>?FRO1xv})v2iDP@9EQ90FVuTEY4-px$T<_I}~DB01J4 zwllkV+@g-!-r%PSKrVnP_wCwD~7hJx2*88(wIr!_02h;+3-+HoZPODOMXZS`G8=D(rqe#h+Pb@~knlmR1!G$39prkI~X&Rlkpg zkrwHZ&rES!Rd<RVVgW$=un-{w2nQ_KL_wG0q|mj%Y9NcZlJ!?3idZM4TT0f7L#( z@W;|(qpI%m$!6}?-;doF3&z5+$yH&#A%U5nk}1;-+bRu>q)?Cqj8 z#T<>zEGFdOv0X>@P8P!c{5g z*NZW?Gy#Xo&8OC#ik+?%jqi2WUWd}kE@&x1iI*JRwQR@1+Xd#m!;x6uI-@G@nH>km zBHePBvr6AiE6UHwouZ(#Eu~Wylpq!Cl&ziEFQ%g0-yM@Ets48?DV;@&KJeM^bgRo- zRQAmAz0t3)M$VIm-`^`*?>)>um#z9-cel@d*1{2Vai=W#v?YC%z-ZvpIK3Y8-f`|a zFF78BB~kX5mTmhgRh?3R$(=_Y+{>{b%06!SJ)TRfBcEBvJEb~X*ot%n`}<9*?uB)Q z->Lg;FS6#J9-+4-@5JlB81>Ia{omf;)PKCH|5L}E`oDLa^}qADQ~%qJi`F_-?^lmw z94)$NL%grFAg75e?U-|Y^RsP%Rlc3&`gS^eJ9aeb8(!`2*aLrdcD#%DUx(jHcy$@(tnd1jUzh(K zza}cb>W;+G`dddNmI066a6NjVgS{ln_2?|+QTs6mtwRc_e#iar$pBK7+nk=UuHzH< z^NYhRd%x|2y?J=-J-1&zsb*OAvG7Zko6(CSb1>F&kFgDnggGrf77TIu|l=_Voe! z+0f?IBPx*p;ql93dFV58r)aJ0h@t#P4kskdhpr255Un!~^Q5J-?SUiCll!S7@h7)) z(qgqR_fF(?wyisIysaMNW@kIs4p$xFQ)VvK1`Sm~_?CkeR~(nE=*#re8AD;5R;;Qn za7R%A&o+*kQI4g@k8#{QdW_@$=3}DuK7?UM@w`PT{V+cqZ)-sBSO-MIs{U7R^Qw{b z*JkQ%I?DL|qwlJ-9gd$c&XewQZU0c&UZiYa0J|x$>r*zPu(>c{vksV2??t&LaZ)Z^4p@~K^1 z^)b!{$uBs;$CI0UQD|>=o~YPks}nZI4%;cIF1a=@mOL`NIT@#{DFgFT%zB($#)3%6 zqV#&vdJYgXvBs#vr$_nG*_O6dcz4&ZeTjFK*j>YOP{WxG4~YTJv!=o18761QqHpn> z^XB-uCgsK+5?66Wz#0XIJNjxJi?uA*!LTr2t%H%S+&r?*IIwI%cKr-lgq~jBG9B?L zEk(0MS^t)zS)y!E%XGw*Ee|0c*76W?&uIBO;gYqjlraab!m0smG|ERFj_Gr1zwuX+f*hAF^6CTH&kNX!!n{sUI#&fw0 z;pGlK2ZC75;u+C;7e+v5+gpcC^!{rPdk|l9m}6oAj^ae?Rcd9+kum25p3~c#NjIvz zA1S%G4)qs+)7iH6uxK6Ik&HRSD9gYZ@>Lyt8V-M6v>tEAT$LW*t=QYJemQ<{FW1hP zf1YQlkK+uP?*>>Fv8~)Z_Uq31tE)4b%_vLQe$-N*_Z&69qJL)|F{4QvBGJli%x%Is zG){Xl*CO8D*|uxDM?GD-?BP5~n@&0>k>bP|uE=Yf38Gfb8x{UtOiRF^b1>73*-$)i(7rqm9t?_Xb5Uxc-A`?e&tGi{uh zUcVpxjB@}nF;LyxVc%P873>N)3_ zWx-~nj6DZC{Q4_f#U&`W&aTg`2Ug)jaAj&llBf6OK61~u;yf2vhA}*wQfp{-G{d!Kt8ogU6qTBRfxGr#w!+M9(S&pnu@ zEPcBuH0q2|I19jPpBOMNOG=CjwT^k--p26}JtB+vt_p9(BDYng*E>CY4EB`qcV2g2 zwF_hNWS#PVtHZr+TxSJz=!a2ew(#B6@K&K_a{9qKot#(nt-J_u&xncgq0L3H3-V8G z9v6Fg|I5a>_5tI>{+DIn*r-_l;idIVkDAptPS@WVdn?vIcKYzr;r&%!;6&cIJ7$%t zJ(ALjyfq8Ykk2e5y}oYtt+V~JbB70G>D5PDCbgSqUYP${3)^l6{45&Px;Zl@`c^d$ zfFGwfKN~%lo|xyh4~$s*UlFOXmuK<$Bxf(3tt>P%=f+v7>Zv7XSD)a9eX|0wsjy%8 zna>lbhg`$2V?T)vjrAK|_?h$!B_})rGKu_fRg5VC=?n8;ZfQl|rcc4T9s7643uEC; zp8db}?8M%!(TKz%tv-2Ht0fES(=K0)-XO}3w(z~9Q;&eFL)o=4>NU^C`&H@dZParY zo_4v?0OgcnQeIqy6(shBmuJUzY;^3fKgIXqYp@fiKO9TWK0I*JSi(LWyYLGzS{p|A zM_EW=9;z3m2bz5+?enRh-Af1?f&G0>4zL++*(UM33rTm*3V>xeaquRPL@ zdQ4+wS>w9)W#KdG&KjB9=xa=A+_*@T)-*RoYrixu|G2$<9m0w`qH`+DA!83+J74aS z!^SlyE&IYeyz{Ac(QxUabG!YrUDV&T@zMOpar)jr`o+fm`A@Wq^+h$zPivPrWi}fR zx5z=zpSK{#*maRFWv3i!HV2pCCPS-{@^v%(na3wiqz*o)_kGlzcd;marA0)xwTH%K z)@6>&Zp<8&)VQtP9@k}b7krxIdLYiI%10coC}_XwwCeVUmpl4#i=l)?QPj9s6j+5< z@4aQ?{Jm>8E=FGouisVkFv5C-O$b{MN)XBsokDL&7|(0UFK?)K#q`BJA_a(a-*;HI;@W;ltN5TQ+hmr83VFYzWkxiL-j!e+4XE0IExPR_gt8h-XP9ABF|5zW^ z35=A^wkfzHU~i*s@;G`r+a`R2CnEQOSNjc~NAb=`@T#x8a>2|Qk%cqo&7XDU+(^Zu z9*ZJZUK$xbWNt6 zs!Z?&#^N}P%NhhG`Bw!x#BqvXBjl?*j2C>CBbYX1*^dpJ8zBfAp9n0B^93u+@>oHZ z&B0CoIFyl<7|O&#{ki!GJ=f0n$!FkPDjz{4!xw~52!0~KR-~(A(av)|RosY}^Vj#; zLzc`&Bt)agQlC{K6QAEF&pKov(EkzVoGa%0AAft+M_*+;JLCK{U!7WXdgj%>OOU1i zGy)m{jetf#BcKt`2xtT}0vZ90fJQ(gpb^jrXaqC@8Uc-fMnEH=5zq)|1T+E~0gZr0 zKqH_L&4rwdV#>6H6j9=&n?-pX)3PjKQn z&S!&b@2{{o-nI8#o)nW^xy7)z5^}1-uBm?i#(2(2ycw?UH0UmIbxqeqqZAREq4_=jq+$G`JaKlQ6Bv`PXG8M8ZsyN_6fStr9D(~|7iatmjql9DnAu?rZ_@#uDt9A zmt1|RtMT2B?QLtn$f&9It{@QG{Pvm1n!R)Ks&dP*U0n&XhL(p9{|Al(CO zDGbwB?-jy30vbxjGvp7dTNWno3wg^ zW9cE%jbDOq2gab6dzN&74h6O#_AKoI@o?Y^mT)8K)If~#H<3;gDW5|q%m@XucF88o zN=-J~@sYL$qVi^9AoE}$ecsH=kS2+)ZjVP-&R9W_74mHGy8H{Zmn$glNUl(l8BdY@ zXUUG5sYL#w5YD2DCoCPuP5J(A;cXf=^vkQ*oS#)Hql{7%S`3nzs9 zU>n$7QxJ7yW=KrSYAycb1X6oSd<)Bz@p`NQCLB>dnQ4Kvyp7N9x_F= zNf!U)YGrS&N)&$uuKXEWDh8^Y`Avmhh|9)dU+jn3V?~BG1XZ7nv1E$Y--*zZck+qLG#+ z7*UPailk~h6Sf?wsgPPtsXzTDF2(1NlIoL?x&%_|DfRZgxKwpq>O4vnvBVjWdW}*$ zRcV*UC2pXEh_El-1QDmNUIsr^Hzand8!``-K{u>Ho&s?rHB~ocS7)`|Zun~4_WR0q zVI#8T?k2=ezjz-Oy6r)p+|5S)0#unAIe%xHB|}aXNd;Bpr_aTUEC{Z6gd`IW*0>+ zqClo1N24S&OOfwVAX|}~Cd?c~4y0@@$$qmDF?iEOlmg;8YB=kWG_*U{BfC%u{+$cj z<19iv&RWCcJV`v#I|{NMkM$5N8_wwS*j6XYc@L4NVjA>4(yoP=c?D$$a7cP=-+4$_ zEr`NKx97m}E}S_}7~Nwag?H}NR|q3Q+Vt_UY7|h!_Wg`>As>xlVKe)82!2LVz%@3J zV*eFHfNWvi&ytNE9@N42E7Z~GsfzG%Ha2>ZZ3_E5NTz~xb#_3f+$bFbj(5g1VU#&u z&6zn@809B<)rYk0OMA$GLY=Buax{j zbUMSgi}YYRlkL>ymvLiQk-Y^>Wo9h|kD@Ab@F|;Cf(p$-pCg;k*(49D)=a_%@d1eB zFo-=MIA5m7ah?S?h@@ih4p6ED1kELtgYY3Fv8SaWBo9W^6`>EZp(n?R8;!R_0o(oN3jzTQW-H1KrOvHw{ z3bAP>+i=Ki1no74A@-RSh;8!;#D4QR!~ydb#6j}`#33^Sagte#7%M@yvWsj+nYMaDbfFW%zXcI}cbechQh`?xWq}psQkm0CNRBfOO3b232T=!^-w3|* zIJ+8HB`b6JG*^j?6k1#g0-{)vje~@;cmY_!pCI~%#qb^AuuIQ?9Ln+7g(7t|cw9ue zX_53BR7LhATWt!P-cA(Tlf}}{@B@fwn#~t?IQ%e}w9ffAZv83nzG0n{q&#Ik*pumU z6Q4h67;gP&do9?iN}q#JAe~cyd{w2tf))G{ky|BM0Y5nRw?SeM0*hP#-fzI88zbYq z8$|lM;BgqIzmF&hAeONE^Dc)h@lv!BiRV|xMIUZ_$ZPr^Kn-UWd=Ry z{VdbGHt@u6k(&Ww*IGn^S8 zPu7skNAO%`@Ro=nFmH+ybU1g6bVXn0(TqY7flt}=XT)A82T6qClJWR=l54}F3b1xK}h-2@QDopVhi|cTHor`D+3i6=~UW0gob+x#LiznNullcVAbfID+M01syS7~Pcf_caw$-S}vKFVb= z8Gy%uJHR^w2~0e1BX&Uvy$RZUl%ic}vI$as@hO|Qk#bCSr46pol76P-hz-~1P6^$a z`+Jt~Y`lcz4V3c@p#>uYtEQlNCTJ8{>&2j4$y~)rSxC}iW|$p-K|GC=Dc**p+Y8?V zm%Z>!*k1&JRrth;g1yJ%^d5tgybOPVObR&vR2a*LaW~-)8r`|v${nHwR%2w@NsPzyIw&WEFw6EY zHm(D&&?^GT5N?97>N=Q%S5EO#bY;B=Wg_br$btrPV$gosBd((AH_Ze@pUKUe$ob$2UMuyj6WTQd4 z5P}6z%w%K|PPTr=KFZn4X;D=sg|mAW8=Jx7%la1t7(CuAc4r2|E3QPQDMCoNO*S4r zvdL!!C1Al( zyF1C@1Hkp8SDa0KKZfnuES-;X6;mlhbmWC3S$WRHvgu}Y$|TkIE+jbt%p~@%3-3Z| zJ5tHYmwHYB#>!1Mq#$n^CI5=N@KTnok}1<;LS91Ky)n_JUpZlF6e9OX3-`$h#KJgp_CpqmTMLFf}8yN`&}y1dAPapD|+ zrNlX5kB43%dN9Qo!$dY*n*!cx30^*C#A|WRGRMs+SOnrSh)i)rETgp;-WSk)YA*b? zW?m_JcHCndYl@(GJ7fykaf{USX^F?dDCYD@0_SFCPv*XeL@!1W&PvyVQ^NU8j#m@Q zbuilvJlPZhv$q@$A{j%~Ih z_M5~hVBUc^XeJ{LnL7|CnU^9?Hv1w@FcJW&i^$5ROA#A?q=R9SGA+q#NI_GhwVs+IN7JW*@BOJ zQ>;O@3!Fd60j?RWDEjqm0{4yn=N=Xs?U0Z;l*2{?Xx@O>JRRyXk0drB?;&p}3b+zU zFZx53@O=c<*SKkUWsZZo|0oD9=KUTNcqxeh zh^t71L0ktSkPG5w5dTc*TM@AC0*fnWy3k()N?oZ8mo-*c6=2;Dsa`Itp9l<=!u|wQ^9j_VOkg$ zQFBy1*fmPCE82p`f}@FT0P72Dwrl1OC(5_xf%gnd|1HkL!Qidn5f5vHtI!$G1n*u~ zAsy!F3o4QHfU6*kIdtOUhmm&NKOa)o6RsKd4&{E42tEs5!u=w+|08P7*pkrXsiw8V z)#TxW(kvFi*H6~OQ_Z*x40-vR3C%aCxzp9;X^qk>5y6j6*2I7|7EyEFu7oC^4c0za z^H5x~H%@U**6a<*~697)aft|s@-s-6Qx@P(5# z2SD?;)ZD%`p?M=UUvV|(#We?t;2%!b90<+5)O@uW?5G;*r*N>o=W0F?*Bm7LAA+Z5 zzg-~I?6(Jmn*BZpp(eerL2%OhC1swxcTlK}WrZ0Voe&ZR2dy;z3fhBZk~%K1eu0$CqcT?*vgszfIzquB0-g7*N0z7Gq|Cir$>_CyvBiM|R z;~NBjF;anX#3hwy0+ta0l>Wy-`t~DYP#UK2V&YiM=YXZv%QS^hjXeM|{)DnTI zlORR~3f?Ij4s%9igK`8`P6zF(2wlXS@U2Kl{6hke`GEG48fd`rmh zO*-4x$-+xX=lkv=T}C?ME5bwEEGJ#;yMi|RkS_5Zpl)B%<-%w~My1*BQ6T2q%8dTZ z5Wa3yP_M&H-)ZCzB;WQ8q)|0xhJ2&RA54DOS5N+_q%(Zw~X{1XWr%xwc?l^r0=?cf`Gf58=#=S6Vo7G%Q`Bu}>;mi=etLQ`xX;T%;-ZYaoXN9YA8$Q&i)yP0$@XrI<^2gEm8Os)g#9Un z)EJO_Ee8itoJvzj5yIvJSuRk;?M(=&G=&r)?1xbwd9D)Tn=QwZN>fM?!u|_|#wwwH z2_coHkRpUVi3pBVLgyufRGLDH5cU*Wx>gDCm6B7ON>fM?!tP5;4?lr?$;jV;2i3!; zV$cM4;L#%=rFw7y5&s2AZ0^_Jjk;+ZUQAyOjQpjso#ss%Mgsr?gbVcyP=jU;~x!BkXXDW$d`k;aCh zcm!tTP17Nh&UdZM%lH!rk;b0FK2b@Do4F#&mDkWwx20HxlespL7ZAvdyDL?BlI7Oqj@ zQ5D=p*EcaEmmQ0_KnfT4xa3N|4Xh3r$cyvy?*S(X<>arUGz%1&R2&RN^eiD?=t?D- z7iV;%hMcJ62Pk>b{XrJmbr6?tMkUrLS?ZQUgcE=F!_4b8Uge#p^3Dex;oRSy%TL+; z3YAx-^5!#-Q*QyMvSmmWGD=(*>i!#6ykL=1i9n@@Yar*vcTo=VykTCjD0+}&DoZM$ z$u|*-MNtaM^QQ4GMda6riZ(*Z=O_eGS)WoUS_COy+OfD4F5h$Zd>pdw65w zgnj#u$d}!UVM5q#q`Q+g?F(oxLfW=hlV3nOWPd@rkaXDo9qA&{8FoL)7n9Dmdy(!z zI^TYhbT85o`zFelkS?~@k?u{p#I`9@M!MWyOny1(3i}rF`;Z%WKbH02y&qonn}^I*DBE|6lD(*3lU5)6dlsV|1F3S ze;xoii~lN!$S~voV)4(Tr}GiKj3nS!r~JMuq-?I4J*hYyfs;4&MEj2bDL!YJ;=U6l ze~z)llE;Cy5VHW!Xw=pJ(8*W--5dVq2uPZz!j5OiW?KH=x#911Z}_*MNS|}VpFXG? z{$8~-^^E74l8;MM&jgi+rK!)k;jfZdn);j@{wm4wz@@2YvPxoU>f_SXbAhAV)%0;` z>bcNK=9@k?w&x-znTM?xkn&7%baPA}m!_VnPBP2%acS!Lfg_x0`nWXp)H}&8rcYfj zndT%jOdpq~o=co$y6JOn_^TwAratF}ze-|hs;(k>W;?=In%cPG|1Wsv=TVI3D)RBd zmafiIG9uvK@aI`vkdH&pT!I*KZuqkaL(UC2|PVV*RzwniFbOTBQg)C&hoy>PJ9 z3kQq)L(@_(94z(1!BQ_AEcL>{QZF2=9LNML^}@kYFB~lO!ogB694z(1!BQ_AQk@qL zY0jF(R2S-4x|z0{KBxp4u_ z0ZUz|v($w;OI@h5)P*`rU8qatL*NE1lR2xi^{w!67OS=?ma|PU)nrbeylF$w5|*0G zEj5{2YBIOfWNxX++)|Tysxz6J5zHsbt}o12mzFw`u+))+>z$&c^#jd$q{A4p{}r;s`>P$8Ho^H@V^m6%TTaBJ(cr%4zgo5U0{;6NqXMIdFrT zm?d)_b$Lk3GufXwI?`WYmBu}RyrW3UrpNJYcB(XvRk|lpr3uVA&LrO_%89H}eh!i~ ztl=ak$1(XHbI)ZHOh+>FO;}bHeiy_TI=KhLWID;IdOC=gPB@u$2)w!A{plp$Vel4% z_o?EImlYz*gMm!eaaZhW)N(vd$1*v>Z$d_90EEIM8N{97WMzOLCgrFHF@SV7SPHMc zAn4O5C(j66JzHdQm$-(S=YV(w1dsI`&8$nnS`XG@mj$0C3^Qg25u9QvWW+34*vHM{~r@l)uq-F|6NQf{F?~)K_(Uc_p9c> zEwap4VafpD9Y`>9IFP(SNKQd5 z9Z23HB)2lDAo&9^xSvS{$sd_~m5^KlQwoxKAQU7^K`2O;fgmK+C?gKZ3h)TYc_;Di z1&{TgrFaUG2VB-tS1bX^gUC>jYy_d|??^cH-vXwp|4SfL{a*(`OR4Gq&i8j_>PI9W-&g%BWL*4dchcwI3ez;rqxWz8lR79bW=6XmaOt3R%3i4UL!0JHH#@2&tGrfMw5iO3z;`QQdsGlU*iLS4b=8l&o2 zlGAk%8{7iC&K#~%)ktM>F5io?v)Vy?31Suuv!h^rpJ@W;W8h^5Kyd4unGYfzL{>2f zZi`pD;tRmztNh;JJrU>KAhN1mIj%QomU6YO+|Coc%+cTtWC`OyoXHZ-12GCjha-hI zyIE6QRubw(W4NoGb18V|LX0go3>(v84y~)@0t(QxCrU2*2&_MV#R^woFxt*>irYhs(N38VD-j9iY8FKqb_T@%i_})M(N5zmxbzi!BVvf zfKZhQgJ5Nv93i%CW(IhyOinitwBVE>^7nBH_>GK!-B-EE7otupyrSIPK{tCMTe;a6 z1l?>q;pRY>6@a2s`K+N}v5>AV3vQkP7TwIM0YUGEI(hW2_#E))oybqgfIA<`abN&M z3@E<(bKLnzx$_a-IS<*&ov9$`&QIKefZ7a~b)UxavKgufH zi)>Ye2SBh2BO&EB{KGD5ipxS39s^5t;wM0;R^AALRaoi>sVY1R9;=YG6$GpBK*DIw zt6+Ts7Rwlbi+t?WvfJ|z=-C5q_TtfyJ&3b#FP=8qgB8i|4tj3pE~tcGaQqD7y+yC9 zAmv&1F%rFg!!6;Rv`~5tEHc>9Qr?JS@Q*0vohJtW$WmTrV({0NegQcKv%K^{d@vYg zrRU=T%V6{_Jp~*~s9_E#-?$248k*FBHmP4==qQDh%EMrT3!E7|hXRvEZ<8O!qy-VZoYy+jG@voEw!=)S101Wn! z()bJDKuKx*+a+d6DUUH3tnsBur~!jjSL%a#2BWl;k0A!5y!1kRFnIfw`r#FWSyuWl za2U+KrHyD!22WGzLU0)Dvr758mcf@(%D2A^zO2#}$Ye13l>P;V7`y|;LCk%o_zd%h zd(z~=8bY1)A)jB>;X&tj&wHGodI)|0Pd%v3?H|ygj5NmoZ*OjW&gRzra)?)LV_B}5 z7Dm_pPqwi>_e@f4W5?0u@BpfQcN^w8_Z<45 z4!Qj5kjp#%W%Pvr54pUFpRow=kjtC+8H)f9xx9&=u?X;x%bWNaivX5t!khRRivSO~ zyceosWWE{TA(!_eCz)pkc*x~V{ES6_hg{yo&sYR_$mLD^j75NlT;6&|w~HCzA(wZW zlguy!+@gCgagym~fQMY(=_-jst^g0YyosN&2uz~xiJ!3u+TMmwP*{kETwdpAEJ8fw z^144`k)#f}yjQ3%jPj7nJC`7)@Q}-UC7UpXhg{zIgirXfdlMn=!m+Rx;vtv!8g>K` z;)2k-h-@=7ob=*z5!)dia(S;M9S~uYp16lxsm{wpR>Qah*=f$pM5fc7mx+pWUM7+h z8GOfr1s0*~qAng@!(J>hdSX<{C6ZqrbNDwZe&$(}ua@#XWMpxYeE!OnuOVB#Y;3B+X+i-ayBRyk2T6e1O9k17cC;1;1k6!XxIyh(9hi71}n@?vq`08u>El`aCjeCA?c;UuX^q5m2VD@xcP4Pfzd zto#RG-SsA^TID?l%VcB^gI~+z#(`yq%j1EMDgapKx;*~MgW~lT#SxeHTAT+wdb+$1 zfET^4HJ?6MZ0P<+>AEq7TJqI8$bqG_z| z%em;3w)_O_230A39itDY=v=I?*Rylw$`7F5n{*i`>b$|o;hYX{KZo8jxb{!C1R(yU z%i<9cOF;v+yQ~bCg*qK~S-o5qDqzAaZ4Gru>w4gYA(yoftX*`|VVMgW z8zZ9EP2lppz!AdyY7RiY`vqS09K9P7&a68HIjT?PH-V@6lzNyxoRGMl5^SUVC%{u} zG+>{q5}yc`QG<8;1G#~dgB`|JGPsSC>L*|FbLj?Ky)9N(Ln z0|v+UB<8@u@%@XHGk7pm7y=oC|9}sMK{}Nc)K=y}0Qvv93!$^%bo@f-)}B}^ zJPt3vc_GwtFN9u0(IV$U=))BK0HT3@eM`w6squTrqO-=|4TgGxuz+Ekpw{u$V7N4Zdl5X+o$m}iNS0ip1`hYtfy#SKaQ4_qXgwum@K=Z9> zNu6=bg-uTyuBLII6v#1fI?$qMtd#iuOYAO{4RuLMti6uiJ?(%jowSI+%~Tq5nc~c|vdv1$#pfvl4C)H&JdB z^+H@9$VUc32em@Z({5#c!OC1hkxVWNMikVk<$rOvB?GOsssEhC0umv5c|I`3)-WHm4^ z|7?{vL*>;mkM&YFyX%oshE0)m4-4)#PRY)R4_{*t9ah8FP&e3fX#6gOZlWvi@R{kn zhn;sbq8!?O(#hhTJu_nqvmQj$^#K^S(q3blqFt|II-SdOmG9oBd>fO!JGxsTz|lPp z0ROvDt-7VYO(~q)l!P~an=-o>#`^c)rX=$5+mtp%)oscObVO6kz+=zFKZK?r{(QIW z{;Z1I42C*nm19PrW0*W-mFHlZa1L1wb;v5m^C4^=X5Ivq7ypuroT!rVFS$rQjD(4` zwEB{ZoUD@ZFS*DI99{QIF7iSr>3+#YUgRX*FS*Dmj;{M97dh2Qx?ggUKX8QIFS*Ei zC+U94MNV^)?w4HTB~H@)l8c!4PalV|jtu_3q^HVZ@R!5`qw%=)jCUCPsqp}R-^?@3 zVemJ^1N_BF&-D(2KiVDO?@oGdb{Ogxz_Y)F0k6rq1oWDtH5BrY=OgoxU{isT%WT+n5h1^V5tv}Sl zR6DB+cYIm#bapXPb|IubKyD?Za5^AlzYK1WWX9J>7^(XaQ-*(;ta^0>t&M~n-sLn} zi>Gr&P-~*HMIO4JkVCuBRu`J;LQDM1y^09b5`l3Myt2ioY~u1YPeGwF@=#*_tQq(* zwp!*~j$~JL&@qZ?4>E~`MI96WLZaKHFhqp9<2;s+#fZ5bY6Zly@IIV4$niZuE`k*P zqqrl;g?;cTl2QbFYVD-n5LF;4ium$YPF)RoXVtvGz=nY;Q>lX2`h>O#*=kKupGs(d z0L`p8Pr-W~I0}*nA&G-Kz?(`)?gT%TuslS9&`^?pMSKG)r*DeH3+ClsRq%ATV79^* zWUGQ_rm_{DgK}1!m(7;g363iGB}l4*hpK}2g0BibLV^WTl7B^fb1h@d5%gmu@sl2C z!XED-$=(KJanL7<_)=D0@nJ&fR0@4^l2CsL@eQk-w=W^YtFS#HkhI+?usGUa%)z%~6ZT@PAnQ1T5&LfCQ z*b8_k7FmxnEB#$&?LgG4oau}`iS%@&tu$_3lxnYPrqjKc0Ui1Gn9kr&wyCr@1t6QE z0;^B=d3qEAe4ci}vh-W)88zQMhKhO882?}SWtUe`@OA{7QA&K3>3pY_b(d+*zOKFm z&#R{)<2!(eL23BiBNW0S3w)?2zVbuS-Eq-;@RX<&0fHq@>7%47C=~+fLiW%J$hzan zLddTE{~u)RX)@J;Z0Em+EZ{=Mo)?GgvG0Y9{ZUppkgZEVmW~c=icCBQ++J4gLiP?J z8;)t+913J*n6dGTWXA3-cK56A>N~fx@FV1S`JG!?L{Z`8cWz}dY17N^+{zxLZ7;ub zD|?a-dEX$v7wNE<_lsl+=?pKg@5tVyv%UPztt=&-@4bt38R>|Z-?^3Lq>H`$&aLc2 zy2Q)x+{(VB%Y~)Bb1VCCx#Zo-jQ-3JUVi6R4j^rMPa}UIY1_-&)N&B%kaslsgGqTE|ekJK_FTZmuhmg*9oE}O#;y68wbg|>~X{1XWr%xwc?l^r0=?cf`Gf58= zmio@EtlmPW`JG!ioEdoOPA6(eo5IJhn0eBF4;jmt(i^^?1-_5pHuDsHthmMGvRV8` zkUoSEV3hcRl1)^6{8E}{V5*u4GpO+ixZ)E$flaaMM7Q{d+bs@7Frh<&-;Vs&B=moN zRB|N>!2)PC9I(}Xxm=OXT8P_K;jU=?empZ}Kjz98DXm2MDUm=8^9D=*k7(_5^5i~5 zLG^q;lj$Iy?Nv(HznwzoD4|?b%MltE7aC6@l=K0GCb2XCHh|_YkvBj(xrG^*P$;RB z8WQ{D zT>MVD3#duA;>Vjfh@W^88cgcR2K-L`Z$SM3wHqRxB>Za&v5&VZ@s)sB1mXb_@4ZEd zZwAB(Af6&|@Gq4(k((^3b0UamNIdnJ5-$}Z2XQTkbfCxn<}Z9SXbwUZ;DRkk7VA`&yO@jM82 zQ~U{FSFrfcp3%%-LS|{ivq%`Oli=b}Lb1>NqjLSDa{Z@peF>}&f$L6kJv<%lnaQ>V zsCP*Jig1rFMbAdUmi-mvI>lUPmLiw1U9D{E$R>*O1i0wIiG6V=xLn0yHBW;2EU91S zpDJ;wUkCMRP|uS3=x;&2n8_*M2R@C`r_ik3j>JUvfSf!WI=1qs_>~?6DAR-QY17WJ zY2Hk#?X$>L&s@tUQDt#Slb->a&-!BD;yJR?vB_91rT>>B;Z0+e@e(_FU+3AAv)G#e zWvWbTrkj`vr`_Ub8gliq>~H@Fxh`gM%4@)X2rN_l7zB?*;uz4M06jSlm|1LUiPpAB zX4mUd;!L(ML^QMgut>nypx zTbgg>aEo$%B;eWzt^$-2``$|B`jm1#9&l|3*F184zgM}QRjz*^m&}OW-~!o;J$H|C z{fBZDUIv1Qxr;TD>jyG6B_K!_uObId0sRK@+CMZ=O1o%J0xqNPlp%gi8C|vv@hj~o zz&kPKcKg!=%f}t#-YKBVTZ;9ptcbsVRm4nvW_A?&13Ij@VC7GtNN9 zpKW>v8!;No>NQMw1*geS5&M0>P@xP}!Z2np8J5ik9WPgCDa(X5_F>XiFilz9OrDcp z0@FBi|4zbcjCP^9e<$TOfES_mRiu3dK+)q!ynx)(n0xI9!886zB+r31LGi^jC*v;1 zk2jHBnOti5H;n%Uz@4B>P+ajjU~Up#Z&Kb@QQqV_U@qnZ{n#Z9$hi?9K5F5kr`UBT zfG870GLE%|xAoVNKh`vjHIZD;39(gb`VPvygs{}|im7D^pIpVZKMq#T_t?q3TGDn~ z68lKtCc7%-e0VAI<%xH8NQR1wXgutgPm<&Q zcZS5XpZwydNO}Z637+!gPvj#;1M{tcya}%mX4V8>4ah5|L)*~#Qb69s(Rmj3%#GMz zl>!^P1I-xwO$-_>5#NUT|8KfDIE2=i`k#O@>3-iVzh_NN%r3*LzF43T)X zxe+_g9C9NT8y}lpSUkoryrgh^Y(jn!%nd$l@(ZsXbIq8Q#ZWmMg*m1Ubm&c9Yz%Ou z#6TG_4{UtXDnBpg7rK;@#ez`Af^k84W5$ddGj_~EWExeB!jxll$U)kei*xdi4XH4v z01C8mg%_c5Y~@9hVsodmQAHy>!s91-bD`D~QnAHTO<(3tU0Ixvb{CIuN|@+!82Tj8 zgYHkp5mfR`pZW3$`ZF5Ic+OtT^nMX zE;fxROL?wD0#>*;(Nil}wsDS|k8PaGy0BEBN$3xn1gGTt(Y*MS4=+UvF0*=qt<&Y9 z%vjQV2Lxkz2riyli&9c9g7T|~#a0%x%E}}LvjQTlVykPX_7oa1L(?6#-E{Hzn5b>2 zsj9SSOrEREBXSipKh!0v-N;bS2o`8FMrkufGXeEWqvrYBi-hi z$%4JPsy^oQ5Cz2_YcVQ6ChX)`r|;w35(8!A*cXNcQ;9B$PMJVzWFBuyxATNza>k$- zL`))%frw|w6vnNgD@O`;@>`OlJC_pB7(a#H2E90smb&_=dP?K!N+P0EBOnXy5C$18 z{Q!h$M&@?6fHo0h5f0mEwUY2LP=DRY6X?XB*etXoKOEwb&JY)b4H*@RnPWOS9GEfM z#IdG{<8nv8(8&k^Toi)FgcOCtb)N1yw&4PT&dZf-RPLx+q-_@j_MsqG`0SxGQtTlt zm6`Kn9HER=0v5DTKH+j^q)?P=uEx_ott?(RW&t#Gf*Iyww>N?6aZXGOg+$AM=a|N* zPF8DnaVRvdX@6L7u8N#@FAy$~y55cqg$j9v0}CAur3GyW}KIx>eBi9F|18>jLsjw&PIl(Ky?7! zb<}BeFrv}T=@@68prd%F$DKCg-dew%ZFJ5c`6$h?hIye_IkVg$MAhz2A{0%*ur2?? z)@ayloEUB+Q0oX47-{*A$f!hOoMDwh+cV0j7H6)t-lAp9Z1*r$>xSRyS)yZ&j3!?6 zmgF4O5?^;?@QUbN*ogCX+}#(TL8cu}hW`W(bBW!vyPUlo%5`*OG z{hzkC)MD{K$87pX83Q-kZ2(*7R>^)utIaitdEur$y!jjv6*T@{q`8-k-UAF>&PIh3 z2+|7mANmot6!R=pDVbwNh|Y~8ZH_aoBGVjy7{HyTxUJPu;Kbqz=B#cLqDu#Ean0g1 z(JV!>dWAQElijrkK1SVqO>r9(s0dXbOIlDNhG3do0O^WAZ;5VwRjNAC4_?`b#9IrAW@hN6N|of>xX zYVrBn+mS{L+(!n}ELwb^hp~WH&ZhTNPblC-GIaF%&&|*Gn97IXv)BO{QDG5xm zTOVFK)84?hk7Vfip>M`&2Rt)IW9Ei9Gfn2HZe7axa@kv2aL%t#u0JnSr>LTh)M0g3 z&|IB!e0!nJjd69(@$XUR#^vf9PNE9?evzwlctzWI|KK(;ETNNcLTIqCtq;|SJ^;t+ zT902SO5XY_9fM=&7h5{C09rQ0$uU)J-n1ZVbQ8v#0PBO8o-?vDP_$#oXSyXsmV{ZZ z(S^3DjfWFZ1wMsA-{E^DNt70$FmhblwqPlp$^%1X>m$?A~G-q%utJCXbpP!sORmw3?n*)fXDrf? zeP=KxIb#+`Au@EFDTS_`fKN@$Ns*chY%-11bGDZlA7cTqwNwGIxJmBVM<%p*Qw_7T zzGCAU%9xWC4ivY0ap)v+_+>yFYY{{=czPbd)4_*oVJaao0*dvao8&DLXQE zt#s4|H+Jq?`PS~D{+HOCP$|G8j*SoZ1oFLP~ zP7{nS{A#3dpX8XRgsOVdsD93KzDRwg?hFhB!QPcQRinwXbJp%T&a4CR4{R&loA3V( ztKY>-Tci0?)&hRgJP^$0LdmeXN5i6iuu%8SeZ1=t%k_4?bp`sbGDUcS!hB2THRF9FbFiG(j8t&p{W34^Rr4)k zxs+`j)XN7ci6IZe13s3Qn*?~0%RvNR#Mtyi4yje0X+B+~|AtQmsKVI&g| zafy2i0!GEI z8J$4L75V!EhGt~|uhz`$cOtk0LbhY)!8uu3)qMy+F?tXox8_p;L$fkOvY!M;+`BT6 zG1*R^2j^rh!26P85n|p(Mi)u4iBS=P?`01SGSMM;zJVKDcn6m4Wi%Qgx4mP5!B<>l zU_=eRAOqh|&1GN~yrjnx7@3c<+zCScuSbY`m4l2|A)ptG)=096(OLxQ1rx^wAN1k? zz{3bE0g+t%M56>D=B;OxLSVbr=P-~d=51h9hTwbshjQpdqTP&m^r)j^l>iqQDx+FM zWmJ#gdzr8xvR;Y6)-hg)FcJsoB{E*gAOYq;GqvT6DRxxMC?UzsjOIwPi4lbs^ETsG zF0~=ZLsLQ-L4=GbB#|3px$5yt9?N>IBsVf5Ys}lkhy~{k;}&3c2tyO28zgxhqaFmQ zfr)*B-^l0|#WO+s6!WfQbi3ja%f+I1B0vU=Sf=mYcxaIM4l96IuI>0`1JHa%EDYu1 zH@IlHL1n~u+D$i@N*My9u7H4qTnYTL*ARff_@SyA)J8_kgL3g3T!lyh#b|;gH!+$h z$sP5GjC40MienC=@X?5#j7yTybCTT5=;sK&x9`y4Z;;@zc1}}}-xj-mCX(|J9F2J$ zWkS7x+RUgx3u0oj!Jojq|EbLVt&c<9o)VSYT#HG+0wLyYMG11T@pe69Ep-U!A)|VP z0*orow0LvcAiI<&U20;)ZiPfb{LUCT8Er@K@mc*~JsPuA8q~z-S_Hu}vBKaPH7TBn z4F=DMe}+r=nRu7M??TjWl8pFyoUkx)*x(u6sdy&dZ}5!nRy-5O4W7|S#WTUbhKQ~( z`mExac+}t-nQ`3uJdzwo2$az|Nj5R!-@cTFF@Y2dWHbsv@H~5n78-a){Bln4OiVL) zM)MWV1do6QKciKOXQI*I88s=Mi5&*ds7LWk^cg&(1Bz$jE`w+E5yexBUNU$_{Mt;~ z$pl~8M|&6*Lc>hIkSG#$YhpB0@k}`Ob*|!R5qV+?L`W_Dq2gafg8C|jwj+@h#cg85 zBUga09Ax5%;5RcmD#<2BA4HI>_abrH;2Hf;@k~5#@Qi+{__Ijx1Y*gIp_5BTpEjMt z|1?6(+s@(tjY;yz-Oq|RHiSLeegg{YLts9}rh&9+a*f6>jS0yqOqzNrYa17B4pYqA z#?K%qS|quov1lYIFEgdLJ&Pno1#BKFfQO1n_-yDrVUmKN43K;j(N|28-DOXKjia*# zVt%&qW$$SwNf8*Mw9T{??XWHMy$_#0MD=1D<6iwBRe_C+dsh!?D}_nhDF7S%wB_O` zv#CPB8)?#5W^YHDLSwPoa$#Z@*mg2$qQb<4Sw}WAqBsOkMQm4qi5n2m8%DP){yrq`H+V)9P47=0;((i}jpi}I zQO9D&h~khMsGY9}zKPK}#WV5mg5S)DIx74SkX-fnHEZt%%7ouwVavy!n-yfkE+l`A z04%#ogN$S$qN`1k7IlM3Qp0Hp!NBMQ0xCOwhzWiP1w6%a&LlYpXxD%ZZNLW3Lv|8a zYW881Jc@{_0_;~1Jz$f4!!`&;TjaG&1KS>>*gck z(D4pOG#Wy4c2RDwa6hnK?{BfQbmT=V8bMQEFz%nwUww}>w1gVP&4kK6? z)k$(aBaSGrj1C(tBaR%fj3Rdq!0OH&Nsgsa@AZ!%$^BFT+6vvxFByCa5KRFjqZR>K zXb(oq;H_swg$I_A(S-G()^DWNpAm-jjLh(F86!59vjz-|o{?m8 zIC+}G$JeT)K_lq?u@q2euw zy;$fG1oT5u^HE2o)U%aQwW*TPLIgA=Hpm1mfK$+;P^&MBhHYkaMv~Vv`X++!y*oC@ z#M1_^b?ruz%`BBgs7Ly=l~ESK_YTGenYhW|HP>l?&Jy*Yp)z_HL2@y{c@JgNT&o}i zI#7UOv>8EInb>ad$~qm11OlYOXbu9r1vd{eAzv?t(&8N~)d{AA5j~9~y&Df7KK%T< z2MMq!BF?-D1k!SaWh2)RA%0p=xP*oaoi#d1#xs)-Q`w%HA1;O9FH%BKyZ=4AzA+=xRxBVno-I zshX)941rO%B%2uVP6|~qx(gw0ZK{>f5C6Qg+u zxxH0}I3_VUW+03Z)4{1C7c-?0AZtc+TA>C;HzDK}R|!m37y{vuajl@57#V8NK;E}q zXQ*1=VSslS6r&Q5;6K8l5Ar1B7&?NOqimB zmNuZkYo*X8M%;Zkv+xu^-XKE^jGoM;J`eB(LuK^4TxtRK7-rkt#ON{vr{om?xkEw8 zjOcfAs5E4w5wMf+>Kn?Eg9Ny}aIE7@fF%f{z%%~MH{aYpZIB?%n!`oQxkFIR?TEMn zJCmPQ=Dy)kd0zdZvcM`n45mkO85QLkamD3kK7)*Ajc#J}973)JJps%&5uBD0W%D2< zm!k5#(Qk&&N?!u)=Lk;Ta{ymM7zv{{=FK;z4H5u@%R_LAu`i6e5KNnxqT#~8FuD-| zPkH%wcbMc(IJfQp4(w|PPW^O5K8z65Pv8**hvzP+;$I-38b)UjOu0-wVF--4iO#K% zIUhox+!;THfKnJSFSLl!*AU3Tm}NN1bqb*23})m6bf6@+6ZvQ_%_4NE5Rvf%1e=G) zf%=+IgZqf*0sb@>MX^sna8-oxWCTYR34mpS+RVs2NZF7=(kSz0Mk|D21EW<4aV*k| z8wI_Y(N0NjV6+P%j$Kzf;w}VK#;8Zo{rEN4orq2%xa}kSAcE7r2LN&}2T?G3#8AJ6 zh$;yzBcrU%&m&nRlH7olT+;zo3#y4x6GHBY64P!N7#$wMK#bW4H8DCdgy9%4yzP{Y zHg=v`H95KDvV*$x(!X>HxcdZOJ;<1=ZQQFFWc*1!foR62c}-!1nzSre zv)salqn}1eM)R5&QK@n}@enXnz7e>^tC=>)Y$@UhJa2A>5`9mjyZx=ap{ z&MyHh*E9J29)crtqPWj=U$F>r;hQKgf#)2;u;so2!mkmWa*6s4g8R})hzrXFi_?^ZiJiRDcO? zYMoK%QuI@)X9K4u^Tlzapck8A#_75+bMUl#BB*r;aQU7NhJuI&gPb2AiCAlp4+Tgf zHW=h10g?#TiZUMyklKuq-hM8K(+0IM0RlPX+KJy?2oNlzE=dNqfA>(-KA=t;E-lpA zU)}`DaiKP!f@mHP;RVY8DIZ0E1Lfc#6J|=ulN_2LCq|GSO2>QnfM?A zCM%=670=pet$#l-DR zj!;9SQ}IG@u`rQNSX4M@b3mMrAadx3qzI%8;;c^u0TvT*XfXjon?+9U2GFcyqU z6jnx#Z3tZU8E;3B%4iD+><|PhBZ5Ge%Yn39&*Jwb1iLHX8J6L0Ly4@9Og~3peT->; zg8Cd|Wa^_aqRj|6ke2Hle)FIj!`7!n)<>p%1lGrxW>QL{>30udz8lWH9wVFh%Jn(Gx18UWWc5d=Y8te-(7j0KE)-(KMwOjUBH2{KJ;43t8)B=r@-teF`LXOo@mmBE-G#mML9HCg1O8EB=1uyWbgKlP{K1 zy5=+ZTPvb?Q{I0Dc)sYn53E9DZo8wNq??<~IKzI;=9yh({a^8Y)145tEpA$$wjKIg)k30C2$oM=0+gaq4 zzY*|e1Sq;JnmZ8pAm~NWz=+FVx2#npV%lIBRq^3YN4x-GAp&{o5cHae#1w=QggFSL zH6ZB4yfYClLzs_1+O-Ic2$<}_MVp=Pj1n&6ta&g??>&0>S9)il9{XVBeo8}n3hVVK9 z$Aud#z1Aa57cfWbKXHgl3>N@|Sf;-%r|~o}vnPc=G-(nY z$nW*_w{^F)_4cPz2JCLhWLvZ8miE?ct3hS5?d|E^Hhr+C&7^S08RroBvts@)yqN!e zFIC}x{?ID_BZpQM`#<+m#jDpB-|lB({zVD@#sq$EF5Z5!^aj5r<|kMA;}VNzmCUM3 z&h+tgttwGoa*e+(;a{5auS@usCH%sK-=CNTu97(w6;)ejd2xIpJdmhhoWCl6|4M&s z!OQ;Am|vIjmnQtLLdQ=4mquP7BAI(3W%S0siglmV0i&X@Z&)I6WjY& zhZ1P~mg0f^#RKt?5FrOYE`Egz3$>!7w>w?YT3LP7g32qZyL#JNyDGB1y}S*tskn(9s9d`+tDvj*ahAw z_>*GEiY5L{i3;?juDm{3Rad^Yj6Epf&m#Uv|4Ny<{EEHk(=vZm?IOP{waA~m8-_6! zt7qfA%lUi3nP;0O?Uw11hXy2LQpOf3qwaiVVCZuvZYJB`*<LbhKTMP?D!;xe zL7Bz!v-uU}`{O12jaqwsO-D3PUQlBDd+# z1`cf$w}3D#F@Idj&m{b@DLPUmdf`>W&Iz<;;k|Nfm>Gj=R7baW>r)! z@)JjdrB;5$ye+Eq@4qGW(9AvmR_A{|wn_+F@)tF(*;+Ect~xoZN-(MXMQhfqsVJ|U zStBs{;;Zugdv3`ty2k%yEag9TOUnNbFR|D^3aj@9YP|DM>e4y$Dv~9OR?kZMU9}DV z0gPp$(jQ2nHU7TT{8E2vVyoYl@=H_YmHzf4eo10=dF9G7|5@)yo&U(8)aqGveqqXA zmh$VM0*m~wpy+RTsk%k}qlb?9Kk^dQt84s&i86@4;^Z3tcVptwK{% zh!c?Z-LYE#K*B#z>o2dJRe4SFYJW$mb}`(2k<|m$j=_CWkr*- z{&QY!a+d$%Evsf#_&22dyJE01L!Svh_`(o{qS;^}72UH z6%Vi5yL$(Euyl9!_Q05DTl=#EeP(KN^3c`bKN*YpEvc!G&o7ye(f7w4i*xTFlU6=0 z9CpgM#ikFmrTbVzOGj%@dsn*O8V}Y$XN6xFTVO7+bnVjM6>p)zIm;XVAhv0E8&=o7 z?Y-StSVNmfr-wfwmV8%L@eRdWr5@9R^IB!M2V6S>t4#k0EXw4g{Uz%rADvlwkNixk zs9cK>&x35{%ckJgx7&mCfh-p2Y=1;d!NS$o9a6-|m<1WO%dWyhS2T*4{u+O3tYp!W zigzXJDwnNUQeE!vtM$j!miw7nzYx*E+J^FV$-44&=U>@sy2>wGR9}t~4CfygFIc=3 zm9sj3pw=H-TQ1AMU&@GgoljoqcP8Dxud^+k$z9{R+I!8gxH}dI0r-}hnkx>N;U~19 z=0FIAT0vKpe+m9%MqaFL(Ha{~-deGpp}1q~3Q_eYY2K>0-8|V&n1+b*xZ8`XL`6RS zw#)46-JkBa$}NR<_L}Yly5Mq%mPCZ(&>bPU*icwhAG+oId9mU}NnG2DZ`fLqoSB5l zO__L9L8=%V^NNaD<;mpCie$3Xzk0Vn5c9_*u>Z!sJee1VbxL46{v)p-3*gfhSQVuI z&C9Owdtz7t{k;i)LSkOQ;Pu5X6z}mT?DqHJkNzf0HqWXk@h{pUvy1oTae238Sj(fg zE;J6V=6LOWU0skB1kaR?3)DqC0desOyuA{se^OweaBXNAJQetfdtZI~gX>Ii8Qfq{ zhuAbnmz<$9s$e^6CW1jlbbz)}bR)1W&ZgMCeWL=`*n%?sL~ZHi{@8n?9Dctt%ml;3 zhAfpYUffuR{hhyWi+{<3rJd#ViP_W3D(kkP;Kr?s%hxT4C9r~Vny=>1t<@!Ut1IgU znD#qUY^Xn-WV5a3KEv$t=P%f^XfZbS_2r-N$D}5`L-zXlwfRkTi|XpIP@}))Qpld# zM()r`4IWs3^eO`^E-i<^n64oAvLt8*M()bR#t#027==gQzTO14BnWS#1?4sw{M>`(KU6rgF3w)tShD={>c@vyv4hmF3%uOJp>6*p%f;uAQui1^dbN_LCpoWW}ea!1W<<-`0D~FoB(;xTb6zKJw(ix={i`=Wu zzxPm9|D=Yo);6>jr9NJP@2ll%1?MddUtE*b$*QQW@<*9Oi zf6AY6Z`sz>i?+_btezX@F^TIZ`D2bu!ZW^D>`z-!#6AyMQJO`0_PlN7;h0Kc5gQTc zhoeT((dMHNzalmbhG155E{#J;vINnr62=D;Ny16o4q~(m8cWTcNNxVkvgygvC@mb{He1}YZUlb64^E5imJeC5eS!Yn`j z_{84gmB~G6%3}7S>x)laE%K{7FzJpplkY3uyA}B_rC2NcQ3VMM;q~)y*ODw*Tv;LR zn*XA8*lIJ^O}-}(_g0@z*5_T>jjCdxxZa1($TnGw=@O26r_NWXyG6*kpKz_9jHB+f zBBSW=Ir}LW8;+?ve;f{DQ@PgiS)V^GCc7;Erd1V5KT)u1;;iIs-0Z;scGSZDm1bc~ z)+9+M%|HUR!69+>mRkQ~u`T6D5RvfjKD7H%ed!583{N4Ygw!-jwnVpRc_0YN=VeWE>*1 zRw&MVHSq8=)mZ>JhJ4=bKMAA!j+$%y-b4osY~iY}UFKL#{C6I4X7Iv-RsOb^ zKR4lTP55(D_-#a%OLz0Z1}rmNKP4&r5?OYn*x!}%XVzAnD7k(O{wMig?t1>fMEc_t z-Oem3`m!|*Eh`&ZmNji!+0eMWWpg$B{_XNm$Lniv>Fyh7>2K|B=}%{kh2_J57IO5b zTiaVY+ckefb2anY(wa9eS=+F5D>1DL!J0vK>jRSF)gR<#I&VtTm+ZZB?WV-Cs;UH@ zf~2Yvd=64sRaKKH?{4cGScIoe+4O<(Zo=7)f$m+1cX#!+X3OzRWKl)Li+_$@L zUrl*?)swmCmNidj`6?%KLiZdr9|~ zw{@i3_GS#79^JNVdGEe-KkC_UkbyfZlhe&5jqBDfmtOI&R9l&KRXZZ&;ZOb|0uI$8Ox~u4&ob+26gtwLjg0#%1Wf6X!U@+m#J! zuX1YNg{Mu@Jg$%}`&zpO(!n5hW{AKj>ZrDNX8KyQ5Jf+VP4~5Ay7!-QUvL zmdUhqr(KoU*KQOAMS1O+N?~v52&A-m`G)3|>(+XF2GxQNn3cM_wXLJGCv1weZlDiU zrKvNvbAi@mdhw`=ilQy;NvGR0Ef}Xgd(c>`cN;b?Yq@qw^F}nI+O*nKrn-mwUyF7g zd{%H9(dHBpB$qcY->AUKwVRhTu51W8=yk%QX$phQw}R^3y&J0TQQ`adrGv>~r-K&N z&|JxxVFlXT*4^d|*Y4J?Oxk0S`&;*>TWpDzS9)VKsLJc?ZRyPPb{QWodxnQtwnc=p zo!#l)0U7Z?XuDf8*+6F9y?Fd*O5B&tw(RLcfqk7lUZw-nEiurO>D<$kZcku#;Q17k zre!HL$+jd{185MSMeUiYVDO3B)787HwaW+z&N$q0I3fs>RbjGPn!3Ng6I#HLrj>Cm zM#x!35ki|}R)LRl@5q?C*qvj4rJx+vGWT}=xN_|&`OIB zRa?yCCOaS;2l}?X9(Y7sFj4v7wxzTGMoyQW-fZXYgI;SlEW14H^#;JduqEqd+WK44 zJ;E!Kxwj{Mpc5);9{qYV(HKd7z&690G8KPP0Q7hyHKG^jZ+5EN8i|(zsZp}I@gED3&y>8>ROInt%S$VCc zty|V~U6V~WF1YqOP1dxmT-&^Hg9V)o%NkcVtlV5vz1ouA*|c(9r4d|PSGpCp6mm0l zHP+a~nx;)QwX%9yrA@9|wr1IeC2N#-!HVV}+1Na!JDlW}zI<(irqGWTw3$|+-}IPw zJ$(GN&}&#r;p(9HHAi+#GRB+Q3?YWx9aC(ZmqwInC&+}+i>C*!qX4QuJ@ zM5oP4*p=Rw?g~oPN5_KAc>C;}GOV5L2W%#G$7F12Y3=j&W`wWD8|cHL=b81Rg%*)b zfzd&;v%P&v=;;+w54sdo96T!qbDP(`zq38t;h`_cls-`1nrZ9oG;k|^V@_B5oi4?CaUemf0>j7)8N zJsP|Itf^bjvb=NPobrRsjy8bL?)p$iW}jlbT?3ijtpi=*)((T*-P+T-2NuFS@PmB% z+jDtD@ZI4vK`*lp6Wp>7WRN+Rzo)x1Aag>kX=$u!ZrRwp%;O^0pSEj6K=Mq=Ok|sG zT-{Q=V9kncmL^lwX5c)wTusfIY+Tl`%wiik*$e|GlSy5-dQ+24HZ(T}4Kd4>Vc|SB zsg=vXVv^XNhNEw9TCJ8(Z*GURS%f}Lm$dop)3qar6jrQ zwS=nzf<=U-TST-E3!;5k5cYu&m&^h+&?k>J_qSpLy&F>+oBq~r-m|ul2aPb#N1fZ@ z3O=H0^BNo0*(EJ3ti5-Cj}#}%uHo3QTv(~JvW0L8W=5=;jDq@cH>iB+Oa`_Kw^=Q+ zyk-=i3qiMFUw3Qe@zW7Y@OVFpn7a$Gaa3~vtF0n>I5L~ zP$VxfSZI3S<;-N;umrVW+hg{^(R!V#ATis0Yzw7YmgiQ7+Y;=vT0q7sh#j)LKjD;M z_FhBonzb!``}@(b?$*p+R3rl!R747Bef{ZuqEe>W+`qE4U7gvCXKpHkzS@ojIrd-) zv+R~c)x@d-6YbfX_#iVF**7WIW+d!nw2}yF2dgy9?XB5|xfrO5+|cCQA87Z7lb+l% zV@1N7q;|ltz<0J8p<78&DZ=Jj$psBVS0q15LJx+mc<4=QP+1?k+_A@MlNNevN`=W2y=Titmx_b$eBVi2e3Wi zO%&vUY3B9q>FsUjy(%R#kfByWZ?h8j_4Y(44NZ-WuxhYbHpSa4+zAW2JWN3qEt&l- zGVfWw@HI3*uHuPe@k$ncBCy=b{d35_vbo9RL3+=3@X8B+td7HB{w-6isnVtAv%+qnZR0*X13l4dkJj287bDwrn~!? z5iHy@DNA&{vYRPWt<7m1CpI&E8k!l!Nt2^O%hFOQzlK}VJkTqI`qeWd%kk%vN zsb>O42nDua5_ERKN!8oW7U{5R-}pk#u7iE88Brk-?u7*INQE8*td*YK3=gf1Me~h7 zm@m9^PoMRW>4EKsZ3ypf;GSxM3(ShoXee=FH0Xbz1It4DvVAHXdBC8KkiZ`&SQSO+ zrfY|dG_N6k7)jA~h@z6?_S}7}Ze^g8*cc{l zqqT0llZ8)4XvQn943=HZnY-n)+%C%2Wix5b@;JicqOEmQUN-WgV3D@kc5xzLXf9BK zJ`27Oi(OfxEbbQ13bq%IDd2R=U{?UgK9=kj<0~+0roF4%waXJe-jl=S)dT-BR6$UE z$jS8)J;!DU&yq6f8wb)oa3klG0B6fV4^|`H-XcQ{c?!dBe9$_M{miMcr@C|Pp-V! zd?ZIKGmjwG)Oh)$N3IzaADuTLZ(?jx>|%dPY-(&;e0pqVyaewI$X`Cf34?FOy!g$9 z_=D0>DKM+z^5J_2EQ^&aVa8vN8l-#{oZqczI#b(Y)LI z+v9h{_KyD6H^1>jp|=F@+DqN!+k%I_%(tvC?`UiVsJOI^M?rJjcoMK%=R*&Xqkzn6HnCz_w~;D=eGO@85HTZummJkvJ+ zif6uFC@^XKY8me)D(#)5v0l=6B;Xc9llHJ);tMb^XzGx5kp492V;wqSo|CEkgeOMe zDGu!Ri{(wH%s`)D>mytVSmo~WQ`U{3^K+|Oq?Z&t?%zSVQ3j+R1|4tc zfnN09yrceYq-oxhprwY%8?>*@ojNgP*yeY0@UbrvUT&lV}?W^MFZyw4(fk&qd(ZBJk@G zm<7A}^CECT1Rm?aZu_Yl8!;~YxP>;uoj1fMfX9Ccgy%c3TMl`s5B#!ap5-UJGXmck zfgb{_wu<~`9M~%> zI5SslS<5+}6PC6Uw1%*}4N~6HF)Q5sr-#X}Wj}TS+S3=Hy>tQExeL$=pq-reZeO%5 z(=R})ya27?0<&4X0`~!4Xz8vllmAJS$IoIkKjAZgw}3|dCOicb zgZjO+a4aqF?XmT;1_$&Npl^5Sow@YWpwk95X#RH@y4L?o&{>zl1^7zCZBGI)>nGi9 zKY8vfz>%t;vwp~yb8Zi(T$!ak2-@pH3)YQz+i2K#5kKsT<+~qBdosee%&-MK4LH~6 zM1-f_@@QLL1kcf29_b6)xf6p(KC1(=rY8WCb^x>ux%twzXz4e3o(2zXt@iC%z^6?f zvp@P^EdI{X&xmlB!A?_N%$J>H@=?Z87vMgm8;rbmSsN9!^DOAxELwS9M|o=h64-d8 zLfS@`#y?HLakBYxXoo?=Fl*l%bU(xP90TpjFrU;H9c#i_Y?Sy>2HT?WOBN=M^w$8h zpG(cYOZG-en|Lv7EojS4zc;b%TFy?;AkF~JM&LUm@JYa|NAo`hnEm6}NIL=Z^bKdf z62Bv&cjWVM))n<`X}GV6tKMn5Pl9I!c&y&+pzK&T^S=xl$I{aFie15;Li29Krs-;- z2l^kh`50(!4Ad41^Qt^j%0g^}Wlz9{`K+JqW%^lN9(xzzM*ViIfX?0g_f`O{$Be zRpK*+C{4?42Mt-tCO?NAI4Yk6)<}Nxa3@$XwtD|^$dwOy%CO-(E<6*RU9V{?>5qZV z_AKYU%?2neF4j-lFG16O5}uAX?%7ABF91BzfmZmfckQCj+EU=6)g;~! z#Khe-Y2d@l_pm5C zmoM58B76@M!MS`nH!MW>9yVsq<-=NQkBQFn{`F1Pzytr zm!Y4H&`%q&fvw1cNe(WpkA5I?#k?dEQvtJFt#7_VXVLT%nJXr?tqL$lLiy_*I{E3p zFqdf`Pq$N;bo!ALX1zNdm~{G;%w_W5<-nxV&*aj}xXF?$CL>1wlEN%M>%h!UACtLE z`6nEhbo!iJI!~ulm~{G}6lVEn9GG7hOpd7Vl^tmVF!&>GbDwpzEdmq2J70 zrv8sQFy(h`V3rO2&cN&%eK_X8Y|jB^lFQI*2 zV7;{c^r@N4wC_2G|EEeqto-zOnal9EQ&@5p<(!hK{Pb;^%kUp{U{+1vmP>!ofk~&Y z%cVc=z@(pXVD|Uv2>hVazE^BXjI@39jhV}|@2mr}eEP^-`l(>qGvh;lmP;QSjDL|i z_0v+dKKe&o{&^0}`sgol>9r0_I{haueWL@DPJfC^?{Z+$>0fc_CnE54XFlY?))1x$ zb$%ZRwiQL*xWE<}c~bu89Q|`oh1CAje`78q@5ddO?V~@(rN8FDq|?9S(i$#=Zz0x(&_IpmudfW2PU2VAD7-3ffJ5==RtmiDL2ZOAH6Pg zRT)90i7e*ylV zMflG-bn??Dr2K4e5qvWWlTN>o!lXAkFzNISnalL2&w)v&ugIkzbzsuzJ96nK9GGGUy~%k+N*{5}ek zP9KxP%zw&(Nv9vlTqgfR5tx5n&0KljV;E1uu6;Zj+(u!&jm#(3cb7BYei)&%e)_4* zW$HiYz@*bh<5l6Hsmt$r{79pw)dz5Ge7-R<}&Sj-hoM{ z|H`Gm>cCRJ1GD@x_??uWbo#UuCVh(olTN>uxlH{B9GGKklUovq`$X zh5MT#@BbcVeCQ{0%TK_sr7-KKk4#~<{}Bf!ojx*inf9M`VAAO)bLmsy$5NPd`ppz( z`IQb#I{j$oGUYcoFzNKGx%9&hOgjB+F8%2U{F<*x@!5vaXC?$F79!hu=dLk>*(BMwaZ(+*7fvkpx9^A1esIdCx&!g!6!)J^demk2&x!0Uvi@j>l`xc){foT&hpU zo$)FLeK`uR1=j@Ba0y_prz~eY;41>+)aEN>EE>N7`x z@NB^NOXuPr8;@3}0IR>a$Q$9+5e7GXPNc6Uztg^LfO)^5?HvF-waoZeSpS~^{ZV8Ap8p8so0-%Sr|>m zJxkc1ae&?OX9Avz##UPX6yRS$SPEYSxE19$SUUguM>q6M>)Q(Ww+Zv?h4r@q9)
{-gk(!2CiNA)h?&8o&KLor9gMYQH{}twkK3!|!{{%dM@!%~K^<%`S-1;X0{ss!u`Ys3jlSHVWS4H?& z08V4SuKb$;S6+s~aj|_n0pIG-`v7ls^!rx8-^P65yAf>93Bcvfcsv5wmCrXEnEc-d z{8jj`wxB-3F9Sa1=+CQwzwFc(L&v*jLtjuI^?d^1#~k~42?c{o>GJ?D$Nc1bn=HQ) za1x!p3p|AD0K4s53%KmkaQ)f=_%WwFT@n8G0G^8WPqgLzS%iKs;ICqQRUbbOcq{l- z-=6@SK!0kHpZ)nB;P-)lo-OZZfIpuI$LE)T-S)l_feWFFr;r(HBIT6p0fbIi1Dkm{1t%R^6CMP zLeTlP9_euX|JOIS1O7bbUpvs$k3PUj1hNsn1&Lq49$jhSy8++o=;tQ@pPm!8_e+4+ z!yd6c%>RAB%N_og0ndOubiMs&z>hon{2Ra@!}X>gsV@b1Jm{*Aive%KT+;De2l#U+U*}^QaJ!@by?|YL9tQjmXkWtC|6#x% zbL{O&z)R40E&mIEPlApnU_A4@Zvnm(nN{Ck0337L|0}?5eQyB%4Cb%OvjBZNi2V33 z@|gtKOHljS$9yjd_(7-rmjV7Xg39|Eq(7Dj=hr&{yY=yXR2S|fjP-m4%IA3A2$=q1 z)=2%NF1!Qn*ZzDM@TVR9`~=`Pu)opqd>HVrFc@lo@hX^izau}qM(VwR!R)g2{TMLs zAGN-J0lX7%y{-QZzz-rnR7&JK8g&-{=2e87hnau}puegg)quH2R(J*AgBYw0;Aj7K z0RA|LI)43t%MlcQFVbg%^7FjA0RPs`U*^9T@QnxyZF!$UI&4p#_b}kU#+d4Od>!!5 zP@l@^B5%l)v5nt!0WMJ-EQl@6!3Qw;e5Ul@W)|) z?zHr40k3h|a~thGAz6Xa5?l9|D}D~0K5I&8-Z^I9PWR8?{2^|pmREYpCCW_qvOYWzX7Lz-z43U z=MModggo$H8*7hmAwl%z5_V2~f>4L#ToDESmtzr^*1o~P#g0W16Wo%ARwybWdk*pWRu z$~hZKUZg^m_S`4)UX?lF$-b9`v!8-9nj%Ls4SV{LJ*r2J(BWYs_yV?HTg~r>@ui#j z4{eeZma1|B_TBIAXJSm5hE977yKSITvE)Jsg z)Oijh82&In)4k|d__jFgMKz=JVT za%2L=+@4u%YwTJP=mtrW<=uVE%nT>N4x0g*ye_|CimD$cS$p+ zlQ;)FTPZkt8a=RxMpe$)%{h!RID`{c6k314G~7<1PUlhbRt9> zCP&|z;%Y(#wks-h!`AH_6{tGKQ6F+zv29p5wQ^5b-80;gi{`vl&L*DzV%svLYs2kk z!d0Dz*M!2waU0h12YX)Iuuz0zyxp@wMfD-Tw!L~f4-cUV&TOGZV0&N>(@;C(9w20@ zaL$3WN)b-Kz|9 zFsrFO?2dCJk}1F*U8>qfbZGlc+kH!{j-DG@6;5S6jm&CPu(ymJO6^)N z+2#gnWDi#g_GF;Sfq!QIAcyHo;pXrpt2NQ7Y!-cKI!+dL&rlb|MVanrLaJgMcU?!! zD$AwKJxkfK5zdKMsvaB|J&8QW=;6pba||%QM;dzaE_ua-+l7u?b`V1q(n@kpinn4O zGHFcis&Jr~+nj9U9@~rVL{4EdBD2Q>Ih$BHAj!5V)GS*~h#2-(ms42oY(g^?{1pI@ zxP+S{id6QtS2^bvg$ro(z&1TF9Yu+uP$TXPkkyjFz8PJz_eF43@E~i=?jFNu@2AWG zc)|UY8aFwQ$B3XhTn(&D!U<#tHKKEdZ9zCf*5KeAFPj>=ueNmsI;zU=-ilkCT_ug( z(H(`V5a8f%NpPzyy2mpjWkX;)5bj;zb9$>6Um=8>3La>^@a*X_O&fn8p(FzN;Bv=R zfDng&O#u0UD^ISu_*Fjo-=2uY#Ybq#;$LL?Dg>6o^5|zRz+cgR3~9ZzJo?v^Po94i z!RhMMtI2C!K7L0x9(?e@>ZN>ZfzUK*tao<5a)q>Q5x#PKd^NLdC|^f}Z#6!``e!;# z?WJ{P0Bf40EAjC2StCs0RN9dUU;ek_-eEi!)Jyq}Mfh%i2JM6YRxjl{Y56KG;q=+K z_X|2b?L|I@PuW=c{%nNrm0KQ0%J&e`Xm+2Hz6d^!rCwU^86Y%G+Bxur-v@+x1-Gfy zDRRLlI?4CGW0LZ#WiZBw2G`-qzSoEG;49^PE<}APLcn9#obq11+V>X0x2l(x_dfwB C6M#qn literal 0 HcmV?d00001 diff --git a/firmware/am43x-evm-scale-data.bin b/firmware/am43x-evm-scale-data.bin new file mode 100644 index 0000000000000000000000000000000000000000..2d71341089816be7989e6dc11b265d9194ac909e GIT binary patch literal 41 hcmd-HXAn+dU{VptW>OLB0@CSBDpG9>aG{wnApnMi2I2q! literal 0 HcmV?d00001 From df931edf42c0f6ca708fb61aeea38230dfbf2a2c Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:42:28 -0500 Subject: [PATCH 010/213] Add BeagleBoard.org DTBS: v4.19.x-ti https://github.com/beagleboard/BeagleBoard-DeviceTrees/tree/v4.19.x-ti https://github.com/beagleboard/BeagleBoard-DeviceTrees/commit/af07ef77cc6f8f94568a4c238cc6d41fb8c81931 Signed-off-by: Robert Nelson --- arch/arm/boot/dts/Makefile | 15 + arch/arm/boot/dts/am335x-abbbi.dts | 169 + .../dts/am335x-bone-common-no-capemgr.dtsi | 381 +++ .../arm/boot/dts/am335x-bone-common-univ.dtsi | 2961 +++++++++++++++++ .../am335x-bone-common-universal-pins.dtsi | 967 ++++++ arch/arm/boot/dts/am335x-bone-common.dtsi | 252 +- arch/arm/boot/dts/am335x-bone-jtag.dtsi | 15 + .../arm/boot/dts/am335x-bone-pinmux-can0.dtsi | 45 + ...335x-bone-pinmux-panel-1024x600-24bit.dtsi | 151 + .../boot/dts/am335x-bone-pinmux-ttyS1.dtsi | 48 + .../boot/dts/am335x-bone-pinmux-ttyS2.dtsi | 48 + .../boot/dts/am335x-bone-pinmux-ttyS4.dtsi | 48 + arch/arm/boot/dts/am335x-bone-uboot-univ.dts | 33 + arch/arm/boot/dts/am335x-bone.dts | 11 +- arch/arm/boot/dts/am335x-boneblack-audio.dts | 50 + .../boot/dts/am335x-boneblack-bbb-exp-c.dts | 47 + .../boot/dts/am335x-boneblack-bbb-exp-r.dts | 47 + .../arm/boot/dts/am335x-boneblack-bbbmini.dts | 209 ++ .../arm/boot/dts/am335x-boneblack-common.dtsi | 65 +- .../dts/am335x-boneblack-roboticscape.dts | 52 + .../boot/dts/am335x-boneblack-uboot-univ.dts | 44 + arch/arm/boot/dts/am335x-boneblack-uboot.dts | 44 + ...am335x-boneblack-wireless-roboticscape.dts | 149 + .../boot/dts/am335x-boneblack-wireless.dts | 43 +- .../dts/am335x-boneblack-wl1835mod-cape.dtsi | 128 + .../boot/dts/am335x-boneblack-wl1835mod.dts | 49 + arch/arm/boot/dts/am335x-boneblack.dts | 19 +- arch/arm/boot/dts/am335x-boneblue.dts | 765 ++++- .../arm/boot/dts/am335x-bonegreen-common.dtsi | 9 +- .../arm/boot/dts/am335x-bonegreen-gateway.dts | 206 ++ ...am335x-bonegreen-wireless-common-univ.dtsi | 2835 ++++++++++++++++ .../am335x-bonegreen-wireless-uboot-univ.dts | 59 + .../boot/dts/am335x-bonegreen-wireless.dts | 43 +- arch/arm/boot/dts/am335x-bonegreen.dts | 15 +- arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi | 227 ++ arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi | 217 ++ arch/arm/boot/dts/am335x-osd3358-sm-red.dts | 191 +- arch/arm/boot/dts/am335x-osd335x-common.dtsi | 4 +- arch/arm/boot/dts/am335x-peripheral-can0.dtsi | 13 + ...m335x-peripheral-panel-1024x600-24bit.dtsi | 50 + .../arm/boot/dts/am335x-peripheral-ttyS1.dtsi | 13 + .../arm/boot/dts/am335x-peripheral-ttyS2.dtsi | 13 + .../arm/boot/dts/am335x-peripheral-ttyS4.dtsi | 13 + arch/arm/boot/dts/am335x-pocketbeagle.dts | 2122 +++++++++++- arch/arm/boot/dts/am335x-revolve.dts | 984 ++++++ arch/arm/boot/dts/am335x-roboticscape.dtsi | 489 +++ arch/arm/boot/dts/am335x-sancloud-bbe.dts | 76 +- arch/arm/boot/dts/am33xx.dtsi | 166 +- .../dts/am5729-beagleboneai-roboticscape.dts | 242 ++ arch/arm/boot/dts/am5729-beagleboneai.dts | 2147 +++++++++++- .../boot/dts/am57xx-beagle-x15-common.dtsi | 5 +- arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts | 11 +- arch/arm/boot/dts/am57xx-beagle-x15-revc.dts | 11 +- arch/arm/boot/dts/am57xx-beagle-x15.dts | 11 +- arch/arm/boot/dts/am57xx-cmem.dtsi | 43 + arch/arm/boot/dts/dra7.dtsi | 2 +- .../dt-bindings/board/am335x-bbw-bbb-base.h | 103 + include/dt-bindings/bus/ti-sysc.h | 1 + include/dt-bindings/clock/am3.h | 12 +- include/dt-bindings/clock/dra7.h | 12 +- include/dt-bindings/clock/omap5.h | 10 +- include/dt-bindings/gpio/gpio.h | 6 + .../interrupt-controller/arm-gic.h | 2 +- .../dt-bindings/interrupt-controller/irq.h | 2 +- include/dt-bindings/media/omap3-isp.h | 10 +- include/dt-bindings/pinctrl/am33xx.h | 130 +- include/dt-bindings/pinctrl/dra.h | 5 +- include/dt-bindings/pinctrl/omap.h | 1 + include/dt-bindings/thermal/thermal.h | 3 +- include/uapi/linux/input-event-codes.h | 17 +- 70 files changed, 16834 insertions(+), 552 deletions(-) create mode 100644 arch/arm/boot/dts/am335x-abbbi.dts create mode 100644 arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-common-univ.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-jtag.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi create mode 100644 arch/arm/boot/dts/am335x-bone-uboot-univ.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-audio.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-bbbmini.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-roboticscape.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-uboot.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts create mode 100644 arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi create mode 100644 arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts create mode 100644 arch/arm/boot/dts/am335x-bonegreen-gateway.dts create mode 100644 arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi create mode 100644 arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts create mode 100644 arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi create mode 100644 arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi create mode 100644 arch/arm/boot/dts/am335x-peripheral-can0.dtsi create mode 100644 arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi create mode 100644 arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi create mode 100644 arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi create mode 100644 arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi create mode 100644 arch/arm/boot/dts/am335x-revolve.dts create mode 100644 arch/arm/boot/dts/am335x-roboticscape.dtsi create mode 100644 arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts create mode 100644 arch/arm/boot/dts/am57xx-cmem.dtsi create mode 100644 include/dt-bindings/board/am335x-bbw-bbb-base.h diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 3b9255163740f..0416e9503d546 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -697,6 +697,20 @@ dtb-$(CONFIG_SOC_AM33XX) += \ am335x-base0033.dtb \ am335x-bone.dtb \ am335x-boneblack.dtb \ + am335x-revolve.dtb \ + am335x-boneblack-wireless-roboticscape.dtb \ + am335x-boneblack-roboticscape.dtb \ + am335x-bonegreen-gateway.dtb \ + am335x-bonegreen-wireless-uboot-univ.dtb \ + am335x-boneblack-uboot-univ.dtb \ + am335x-boneblack-uboot.dtb \ + am335x-bone-uboot-univ.dtb \ + am335x-boneblack-audio.dtb \ + am335x-boneblack-bbb-exp-r.dtb \ + am335x-boneblack-bbb-exp-c.dtb \ + am335x-boneblack-bbbmini.dtb \ + am335x-boneblack-wl1835mod.dtb \ + am335x-abbbi.dtb \ am335x-boneblack-wireless.dtb \ am335x-boneblack-prusuart.dtb \ am335x-boneblue.dtb \ @@ -748,6 +762,7 @@ dtb-$(CONFIG_SOC_OMAP5) += \ omap5-uevm.dtb dtb-$(CONFIG_SOC_DRA7XX) += \ am57xx-beagle-x15.dtb \ + am5729-beagleboneai-roboticscape.dtb \ am57xx-beagle-x15-revb1.dtb \ am57xx-beagle-x15-revc.dtb \ am57xx-cl-som-am57x.dtb \ diff --git a/arch/arm/boot/dts/am335x-abbbi.dts b/arch/arm/boot/dts/am335x-abbbi.dts new file mode 100644 index 0000000000000..8be51c7aadccf --- /dev/null +++ b/arch/arm/boot/dts/am335x-abbbi.dts @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * Copyright 2015 Konsulko Group + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" + +&sgx { + status = "okay"; +}; + + +/ { + model = "Arrow BeagleBone Black Industrial"; + compatible = "arrow,am335x-abbbi", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-abbbi.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&am33xx_pinmux { + adi_hdmi_bbbi_pins: adi_hdmi_bbbi_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + >; + }; + + mcasp0_pins: mcasp0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ + AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */ + >; + }; +}; + +&lcdc { + status = "okay"; + + /* If you want to get 24 bit RGB and 16 BGR mode instead of + * current 16 bit RGB and 24 BGR modes, set the propety + * below to "crossed" and uncomment the video-ports -property + * in tda19988 node. + */ + blue-and-red-wiring = "straight"; + + port { + lcdc_0: endpoint@0 { + remote-endpoint = <&hdmi_0>; + }; + }; +}; + +&i2c0 { + adv7511: adv7511@39 { + compatible = "adi,adv7511"; + reg = <0x39>; + + pinctrl-names = "default"; + pinctrl-0 = <&adi_hdmi_bbbi_pins>; + + /* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */ + /* video-ports = <0x234501>; */ + + #sound-dai-cells = <0>; + + ports { + port@0 { + hdmi_0: endpoint@0 { + remote-endpoint = <&lcdc_0>; + }; + }; + }; + }; +}; + +&rtc { + system-power-controller; +}; + +&mcasp0 { + #sound-dai-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&mcasp0_pins>; + status = "okay"; + op-mode = <0>; /* MCASP_IIS_MODE */ + tdm-slots = <2>; + serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */ + 0 0 1 0 + >; + tx-num-evt = <1>; + rx-num-evt = <1>; +}; + +/ { + clk_mcasp0_fixed: clk_mcasp0_fixed { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24576000>; + }; + + clk_mcasp0: clk_mcasp0 { + #clock-cells = <0>; + compatible = "gpio-gate-clock"; + clocks = <&clk_mcasp0_fixed>; + enable-gpios = <&gpio1 27 0>; /* BeagleBone Black Clk enable on GPIO1_27 */ + }; + + sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "TI BeagleBone Black"; + simple-audio-card,format = "i2s"; + simple-audio-card,bitclock-master = <&dailink0_master>; + simple-audio-card,frame-master = <&dailink0_master>; + + dailink0_master: simple-audio-card,cpu { + sound-dai = <&mcasp0>; + clocks = <&clk_mcasp0>; + }; + + simple-audio-card,codec { + sound-dai = <&adv7511>; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi b/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi new file mode 100644 index 0000000000000..8fdafb8af3c8c --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-common-no-capemgr.dtsi @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +/ { + cpus { + cpu@0 { + cpu0-supply = <&dcdc2_reg>; + }; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x10000000>; /* 256 MB */ + }; + + chosen { + stdout-path = &uart0; + }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&user_leds_s0>; + + compatible = "gpio-leds"; + + led2 { + label = "beaglebone:green:usr0"; + gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led3 { + label = "beaglebone:green:usr1"; + gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led4 { + label = "beaglebone:green:usr2"; + gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "cpu0"; + default-state = "off"; + }; + + led5 { + label = "beaglebone:green:usr3"; + gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + }; + + vmmcsd_fixed: fixedregulator0 { + compatible = "regulator-fixed"; + regulator-name = "vmmcsd_fixed"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&am33xx_pinmux { + user_leds_s0: user_leds_s0 { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a8.gpio1_24 */ + >; + }; + + i2c0_pins: pinmux_i2c0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_sda.i2c0_sda */ + AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_scl.i2c0_scl */ + >; + }; + + i2c2_pins: pinmux_i2c2_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_ctsn.i2c2_sda */ + AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_rtsn.i2c2_scl */ + >; + }; + + uart0_pins: pinmux_uart0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + >; + }; + + cpsw_default: cpsw_default { + pinctrl-single,pins = < + /* Slave 1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLUP, MUX_MODE0) + >; + }; + + cpsw_sleep: cpsw_sleep { + pinctrl-single,pins = < + /* Slave 1 reset value */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + >; + }; + + davinci_mdio_default: davinci_mdio_default { + pinctrl-single,pins = < + /* MDIO */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0) + >; + }; + + davinci_mdio_sleep: davinci_mdio_sleep { + pinctrl-single,pins = < + /* MDIO reset value */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7) + >; + }; + + mmc1_pins: pinmux_mmc1_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* spio0_cs1.gpio0_6 */ + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + >; + }; + + emmc_pins: pinmux_emmc_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + >; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_pins>; + + status = "okay"; +}; + +&usb { + status = "okay"; +}; + +&usb_ctrl_mod { + status = "okay"; +}; + +&usb0_phy { + status = "okay"; +}; + +&usb1_phy { + status = "okay"; +}; + +&usb0 { + status = "okay"; + dr_mode = "peripheral"; + interrupts-extended = <&intc 18 &tps 0>; + interrupt-names = "mc", "vbus"; +}; + +&usb1 { + status = "okay"; + dr_mode = "host"; +}; + +&cppi41dma { + status = "okay"; +}; + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + + status = "okay"; + clock-frequency = <400000>; + + tps: tps@24 { + reg = <0x24>; + }; + + baseboard_eeprom: baseboard_eeprom@50 { + compatible = "atmel,24c256"; + reg = <0x50>; + + #address-cells = <1>; + #size-cells = <1>; + baseboard_data: baseboard_data@0 { + reg = <0 0x100>; + }; + }; +}; + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins>; + + status = "okay"; + clock-frequency = <100000>; +}; + + +/include/ "tps65217.dtsi" + +&tps { + /* + * Configure pmic to enter OFF-state instead of SLEEP-state ("RTC-only + * mode") at poweroff. Most BeagleBone versions do not support RTC-only + * mode and risk hardware damage if this mode is entered. + * + * For details, see linux-omap mailing list May 2015 thread + * [PATCH] ARM: dts: am335x-bone* enable pmic-shutdown-controller + * In particular, messages: + * http://www.spinics.net/lists/linux-omap/msg118585.html + * http://www.spinics.net/lists/linux-omap/msg118615.html + * + * You can override this later with + * &tps { /delete-property/ ti,pmic-shutdown-controller; } + * if you want to use RTC-only mode and made sure you are not affected + * by the hardware problems. (Tip: double-check by performing a current + * measurement after shutdown: it should be less than 1 mA.) + */ + + interrupts = <7>; /* NMI */ + interrupt-parent = <&intc>; + + ti,pmic-shutdown-controller; + + charger { + status = "okay"; + }; + + pwrbutton { + status = "okay"; + }; + + regulators { + dcdc1_reg: regulator@0 { + regulator-name = "vdds_dpr"; + regulator-always-on; + }; + + dcdc2_reg: regulator@1 { + /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */ + regulator-name = "vdd_mpu"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1351500>; + regulator-boot-on; + regulator-always-on; + }; + + dcdc3_reg: regulator@2 { + /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */ + regulator-name = "vdd_core"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1150000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo1_reg: regulator@3 { + regulator-name = "vio,vrtc,vdds"; + regulator-always-on; + }; + + ldo2_reg: regulator@4 { + regulator-name = "vdd_3v3aux"; + regulator-always-on; + }; + + ldo3_reg: regulator@5 { + regulator-name = "vdd_1v8"; + regulator-always-on; + }; + + ldo4_reg: regulator@6 { + regulator-name = "vdd_3v3a"; + regulator-always-on; + }; + }; +}; + +&cpsw_emac0 { + phy_id = <&davinci_mdio>, <0>; + phy-mode = "mii"; +}; + +&mac { + slaves = <1>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&cpsw_default>; + pinctrl-1 = <&cpsw_sleep>; + status = "okay"; +}; + +&davinci_mdio { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&davinci_mdio_default>; + pinctrl-1 = <&davinci_mdio_sleep>; + status = "okay"; +}; + +&mmc1 { + status = "okay"; + bus-width = <0x4>; + pinctrl-names = "default"; + pinctrl-0 = <&mmc1_pins>; + cd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>; +}; + +&aes { + status = "okay"; +}; + +&sham { + status = "okay"; +}; + +&rtc { + clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>; + clock-names = "ext-clk", "int-clk"; + system-power-controller; +}; + +&wkup_m3_ipc { + ti,scale-data-fw = "am335x-bone-scale-data.bin"; +}; + +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@4a300000 { +// status = "okay"; +// }; +//}; diff --git a/arch/arm/boot/dts/am335x-bone-common-univ.dtsi b/arch/arm/boot/dts/am335x-bone-common-univ.dtsi new file mode 100644 index 0000000000000..c0f5bf58cf340 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-common-univ.dtsi @@ -0,0 +1,2961 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +&am33xx_pinmux { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + + /* P8_07 (ZCZ ball R7) gpio2_2 */ + P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_advn_ale.timer4 */ + + /* P8_08 (ZCZ ball T7) gpio2_3 */ + P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_oen_ren.timer7 */ + + /* P8_09 (ZCZ ball T6) gpio2_5 */ + P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_be0n_cle.timer5 */ + + /* P8_10 (ZCZ ball U6) gpio2_4 */ + P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_wen.timer6 */ + + /* P8_11 (ZCZ ball R12) gpio1_13 */ + P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P8_12 (ZCZ ball T12) gpio1_12 */ + P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P8_13 (ZCZ ball T10) gpio0_23 */ + P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P8_14 (ZCZ ball T11) gpio0_26 */ + P8_14_default_pin: pinmux_P8_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pin: pinmux_P8_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_gpio_input_pin: pinmux_P8_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P8_14_pwm_pin: pinmux_P8_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad10.ehrpwm2_tripzone_input */ + + /* P8_15 (ZCZ ball U13) gpio1_15 */ + P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P8_16 (ZCZ ball V13) gpio1_14 */ + P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P8_17 (ZCZ ball U12) gpio0_27 */ + P8_17_default_pin: pinmux_P8_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pin: pinmux_P8_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_gpio_input_pin: pinmux_P8_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P8_17_pwm_pin: pinmux_P8_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad11.ehrpwm0_synco */ + + /* P8_18 (ZCZ ball V12) gpio2_1 */ + P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P8_19 (ZCZ ball U10) gpio0_22 */ + P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad8.ehrpwm2a */ + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn2.pru1_out13 */ + P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn2.pru1_in13 */ + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn1.pru1_out12 */ + P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn1.pru1_in12 */ + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + + /* P8_26 (ZCZ ball V6) gpio1_29 */ + P8_26_default_pin: pinmux_P8_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pin: pinmux_P8_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + P8_26_gpio_input_pin: pinmux_P8_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x087c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn0.gpio1_29 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data14.eqep1_index */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data14.uart5_rxd */ + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data15.eqep1_strobe */ + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data13.eqep1b_in */ + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data11.ehrpwm1b */ + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data12.eqep1a_in */ + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data10.ehrpwm1a */ + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data8.ehrpwm1_tripzone_input */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data8.uart5_txd */ + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data9.ehrpwm0_synco */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data9.uart5_rxd */ + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data6.eqep2_index */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data6.pru1_out6 */ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data6.pru1_in6 */ + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data7.eqep2_strobe */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data7.pru1_out7 */ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data7.pru1_in7 */ + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data4.eqep2a_in */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data4.pru1_out4 */ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data4.pru1_in4 */ + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data5.eqep2b_in */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data5.pru1_out5 */ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data5.pru1_in5 */ + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data2.ehrpwm2_tripzone_input */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data2.pru1_out2 */ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data2.pru1_in2 */ + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data3.ehrpwm0_synco */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data3.pru1_out3 */ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data3.pru1_in3 */ + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data0.ehrpwm2a */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data0.pru1_out0 */ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data0.pru1_in0 */ + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data1.ehrpwm2b */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data1.pru1_out1 */ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data1.pru1_in1 */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) gpio0_30 */ + P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P9_12 (ZCZ ball U18) gpio1_28 */ + P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P9_13 (ZCZ ball U17) gpio0_31 */ + P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P9_14 (ZCZ ball U14) gpio1_18 */ + P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P9_15 (ZCZ ball R13) gpio1_16 */ + P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a0.ehrpwm1_tripzone_input */ + + /* P9_16 (ZCZ ball T14) gpio1_19 */ + P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a3.ehrpwm1b */ + + /* P9_17 (ZCZ ball A16) gpio0_5 */ + P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P9_18 (ZCZ ball B16) gpio0_4 */ + P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P9_21 (ZCZ ball B17) gpio0_3 */ + P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P9_22 (ZCZ ball A17) gpio0_2 */ + P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P9_23 (ZCZ ball V14) gpio1_17 */ + P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a1.ehrpwm0_synco */ + + /* P9_24 (ZCZ ball D15) gpio0_15 */ + P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P9_26 (ZCZ ball D16) gpio0_14 */ + P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P9_27 (ZCZ ball C13) gpio3_19 */ + P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* mcasp0_ahclkr.ecap2_in_pwm2_out */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P9_30 (ZCZ ball D12) gpio3_16 */ + P9_30_default_pin: pinmux_P9_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pin: pinmux_P9_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_gpio_input_pin: pinmux_P9_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P9_30_pwm_pin: pinmux_P9_30_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr0.ehrpwm0_tripzone_input */ + P9_30_spi_pin: pinmux_P9_30_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_axr0.spi1_d1 */ + P9_30_pruout_pin: pinmux_P9_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr0.pru0_out2 */ + P9_30_pruin_pin: pinmux_P9_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) gpio0_20 */ + P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr1.timer7 */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) gpio3_20 */ + P9_91_default_pin: pinmux_P9_91_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pin: pinmux_P9_91_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_input_pin: pinmux_P9_91_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_qep_pin: pinmux_P9_91_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P9_91_pruout_pin: pinmux_P9_91_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P9_91_pruin_pin: pinmux_P9_91_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P9_42 (ZCZ ball C18) gpio0_7 */ + P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) gpio3_18 */ + P9_92_default_pin: pinmux_P9_92_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pin: pinmux_P9_92_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_input_pin: pinmux_P9_92_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_qep_pin: pinmux_P9_92_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P9_92_pruout_pin: pinmux_P9_92_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P9_92_pruin_pin: pinmux_P9_92_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_default_pin: pinmux_A15_default_pin { + pinctrl-single,pins = <0x1b0 (PIN_OUTPUT_PULLUP | MUX_MODE7)>; }; /* Mode 7, Pull-Up */ + A15_clkout_pin: pinmux_A15_clkout_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_gpio_pin: pinmux_A15_gpio_pin { + pinctrl-single,pins = <0x1b0 0x2f>; }; /* Mode 7, RxActive */ + A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin { + pinctrl-single,pins = <0x1b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin { + pinctrl-single,pins = <0x1b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/1"; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/2"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/1"; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/2"; +}; + +&uart3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/3"; +}; + +&uart4 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/4"; +}; + +&uart5 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/5"; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/0"; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/1"; +}; + +&eqep0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/0"; +}; + +&eqep1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/1"; +}; + +&eqep2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/2"; +}; + +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&spi0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +&spi1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +/**********************************************************************/ +/* Pin Multiplex Helpers */ +/* */ +/* These provide userspace runtime pin configuration for the */ +/* BeagleBone cape expansion headers */ +/**********************************************************************/ + +&ocp { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_03_default_pin>; + pinctrl-1 = <&P8_03_gpio_pin>; + pinctrl-2 = <&P8_03_gpio_pu_pin>; + pinctrl-3 = <&P8_03_gpio_pd_pin>; + pinctrl-4 = <&P8_03_gpio_input_pin>; + }; + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_04_default_pin>; + pinctrl-1 = <&P8_04_gpio_pin>; + pinctrl-2 = <&P8_04_gpio_pu_pin>; + pinctrl-3 = <&P8_04_gpio_pd_pin>; + pinctrl-4 = <&P8_04_gpio_input_pin>; + }; + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_05_default_pin>; + pinctrl-1 = <&P8_05_gpio_pin>; + pinctrl-2 = <&P8_05_gpio_pu_pin>; + pinctrl-3 = <&P8_05_gpio_pd_pin>; + pinctrl-4 = <&P8_05_gpio_input_pin>; + }; + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_06_default_pin>; + pinctrl-1 = <&P8_06_gpio_pin>; + pinctrl-2 = <&P8_06_gpio_pu_pin>; + pinctrl-3 = <&P8_06_gpio_pd_pin>; + pinctrl-4 = <&P8_06_gpio_input_pin>; + }; + + /* P8_07 (ZCZ ball R7) */ + P8_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_07_default_pin>; + pinctrl-1 = <&P8_07_gpio_pin>; + pinctrl-2 = <&P8_07_gpio_pu_pin>; + pinctrl-3 = <&P8_07_gpio_pd_pin>; + pinctrl-4 = <&P8_07_gpio_input_pin>; + pinctrl-5 = <&P8_07_timer_pin>; + }; + + /* P8_08 (ZCZ ball T7) */ + P8_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_08_default_pin>; + pinctrl-1 = <&P8_08_gpio_pin>; + pinctrl-2 = <&P8_08_gpio_pu_pin>; + pinctrl-3 = <&P8_08_gpio_pd_pin>; + pinctrl-4 = <&P8_08_gpio_input_pin>; + pinctrl-5 = <&P8_08_timer_pin>; + }; + + /* P8_09 (ZCZ ball T6) */ + P8_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_09_default_pin>; + pinctrl-1 = <&P8_09_gpio_pin>; + pinctrl-2 = <&P8_09_gpio_pu_pin>; + pinctrl-3 = <&P8_09_gpio_pd_pin>; + pinctrl-4 = <&P8_09_gpio_input_pin>; + pinctrl-5 = <&P8_09_timer_pin>; + }; + + /* P8_10 (ZCZ ball U6) */ + P8_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_10_default_pin>; + pinctrl-1 = <&P8_10_gpio_pin>; + pinctrl-2 = <&P8_10_gpio_pu_pin>; + pinctrl-3 = <&P8_10_gpio_pd_pin>; + pinctrl-4 = <&P8_10_gpio_input_pin>; + pinctrl-5 = <&P8_10_timer_pin>; + }; + + /* P8_11 (ZCZ ball R12) */ + P8_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_11_default_pin>; + pinctrl-1 = <&P8_11_gpio_pin>; + pinctrl-2 = <&P8_11_gpio_pu_pin>; + pinctrl-3 = <&P8_11_gpio_pd_pin>; + pinctrl-4 = <&P8_11_gpio_input_pin>; + pinctrl-5 = <&P8_11_qep_pin>; + pinctrl-6 = <&P8_11_pruout_pin>; + }; + + /* P8_12 (ZCZ ball T12) */ + P8_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_12_default_pin>; + pinctrl-1 = <&P8_12_gpio_pin>; + pinctrl-2 = <&P8_12_gpio_pu_pin>; + pinctrl-3 = <&P8_12_gpio_pd_pin>; + pinctrl-4 = <&P8_12_gpio_input_pin>; + pinctrl-5 = <&P8_12_qep_pin>; + pinctrl-6 = <&P8_12_pruout_pin>; + }; + + /* P8_13 (ZCZ ball T10) */ + P8_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_13_default_pin>; + pinctrl-1 = <&P8_13_gpio_pin>; + pinctrl-2 = <&P8_13_gpio_pu_pin>; + pinctrl-3 = <&P8_13_gpio_pd_pin>; + pinctrl-4 = <&P8_13_gpio_input_pin>; + pinctrl-5 = <&P8_13_pwm_pin>; + }; + + /* P8_14 (ZCZ ball T11) */ + P8_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_14_default_pin>; + pinctrl-1 = <&P8_14_gpio_pin>; + pinctrl-2 = <&P8_14_gpio_pu_pin>; + pinctrl-3 = <&P8_14_gpio_pd_pin>; + pinctrl-4 = <&P8_14_gpio_input_pin>; + pinctrl-5 = <&P8_14_pwm_pin>; + }; + + /* P8_15 (ZCZ ball U13) */ + P8_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P8_15_default_pin>; + pinctrl-1 = <&P8_15_gpio_pin>; + pinctrl-2 = <&P8_15_gpio_pu_pin>; + pinctrl-3 = <&P8_15_gpio_pd_pin>; + pinctrl-4 = <&P8_15_gpio_input_pin>; + pinctrl-5 = <&P8_15_qep_pin>; + pinctrl-6 = <&P8_15_pru_ecap_pin>; + pinctrl-7 = <&P8_15_pruin_pin>; + }; + + /* P8_16 (ZCZ ball V13) */ + P8_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P8_16_default_pin>; + pinctrl-1 = <&P8_16_gpio_pin>; + pinctrl-2 = <&P8_16_gpio_pu_pin>; + pinctrl-3 = <&P8_16_gpio_pd_pin>; + pinctrl-4 = <&P8_16_gpio_input_pin>; + pinctrl-5 = <&P8_16_qep_pin>; + pinctrl-6 = <&P8_16_pruin_pin>; + }; + + /* P8_17 (ZCZ ball U12) */ + P8_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_17_default_pin>; + pinctrl-1 = <&P8_17_gpio_pin>; + pinctrl-2 = <&P8_17_gpio_pu_pin>; + pinctrl-3 = <&P8_17_gpio_pd_pin>; + pinctrl-4 = <&P8_17_gpio_input_pin>; + pinctrl-5 = <&P8_17_pwm_pin>; + }; + + /* P8_18 (ZCZ ball V12) */ + P8_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_18_default_pin>; + pinctrl-1 = <&P8_18_gpio_pin>; + pinctrl-2 = <&P8_18_gpio_pu_pin>; + pinctrl-3 = <&P8_18_gpio_pd_pin>; + pinctrl-4 = <&P8_18_gpio_input_pin>; + }; + + /* P8_19 (ZCZ ball U10) */ + P8_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_19_default_pin>; + pinctrl-1 = <&P8_19_gpio_pin>; + pinctrl-2 = <&P8_19_gpio_pu_pin>; + pinctrl-3 = <&P8_19_gpio_pd_pin>; + pinctrl-4 = <&P8_19_gpio_input_pin>; + pinctrl-5 = <&P8_19_pwm_pin>; + }; + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_20_default_pin>; + pinctrl-1 = <&P8_20_gpio_pin>; + pinctrl-2 = <&P8_20_gpio_pu_pin>; + pinctrl-3 = <&P8_20_gpio_pd_pin>; + pinctrl-4 = <&P8_20_gpio_input_pin>; + pinctrl-5 = <&P8_20_pruout_pin>; + pinctrl-6 = <&P8_20_pruin_pin>; + }; + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_21_default_pin>; + pinctrl-1 = <&P8_21_gpio_pin>; + pinctrl-2 = <&P8_21_gpio_pu_pin>; + pinctrl-3 = <&P8_21_gpio_pd_pin>; + pinctrl-4 = <&P8_21_gpio_input_pin>; + pinctrl-5 = <&P8_21_pruout_pin>; + pinctrl-6 = <&P8_21_pruin_pin>; + }; + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_22_default_pin>; + pinctrl-1 = <&P8_22_gpio_pin>; + pinctrl-2 = <&P8_22_gpio_pu_pin>; + pinctrl-3 = <&P8_22_gpio_pd_pin>; + pinctrl-4 = <&P8_22_gpio_input_pin>; + }; + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_23_default_pin>; + pinctrl-1 = <&P8_23_gpio_pin>; + pinctrl-2 = <&P8_23_gpio_pu_pin>; + pinctrl-3 = <&P8_23_gpio_pd_pin>; + pinctrl-4 = <&P8_23_gpio_input_pin>; + }; + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_24_default_pin>; + pinctrl-1 = <&P8_24_gpio_pin>; + pinctrl-2 = <&P8_24_gpio_pu_pin>; + pinctrl-3 = <&P8_24_gpio_pd_pin>; + pinctrl-4 = <&P8_24_gpio_input_pin>; + }; + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_25_default_pin>; + pinctrl-1 = <&P8_25_gpio_pin>; + pinctrl-2 = <&P8_25_gpio_pu_pin>; + pinctrl-3 = <&P8_25_gpio_pd_pin>; + pinctrl-4 = <&P8_25_gpio_input_pin>; + }; + + /* P8_26 (ZCZ ball V6) */ + P8_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_26_default_pin>; + pinctrl-1 = <&P8_26_gpio_pin>; + pinctrl-2 = <&P8_26_gpio_pu_pin>; + pinctrl-3 = <&P8_26_gpio_pd_pin>; + pinctrl-4 = <&P8_26_gpio_input_pin>; + }; + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_27_default_pin>; + pinctrl-1 = <&P8_27_gpio_pin>; + pinctrl-2 = <&P8_27_gpio_pu_pin>; + pinctrl-3 = <&P8_27_gpio_pd_pin>; + pinctrl-4 = <&P8_27_gpio_input_pin>; + pinctrl-5 = <&P8_27_pruout_pin>; + pinctrl-6 = <&P8_27_pruin_pin>; + }; + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_28_default_pin>; + pinctrl-1 = <&P8_28_gpio_pin>; + pinctrl-2 = <&P8_28_gpio_pu_pin>; + pinctrl-3 = <&P8_28_gpio_pd_pin>; + pinctrl-4 = <&P8_28_gpio_input_pin>; + pinctrl-5 = <&P8_28_pruout_pin>; + pinctrl-6 = <&P8_28_pruin_pin>; + }; + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_29_default_pin>; + pinctrl-1 = <&P8_29_gpio_pin>; + pinctrl-2 = <&P8_29_gpio_pu_pin>; + pinctrl-3 = <&P8_29_gpio_pd_pin>; + pinctrl-4 = <&P8_29_gpio_input_pin>; + pinctrl-5 = <&P8_29_pruout_pin>; + pinctrl-6 = <&P8_29_pruin_pin>; + }; + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_30_default_pin>; + pinctrl-1 = <&P8_30_gpio_pin>; + pinctrl-2 = <&P8_30_gpio_pu_pin>; + pinctrl-3 = <&P8_30_gpio_pd_pin>; + pinctrl-4 = <&P8_30_gpio_input_pin>; + pinctrl-5 = <&P8_30_pruout_pin>; + pinctrl-6 = <&P8_30_pruin_pin>; + }; + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "qep"; + pinctrl-0 = <&P8_31_default_pin>; + pinctrl-1 = <&P8_31_gpio_pin>; + pinctrl-2 = <&P8_31_gpio_pu_pin>; + pinctrl-3 = <&P8_31_gpio_pd_pin>; + pinctrl-4 = <&P8_31_gpio_input_pin>; + pinctrl-5 = <&P8_31_uart_pin>; + pinctrl-6 = <&P8_31_qep_pin>; + }; + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_32_default_pin>; + pinctrl-1 = <&P8_32_gpio_pin>; + pinctrl-2 = <&P8_32_gpio_pu_pin>; + pinctrl-3 = <&P8_32_gpio_pd_pin>; + pinctrl-4 = <&P8_32_gpio_input_pin>; + pinctrl-5 = <&P8_32_qep_pin>; + }; + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_33_default_pin>; + pinctrl-1 = <&P8_33_gpio_pin>; + pinctrl-2 = <&P8_33_gpio_pu_pin>; + pinctrl-3 = <&P8_33_gpio_pd_pin>; + pinctrl-4 = <&P8_33_gpio_input_pin>; + pinctrl-5 = <&P8_33_qep_pin>; + }; + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_34_default_pin>; + pinctrl-1 = <&P8_34_gpio_pin>; + pinctrl-2 = <&P8_34_gpio_pu_pin>; + pinctrl-3 = <&P8_34_gpio_pd_pin>; + pinctrl-4 = <&P8_34_gpio_input_pin>; + pinctrl-5 = <&P8_34_pwm_pin>; + }; + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_35_default_pin>; + pinctrl-1 = <&P8_35_gpio_pin>; + pinctrl-2 = <&P8_35_gpio_pu_pin>; + pinctrl-3 = <&P8_35_gpio_pd_pin>; + pinctrl-4 = <&P8_35_gpio_input_pin>; + pinctrl-5 = <&P8_35_qep_pin>; + }; + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_36_default_pin>; + pinctrl-1 = <&P8_36_gpio_pin>; + pinctrl-2 = <&P8_36_gpio_pu_pin>; + pinctrl-3 = <&P8_36_gpio_pd_pin>; + pinctrl-4 = <&P8_36_gpio_input_pin>; + pinctrl-5 = <&P8_36_pwm_pin>; + }; + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_37_default_pin>; + pinctrl-1 = <&P8_37_gpio_pin>; + pinctrl-2 = <&P8_37_gpio_pu_pin>; + pinctrl-3 = <&P8_37_gpio_pd_pin>; + pinctrl-4 = <&P8_37_gpio_input_pin>; + pinctrl-5 = <&P8_37_uart_pin>; + pinctrl-6 = <&P8_37_pwm_pin>; + }; + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_38_default_pin>; + pinctrl-1 = <&P8_38_gpio_pin>; + pinctrl-2 = <&P8_38_gpio_pu_pin>; + pinctrl-3 = <&P8_38_gpio_pd_pin>; + pinctrl-4 = <&P8_38_gpio_input_pin>; + pinctrl-5 = <&P8_38_uart_pin>; + pinctrl-6 = <&P8_38_pwm_pin>; + }; + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_39_default_pin>; + pinctrl-1 = <&P8_39_gpio_pin>; + pinctrl-2 = <&P8_39_gpio_pu_pin>; + pinctrl-3 = <&P8_39_gpio_pd_pin>; + pinctrl-4 = <&P8_39_gpio_input_pin>; + pinctrl-5 = <&P8_39_qep_pin>; + pinctrl-6 = <&P8_39_pruout_pin>; + pinctrl-7 = <&P8_39_pruin_pin>; + }; + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_40_default_pin>; + pinctrl-1 = <&P8_40_gpio_pin>; + pinctrl-2 = <&P8_40_gpio_pu_pin>; + pinctrl-3 = <&P8_40_gpio_pd_pin>; + pinctrl-4 = <&P8_40_gpio_input_pin>; + pinctrl-5 = <&P8_40_qep_pin>; + pinctrl-6 = <&P8_40_pruout_pin>; + pinctrl-7 = <&P8_40_pruin_pin>; + }; + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_41_default_pin>; + pinctrl-1 = <&P8_41_gpio_pin>; + pinctrl-2 = <&P8_41_gpio_pu_pin>; + pinctrl-3 = <&P8_41_gpio_pd_pin>; + pinctrl-4 = <&P8_41_gpio_input_pin>; + pinctrl-5 = <&P8_41_qep_pin>; + pinctrl-6 = <&P8_41_pruout_pin>; + pinctrl-7 = <&P8_41_pruin_pin>; + }; + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_42_default_pin>; + pinctrl-1 = <&P8_42_gpio_pin>; + pinctrl-2 = <&P8_42_gpio_pu_pin>; + pinctrl-3 = <&P8_42_gpio_pd_pin>; + pinctrl-4 = <&P8_42_gpio_input_pin>; + pinctrl-5 = <&P8_42_qep_pin>; + pinctrl-6 = <&P8_42_pruout_pin>; + pinctrl-7 = <&P8_42_pruin_pin>; + }; + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_43_default_pin>; + pinctrl-1 = <&P8_43_gpio_pin>; + pinctrl-2 = <&P8_43_gpio_pu_pin>; + pinctrl-3 = <&P8_43_gpio_pd_pin>; + pinctrl-4 = <&P8_43_gpio_input_pin>; + pinctrl-5 = <&P8_43_pwm_pin>; + pinctrl-6 = <&P8_43_pruout_pin>; + pinctrl-7 = <&P8_43_pruin_pin>; + }; + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_44_default_pin>; + pinctrl-1 = <&P8_44_gpio_pin>; + pinctrl-2 = <&P8_44_gpio_pu_pin>; + pinctrl-3 = <&P8_44_gpio_pd_pin>; + pinctrl-4 = <&P8_44_gpio_input_pin>; + pinctrl-5 = <&P8_44_pwm_pin>; + pinctrl-6 = <&P8_44_pruout_pin>; + pinctrl-7 = <&P8_44_pruin_pin>; + }; + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_45_default_pin>; + pinctrl-1 = <&P8_45_gpio_pin>; + pinctrl-2 = <&P8_45_gpio_pu_pin>; + pinctrl-3 = <&P8_45_gpio_pd_pin>; + pinctrl-4 = <&P8_45_gpio_input_pin>; + pinctrl-5 = <&P8_45_pwm_pin>; + pinctrl-6 = <&P8_45_pruout_pin>; + pinctrl-7 = <&P8_45_pruin_pin>; + }; + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_46_default_pin>; + pinctrl-1 = <&P8_46_gpio_pin>; + pinctrl-2 = <&P8_46_gpio_pu_pin>; + pinctrl-3 = <&P8_46_gpio_pd_pin>; + pinctrl-4 = <&P8_46_gpio_input_pin>; + pinctrl-5 = <&P8_46_pwm_pin>; + pinctrl-6 = <&P8_46_pruout_pin>; + pinctrl-7 = <&P8_46_pruin_pin>; + }; + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_11_default_pin>; + pinctrl-1 = <&P9_11_gpio_pin>; + pinctrl-2 = <&P9_11_gpio_pu_pin>; + pinctrl-3 = <&P9_11_gpio_pd_pin>; + pinctrl-4 = <&P9_11_gpio_input_pin>; + pinctrl-5 = <&P9_11_uart_pin>; + }; + + /* P9_12 (ZCZ ball U18) */ + P9_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P9_12_default_pin>; + pinctrl-1 = <&P9_12_gpio_pin>; + pinctrl-2 = <&P9_12_gpio_pu_pin>; + pinctrl-3 = <&P9_12_gpio_pd_pin>; + pinctrl-4 = <&P9_12_gpio_input_pin>; + }; + + /* P9_13 (ZCZ ball U17) */ + P9_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_13_default_pin>; + pinctrl-1 = <&P9_13_gpio_pin>; + pinctrl-2 = <&P9_13_gpio_pu_pin>; + pinctrl-3 = <&P9_13_gpio_pd_pin>; + pinctrl-4 = <&P9_13_gpio_input_pin>; + pinctrl-5 = <&P9_13_uart_pin>; + }; + + /* P9_14 (ZCZ ball U14) */ + P9_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_14_default_pin>; + pinctrl-1 = <&P9_14_gpio_pin>; + pinctrl-2 = <&P9_14_gpio_pu_pin>; + pinctrl-3 = <&P9_14_gpio_pd_pin>; + pinctrl-4 = <&P9_14_gpio_input_pin>; + pinctrl-5 = <&P9_14_pwm_pin>; + }; + + /* P9_15 (ZCZ ball R13) */ + P9_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_15_default_pin>; + pinctrl-1 = <&P9_15_gpio_pin>; + pinctrl-2 = <&P9_15_gpio_pu_pin>; + pinctrl-3 = <&P9_15_gpio_pd_pin>; + pinctrl-4 = <&P9_15_gpio_input_pin>; + pinctrl-5 = <&P9_15_pwm_pin>; + }; + + /* P9_16 (ZCZ ball T14) */ + P9_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_16_default_pin>; + pinctrl-1 = <&P9_16_gpio_pin>; + pinctrl-2 = <&P9_16_gpio_pu_pin>; + pinctrl-3 = <&P9_16_gpio_pd_pin>; + pinctrl-4 = <&P9_16_gpio_input_pin>; + pinctrl-5 = <&P9_16_pwm_pin>; + }; + + /* P9_17 (ZCZ ball A16) */ + P9_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_17_default_pin>; + pinctrl-1 = <&P9_17_gpio_pin>; + pinctrl-2 = <&P9_17_gpio_pu_pin>; + pinctrl-3 = <&P9_17_gpio_pd_pin>; + pinctrl-4 = <&P9_17_gpio_input_pin>; + pinctrl-5 = <&P9_17_spi_cs_pin>; + pinctrl-6 = <&P9_17_i2c_pin>; + pinctrl-7 = <&P9_17_pwm_pin>; + pinctrl-8 = <&P9_17_pru_uart_pin>; + }; + + /* P9_18 (ZCZ ball B16) */ + P9_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_18_default_pin>; + pinctrl-1 = <&P9_18_gpio_pin>; + pinctrl-2 = <&P9_18_gpio_pu_pin>; + pinctrl-3 = <&P9_18_gpio_pd_pin>; + pinctrl-4 = <&P9_18_gpio_input_pin>; + pinctrl-5 = <&P9_18_spi_pin>; + pinctrl-6 = <&P9_18_i2c_pin>; + pinctrl-7 = <&P9_18_pwm_pin>; + pinctrl-8 = <&P9_18_pru_uart_pin>; + }; + + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + /* P9_21 (ZCZ ball B17) */ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_21_default_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_gpio_input_pin>; + pinctrl-5 = <&P9_21_spi_pin>; + pinctrl-6 = <&P9_21_uart_pin>; + pinctrl-7 = <&P9_21_i2c_pin>; + pinctrl-8 = <&P9_21_pwm_pin>; + pinctrl-9 = <&P9_21_pru_uart_pin>; + }; + + /* P9_22 (ZCZ ball A17) */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_22_default_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_gpio_input_pin>; + pinctrl-5 = <&P9_22_spi_sclk_pin>; + pinctrl-6 = <&P9_22_uart_pin>; + pinctrl-7 = <&P9_22_i2c_pin>; + pinctrl-8 = <&P9_22_pwm_pin>; + pinctrl-9 = <&P9_22_pru_uart_pin>; + }; + + /* P9_23 (ZCZ ball V14) */ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_23_default_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_gpio_input_pin>; + pinctrl-5 = <&P9_23_pwm_pin>; + }; + + /* P9_24 (ZCZ ball D15) */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_24_default_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_gpio_input_pin>; + pinctrl-5 = <&P9_24_uart_pin>; + pinctrl-6 = <&P9_24_can_pin>; + pinctrl-7 = <&P9_24_i2c_pin>; + pinctrl-8 = <&P9_24_pru_uart_pin>; + pinctrl-9 = <&P9_24_pruin_pin>; + }; + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_25_default_pin>; + pinctrl-1 = <&P9_25_gpio_pin>; + pinctrl-2 = <&P9_25_gpio_pu_pin>; + pinctrl-3 = <&P9_25_gpio_pd_pin>; + pinctrl-4 = <&P9_25_gpio_input_pin>; + pinctrl-5 = <&P9_25_qep_pin>; + pinctrl-6 = <&P9_25_pruout_pin>; + pinctrl-7 = <&P9_25_pruin_pin>; + }; + + /* P9_26 (ZCZ ball D16) */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_26_default_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_gpio_input_pin>; + pinctrl-5 = <&P9_26_uart_pin>; + pinctrl-6 = <&P9_26_can_pin>; + pinctrl-7 = <&P9_26_i2c_pin>; + pinctrl-8 = <&P9_26_pru_uart_pin>; + pinctrl-9 = <&P9_26_pruin_pin>; + }; + + /* P9_27 (ZCZ ball C13) */ + P9_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_27_default_pin>; + pinctrl-1 = <&P9_27_gpio_pin>; + pinctrl-2 = <&P9_27_gpio_pu_pin>; + pinctrl-3 = <&P9_27_gpio_pd_pin>; + pinctrl-4 = <&P9_27_gpio_input_pin>; + pinctrl-5 = <&P9_27_qep_pin>; + pinctrl-6 = <&P9_27_pruout_pin>; + pinctrl-7 = <&P9_27_pruin_pin>; + }; + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_default_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_gpio_input_pin>; + pinctrl-5 = <&P9_28_spi_cs_pin>; + pinctrl-6 = <&P9_28_pwm_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_29_default_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_gpio_input_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pwm_pin>; + pinctrl-7 = <&P9_29_pruout_pin>; + pinctrl-8 = <&P9_29_pruin_pin>; + }; + + /* P9_30 (ZCZ ball D12) */ + P9_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_30_default_pin>; + pinctrl-1 = <&P9_30_gpio_pin>; + pinctrl-2 = <&P9_30_gpio_pu_pin>; + pinctrl-3 = <&P9_30_gpio_pd_pin>; + pinctrl-4 = <&P9_30_gpio_input_pin>; + pinctrl-5 = <&P9_30_spi_pin>; + pinctrl-6 = <&P9_30_pwm_pin>; + pinctrl-7 = <&P9_30_pruout_pin>; + pinctrl-8 = <&P9_30_pruin_pin>; + }; + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_31_default_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_gpio_input_pin>; + pinctrl-5 = <&P9_31_spi_sclk_pin>; + pinctrl-6 = <&P9_31_pwm_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) */ + P9_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer", "pruin"; + pinctrl-0 = <&P9_41_default_pin>; + pinctrl-1 = <&P9_41_gpio_pin>; + pinctrl-2 = <&P9_41_gpio_pu_pin>; + pinctrl-3 = <&P9_41_gpio_pd_pin>; + pinctrl-4 = <&P9_41_gpio_input_pin>; + pinctrl-5 = <&P9_41_timer_pin>; + pinctrl-6 = <&P9_41_pruin_pin>; + }; + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) */ + P9_91_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_91_default_pin>; + pinctrl-1 = <&P9_91_gpio_pin>; + pinctrl-2 = <&P9_91_gpio_pu_pin>; + pinctrl-3 = <&P9_91_gpio_pd_pin>; + pinctrl-4 = <&P9_91_gpio_input_pin>; + pinctrl-5 = <&P9_91_qep_pin>; + pinctrl-6 = <&P9_91_pruout_pin>; + pinctrl-7 = <&P9_91_pruin_pin>; + }; + + /* P9_42 (ZCZ ball C18) */ + P9_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P9_42_default_pin>; + pinctrl-1 = <&P9_42_gpio_pin>; + pinctrl-2 = <&P9_42_gpio_pu_pin>; + pinctrl-3 = <&P9_42_gpio_pd_pin>; + pinctrl-4 = <&P9_42_gpio_input_pin>; + pinctrl-5 = <&P9_42_spi_cs_pin>; + pinctrl-6 = <&P9_42_spi_sclk_pin>; + pinctrl-7 = <&P9_42_uart_pin>; + pinctrl-8 = <&P9_42_pwm_pin>; + pinctrl-9 = <&P9_42_pru_ecap_pin>; + }; + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) */ + P9_92_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_92_default_pin>; + pinctrl-1 = <&P9_92_gpio_pin>; + pinctrl-2 = <&P9_92_gpio_pu_pin>; + pinctrl-3 = <&P9_92_gpio_pd_pin>; + pinctrl-4 = <&P9_92_gpio_input_pin>; + pinctrl-5 = <&P9_92_qep_pin>; + pinctrl-6 = <&P9_92_pruout_pin>; + pinctrl-7 = <&P9_92_pruin_pin>; + }; + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "clkout", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&A15_default_pin>; + pinctrl-1 = <&A15_clkout_pin>; + pinctrl-2 = <&A15_gpio_pin>; + pinctrl-3 = <&A15_gpio_pu_pin>; + pinctrl-4 = <&A15_gpio_pd_pin>; + }; + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P8_03 { + gpio-name = "P8_03"; + gpio = <&gpio1 6 0>; + input; + dir-changeable; + }; + + P8_04 { + gpio-name = "P8_04"; + gpio = <&gpio1 7 0>; + input; + dir-changeable; + }; + + P8_05 { + gpio-name = "P8_05"; + gpio = <&gpio1 2 0>; + input; + dir-changeable; + }; + + P8_06 { + gpio-name = "P8_06"; + gpio = <&gpio1 3 0>; + input; + dir-changeable; + }; + + P8_07 { + gpio-name = "P8_07"; + gpio = <&gpio2 2 0>; + input; + dir-changeable; + }; + + P8_08 { + gpio-name = "P8_08"; + gpio = <&gpio2 3 0>; + input; + dir-changeable; + }; + + P8_09 { + gpio-name = "P8_09"; + gpio = <&gpio2 5 0>; + input; + dir-changeable; + }; + + P8_10 { + gpio-name = "P8_10"; + gpio = <&gpio2 4 0>; + input; + dir-changeable; + }; + + P8_11 { + gpio-name = "P8_11"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P8_12 { + gpio-name = "P8_12"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P8_13 { + gpio-name = "P8_13"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P8_14 { + gpio-name = "P8_14"; + gpio = <&gpio0 26 0>; + input; + dir-changeable; + }; + + P8_15 { + gpio-name = "P8_15"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P8_16 { + gpio-name = "P8_16"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P8_17 { + gpio-name = "P8_17"; + gpio = <&gpio0 27 0>; + input; + dir-changeable; + }; + + P8_18 { + gpio-name = "P8_18"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P8_19 { + gpio-name = "P8_19"; + gpio = <&gpio0 22 0>; + input; + dir-changeable; + }; + + P8_20 { + gpio-name = "P8_20"; + gpio = <&gpio1 31 0>; + input; + dir-changeable; + }; + + P8_21 { + gpio-name = "P8_21"; + gpio = <&gpio1 30 0>; + input; + dir-changeable; + }; + + P8_22 { + gpio-name = "P8_22"; + gpio = <&gpio1 5 0>; + input; + dir-changeable; + }; + + P8_23 { + gpio-name = "P8_23"; + gpio = <&gpio1 4 0>; + input; + dir-changeable; + }; + + P8_24 { + gpio-name = "P8_24"; + gpio = <&gpio1 1 0>; + input; + dir-changeable; + }; + + P8_25 { + gpio-name = "P8_25"; + gpio = <&gpio1 0 0>; + input; + dir-changeable; + }; + + P8_26 { + gpio-name = "P8_26"; + gpio = <&gpio1 29 0>; + input; + dir-changeable; + }; + + P8_27 { + gpio-name = "P8_27"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + P8_28 { + gpio-name = "P8_28"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P8_29 { + gpio-name = "P8_29"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P8_30 { + gpio-name = "P8_30"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P8_31 { + gpio-name = "P8_31"; + gpio = <&gpio0 10 0>; + input; + dir-changeable; + }; + + P8_32 { + gpio-name = "P8_32"; + gpio = <&gpio0 11 0>; + input; + dir-changeable; + }; + + P8_33 { + gpio-name = "P8_33"; + gpio = <&gpio0 9 0>; + input; + dir-changeable; + }; + + P8_34 { + gpio-name = "P8_34"; + gpio = <&gpio2 17 0>; + input; + dir-changeable; + }; + + P8_35 { + gpio-name = "P8_35"; + gpio = <&gpio0 8 0>; + input; + dir-changeable; + }; + + P8_36 { + gpio-name = "P8_36"; + gpio = <&gpio2 16 0>; + input; + dir-changeable; + }; + + P8_37 { + gpio-name = "P8_37"; + gpio = <&gpio2 14 0>; + input; + dir-changeable; + }; + + P8_38 { + gpio-name = "P8_38"; + gpio = <&gpio2 15 0>; + input; + dir-changeable; + }; + + P8_39 { + gpio-name = "P8_39"; + gpio = <&gpio2 12 0>; + input; + dir-changeable; + }; + + P8_40 { + gpio-name = "P8_40"; + gpio = <&gpio2 13 0>; + input; + dir-changeable; + }; + + P8_41 { + gpio-name = "P8_41"; + gpio = <&gpio2 10 0>; + input; + dir-changeable; + }; + + P8_42 { + gpio-name = "P8_42"; + gpio = <&gpio2 11 0>; + input; + dir-changeable; + }; + + P8_43 { + gpio-name = "P8_43"; + gpio = <&gpio2 8 0>; + input; + dir-changeable; + }; + + P8_44 { + gpio-name = "P8_44"; + gpio = <&gpio2 9 0>; + input; + dir-changeable; + }; + + P8_45 { + gpio-name = "P8_45"; + gpio = <&gpio2 6 0>; + input; + dir-changeable; + }; + + P8_46 { + gpio-name = "P8_46"; + gpio = <&gpio2 7 0>; + input; + dir-changeable; + }; + + P9_11 { + gpio-name = "P9_11"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P9_12 { + gpio-name = "P9_12"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P9_13 { + gpio-name = "P9_13"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P9_14 { + gpio-name = "P9_14"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P9_15 { + gpio-name = "P9_15"; + gpio = <&gpio1 16 0>; + input; + dir-changeable; + }; + + P9_16 { + gpio-name = "P9_16"; + gpio = <&gpio1 19 0>; + input; + dir-changeable; + }; + + P9_17 { + gpio-name = "P9_17"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P9_18 { + gpio-name = "P9_18"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P9_21 { + gpio-name = "P9_21"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P9_22 { + gpio-name = "P9_22"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P9_23 { + gpio-name = "P9_23"; + gpio = <&gpio1 17 0>; + input; + dir-changeable; + }; + + P9_24 { + gpio-name = "P9_24"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P9_25 { + gpio-name = "P9_25"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P9_26 { + gpio-name = "P9_26"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P9_27 { + gpio-name = "P9_27"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P9_28 { + gpio-name = "P9_28"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P9_29 { + gpio-name = "P9_29"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P9_30 { + gpio-name = "P9_30"; + gpio = <&gpio3 16 0>; + input; + dir-changeable; + }; + + P9_31 { + gpio-name = "P9_31"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P9_41 { + gpio-name = "P9_41"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P9_91 { + gpio-name = "P9_91"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P9_42 { + gpio-name = "P9_42"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P9_92 { + gpio-name = "P9_92"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + A15 { + gpio-name = "A15"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi b/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi new file mode 100644 index 0000000000000..c87e3732db468 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-common-universal-pins.dtsi @@ -0,0 +1,967 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&am33xx_pinmux { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + /* P8_02 GND */ + /* P8_03 (ZCZ ball R9 ) emmc */ + /* P8_04 (ZCZ ball T9 ) emmc */ + /* P8_05 (ZCZ ball R8 ) emmc */ + /* P8_06 (ZCZ ball T8 ) emmc */ + + /* P8_07 (ZCZ ball R7 ) */ + P8_07_default_pin: pinmux_P8_07_default_pin { + pinctrl-single,pins = <0x090 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { + pinctrl-single,pins = <0x090 0x2F>; }; /* Mode 7, RxActive */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { + pinctrl-single,pins = <0x090 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { + pinctrl-single,pins = <0x090 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { + pinctrl-single,pins = <0x090 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + + /* P8_08 (ZCZ ball T7 ) */ + P8_08_default_pin: pinmux_P8_08_default_pin { + pinctrl-single,pins = <0x094 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { + pinctrl-single,pins = <0x094 0x2F>; }; /* Mode 7, RxActive */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { + pinctrl-single,pins = <0x094 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { + pinctrl-single,pins = <0x094 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { + pinctrl-single,pins = <0x094 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + + /* P8_09 (ZCZ ball T6 ) */ + P8_09_default_pin: pinmux_P8_09_default_pin { + pinctrl-single,pins = <0x09c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { + pinctrl-single,pins = <0x09c 0x2F>; }; /* Mode 7, RxActive */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { + pinctrl-single,pins = <0x09c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { + pinctrl-single,pins = <0x09c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { + pinctrl-single,pins = <0x09c 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + + /* P8_10 (ZCZ ball U6 ) */ + P8_10_default_pin: pinmux_P8_10_default_pin { + pinctrl-single,pins = <0x098 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { + pinctrl-single,pins = <0x098 0x2F>; }; /* Mode 7, RxActive */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { + pinctrl-single,pins = <0x098 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { + pinctrl-single,pins = <0x098 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { + pinctrl-single,pins = <0x098 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + + /* P8_11 (ZCZ ball R12) */ + P8_11_default_pin: pinmux_P8_11_default_pin { + pinctrl-single,pins = <0x034 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { + pinctrl-single,pins = <0x034 0x2F>; }; /* Mode 7, RxActive */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { + pinctrl-single,pins = <0x034 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { + pinctrl-single,pins = <0x034 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { + pinctrl-single,pins = <0x034 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { + pinctrl-single,pins = <0x034 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_12 (ZCZ ball T12) */ + P8_12_default_pin: pinmux_P8_12_default_pin { + pinctrl-single,pins = <0x030 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { + pinctrl-single,pins = <0x030 0x2F>; }; /* Mode 7, RxActive */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { + pinctrl-single,pins = <0x030 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { + pinctrl-single,pins = <0x030 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { + pinctrl-single,pins = <0x030 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { + pinctrl-single,pins = <0x030 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_13 (ZCZ ball T10) */ + P8_13_default_pin: pinmux_P8_13_default_pin { + pinctrl-single,pins = <0x024 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { + pinctrl-single,pins = <0x024 0x2F>; }; /* Mode 7, RxActive */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { + pinctrl-single,pins = <0x024 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { + pinctrl-single,pins = <0x024 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { + pinctrl-single,pins = <0x024 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_14 (ZCZ ball T11) */ + P8_14_default_pin: pinmux_P8_14_default_pin { + pinctrl-single,pins = <0x028 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_14_gpio_pin: pinmux_P8_14_gpio_pin { + pinctrl-single,pins = <0x028 0x2F>; }; /* Mode 7, RxActive */ + P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin { + pinctrl-single,pins = <0x028 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin { + pinctrl-single,pins = <0x028 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_14_pwm_pin: pinmux_P8_14_pwm_pin { + pinctrl-single,pins = <0x028 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_15 (ZCZ ball U13) */ + P8_15_default_pin: pinmux_P8_15_default_pin { + pinctrl-single,pins = <0x03c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { + pinctrl-single,pins = <0x03c 0x2F>; }; /* Mode 7, RxActive */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { + pinctrl-single,pins = <0x03c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { + pinctrl-single,pins = <0x03c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { + pinctrl-single,pins = <0x03c 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_15_qep_pin: pinmux_P8_15_qep_pin { + pinctrl-single,pins = <0x03c 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P8_15_pruin_pu_pin: pinmux_P8_15_pruin_pu_pin { + pinctrl-single,pins = <0x03c 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + P8_15_pruecapin_pu_pin: pinmux_P8_15_pruecapin_pu_pin { + pinctrl-single,pins = <0x03c 0x35>; }; /* Mode 5, Pull-Up, RxActive */ + + /* P8_16 (ZCZ ball V13) */ + P8_16_default_pin: pinmux_P8_16_default_pin { + pinctrl-single,pins = <0x038 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { + pinctrl-single,pins = <0x038 0x2F>; }; /* Mode 7, RxActive */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { + pinctrl-single,pins = <0x038 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { + pinctrl-single,pins = <0x038 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { + pinctrl-single,pins = <0x038 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_16_qep_pin: pinmux_P8_16_qep_pin { + pinctrl-single,pins = <0x038 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_17 (ZCZ ball U12) */ + P8_17_default_pin: pinmux_P8_17_default_pin { + pinctrl-single,pins = <0x02c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_17_gpio_pin: pinmux_P8_17_gpio_pin { + pinctrl-single,pins = <0x02c 0x2F>; }; /* Mode 7, RxActive */ + P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin { + pinctrl-single,pins = <0x02c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin { + pinctrl-single,pins = <0x02c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_17_pwm_pin: pinmux_P8_17_pwm_pin { + pinctrl-single,pins = <0x02c 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_18 (ZCZ ball V12) */ + P8_18_default_pin: pinmux_P8_18_default_pin { + pinctrl-single,pins = <0x08c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { + pinctrl-single,pins = <0x08c 0x2F>; }; /* Mode 7, RxActive */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { + pinctrl-single,pins = <0x08c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { + pinctrl-single,pins = <0x08c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + + /* P8_19 (ZCZ ball U10) */ + P8_19_default_pin: pinmux_P8_19_default_pin { + pinctrl-single,pins = <0x020 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { + pinctrl-single,pins = <0x020 0x2F>; }; /* Mode 7, RxActive */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { + pinctrl-single,pins = <0x020 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { + pinctrl-single,pins = <0x020 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { + pinctrl-single,pins = <0x020 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + + /* P8_20 (ZCZ ball V9 ) emmc */ + /* P8_21 (ZCZ ball U9 ) emmc */ + /* P8_22 (ZCZ ball V8 ) emmc */ + /* P8_23 (ZCZ ball U8 ) emmc */ + /* P8_24 (ZCZ ball V7 ) emmc */ + /* P8_25 (ZCZ ball U7 ) emmc */ + + /* P8_26 (ZCZ ball V6 ) */ + P8_26_default_pin: pinmux_P8_26_default_pin { + pinctrl-single,pins = <0x07c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_26_gpio_pin: pinmux_P8_26_gpio_pin { + pinctrl-single,pins = <0x07c 0x2F>; }; /* Mode 7, RxActive */ + P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin { + pinctrl-single,pins = <0x07c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin { + pinctrl-single,pins = <0x07c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + + /* P8_27 (ZCZ ball U5 ) hdmi */ + P8_27_default_pin: pinmux_P8_27_default_pin { + pinctrl-single,pins = <0x0e0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { + pinctrl-single,pins = <0x0e0 0x2F>; }; /* Mode 7, RxActive */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { + pinctrl-single,pins = <0x0e0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { + pinctrl-single,pins = <0x0e0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { + pinctrl-single,pins = <0x0e0 0x05>; }; /* Mode 5, Pull-Down*/ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { + pinctrl-single,pins = <0x0e0 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_27_hdmi_pin: pinmux_P8_27_hdmi_pin { + pinctrl-single,pins = <0x0e0 0x00>; }; /* lcd_vsync.lcd_vsync, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */ + + /* P8_28 (ZCZ ball V5 ) hdmi */ + P8_28_default_pin: pinmux_P8_28_default_pin { + pinctrl-single,pins = <0x0e8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { + pinctrl-single,pins = <0x0e8 0x2F>; }; /* Mode 7, RxActive */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { + pinctrl-single,pins = <0x0e8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { + pinctrl-single,pins = <0x0e8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { + pinctrl-single,pins = <0x0e8 0x05>; }; /* Mode 5, Pull-Down */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { + pinctrl-single,pins = <0x0e8 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_28_hdmi_pin: pinmux_P8_28_hdmi_pin { + pinctrl-single,pins = <0x0e8 0x00>; }; /* lcd_pclk.lcd_pclk, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */ + + /* P8_29 (ZCZ ball R5 ) hdmi */ + P8_29_default_pin: pinmux_P8_29_default_pin { + pinctrl-single,pins = <0x0e4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { + pinctrl-single,pins = <0x0e4 0x2F>; }; /* Mode 7, RxActive */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { + pinctrl-single,pins = <0x0e4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { + pinctrl-single,pins = <0x0e4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { + pinctrl-single,pins = <0x0e4 0x05>; }; /* Mode 5, Pull-Down*/ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { + pinctrl-single,pins = <0x0e4 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_29_hdmi_pin: pinmux_P8_29_hdmi_pin { + pinctrl-single,pins = <0x0e4 0x00>; }; /* lcd_hsync.lcd_hsync, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */ + + /* P8_30 (ZCZ ball R6 ) hdmi */ + P8_30_default_pin: pinmux_P8_30_default_pin { + pinctrl-single,pins = <0x0ec 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { + pinctrl-single,pins = <0x0ec 0x2F>; }; /* Mode 7, RxActive */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { + pinctrl-single,pins = <0x0ec 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { + pinctrl-single,pins = <0x0ec 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { + pinctrl-single,pins = <0x0ec 0x05>; }; /* Mode 5, Pull-Down*/ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { + pinctrl-single,pins = <0x0ec 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_30_hdmi_pin: pinmux_P8_30_hdmi_pin { + pinctrl-single,pins = <0x0ec 0x00>; }; /* lcd_ac_bias_en.lcd_ac_bias_en, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT */ + + /* P8_31 (ZCZ ball V4 ) hdmi */ + P8_31_default_pin: pinmux_P8_31_default_pin { + pinctrl-single,pins = <0x0d8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { + pinctrl-single,pins = <0x0d8 0x2F>; }; /* Mode 7, RxActive */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { + pinctrl-single,pins = <0x0d8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { + pinctrl-single,pins = <0x0d8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { + pinctrl-single,pins = <0x0d8 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P8_31_hdmi_pin: pinmux_P8_31_hdmi_pin { + pinctrl-single,pins = <0x0d8 0x08>; }; /* lcd_data14.lcd_data14, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_32 (ZCZ ball T5 ) hdmi */ + P8_32_default_pin: pinmux_P8_32_default_pin { + pinctrl-single,pins = <0x0dc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { + pinctrl-single,pins = <0x0dc 0x2F>; }; /* Mode 7, RxActive */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { + pinctrl-single,pins = <0x0dc 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { + pinctrl-single,pins = <0x0dc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_32_uart_pin: pinmux_P8_32_uart_pin { + pinctrl-single,pins = <0x0dc 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_32_hdmi_pin: pinmux_P8_32_hdmi_pin { + pinctrl-single,pins = <0x0dc 0x08>; }; /* lcd_data15.lcd_data15, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_33 (ZCZ ball V3 ) hdmi */ + P8_33_default_pin: pinmux_P8_33_default_pin { + pinctrl-single,pins = <0x0d4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { + pinctrl-single,pins = <0x0d4 0x2F>; }; /* Mode 7, RxActive */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { + pinctrl-single,pins = <0x0d4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { + pinctrl-single,pins = <0x0d4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_33_hdmi_pin: pinmux_P8_33_hdmi_pin { + pinctrl-single,pins = <0x0d4 0x08>; }; /* lcd_data13.lcd_data13, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { + pinctrl-single,pins = <0x0d4 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + + /* P8_34 (ZCZ ball U4 ) hdmi */ + P8_34_default_pin: pinmux_P8_34_default_pin { + pinctrl-single,pins = <0x0cc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { + pinctrl-single,pins = <0x0cc 0x2F>; }; /* Mode 7, RxActive */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { + pinctrl-single,pins = <0x0cc 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { + pinctrl-single,pins = <0x0cc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { + pinctrl-single,pins = <0x0cc 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + P8_34_hdmi_pin: pinmux_P8_34_hdmi_pin { + pinctrl-single,pins = <0x0cc 0x08>; }; /* lcd_data11.lcd_data11, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_35 (ZCZ ball V2 ) hdmi */ + P8_35_default_pin: pinmux_P8_35_default_pin { + pinctrl-single,pins = <0x0d0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { + pinctrl-single,pins = <0x0d0 0x2F>; }; /* Mode 7, RxActive */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { + pinctrl-single,pins = <0x0d0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { + pinctrl-single,pins = <0x0d0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_35_hdmi_pin: pinmux_P8_35_hdmi_pin { + pinctrl-single,pins = <0x0d0 0x08>; }; /* lcd_data12.lcd_data12, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { + pinctrl-single,pins = <0x0d0 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + + /* P8_36 (ZCZ ball U3 ) hdmi */ + P8_36_default_pin: pinmux_P8_36_default_pin { + pinctrl-single,pins = <0x0c8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { + pinctrl-single,pins = <0x0c8 0x2F>; }; /* Mode 7, RxActive */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { + pinctrl-single,pins = <0x0c8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { + pinctrl-single,pins = <0x0c8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { + pinctrl-single,pins = <0x0c8 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + P8_36_hdmi_pin: pinmux_P8_36_hdmi_pin { + pinctrl-single,pins = <0x0c8 0x08>; }; /* lcd_data10.lcd_data10, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_37 (ZCZ ball U1 ) hdmi */ + P8_37_default_pin: pinmux_P8_37_default_pin { + pinctrl-single,pins = <0x0c0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { + pinctrl-single,pins = <0x0c0 0x2F>; }; /* Mode 7, RxActive */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { + pinctrl-single,pins = <0x0c0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { + pinctrl-single,pins = <0x0c0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { + pinctrl-single,pins = <0x0c0 0x04>; }; /* Mode 4, Pull-Down*/ + P8_37_pwm_pin: pinmux_P8_37_pwm_pin { + pinctrl-single,pins = <0x0c0 0x02>; }; /* Mode 2, Pull-Down*/ + P8_37_hdmi_pin: pinmux_P8_37_hdmi_pin { + pinctrl-single,pins = <0x0c0 0x08>; }; /* lcd_data8.lcd_data8, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + + /* P8_38 (ZCZ ball U2 ) hdmi */ + P8_38_default_pin: pinmux_P8_38_default_pin { + pinctrl-single,pins = <0x0c4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { + pinctrl-single,pins = <0x0c4 0x2F>; }; /* Mode 7, RxActive */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { + pinctrl-single,pins = <0x0c4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { + pinctrl-single,pins = <0x0c4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { + pinctrl-single,pins = <0x0c4 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P8_38_pwm_pin: pinmux_P8_38_pwm_pin { + pinctrl-single,pins = <0x0c4 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + P8_38_hdmi_pin: pinmux_P8_38_hdmi_pin { + pinctrl-single,pins = <0x0c4 0x08>; }; /* lcd_data9.lcd_data9, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + + /* P8_39 (ZCZ ball T3 ) hdmi */ + P8_39_default_pin: pinmux_P8_39_default_pin { + pinctrl-single,pins = <0x0b8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { + pinctrl-single,pins = <0x0b8 0x2F>; }; /* Mode 7, RxActive */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { + pinctrl-single,pins = <0x0b8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { + pinctrl-single,pins = <0x0b8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { + pinctrl-single,pins = <0x0b8 0x05>; }; /* Mode 5, Pull-Down*/ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { + pinctrl-single,pins = <0x0b8 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_39_hdmi_pin: pinmux_P8_39_hdmi_pin { + pinctrl-single,pins = <0x0b8 0x08>; }; /* lcd_data6.lcd_data6, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_40 (ZCZ ball T4 ) hdmi */ + P8_40_default_pin: pinmux_P8_40_default_pin { + pinctrl-single,pins = <0x0bc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { + pinctrl-single,pins = <0x0bc 0x2F>; }; /* Mode 7, RxActive */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { + pinctrl-single,pins = <0x0bc 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { + pinctrl-single,pins = <0x0bc 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { + pinctrl-single,pins = <0x0bc 0x05>; }; /* Mode 5, Pull-Down*/ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { + pinctrl-single,pins = <0x0bc 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_40_hdmi_pin: pinmux_P8_40_hdmi_pin { + pinctrl-single,pins = <0x0bc 0x08>; }; /* lcd_data7.lcd_data7, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_41 (ZCZ ball T1 ) hdmi */ + P8_41_default_pin: pinmux_P8_41_default_pin { + pinctrl-single,pins = <0x0b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { + pinctrl-single,pins = <0x0b0 0x2F>; }; /* Mode 7, RxActive */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { + pinctrl-single,pins = <0x0b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { + pinctrl-single,pins = <0x0b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { + pinctrl-single,pins = <0x0b0 0x05>; }; /* Mode 5, Pull-Down*/ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { + pinctrl-single,pins = <0x0b0 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_41_hdmi_pin: pinmux_P8_41_hdmi_pin { + pinctrl-single,pins = <0x0b0 0x08>; }; /* lcd_data4.lcd_data4, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_42 (ZCZ ball T2 ) hdmi */ + P8_42_default_pin: pinmux_P8_42_default_pin { + pinctrl-single,pins = <0x0b4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { + pinctrl-single,pins = <0x0b4 0x2F>; }; /* Mode 7, RxActive */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { + pinctrl-single,pins = <0x0b4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { + pinctrl-single,pins = <0x0b4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { + pinctrl-single,pins = <0x0b4 0x05>; }; /* Mode 5, Pull-Down*/ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { + pinctrl-single,pins = <0x0b4 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_42_hdmi_pin: pinmux_P8_42_hdmi_pin { + pinctrl-single,pins = <0x0b4 0x08>; }; /* lcd_data5.lcd_data5, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_43 (ZCZ ball R3 ) hdmi */ + P8_43_default_pin: pinmux_P8_43_default_pin { + pinctrl-single,pins = <0x0a8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { + pinctrl-single,pins = <0x0a8 0x2F>; }; /* Mode 7, RxActive */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { + pinctrl-single,pins = <0x0a8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { + pinctrl-single,pins = <0x0a8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { + pinctrl-single,pins = <0x0a8 0x05>; }; /* Mode 5, Pull-Down*/ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { + pinctrl-single,pins = <0x0a8 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_43_pwm_pin: pinmux_P8_43_pwm_pin { + pinctrl-single,pins = <0x0a8 0x03>; }; /* Mode 3, Pull-Down */ + P8_43_hdmi_pin: pinmux_P8_43_hdmi_pin { + pinctrl-single,pins = <0x0a8 0x08>; }; /* lcd_data2.lcd_data2, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_44 (ZCZ ball R4 ) hdmi */ + P8_44_default_pin: pinmux_P8_44_default_pin { + pinctrl-single,pins = <0x0ac 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { + pinctrl-single,pins = <0x0ac 0x2F>; }; /* Mode 7, RxActive */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { + pinctrl-single,pins = <0x0ac 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { + pinctrl-single,pins = <0x0ac 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { + pinctrl-single,pins = <0x0ac 0x05>; }; /* Mode 5, Pull-Down*/ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { + pinctrl-single,pins = <0x0ac 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_44_pwm_pin: pinmux_P8_44_pwm_pin { + pinctrl-single,pins = <0x0ac 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P8_44_hdmi_pin: pinmux_P8_44_hdmi_pin { + pinctrl-single,pins = <0x0ac 0x08>; }; /* lcd_data3.lcd_data3, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_45 (ZCZ ball R1 ) hdmi */ + P8_45_default_pin: pinmux_P8_45_default_pin { + pinctrl-single,pins = <0x0a0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { + pinctrl-single,pins = <0x0a0 0x2F>; }; /* Mode 7, RxActive */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { + pinctrl-single,pins = <0x0a0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { + pinctrl-single,pins = <0x0a0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { + pinctrl-single,pins = <0x0a0 0x05>; }; /* Mode 5, Pull-Down*/ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { + pinctrl-single,pins = <0x0a0 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_45_pwm_pin: pinmux_P8_45_pwm_pin { + pinctrl-single,pins = <0x0a0 0x03>; }; /* Mode 3, Pull-Down*/ + P8_45_hdmi_pin: pinmux_P8_45_hdmi_pin { + pinctrl-single,pins = <0x0a0 0x08>; }; /* lcd_data0.lcd_data0, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /* P8_46 (ZCZ ball R2 ) hdmi */ + P8_46_default_pin: pinmux_P8_46_default_pin { + pinctrl-single,pins = <0x0a4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { + pinctrl-single,pins = <0x0a4 0x2F>; }; /* Mode 7, RxActive */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { + pinctrl-single,pins = <0x0a4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { + pinctrl-single,pins = <0x0a4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { + pinctrl-single,pins = <0x0a4 0x05>; }; /* Mode 5, Pull-Down*/ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { + pinctrl-single,pins = <0x0a4 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P8_46_pwm_pin: pinmux_P8_46_pwm_pin { + pinctrl-single,pins = <0x0a4 0x03>; }; /* Mode 3, Pull-Down*/ + P8_46_hdmi_pin: pinmux_P8_46_hdmi_pin { + pinctrl-single,pins = <0x0a4 0x08>; }; /* lcd_data1.lcd_data1, OMAP_MUX_MODE0 | AM33XX_PIN_OUTPUT | AM33XX_PULL_DISA */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + /* P9_02 GND */ + /* P9_03 3.3V */ + /* P9_04 3.3V */ + /* P9_05 VDD_5V */ + /* P9_06 VDD_5V */ + /* P9_07 SYS_5V */ + /* P9_08 SYS_5V */ + /* P9_09 PWR_BUT */ + /* P9_10 (ZCZ ball A10) RESETn */ + + /* P9_11 (ZCZ ball T17) */ + P9_11_default_pin: pinmux_P9_11_default_pin { + pinctrl-single,pins = <0x070 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { + pinctrl-single,pins = <0x070 0x2F>; }; /* Mode 7, RxActive */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { + pinctrl-single,pins = <0x070 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { + pinctrl-single,pins = <0x070 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { + pinctrl-single,pins = <0x070 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + + /* P9_12 (ZCZ ball U18) */ + P9_12_default_pin: pinmux_P9_12_default_pin { + pinctrl-single,pins = <0x078 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { + pinctrl-single,pins = <0x078 0x2F>; }; /* Mode 7, RxActive */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { + pinctrl-single,pins = <0x078 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { + pinctrl-single,pins = <0x078 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + + /* P9_13 (ZCZ ball U17) */ + P9_13_default_pin: pinmux_P9_13_default_pin { + pinctrl-single,pins = <0x074 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { + pinctrl-single,pins = <0x074 0x2F>; }; /* Mode 7, RxActive */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { + pinctrl-single,pins = <0x074 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { + pinctrl-single,pins = <0x074 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { + pinctrl-single,pins = <0x074 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + + /* P9_14 (ZCZ ball U14) */ + P9_14_default_pin: pinmux_P9_14_default_pin { + pinctrl-single,pins = <0x048 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { + pinctrl-single,pins = <0x048 0x2F>; }; /* Mode 7, RxActive */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { + pinctrl-single,pins = <0x048 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { + pinctrl-single,pins = <0x048 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { + pinctrl-single,pins = <0x048 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_15 (ZCZ ball R13) */ + P9_15_default_pin: pinmux_P9_15_default_pin { + pinctrl-single,pins = <0x040 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { + pinctrl-single,pins = <0x040 0x2F>; }; /* Mode 7, RxActive */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { + pinctrl-single,pins = <0x040 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { + pinctrl-single,pins = <0x040 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_15_pwm_pin: pinmux_P9_15_pwm_pin { + pinctrl-single,pins = <0x040 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_16 (ZCZ ball T14) */ + P9_16_default_pin: pinmux_P9_16_default_pin { + pinctrl-single,pins = <0x04c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { + pinctrl-single,pins = <0x04c 0x2F>; }; /* Mode 7, RxActive */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { + pinctrl-single,pins = <0x04c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { + pinctrl-single,pins = <0x04c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { + pinctrl-single,pins = <0x04c 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_17 (ZCZ ball A16) */ + P9_17_default_pin: pinmux_P9_17_default_pin { + pinctrl-single,pins = <0x15c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { + pinctrl-single,pins = <0x15c 0x2F>; }; /* Mode 7, RxActive */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { + pinctrl-single,pins = <0x15c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { + pinctrl-single,pins = <0x15c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_17_spi_pin: pinmux_P9_17_spi_pin { + pinctrl-single,pins = <0x15c 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { + pinctrl-single,pins = <0x15c 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_17_pwm_pin: pinmux_P9_17_pwm_pin { + pinctrl-single,pins = <0x15c 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + + /* P9_18 (ZCZ ball B16) */ + P9_18_default_pin: pinmux_P9_18_default_pin { + pinctrl-single,pins = <0x158 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { + pinctrl-single,pins = <0x158 0x2F>; }; /* Mode 7, RxActive */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { + pinctrl-single,pins = <0x158 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { + pinctrl-single,pins = <0x158 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { + pinctrl-single,pins = <0x158 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { + pinctrl-single,pins = <0x158 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_18_pwm_pin: pinmux_P9_18_pwm_pin { + pinctrl-single,pins = <0x158 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + + /* P9_19 (ZCZ ball D17) */ + P9_19_default_pin: pinmux_P9_19_default_pin { + pinctrl-single,pins = <0x17c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { + pinctrl-single,pins = <0x17c 0x2F>; }; /* Mode 7, RxActive */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { + pinctrl-single,pins = <0x17c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { + pinctrl-single,pins = <0x17c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_19_can_pin: pinmux_P9_19_can_pin { + pinctrl-single,pins = <0x17c 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { + pinctrl-single,pins = <0x17c 0x73>; }; /* (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) */ + + /* P9_20 (ZCZ ball D18) */ + P9_20_default_pin: pinmux_P9_20_default_pin { + pinctrl-single,pins = <0x178 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { + pinctrl-single,pins = <0x178 0x2F>; }; /* Mode 7, RxActive */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { + pinctrl-single,pins = <0x178 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { + pinctrl-single,pins = <0x178 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_20_can_pin: pinmux_P9_20_can_pin { + pinctrl-single,pins = <0x178 0x12>; }; /* Mode 2, Pull-Up, RxActive */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { + pinctrl-single,pins = <0x178 0x73>; }; /* (SLEWCTRL_SLOW | PIN_INPUT_PULLUP | MUX_MODE3) */ + + /* P9_21 (ZCZ ball B17) */ + P9_21_default_pin: pinmux_P9_21_default_pin { + pinctrl-single,pins = <0x154 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { + pinctrl-single,pins = <0x154 0x2F>; }; /* Mode 7, RxActive */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { + pinctrl-single,pins = <0x154 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { + pinctrl-single,pins = <0x154 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { + pinctrl-single,pins = <0x154 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { + pinctrl-single,pins = <0x154 0x31>; }; /* Mode 1, Pull-Up, RxActive */ + P9_21_i2c_pin: pinmux_P9_21_i2c_pin { + pinctrl-single,pins = <0x154 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_21_pwm_pin: pinmux_P9_21_pwm_pin { + pinctrl-single,pins = <0x154 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + + /* P9_22 (ZCZ ball A17) */ + P9_22_default_pin: pinmux_P9_22_default_pin { + pinctrl-single,pins = <0x150 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { + pinctrl-single,pins = <0x150 0x2F>; }; /* Mode 7, RxActive */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { + pinctrl-single,pins = <0x150 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { + pinctrl-single,pins = <0x150 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_22_spi_pin: pinmux_P9_22_spi_pin { + pinctrl-single,pins = <0x150 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { + pinctrl-single,pins = <0x150 0x31>; }; /* Mode 1, Pull-Up, RxActive */ + P9_22_i2c_pin: pinmux_P9_22_i2c_pin { + pinctrl-single,pins = <0x150 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_22_pwm_pin: pinmux_P9_22_pwm_pin { + pinctrl-single,pins = <0x150 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + + /* P9_23 (ZCZ ball V14) */ + P9_23_default_pin: pinmux_P9_23_default_pin { + pinctrl-single,pins = <0x044 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { + pinctrl-single,pins = <0x044 0x2F>; }; /* Mode 7, RxActive */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { + pinctrl-single,pins = <0x044 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { + pinctrl-single,pins = <0x044 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_23_pwm_pin: pinmux_P9_23_pwm_pin { + pinctrl-single,pins = <0x044 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_24 (ZCZ ball D15) */ + P9_24_default_pin: pinmux_P9_24_default_pin { + pinctrl-single,pins = <0x184 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { + pinctrl-single,pins = <0x184 0x2F>; }; /* Mode 7, RxActive */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { + pinctrl-single,pins = <0x184 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { + pinctrl-single,pins = <0x184 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { + pinctrl-single,pins = <0x184 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_24_can_pin: pinmux_P9_24_can_pin { + pinctrl-single,pins = <0x184 0x32>; }; /* Mode 2, Pull-Up, RxActive */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { + pinctrl-single,pins = <0x184 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + P9_24_pruin_pin: pinmux_P9_24_pruin_pin { + pinctrl-single,pins = <0x184 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + + /* P9_25 (ZCZ ball A14) Audio */ + P9_25_default_pin: pinmux_P9_25_default_pin { + pinctrl-single,pins = <0x1ac 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { + pinctrl-single,pins = <0x1ac 0x2F>; }; /* Mode 7, RxActive */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { + pinctrl-single,pins = <0x1ac 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { + pinctrl-single,pins = <0x1ac 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_25_qep_pin: pinmux_P9_25_qep_pin { + pinctrl-single,pins = <0x1ac 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { + pinctrl-single,pins = <0x1ac 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { + pinctrl-single,pins = <0x1ac 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P9_25_audio_pin: pinmux_P9_25_audio_pin { + pinctrl-single,pins = <0x1ac (PIN_INPUT_PULLUP | MUX_MODE0)>; }; /* mcasp0_ahclkx.mcasp0_ahclkx */ + + /* P9_26 (ZCZ ball D16) */ + P9_26_default_pin: pinmux_P9_26_default_pin { + pinctrl-single,pins = <0x180 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { + pinctrl-single,pins = <0x180 0x2F>; }; /* Mode 7, RxActive */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { + pinctrl-single,pins = <0x180 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { + pinctrl-single,pins = <0x180 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { + pinctrl-single,pins = <0x180 0x30>; }; /* Mode 0, Pull-Up, RxActive */ + P9_26_can_pin: pinmux_P9_26_can_pin { + pinctrl-single,pins = <0x180 0x12>; }; /* Mode 2, Pull-Up, RxActive */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { + pinctrl-single,pins = <0x180 0x33>; }; /* Mode 3, Pull-Up, RxActive */ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { + pinctrl-single,pins = <0x180 0x36>; }; /* Mode 6, Pull-Up, RxActive */ + + /* P9_27 (ZCZ ball C13) */ + P9_27_default_pin: pinmux_P9_27_default_pin { + pinctrl-single,pins = <0x1a4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { + pinctrl-single,pins = <0x1a4 0x2F>; }; /* Mode 7, RxActive */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { + pinctrl-single,pins = <0x1a4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { + pinctrl-single,pins = <0x1a4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { + pinctrl-single,pins = <0x1a4 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { + pinctrl-single,pins = <0x1a4 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { + pinctrl-single,pins = <0x1a4 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_28 (ZCZ ball C12) Audio */ + P9_28_default_pin: pinmux_P9_28_default_pin { + pinctrl-single,pins = <0x19c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { + pinctrl-single,pins = <0x19c 0x2F>; }; /* Mode 7, RxActive */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { + pinctrl-single,pins = <0x19c 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { + pinctrl-single,pins = <0x19c 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_28_pwm_pin: pinmux_P9_28_pwm_pin { + pinctrl-single,pins = <0x19c 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_28_spi_pin: pinmux_P9_28_spi_pin { + pinctrl-single,pins = <0x19c 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { + pinctrl-single,pins = <0x19c 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { + pinctrl-single,pins = <0x19c 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { + pinctrl-single,pins = <0x19c 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P9_28_audio_pin: pinmux_P9_28_audio_pin { + pinctrl-single,pins = <0x19c (PIN_OUTPUT_PULLDOWN | MUX_MODE2)>; }; /* mcasp0_ahclkr.mcasp0_axr2 */ + + /* P9_29 (ZCZ ball B13) Audio */ + P9_29_default_pin: pinmux_P9_29_default_pin { + pinctrl-single,pins = <0x194 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { + pinctrl-single,pins = <0x194 0x2F>; }; /* Mode 7, RxActive */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { + pinctrl-single,pins = <0x194 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { + pinctrl-single,pins = <0x194 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_29_pwm_pin: pinmux_P9_29_pwm_pin { + pinctrl-single,pins = <0x194 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { + pinctrl-single,pins = <0x194 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { + pinctrl-single,pins = <0x194 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { + pinctrl-single,pins = <0x194 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P9_29_audio_pin: pinmux_P9_29_audio_pin { + pinctrl-single,pins = <0x194 (PIN_OUTPUT_PULLUP | MUX_MODE0)>; }; /* mcasp0_fsx.mcasp0_fsx */ + + /* P9_30 (ZCZ ball D12) */ + P9_30_default_pin: pinmux_P9_30_default_pin { + pinctrl-single,pins = <0x198 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_30_gpio_pin: pinmux_P9_30_gpio_pin { + pinctrl-single,pins = <0x198 0x2F>; }; /* Mode 7, RxActive */ + P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin { + pinctrl-single,pins = <0x198 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin { + pinctrl-single,pins = <0x198 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_30_pwm_pin: pinmux_P9_30_pwm_pin { + pinctrl-single,pins = <0x198 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_30_spi_pin: pinmux_P9_30_spi_pin { + pinctrl-single,pins = <0x198 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P9_30_pruout_pin: pinmux_P9_30_pruout_pin { + pinctrl-single,pins = <0x198 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_30_pruin_pin: pinmux_P9_30_pruin_pin { + pinctrl-single,pins = <0x198 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_31 (ZCZ ball A13) Audio */ + P9_31_default_pin: pinmux_P9_31_default_pin { + pinctrl-single,pins = <0x190 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { + pinctrl-single,pins = <0x190 0x2F>; }; /* Mode 7, RxActive */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { + pinctrl-single,pins = <0x190 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { + pinctrl-single,pins = <0x190 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_31_pwm_pin: pinmux_P9_31_pwm_pin { + pinctrl-single,pins = <0x190 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_31_spi_pin: pinmux_P9_31_spi_pin { + pinctrl-single,pins = <0x190 0x23>; }; /* Mode 3, Pull-Down, RxActive */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { + pinctrl-single,pins = <0x190 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { + pinctrl-single,pins = <0x190 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + P9_31_audio_pin: pinmux_P9_31_audio_pin { + pinctrl-single,pins = <0x190 (PIN_OUTPUT_PULLDOWN | MUX_MODE0)>; }; /* mcasp0_aclkx.mcasp0_aclkx */ + + /* P9_32 VADC */ + /* P9_33 (ZCZ ball C8 ) AIN4 */ + /* P9_34 AGND */ + /* P9_35 (ZCZ ball A8 ) AIN6 */ + /* P9_36 (ZCZ ball B8 ) AIN5 */ + /* P9_37 (ZCZ ball B7 ) AIN2 */ + /* P9_38 (ZCZ ball A7 ) AIN3 */ + /* P9_39 (ZCZ ball B6 ) AIN0 */ + /* P9_40 (ZCZ ball C7 ) AIN1 */ + + /* P9_41 (ZCZ ball D14) */ + P9_41_default_pin: pinmux_P9_41_default_pin { + pinctrl-single,pins = <0x1b4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { + pinctrl-single,pins = <0x1b4 0x2F>; }; /* Mode 7, RxActive */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { + pinctrl-single,pins = <0x1b4 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { + pinctrl-single,pins = <0x1b4 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_41_timer_pin: pinmux_P9_41_timer_pin { + pinctrl-single,pins = <0x1b4 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { + pinctrl-single,pins = <0x1b4 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) */ + P9_91_default_pin: pinmux_P9_91_default_pin { + pinctrl-single,pins = <0x1a8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_91_gpio_pin: pinmux_P9_91_gpio_pin { + pinctrl-single,pins = <0x1a8 0x2F>; }; /* Mode 7, RxActive */ + P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { + pinctrl-single,pins = <0x1a8 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { + pinctrl-single,pins = <0x1a8 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_91_qep_pin: pinmux_P9_91_qep_pin { + pinctrl-single,pins = <0x1a8 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_91_pruout_pin: pinmux_P9_91_pruout_pin { + pinctrl-single,pins = <0x1a8 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_91_pruin_pin: pinmux_P9_91_pruin_pin { + pinctrl-single,pins = <0x1a8 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_42 (ZCZ ball C18) */ + P9_42_default_pin: pinmux_P9_42_default_pin { + pinctrl-single,pins = <0x164 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { + pinctrl-single,pins = <0x164 0x2F>; }; /* Mode 7, RxActive */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { + pinctrl-single,pins = <0x164 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { + pinctrl-single,pins = <0x164 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_42_pwm_pin: pinmux_P9_42_pwm_pin { + pinctrl-single,pins = <0x164 0x20>; }; /* Mode 0, Pull-Down, RxActive */ + P9_42_uart_pin: pinmux_P9_42_uart_pin { + pinctrl-single,pins = <0x164 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_42_spics_pin: pinmux_P9_42_spics_pin { + pinctrl-single,pins = <0x164 0x22>; }; /* Mode 2, Pull-Down, RxActive */ + P9_42_spiclk_pin: pinmux_P9_42_spiclk_pin { + pinctrl-single,pins = <0x164 0x24>; }; /* Mode 4, Pull-Down, RxActive */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) */ + P9_92_default_pin: pinmux_P9_92_default_pin { + pinctrl-single,pins = <0x1a0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_92_gpio_pin: pinmux_P9_92_gpio_pin { + pinctrl-single,pins = <0x1a0 0x2F>; }; /* Mode 7, RxActive */ + P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { + pinctrl-single,pins = <0x1a0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { + pinctrl-single,pins = <0x1a0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ + P9_92_qep_pin: pinmux_P9_92_qep_pin { + pinctrl-single,pins = <0x1a0 0x21>; }; /* Mode 1, Pull-Down, RxActive */ + P9_92_pruout_pin: pinmux_P9_92_pruout_pin { + pinctrl-single,pins = <0x1a0 0x25>; }; /* Mode 5, Pull-Down, RxActive */ + P9_92_pruin_pin: pinmux_P9_92_pruin_pin { + pinctrl-single,pins = <0x1a0 0x26>; }; /* Mode 6, Pull-Down, RxActive */ + + /* P9_43 GND */ + /* P9_44 GND */ + /* P9_45 GND */ + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_default_pin: pinmux_A15_default_pin { + pinctrl-single,pins = <0x1b0 (PIN_OUTPUT_PULLUP | MUX_MODE7)>; }; /* Mode 7, Pull-Up */ + A15_clkout_pin: pinmux_A15_clkout_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_gpio_pin: pinmux_A15_gpio_pin { + pinctrl-single,pins = <0x1b0 0x2f>; }; /* Mode 7, RxActive */ + A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin { + pinctrl-single,pins = <0x1b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin { + pinctrl-single,pins = <0x1b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ +}; diff --git a/arch/arm/boot/dts/am335x-bone-common.dtsi b/arch/arm/boot/dts/am335x-bone-common.dtsi index 362746729583f..8952fe1b195a8 100644 --- a/arch/arm/boot/dts/am335x-bone-common.dtsi +++ b/arch/arm/boot/dts/am335x-bone-common.dtsi @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ / { @@ -29,14 +26,14 @@ compatible = "gpio-leds"; led2 { - label = "beaglebone:green:heartbeat"; + label = "beaglebone:green:usr0"; gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>; linux,default-trigger = "heartbeat"; default-state = "off"; }; led3 { - label = "beaglebone:green:mmc0"; + label = "beaglebone:green:usr1"; gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; linux,default-trigger = "mmc0"; default-state = "off"; @@ -66,125 +63,160 @@ }; &am33xx_pinmux { - pinctrl-names = "default"; - pinctrl-0 = <&clkout2_pin>; - user_leds_s0: user_leds_s0 { pinctrl-single,pins = < - AM33XX_IOPAD(0x854, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */ - AM33XX_IOPAD(0x858, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a6.gpio1_22 */ - AM33XX_IOPAD(0x85c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */ - AM33XX_IOPAD(0x860, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a8.gpio1_24 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a8.gpio1_24 */ >; }; i2c0_pins: pinmux_i2c0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_sda.i2c0_sda */ - AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_scl.i2c0_scl */ + AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_sda.i2c0_sda */ + AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0) /* i2c0_scl.i2c0_scl */ >; }; - i2c2_pins: pinmux_i2c2_pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_ctsn.i2c2_sda */ - AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_rtsn.i2c2_scl */ - >; - }; +// i2c2_pins: pinmux_i2c2_pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_ctsn.i2c2_sda */ +// AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_rtsn.i2c2_scl */ +// >; +// }; uart0_pins: pinmux_uart0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0) /* uart0_rxd.uart0_rxd */ - AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* uart0_txd.uart0_txd */ - >; - }; - - clkout2_pin: pinmux_clkout2_pin { - pinctrl-single,pins = < - AM33XX_IOPAD(0x9b4, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr1.clkout2 */ + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) >; }; cpsw_default: cpsw_default { pinctrl-single,pins = < /* Slave 1 */ - AM33XX_IOPAD(0x910, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxerr.mii1_rxerr */ - AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txen.mii1_txen */ - AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxdv.mii1_rxdv */ - AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd3.mii1_txd3 */ - AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd2.mii1_txd2 */ - AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd1.mii1_txd1 */ - AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd0.mii1_txd0 */ - AM33XX_IOPAD(0x92c, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_txclk.mii1_txclk */ - AM33XX_IOPAD(0x930, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxclk.mii1_rxclk */ - AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd3.mii1_rxd3 */ - AM33XX_IOPAD(0x938, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd2.mii1_rxd2 */ - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd1.mii1_rxd1 */ - AM33XX_IOPAD(0x940, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd0.mii1_rxd0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLUP, MUX_MODE0) >; }; cpsw_sleep: cpsw_sleep { pinctrl-single,pins = < /* Slave 1 reset value */ - AM33XX_IOPAD(0x910, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) >; }; davinci_mdio_default: davinci_mdio_default { pinctrl-single,pins = < /* MDIO */ - AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0) >; }; davinci_mdio_sleep: davinci_mdio_sleep { pinctrl-single,pins = < /* MDIO reset value */ - AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7) >; }; mmc1_pins: pinmux_mmc1_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* spio0_cs1.gpio0_6 */ - AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat0.mmc0_dat0 */ - AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat1.mmc0_dat1 */ - AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat2.mmc0_dat2 */ - AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_dat3.mmc0_dat3 */ - AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_cmd.mmc0_cmd */ - AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* mmc0_clk.mmc0_clk */ + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* spio0_cs1.gpio0_6 */ + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0) >; }; emmc_pins: pinmux_emmc_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */ - AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ - AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ - AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ - AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ - AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ - AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ - AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ - AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ - AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ >; }; + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT_PULLUP | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT_PULLUP | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ }; &uart0 { @@ -251,7 +283,8 @@ &i2c2 { pinctrl-names = "default"; - pinctrl-0 = <&i2c2_pins>; + //pinctrl-0 = <&i2c2_pins>; + pinctrl-0 = <>; status = "okay"; clock-frequency = <100000>; @@ -424,10 +457,69 @@ ti,scale-data-fw = "am335x-bone-scale-data.bin"; }; -&pruss_soc_bus { - status = "okay"; +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@4a300000 { +// status = "okay"; +// }; +//}; + +&ocp { + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + - pruss: pruss@4a300000 { + cape-universal { + compatible = "gpio-of-helper"; status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + }; }; diff --git a/arch/arm/boot/dts/am335x-bone-jtag.dtsi b/arch/arm/boot/dts/am335x-bone-jtag.dtsi new file mode 100644 index 0000000000000..2cdca01417121 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-jtag.dtsi @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2015 Robert Nelson + */ + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&clkout2_pin>; + + clkout2_pin: pinmux_clkout2_pin { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR1, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* xdma_event_intr1.clkout2 */ + >; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi new file mode 100644 index 0000000000000..09612160b10d4 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-can0.dtsi @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-can0.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_19_pinmux { + * mode = "can"; + * }; + * P9_20_pinmux { + * mode = "can"; + * }; + *}; + * + *&dcan0 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + dcan0_pins: pinmux_dcan0_pins { + pinctrl-single,pins = < + /* P9_20: uart1_ctsn.d_can0_tx */ + BONE_P9_20 (PIN_OUTPUT_PULLUP | MUX_MODE2) + /* P9_19: uart1_rtsn.d_can0_rx */ + BONE_P9_19 (PIN_INPUT_PULLUP | MUX_MODE2) + >; + }; +}; + +&dcan0 { + pinctrl-0 = <&dcan0_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi new file mode 100644 index 0000000000000..65e5fbb0cfcc1 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-panel-1024x600-24bit.dtsi @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include "am335x-peripheral-panel-1024x600-24bit.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P8_27_pinmux { + * state = "disabled"; + * }; + * P8_28_pinmux { + * state = "disabled"; + * }; + * P8_29_pinmux { + * state = "disabled"; + * }; + * P8_30_pinmux { + * state = "disabled"; + * }; + * P8_31_pinmux { + * state = "disabled"; + * }; + * P8_32_pinmux { + * state = "disabled"; + * }; + * P8_33_pinmux { + * state = "disabled"; + * }; + * P8_34_pinmux { + * state = "disabled"; + * }; + * P8_35_pinmux { + * state = "disabled"; + * }; + * P8_36_pinmux { + * state = "disabled"; + * }; + * P8_37_pinmux { + * state = "disabled"; + * }; + * P8_38_pinmux { + * state = "disabled"; + * }; + * P8_39_pinmux { + * state = "disabled"; + * }; + * P8_40_pinmux { + * state = "disabled"; + * }; + * P8_41_pinmux { + * state = "disabled"; + * }; + * P8_42_pinmux { + * state = "disabled"; + * }; + * P8_43_pinmux { + * state = "disabled"; + * }; + * P8_44_pinmux { + * state = "disabled"; + * }; + * P8_45_pinmux { + * state = "disabled"; + * }; + * P8_46_pinmux { + * state = "disabled"; + * }; + *}; + */ + +/* standard */ + +&am33xx_pinmux { + lcd_24bit_pins: pinmux_lcd_24bit_pins { + pinctrl-single,pins = < + + /* P8_45: lcd_data0.lcd_data0 */ + BONE_P8_45 (PIN_OUTPUT | MUX_MODE0) + /* P8_46: lcd_data1.lcd_data1 */ + BONE_P8_46 (PIN_OUTPUT | MUX_MODE0) + /* P8_43: lcd_data2.lcd_data2 */ + BONE_P8_43 (PIN_OUTPUT | MUX_MODE0) + /* P8_44: lcd_data3.lcd_data3 */ + BONE_P8_44 (PIN_OUTPUT | MUX_MODE0) + /* P8_41: lcd_data4.lcd_data4 */ + BONE_P8_41 (PIN_OUTPUT | MUX_MODE0) + /* P8_42: lcd_data5.lcd_data5 */ + BONE_P8_42 (PIN_OUTPUT | MUX_MODE0) + /* P8_39: lcd_data6.lcd_data6 */ + BONE_P8_39 (PIN_OUTPUT | MUX_MODE0) + /* P8_40: lcd_data7.lcd_data7 */ + BONE_P8_40 (PIN_OUTPUT | MUX_MODE0) + /* P8_37: lcd_data8.lcd_data8 */ + BONE_P8_37 (PIN_OUTPUT | MUX_MODE0) + /* P8_38: lcd_data9.lcd_data9 */ + BONE_P8_38 (PIN_OUTPUT | MUX_MODE0) + /* P8_36: lcd_data10.lcd_data10 */ + BONE_P8_36 (PIN_OUTPUT | MUX_MODE0) + /* P8_34: lcd_data11.lcd_data11 */ + BONE_P8_34 (PIN_OUTPUT | MUX_MODE0) + /* P8_35: lcd_data12.lcd_data12 */ + BONE_P8_35 (PIN_OUTPUT | MUX_MODE0) + /* P8_33: lcd_data13.lcd_data13 */ + BONE_P8_33 (PIN_OUTPUT | MUX_MODE0) + /* P8_31: lcd_data14.lcd_data14 */ + BONE_P8_31 (PIN_OUTPUT | MUX_MODE0) + /* P8_32: lcd_data15.lcd_data15 */ + BONE_P8_32 (PIN_OUTPUT | MUX_MODE0) + + /* gpmc_ad15.lcd_data16 */ + BONE_P8_15 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad14.lcd_data17 */ + BONE_P8_16 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad13.lcd_data18 */ + BONE_P8_11 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad12.lcd_data19 */ + BONE_P8_12 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad11.lcd_data20 */ + BONE_P8_17 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad10.lcd_data21 */ + BONE_P8_14 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad9.lcd_data22 */ + BONE_P8_13 (PIN_OUTPUT | MUX_MODE1) + /* gpmc_ad8.lcd_data23 */ + BONE_P8_19 (PIN_OUTPUT | MUX_MODE1) + + /* P8_27: lcd_vsync.lcd_vsync */ + BONE_P8_27 (PIN_OUTPUT | MUX_MODE0) + /* P8_29: lcd_hsync.lcd_hsync */ + BONE_P8_29 (PIN_OUTPUT | MUX_MODE0) + /* P8_28: lcd_pclk.lcd_pclk*/ + BONE_P8_28 (PIN_OUTPUT | MUX_MODE0) + /* P8_30: lcd_ac_bias_en.lcd_ac_bias_en */ + BONE_P8_30 (PIN_OUTPUT | MUX_MODE0) + >; + }; +}; + +/ { + panel { + pinctrl-0 = <&lcd_24bit_pins>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi new file mode 100644 index 0000000000000..ae5b813f007ab --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS1.dtsi @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Testing */ +/* sudo /sbin/getty -L ttyS1 115200 vt102 */ + +#include +#include "am335x-peripheral-ttyS1.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_24_pinmux { + * mode = "uart"; + * }; + * P9_26_pinmux { + * mode = "uart"; + * }; + *}; + * + *&uart1 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + uart1_pins: pinmux_uart1_pins { + pinctrl-single,pins = < + /* P9_24: uart1_txd.uart1_txd */ + BONE_P9_24 (PIN_OUTPUT_PULLDOWN | MUX_MODE0) + /* P9_26: uart1_rxd.uart1_rxd */ + BONE_P9_26 (PIN_INPUT_PULLUP | MUX_MODE0) + >; + }; +}; + +&uart1 { + pinctrl-0 = <&uart1_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi new file mode 100644 index 0000000000000..5fa593ac31a55 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS2.dtsi @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Testing */ +/* sudo /sbin/getty -L ttyS2 115200 vt102 */ + +#include +#include "am335x-peripheral-ttyS2.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_21_pinmux { + * mode = "uart"; + * }; + * P9_22_pinmux { + * mode = "uart"; + * }; + *}; + * + *&uart2 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + uart2_pins: pinmux_uart2_pins { + pinctrl-single,pins = < + /* P9_21: spi0_d0.uart2_txd */ + BONE_P9_21 (PIN_OUTPUT_PULLDOWN | MUX_MODE1) + /* P9_22: spi0_sclk.uart2_rxd */ + BONE_P9_22 (PIN_INPUT_PULLUP | MUX_MODE1) + >; + }; +}; + +&uart2 { + pinctrl-0 = <&uart2_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi new file mode 100644 index 0000000000000..1d22a95a23a23 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-pinmux-ttyS4.dtsi @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Testing */ +/* sudo /sbin/getty -L ttyS4 115200 vt102 */ + +#include +#include "am335x-peripheral-ttyS4.dtsi" + +/* cape universal */ + +/* + *&ocp { + * P9_11_pinmux { + * mode = "uart"; + * }; + * P9_13_pinmux { + * mode = "uart"; + * }; + *}; + * + *&uart4 { + * pinctrl-0 = <>; + *}; + * + */ + +/* standard */ + +&am33xx_pinmux { + uart4_pins: pinmux_uart4_pins { + pinctrl-single,pins = < + /* P9_11: gpmc_wait0.uart4_rxd_mux2 */ + BONE_P9_11 (PIN_INPUT_PULLUP | MUX_MODE6) + /* P9_13: gpmc_wpn.uart4_txd_mux2 */ + BONE_P9_13 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) + >; + }; +}; + +&uart4 { + pinctrl-0 = <&uart4_pins>; +}; diff --git a/arch/arm/boot/dts/am335x-bone-uboot-univ.dts b/arch/arm/boot/dts/am335x-bone-uboot-univ.dts new file mode 100644 index 0000000000000..b31b0988e8553 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bone-uboot-univ.dts @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bone-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone"; + compatible = "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bone-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&ldo3_reg>; +}; diff --git a/arch/arm/boot/dts/am335x-bone.dts b/arch/arm/boot/dts/am335x-bone.dts index ce1b68a763637..e09a54f18582e 100644 --- a/arch/arm/boot/dts/am335x-bone.dts +++ b/arch/arm/boot/dts/am335x-bone.dts @@ -1,18 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone"; compatible = "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bone.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &ldo3_reg { diff --git a/arch/arm/boot/dts/am335x-boneblack-audio.dts b/arch/arm/boot/dts/am335x-boneblack-audio.dts new file mode 100644 index 0000000000000..663d549a99456 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-audio.dts @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + clk_mcasp0_fixed: clk_mcasp0_fixed { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <24576000>; + }; + + clk_mcasp0: clk_mcasp0 { + #clock-cells = <0>; + compatible = "gpio-gate-clock"; + clocks = <&clk_mcasp0_fixed>; + enable-gpios = <&gpio1 27 GPIO_ACTIVE_HIGH>; /* BeagleBone Black Clk enable on GPIO1_27 */ + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts new file mode 100644 index 0000000000000..60195222cd03e --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-c.dts @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +#include "am335x-cape-bbb-exp-c.dtsi" diff --git a/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts new file mode 100644 index 0000000000000..a004914041974 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-bbb-exp-r.dts @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +#include "am335x-cape-bbb-exp-r.dtsi" diff --git a/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts b/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts new file mode 100644 index 0000000000000..1ccb5ef64e1d2 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-bbbmini.dts @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * Modified by Mirko Denecke + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include +#include + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&am33xx_pinmux { + dcan1_pins: pinmux_dcan1_pins { + pinctrl-single,pins = < + /* P9_26: uart1_rxd.d_can1_tx */ + BONE_P9_26 (PIN_OUTPUT_PULLUP | MUX_MODE2) + /* P9_24: uart1_txd.d_can1_rx */ + BONE_P9_24 (PIN_INPUT_PULLUP | MUX_MODE2) + >; + }; + + pru_pins: pinmux_pru_pins { + pinctrl-single,pins = < + 0x03c 0x35 /* ecap0_in_pwm0_out.pr1_ecap0_ecap_capin, MODE5 | INPUT_PULLUP | PRU, PPM-sum, SBUS, DSM */ + + 0x0e8 0x25 /* lcd_pclk.pr1_pru1_pru_r30_10, MODE5 | OUTPUT | PRU, CH_1 */ + 0x0e0 0x25 /* lcd_vsync.pr1_pru1_pru_r30_8, MODE5 | OUTPUT | PRU, CH_2 */ + 0x0ec 0x25 /* lcd_ac_bias_en.pr1_pru1_pru_r30_11, MODE5 | OUTPUT | PRU, CH_3 */ + 0x0e4 0x25 /* lcd_hsync.pr1_pru1_pru_r30_9, MODE5 | OUTPUT | PRU, CH_4 */ + 0x0bc 0x25 /* lcd_data7.pr1_pru1_pru_r30_7, MODE5 | OUTPUT | PRU, CH_5 */ + 0x0b8 0x25 /* lcd_data6.pr1_pru1_pru_r30_6, MODE5 | OUTPUT | PRU, CH_6 */ + 0x0b4 0x25 /* lcd_data5.pr1_pru1_pru_r30_5, MODE5 | OUTPUT | PRU, CH_7 */ + 0x0b0 0x25 /* lcd_data4.pr1_pru1_pru_r30_4, MODE5 | OUTPUT | PRU, CH_8 */ + 0x0ac 0x25 /* lcd_data3.pr1_pru1_pru_r30_3, MODE5 | OUTPUT | PRU, CH_9 */ + 0x0a8 0x25 /* lcd_data2.pr1_pru1_pru_r30_2, MODE5 | OUTPUT | PRU, CH_10 */ + 0x0a4 0x25 /* lcd_data1.pr1_pru1_pru_r30_1, MODE5 | OUTPUT | PRU, CH_11 */ + 0x0a0 0x25 /* lcd_data0.pr1_pru1_pru_r30_0, MODE5 | OUTPUT | PRU, CH_12 */ + + BONE_P8_12 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* HC-SR04 TRIG */ + BONE_P8_16 (PIN_INPUT_PULLDOWN | MUX_MODE6) /* HC-SR04 ECHO */ + + BONE_P9_25 (PIN_INPUT_PULLDOWN | MUX_MODE6) /* MPU9250 INT */ + >; + }; + + spi0_pins: pinmux_spi0_pins { + pinctrl-single,pins = < + /* P9_22: spi0_sclk.spi0_sclk */ + BONE_P9_22 (PIN_INPUT_PULLUP | MUX_MODE0) + /* P9_21: spi0_d0.spi0_d0 */ + BONE_P9_21 (PIN_INPUT_PULLUP | MUX_MODE0) + /* P9_18: spi0_d1.spi0_d1 */ + BONE_P9_18 (PIN_OUTPUT_PULLUP | MUX_MODE0) + /* P9_17: spi0_cs0.spi0_cs0 */ + BONE_P9_17 (PIN_OUTPUT_PULLUP | MUX_MODE0) + >; + }; + + spi1_pins: pinmux_spi1_pins { + pinctrl-single,pins = < + /* P9_31: mcasp0_aclkx.spi1_sclk */ + BONE_P9_31 (PIN_INPUT_PULLUP | MUX_MODE3) + + /* P9_29: mcasp0_fsx.spi1_d0 */ + BONE_P9_29 (PIN_INPUT_PULLUP | MUX_MODE3) + + /* P9_30: mcasp0_axr0.spi1_d1 */ + BONE_P9_30 (PIN_OUTPUT_PULLUP | MUX_MODE3) + + /* P9_28: mcasp0_ahclkr.spi1_cs0 */ + BONE_P9_28 (PIN_OUTPUT_PULLUP | MUX_MODE3) + + /* P9_19: uart1_rtsn.spi1_cs1 */ +/* BONE_P9_19 (PIN_OUTPUT_PULLUP | MUX_MODE4)*/ + + /* P9_42: ecap0_in_pwm0_out.spi1_cs1 */ + BONE_P9_42A (PIN_OUTPUT_PULLUP | MUX_MODE2) + >; + }; + + uart4_pins: pinmux_uart4_pins { + pinctrl-single,pins = < + /* P9_11: gpmc_wait0.uart4_rxd_mux2 */ + BONE_P9_11 (PIN_INPUT_PULLUP | MUX_MODE6) + /* P9_13: gpmc_wpn.uart4_txd_mux2 */ + BONE_P9_13 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) + >; + }; + + uart5_pins: pinmux_uart5_pins { + pinctrl-single,pins = < + /* P8_38: lcd_data9.uart5_rxd */ + BONE_P8_38 (PIN_INPUT_PULLUP | MUX_MODE4) + /* P8_37: lcd_data8.uart5_txd */ + BONE_P8_37 (PIN_OUTPUT_PULLDOWN | MUX_MODE4) + >; + }; +}; + +&dcan1 { + pinctrl-names = "default"; + pinctrl-0 = <&dcan1_pins>; + status = "okay"; +}; + +&i2c2 { + clock-frequency = <400000>; +}; + +&spi0 { + pinctrl-names = "default"; + pinctrl-0 = <&spi0_pins>; + status = "okay"; + + spi0@0 { + #address-cells = <1>; + #size-cells = <0>; + spi-max-frequency = <24000000>; + reg = <0>; + compatible = "spidev"; + symlink = "spi/0.0"; + }; +}; + +&spi1 { + pinctrl-names = "default"; + pinctrl-0 = <&spi1_pins>; + status = "okay"; + + spi1@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + spi-max-frequency = <24000000>; + compatible = "spidev"; + symlink = "spi/1.0"; + }; + + spi1@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + spi-max-frequency = <24000000>; + compatible = "spidev"; + symlink = "spi/1.1"; + }; +}; + +&tscadc { + adc { + ti,adc-channels = <0 1>; + }; +}; + +//&pruss { +// pinctrl-names = "default"; +// pinctrl-0 = <&pru_pins>; +// status = "okay"; +//}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_pins>; + status = "okay"; +}; + +&uart5 { + pinctrl-names = "default"; + pinctrl-0 = <&uart5_pins>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-common.dtsi b/arch/arm/boot/dts/am335x-boneblack-common.dtsi index 21bc1173fa6b9..2d01324494591 100644 --- a/arch/arm/boot/dts/am335x-boneblack-common.dtsi +++ b/arch/arm/boot/dts/am335x-boneblack-common.dtsi @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include @@ -29,43 +26,36 @@ &am33xx_pinmux { nxp_hdmi_bonelt_pins: nxp_hdmi_bonelt_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr0 */ - AM33XX_IOPAD(0x8a0, PIN_OUTPUT | MUX_MODE0) /* lcd_data0.lcd_data0 */ - AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE0) /* lcd_data1.lcd_data1 */ - AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE0) /* lcd_data2.lcd_data2 */ - AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE0) /* lcd_data3.lcd_data3 */ - AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE0) /* lcd_data4.lcd_data4 */ - AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE0) /* lcd_data5.lcd_data5 */ - AM33XX_IOPAD(0x8b8, PIN_OUTPUT | MUX_MODE0) /* lcd_data6.lcd_data6 */ - AM33XX_IOPAD(0x8bc, PIN_OUTPUT | MUX_MODE0) /* lcd_data7.lcd_data7 */ - AM33XX_IOPAD(0x8c0, PIN_OUTPUT | MUX_MODE0) /* lcd_data8.lcd_data8 */ - AM33XX_IOPAD(0x8c4, PIN_OUTPUT | MUX_MODE0) /* lcd_data9.lcd_data9 */ - AM33XX_IOPAD(0x8c8, PIN_OUTPUT | MUX_MODE0) /* lcd_data10.lcd_data10 */ - AM33XX_IOPAD(0x8cc, PIN_OUTPUT | MUX_MODE0) /* lcd_data11.lcd_data11 */ - AM33XX_IOPAD(0x8d0, PIN_OUTPUT | MUX_MODE0) /* lcd_data12.lcd_data12 */ - AM33XX_IOPAD(0x8d4, PIN_OUTPUT | MUX_MODE0) /* lcd_data13.lcd_data13 */ - AM33XX_IOPAD(0x8d8, PIN_OUTPUT | MUX_MODE0) /* lcd_data14.lcd_data14 */ - AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE0) /* lcd_data15.lcd_data15 */ - AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_vsync.lcd_vsync */ - AM33XX_IOPAD(0x8e4, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_hsync.lcd_hsync */ - AM33XX_IOPAD(0x8e8, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_pclk.lcd_pclk */ - AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_ac_bias_en.lcd_ac_bias_en */ - >; - }; - - nxp_hdmi_bonelt_off_pins: nxp_hdmi_bonelt_off_pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr0 */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) >; }; mcasp0_pins: mcasp0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x9ac, PIN_INPUT_PULLUP | MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ - AM33XX_IOPAD(0x99c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ - AM33XX_IOPAD(0x994, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mcasp0_fsx.mcasp0_fsx */ - AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mcasp0_aclkx.mcasp0_aclkx */ - AM33XX_IOPAD(0x86c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a11.GPIO1_27 */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ + AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */ >; }; }; @@ -92,9 +82,8 @@ compatible = "nxp,tda998x"; reg = <0x70>; - pinctrl-names = "default", "off"; + pinctrl-names = "default"; pinctrl-0 = <&nxp_hdmi_bonelt_pins>; - pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>; /* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */ /* video-ports = <0x234501>; */ diff --git a/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts b/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts new file mode 100644 index 0000000000000..0a8007794f7fe --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-roboticscape.dts @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" +/* #include "am335x-boneblack-common.dtsi" */ +/* #include "am335x-bone-jtag.dtsi" */ + +/* RoboticsCape */ +#include "am335x-bone-common-universal-pins.dtsi" +#include "am335x-roboticscape.dtsi" + +/ { + model = "TI AM335x BeagleBone Black RoboticsCape"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +/* needed from: am335x-boneblack-common.dtsi */ +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts b/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts new file mode 100644 index 0000000000000..2dd94e621ac65 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-uboot-univ.dts @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bone-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-uboot.dts b/arch/arm/boot/dts/am335x-boneblack-uboot.dts new file mode 100644 index 0000000000000..a1d7e92f5365b --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-uboot.dts @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack-uboot.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts b/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts new file mode 100644 index 0000000000000..a687664858a85 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-wireless-roboticscape.dts @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" +/* #include "am335x-boneblack-common.dtsi" */ +/* #include "am335x-bone-jtag.dtsi" */ +#include + +/* RoboticsCape */ +#include "am335x-bone-common-universal-pins.dtsi" +#include "am335x-roboticscape.dtsi" + +/ { + model = "TI AM335x BeagleBone Black Wireless RoboticsCape"; + compatible = "ti,am335x-bone-black-wireless", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + wlan_en_reg: fixedregulator@2 { + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us= <70000>; + + /* WL_EN */ + gpio = <&gpio3 9 0>; + enable-active-high; + }; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&am33xx_pinmux { + bt_pins: pinmux_bt_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_txd0.gpio0_28 - BT_EN */ + >; + }; + + mmc3_pins: pinmux_mmc3_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ + AM33XX_IOPAD(0x914, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ + AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ + AM33XX_IOPAD(0x91c, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ + AM33XX_IOPAD(0x920, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ + AM33XX_IOPAD(0x908, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ + >; + }; + + uart3_pins: pinmux_uart3_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ + AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gmii1_rxd2.uart3_txd */ + AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3) /* mdio_data.uart3_ctsn */ + AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* mdio_clk.uart3_rtsn */ + >; + }; + + wl18xx_pins: pinmux_wl18xx_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gmii1_txclk.gpio3_9 WL_EN */ + AM33XX_IOPAD(0x944, PIN_INPUT_PULLDOWN | MUX_MODE7) /* rmii1_refclk.gpio0_29 WL_IRQ */ + AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_rxclk.gpio3_10 LS_BUF_EN */ + >; + }; +}; + +&mac { + status = "disabled"; +}; + +&mmc3 { + dmas = <&edma_xbar 12 0 1 + &edma_xbar 13 0 2>; + dma-names = "tx", "rx"; + status = "okay"; + vmmc-supply = <&wlan_en_reg>; + bus-width = <4>; + non-removable; + cap-power-off-card; + ti,needs-special-hs-handling; + keep-power-in-suspend; + pinctrl-names = "default"; + pinctrl-0 = <&mmc3_pins &wl18xx_pins>; + + #address-cells = <1>; + #size-cells = <0>; + wlcore: wlcore@2 { + compatible = "ti,wl1835"; + reg = <2>; + interrupt-parent = <&gpio0>; + interrupts = <29 IRQ_TYPE_EDGE_RISING>; + }; +}; + +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&uart3_pins &bt_pins>; + status = "okay"; + + bluetooth { + compatible = "ti,wl1835-st"; + enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; + }; +}; + +&gpio3 { + ls_buf_en { + gpio-hog; + gpios = <10 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "LS_BUF_EN"; + }; +}; + +/* needed from: am335x-boneblack-common.dtsi */ +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-wireless.dts b/arch/arm/boot/dts/am335x-boneblack-wireless.dts index 83f49f616b19c..01a940a71abf4 100644 --- a/arch/arm/boot/dts/am335x-boneblack-wireless.dts +++ b/arch/arm/boot/dts/am335x-boneblack-wireless.dts @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; @@ -11,11 +8,17 @@ #include "am335x-bone-common.dtsi" #include "am335x-boneblack-common.dtsi" #include +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Black Wireless"; compatible = "ti,am335x-bone-black-wireless", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + chosen { + base_dtb = "am335x-boneblack-wireless.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + wlan_en_reg: fixedregulator@2 { compatible = "regulator-fixed"; regulator-name = "wlan-en-regulator"; @@ -29,38 +32,42 @@ }; }; +&sgx { + status = "okay"; +}; + &am33xx_pinmux { bt_pins: pinmux_bt_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_txd0.gpio0_28 - BT_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_txd0.gpio0_28 - BT_EN */ >; }; mmc3_pins: pinmux_mmc3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ - AM33XX_IOPAD(0x914, PIN_INPUT_PULLUP | MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ - AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ - AM33XX_IOPAD(0x91c, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ - AM33XX_IOPAD(0x920, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ - AM33XX_IOPAD(0x908, PIN_INPUT_PULLUP | MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ >; }; uart3_pins: pinmux_uart3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ - AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gmii1_rxd2.uart3_txd */ - AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3) /* mdio_data.uart3_ctsn */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* mdio_clk.uart3_rtsn */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* mdio_clk.uart3_rtsn */ >; }; wl18xx_pins: pinmux_wl18xx_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gmii1_txclk.gpio3_9 WL_EN */ - AM33XX_IOPAD(0x944, PIN_INPUT_PULLDOWN | MUX_MODE7) /* rmii1_refclk.gpio0_29 WL_IRQ */ - AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_rxclk.gpio3_10 LS_BUF_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gmii1_txclk.gpio3_9 WL_EN */ + AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) /* rmii1_refclk.gpio0_29 WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_rxclk.gpio3_10 LS_BUF_EN */ >; }; }; diff --git a/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi b/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi new file mode 100644 index 0000000000000..94caa22903017 --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-wl1835mod-cape.dtsi @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include + +/ { + wlan_en_reg: fixedregulator@2 { + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + + /* WL_EN */ + gpio = <&gpio0 26 0>; + enable-active-high; + }; + + kim { + compatible = "kim"; + nshutdown_gpio = <44>; /* Bank1, pin12 */ + dev_name = "/dev/ttyO4"; + flow_cntrl = <1>; + baud_rate = <3000000>; + }; + + btwilink { + compatible = "btwilink"; + }; +}; + +&am33xx_pinmux { + bt_pins: pinmux_bt_pins { + pinctrl-single,pins = < + 0x30 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_ad12.gpio1_12 */ + >; + }; + + mmc2_pins: pinmux_mmc2_pins { + pinctrl-single,pins = < + 0x80 (PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + 0x84 (PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + 0x00 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + 0x04 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + 0x08 (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + 0x0c (PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + >; + }; + + mmc2_pins_sleep: pinmux_mmc2_pins_sleep { + pinctrl-single,pins = < + 0x80 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_csn1.mmc1_clk */ + 0x84 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_csn2.mmc1_cmd */ + 0x00 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad0.mmc1_dat0 */ + 0x04 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad1.mmc1_dat1 */ + 0x08 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad2.mmc1_dat2 */ + 0x0c (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad3.mmc1_dat3 */ + >; + }; + + /* wl18xx card enable/irq GPIOs. */ + wlan_pins: pinmux_wlan_pins { + pinctrl-single,pins = < + 0x28 (PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad10.gpio0_26 WL_EN*/ + 0x2C (PIN_INPUT_PULLUP | MUX_MODE7) /* gpmc_ad11.gpio0_27 WL_IRQ*/ + 0x7C (PIN_OUTPUT_PULLUP | MUX_MODE0) /* gpmc_csn0.gpio1_29 BF_EN*/ + >; + }; + + /* wl18xx card enable/irq GPIOs. */ + wlan_pins_sleep: pinmux_wlan_pins_sleep { + pinctrl-single,pins = < + 0x28 (PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_ad10.gpio0_26 WL_EN*/ + 0x2C (PIN_INPUT_PULLUP | MUX_MODE7) /* gpmc_ad11.gpio0_27 WL_IRQ*/ + 0x7C (PIN_OUTPUT_PULLUP | MUX_MODE0) /* gpmc_csn0.gpio1_29 BF_EN*/ + >; + }; + + uart4_pins_default: pinmux_uart4_pins_default { + pinctrl-single,pins = < + 0xD0 (PIN_INPUT | MUX_MODE6) /* lcd_data12.uart4_cts */ + 0xD4 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* lcd_data13.uart4_rts */ + 0x70 (PIN_INPUT_PULLUP | MUX_MODE6) /* gpmc_wait0.uart4_rxd */ + 0x74 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* gpmc_wpn.uart4_txd */ + >; + }; + + uart4_pins_sleep: pinmux_uart4_pins_sleep { + pinctrl-single,pins = < + 0xD0 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* lcd_data12.uart4_cts */ + 0xD4 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* lcd_data13.uart4_rts */ + 0x70 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_wait0.uart4_rxd */ + 0x74 (PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_wpn.uart4_txd */ + >; + }; +}; + +&mmc2 { + status = "okay"; + vmmc-supply = <&wlan_en_reg>; + bus-width = <4>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&mmc2_pins &wlan_pins>; + pinctrl-1 = <&mmc2_pins_sleep &wlan_pins_sleep>; + ti,non-removable; + ti,needs-special-hs-handling; + cap-power-off-card; + keep-power-in-suspend; + + #address-cells = <1>; + #size-cells = <0>; + wlcore: wlcore@0 { + compatible = "ti,wl1835"; + reg = <2>; + interrupt-parent = <&gpio0>; + interrupts = <27 IRQ_TYPE_LEVEL_HIGH>; + }; +}; + +&uart4 { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&uart4_pins_default>; + pinctrl-1 = <&uart4_pins_sleep>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts b/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts new file mode 100644 index 0000000000000..78abd6c34821d --- /dev/null +++ b/arch/arm/boot/dts/am335x-boneblack-wl1835mod.dts @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common-no-capemgr.dtsi" + +/ { + model = "TI AM335x BeagleBone Black"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; +}; + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +/* EMMC in reset */ +&gpio1 { + emmc_rst { + gpio-hog; + gpios = <20 0>; + output-high; + line-name = "EMMC ResetN"; + }; +}; + +#include "am335x-boneblack-wl1835mod-cape.dtsi" diff --git a/arch/arm/boot/dts/am335x-boneblack.dts b/arch/arm/boot/dts/am335x-boneblack.dts index 5497718f5d44f..7bcf13bad47dd 100644 --- a/arch/arm/boot/dts/am335x-boneblack.dts +++ b/arch/arm/boot/dts/am335x-boneblack.dts @@ -1,23 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" #include "am335x-boneblack-common.dtsi" - -&sgx { - status = "okay"; -}; +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Black"; compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-boneblack.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; }; &cpu0_opp_table { diff --git a/arch/arm/boot/dts/am335x-boneblue.dts b/arch/arm/boot/dts/am335x-boneblue.dts index 7bcd72691f06d..b879d467cdceb 100644 --- a/arch/arm/boot/dts/am335x-boneblue.dts +++ b/arch/arm/boot/dts/am335x-boneblue.dts @@ -1,19 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; #include "am33xx.dtsi" +#include "am335x-bone-common-universal-pins.dtsi" #include / { model = "TI AM335x BeagleBone Blue"; compatible = "ti,am335x-bone-blue", "ti,am33xx"; + chosen { + stdout-path = &uart0; + base_dtb = "am335x-boneblue.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + cpus { cpu@0 { cpu0-supply = <&dcdc2_reg>; @@ -25,10 +29,6 @@ reg = <0x80000000 0x20000000>; /* 512 MB */ }; - chosen { - stdout-path = &uart0; - }; - leds { pinctrl-names = "default"; pinctrl-0 = <&user_leds_s0>; @@ -125,142 +125,329 @@ gpio = <&gpio3 9 0>; enable-active-high; }; + + bt_en { + pinctrl-names = "default"; + pinctrl-0 = <&bt_pins>; + compatible = "gpio-leds"; + + wl18xx_bt_en { + label = "wl18xx_bt_en"; + gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; }; -&am33xx_pinmux { - user_leds_s0: user_leds_s0 { - pinctrl-single,pins = < - AM33XX_IOPAD(0x854, PIN_OUTPUT | MUX_MODE7) /* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */ - AM33XX_IOPAD(0x858, PIN_OUTPUT | MUX_MODE7) /* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */ - AM33XX_IOPAD(0x85c, PIN_OUTPUT | MUX_MODE7) /* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */ - AM33XX_IOPAD(0x860, PIN_OUTPUT | MUX_MODE7) /* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */ - AM33XX_IOPAD(0x9b0, PIN_OUTPUT | MUX_MODE7) /* (A15) xdma_event_intr0.gpio0[19] - WIFI_LED */ - AM33XX_IOPAD(0x890, PIN_OUTPUT | MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */ - AM33XX_IOPAD(0x894, PIN_OUTPUT | MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */ - AM33XX_IOPAD(0x82c, PIN_OUTPUT | MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */ - AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE7) /* (T5) lcd_data15.gpio0[11] - P8.32, BATT_LED_2 */ - AM33XX_IOPAD(0x87c, PIN_OUTPUT | MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */ - AM33XX_IOPAD(0x828, PIN_OUTPUT | MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */ +&aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + serial4 = &uart4; + serial5 = &uart5; + d-can0 = &dcan0; + d-can1 = &dcan1; + usb0 = &usb0; + usb1 = &usb1; + phy0 = &usb0_phy; + phy1 = &usb1_phy; + ethernet0 = &cpsw_emac0; + ethernet1 = &cpsw_emac1; + spi0 = &spi0; + spi1 = &spi1; +}; - >; +&sgx { + status = "okay"; +}; + +&cpu0_opp_table { + /* + * Octavo Systems: + * The EFUSE_SMA register is not programmed for any of the AM335x wafers + * we get and we are not programming them during our production test. + * Therefore, from a DEVICE_ID revision point of view, the silicon looks + * like it is Revision 2.1. However, from an EFUSE_SMA point of view for + * the HW OPP table, the silicon looks like it is Revision 1.0 (ie the + * EFUSE_SMA register reads as all zeros). + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; }; +}; - i2c0_pins: pinmux_i2c0_pins { +&am33xx_pinmux { + /*************************************************************************** + * Static Pinmux + ***************************************************************************/ + mux_helper_pins: pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0) /* (C17) I2C0_SDA.I2C0_SDA */ - AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0) /* (C16) I2C0_SCL.I2C0_SCL */ + + /* GPIO Inputs */ + 0x09c 0x37 /*P8.9 Pause BUTTON, input pullup*/ + 0x098 0x37 /*P8.10 MODE BUTTON input pullup*/ + 0x1AC 0x37 /*P9.25 MPU-9150 INTERRUPT IN*/ + + /* Motor Control GPIO Out*/ + 0x088 ( PIN_OUTPUT | MUX_MODE7 ) /* (T13) gpmc_csn3.gpio2[0] - MDIR_1A different from cape! */ + 0x074 ( PIN_OUTPUT | MUX_MODE7 ) /* (U17) gpmc_wpn.gpio0[31] - P9.13, MDIR_1B */ + 0x040 ( PIN_OUTPUT | MUX_MODE7 ) /* (R13) gpmc_a0.gpio1[16] - P9.15, MDIR_2A */ + 0x0D8 ( PIN_OUTPUT | MUX_MODE7 ) /* (V4) lcd_data14.gpio0[10] - P8.31, MDIR_2B different from cape! */ + 0x0AC ( PIN_OUTPUT | MUX_MODE7 ) /* (R4) lcd_data3.gpio2[9] - P8.44, MDIR_3A */ + 0x0A8 ( PIN_OUTPUT | MUX_MODE7 ) /* (R3) lcd_data2.gpio2[8] - P8.43, MDIR_3B */ + 0x0A0 ( PIN_OUTPUT | MUX_MODE7 ) /* (R1) lcd_data0.gpio2[6] - P8.45, MDIR_4A */ + 0x0A4 ( PIN_OUTPUT | MUX_MODE7 ) /* (R2) lcd_data1.gpio2[7] - P8.46, MDIR_4B */ + 0x1B4 ( PIN_OUTPUT | MUX_MODE7 ) /* (D14) xdma_event_intr1.gpio0[20] - P9.41, MOT_STBY */ + + /* PRU encoder input */ + 0x038 0x36 /* P8_16,PRU0_r31_16,MODE6 */ + + /* PRU Servo output */ + 0x0e0 0x05 /*pru1_pru_r30_8, MODE5*/ + 0x0e8 0x05 /*pru1_pru_r30_10, MODE5 */ + 0x0e4 0x05 /*pr1_pru1_pru_r30_9, MODE5 */ + 0x0ec 0x05 /*pru1_pru_r30_11, MODE5 */ + 0x0b8 0x05 /*pru1_pru_r30_6, MODE5 */ + 0x0bc 0x05 /*pru1_pru_r30_7, MODE5 */ + 0x0b0 0x05 /*pru1_pru_r30_4, MODE5 */ + 0x0b4 0x05 /*pru1_pru_r30_5, MODE5 */ + 0x0C8 0x0F /*P8.36, SERVO_PWR GPIO OUT*/ + + /* WILINK 8 */ + 0x08c 0x0F /*P8.18 V12 A2DP FSYNC */ + 0x078 0x0F /*P9.12 A2DP_CLOCK*/ >; }; - i2c2_pins: pinmux_i2c2_pins { + user_leds_s0: user_leds_s0 { pinctrl-single,pins = < - AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3) /* (D18) uart1_ctsn.I2C2_SDA */ - AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3) /* (D17) uart1_rtsn.I2C2_SCL */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT, MUX_MODE7) /* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT, MUX_MODE7) /* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT, MUX_MODE7) /* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT, MUX_MODE7) /* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */ + AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_OUTPUT, MUX_MODE7) /* (A15) xdma_event_intr0.gpio0[19] - WIFI_LED */ + AM33XX_PADCONF(AM335X_PIN_GPMC_ADVN_ALE, PIN_OUTPUT, MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_OEN_REN, PIN_OUTPUT, MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD11, PIN_OUTPUT, MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE7) /* (T5) lcd_data15.gpio0[11] - P8.32, BATT_LED_2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN0, PIN_OUTPUT, MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD10, PIN_OUTPUT, MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */ + >; }; - /* UT0 */ - uart0_pins: pinmux_uart0_pins { + i2c0_pins: pinmux_i2c0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0) /* (E15) uart0_rxd.uart0_rxd */ - AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* (E16) uart0_txd.uart0_txd */ + AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0) /* (C17) I2C0_SDA.I2C0_SDA */ + AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0) /* (C16) I2C0_SCL.I2C0_SCL */ >; }; - /* UT1 */ - uart1_pins: pinmux_uart1_pins { + i2c1_pins: pinmux_i2c1_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x980, PIN_INPUT_PULLUP | MUX_MODE0) /* (D16) uart1_rxd.uart1_rxd */ - AM33XX_IOPAD(0x984, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* (D15) uart1_txd.uart1_txd */ + 0x15C 0x32 /* P9.17,i2c1_scl,INPUT_PULLUP,MODE2 */ + 0x158 0x32 /* P9.18,i2c1_sda,INPUT_PULLUP,MODE2 */ >; }; - /* GPS */ - uart2_pins: pinmux_uart2_pins { + i2c2_pins: pinmux_i2c2_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x950, PIN_INPUT_PULLUP | MUX_MODE1) /* (A17) spi0_sclk.uart2_rxd */ - AM33XX_IOPAD(0x954, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* (B17) spi0_d0.uart2_txd */ + AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D18) uart1_ctsn.I2C2_SDA */ + AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D17) uart1_rtsn.I2C2_SCL */ >; }; - /* DSM2 */ - uart4_pins: pinmux_uart4_pins { + /* UT0 */ + uart0_pins: pinmux_uart0_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x870, PIN_INPUT_PULLUP | MUX_MODE6) /* (T17) gpmc_wait0.uart4_rxd */ + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) >; }; /* UT5 */ uart5_pins: pinmux_uart5_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x8C4, PIN_INPUT_PULLUP | MUX_MODE4) /* (U2) lcd_data9.uart5_rxd */ - AM33XX_IOPAD(0x8C0, PIN_OUTPUT_PULLDOWN | MUX_MODE4) /* (U1) lcd_data8.uart5_txd */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_INPUT_PULLUP, MUX_MODE4) /* (U2) lcd_data9.uart5_rxd */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT_PULLDOWN, MUX_MODE4) /* (U1) lcd_data8.uart5_txd */ >; }; mmc1_pins: pinmux_mmc1_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */ + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */ >; }; mmc2_pins: pinmux_mmc2_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* (U9) gpmc_csn1.mmc1_clk */ - AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* (V9) gpmc_csn2.mmc1_cmd */ - AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* (U7) gpmc_ad0.mmc1_dat0 */ - AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* (V7) gpmc_ad1.mmc1_dat1 */ - AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* (R8) gpmc_ad2.mmc1_dat2 */ - AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* (T8) gpmc_ad3.mmc1_dat3 */ - AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* (U8) gpmc_ad4.mmc1_dat4 */ - AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* (V8) gpmc_ad5.mmc1_dat5 */ - AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* (R9) gpmc_ad6.mmc1_dat6 */ - AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* (T9) gpmc_ad7.mmc1_dat7 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* (U9) gpmc_csn1.mmc1_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* (V9) gpmc_csn2.mmc1_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* (U7) gpmc_ad0.mmc1_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* (V7) gpmc_ad1.mmc1_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* (R8) gpmc_ad2.mmc1_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* (T8) gpmc_ad3.mmc1_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* (U8) gpmc_ad4.mmc1_dat4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* (V8) gpmc_ad5.mmc1_dat5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* (R9) gpmc_ad6.mmc1_dat6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* (T9) gpmc_ad7.mmc1_dat7 */ >; }; mmc3_pins: pinmux_mmc3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE6) /* (L15) gmii1_rxd1.mmc2_clk */ - AM33XX_IOPAD(0x914, PIN_INPUT_PULLUP | MUX_MODE6) /* (J16) gmii1_txen.mmc2_cmd */ - AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE5) /* (J17) gmii1_rxdv.mmc2_dat0 */ - AM33XX_IOPAD(0x91c, PIN_INPUT_PULLUP | MUX_MODE5) /* (J18) gmii1_txd3.mmc2_dat1 */ - AM33XX_IOPAD(0x920, PIN_INPUT_PULLUP | MUX_MODE5) /* (K15) gmii1_txd2.mmc2_dat2 */ - AM33XX_IOPAD(0x908, PIN_INPUT_PULLUP | MUX_MODE5) /* (H16) gmii1_col.mmc2_dat3 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6) /* (L15) gmii1_rxd1.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6) /* (J16) gmii1_txen.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5) /* (J17) gmii1_rxdv.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5) /* (J18) gmii1_txd3.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5) /* (K15) gmii1_txd2.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5) /* (H16) gmii1_col.mmc2_dat3 */ >; }; bt_pins: pinmux_bt_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLUP | MUX_MODE7) /* (K17) gmii1_txd0.gpio0[28] - BT_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* (K17) gmii1_txd0.gpio0[28] - BT_EN */ >; }; uart3_pins: pinmux_uart3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* (L17) gmii1_rxd3.uart3_rxd */ - AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* (L16) gmii1_rxd2.uart3_txd */ - AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3) /* (M17) mdio_data.uart3_ctsn */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* (M18) mdio_clk.uart3_rtsn */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* (L17) gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* (L16) gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* (M17) mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* (M18) mdio_clk.uart3_rtsn */ >; }; wl18xx_pins: pinmux_wl18xx_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* (K18) gmii1_txclk.gpio3[9] - WL_EN */ - AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7) /* (K16) gmii1_txd1.gpio0[21] - WL_IRQ */ - AM33XX_IOPAD(0x930, PIN_OUTPUT_PULLUP | MUX_MODE7) /* (L18) gmii1_rxclk.gpio3[10] - LS_BUF_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* (K18) gmii1_txclk.gpio3[9] - WL_EN */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) /* (K16) gmii1_txd1.gpio0[21] - WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* (L18) gmii1_rxclk.gpio3[10] - LS_BUF_EN */ >; }; /* DCAN */ dcan1_pins: pinmux_dcan1_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x96c, PIN_INPUT | MUX_MODE2) /* (E17) uart0_rtsn.dcan1_rx */ - AM33XX_IOPAD(0x968, PIN_OUTPUT | MUX_MODE2) /* (E18) uart0_ctsn.dcan1_tx */ - AM33XX_IOPAD(0x940, PIN_OUTPUT | MUX_MODE7) /* (M16) gmii1_rxd0.gpio2[21] */ + AM33XX_PADCONF(AM335X_PIN_UART0_RTSN, PIN_INPUT, MUX_MODE2) /* (E17) uart0_rtsn.dcan1_rx */ + AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_OUTPUT, MUX_MODE2) /* (E18) uart0_ctsn.dcan1_tx */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_OUTPUT, MUX_MODE7) /* (M16) gmii1_rxd0.gpio2[21] */ + >; + }; + + /* E1 */ + eqep0_pins: pinmux_eqep0_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MCASP0_AXR0, PIN_INPUT, MUX_MODE1) /* (B12) mcasp0_aclkr.eQEP0A_in */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_FSR, PIN_INPUT, MUX_MODE1) /* (C13) mcasp0_fsr.eQEP0B_in */ + >; + }; + + /* E2 */ + eqep1_pins: pinmux_eqep1_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_INPUT, MUX_MODE2) /* (V2) lcd_data12.eQEP1A_in */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_INPUT, MUX_MODE2) /* (V3) lcd_data13.eQEP1B_in */ + >; + }; + + /* E3 */ + eqep2_pins: pinmux_eqep2_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_GPMC_AD12, PIN_INPUT, MUX_MODE4) /* (T12) gpmc_ad12.eQEP2A_in */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD13, PIN_INPUT, MUX_MODE4) /* (R12) gpmc_ad13.eQEP2B_in */ + >; + }; + + ehrpwm1_pins: pinmux_ehrpwm1_pins { + pinctrl-single,pins = < + 0x020 0x24 /* P8_19_pwm_pin */ + 0x024 0x24 /* P8_13_pwm_pin */ >; }; + + ehrpwm2_pins: pinmux_ehrpwm2_pins { + pinctrl-single,pins = < + 0x048 0x26 /* P9_14_pwm_pin */ + 0x04c 0x26 /* P9_16_pwm_pin */ + >; + }; + + /* H18 SPI1_SS1 default SPI*/ + H18_default_pin: pinmux_H18_default_pin { + pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE2 ) >; }; + H18_gpio_pin: pinmux_H18_gpio_pin { + pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + H18_gpio_pu_pin: pinmux_H18_gpio_pu_pin { + pinctrl-single,pins = < 0x144 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + H18_gpio_pd_pin: pinmux_H18_gpio_pd_pin { + pinctrl-single,pins = < 0x144 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + H18_spi_pin: pinmux_H18_spi_pin { + pinctrl-single,pins = < 0x144 ( PIN_OUTPUT | MUX_MODE2 ) >; }; + + + /* C18 SPI1_SS2 default spi*/ + C18_default_pin: pinmux_C18_default_pin { + pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE2 ) >; }; + C18_gpio_pin: pinmux_C18_gpio_pin { + pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + C18_gpio_pu_pin: pinmux_C18_gpio_pu_pin { + pinctrl-single,pins = < 0x164 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + C18_gpio_pd_pin: pinmux_C18_gpio_pd_pin { + pinctrl-single,pins = < 0x164 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + C18_spi_pin: pinmux_C18_spi_pin { + pinctrl-single,pins = < 0x164 ( PIN_OUTPUT | MUX_MODE2 ) >; }; + + + + /* D13 BLUE_GP0_PIN_5 gpio 3_20 */ + D13_default_pin: pinmux_D13_default_pin { + pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + D13_gpio_pin: pinmux_D13_gpio_pin { + pinctrl-single,pins = < 0x1A8 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + D13_gpio_pu_pin: pinmux_D13_gpio_pu_pin { + pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + D13_gpio_pd_pin: pinmux_D13_gpio_pd_pin { + pinctrl-single,pins = < 0x1A8 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + + /* H17 BLUE_GP1_PIN_4 gpio 3_1 */ + H17_default_pin: pinmux_H17_default_pin { + pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + H17_gpio_pin: pinmux_H17_gpio_pin { + pinctrl-single,pins = < 0x10C ( PIN_OUTPUT | MUX_MODE7 ) >; }; + H17_gpio_pu_pin: pinmux_H17_gpio_pu_pin { + pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + H17_gpio_pd_pin: pinmux_H17_gpio_pd_pin { + pinctrl-single,pins = < 0x10C ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + + /* J15 BLUE_GP1_PIN_3 gpio 3_2 */ + J15_default_pin: pinmux_J15_default_pin { + pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + J15_gpio_pin: pinmux_J15_gpio_pin { + pinctrl-single,pins = < 0x110 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + J15_gpio_pu_pin: pinmux_J15_gpio_pu_pin { + pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + J15_gpio_pd_pin: pinmux_J15_gpio_pd_pin { + pinctrl-single,pins = < 0x110 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + + + /* U16 BLUE_GP0_PIN_3 gpio 1_25 */ + U16_default_pin: pinmux_U16_default_pin { + pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + U16_gpio_pin: pinmux_U16_gpio_pin { + pinctrl-single,pins = < 0x064 ( PIN_OUTPUT | MUX_MODE7 ) >; }; + U16_gpio_pu_pin: pinmux_U16_gpio_pu_pin { + pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLUP | MUX_MODE7 ) >; }; + U16_gpio_pd_pin: pinmux_U16_gpio_pd_pin { + pinctrl-single,pins = < 0x064 ( PIN_INPUT_PULLDOWN | MUX_MODE7 ) >; }; + + }; &uart0 { @@ -271,23 +458,16 @@ }; &uart1 { - pinctrl-names = "default"; - pinctrl-0 = <&uart1_pins>; - status = "okay"; }; &uart2 { - pinctrl-names = "default"; - pinctrl-0 = <&uart2_pins>; status = "okay"; }; &uart4 { pinctrl-names = "default"; - pinctrl-0 = <&uart4_pins>; - status = "okay"; }; @@ -353,32 +533,19 @@ }; }; +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + status = "okay"; + clock-frequency = <400000>; +}; + &i2c2 { pinctrl-names = "default"; pinctrl-0 = <&i2c2_pins>; status = "okay"; clock-frequency = <400000>; - - mpu9250@68 { - compatible = "invensense,mpu9250"; - reg = <0x68>; - interrupt-parent = <&gpio3>; - interrupts = <21 IRQ_TYPE_EDGE_RISING>; - i2c-gate { - #address-cells = <1>; - #size-cells = <0>; - ax8975@c { - compatible = "ak,ak8975"; - reg = <0x0c>; - }; - }; - }; - - pressure@76 { - compatible = "bosch,bmp280"; - reg = <0x76>; - }; }; /include/ "tps65217.dtsi" @@ -387,6 +554,8 @@ interrupts = <7>; /* NMI */ interrupt-parent = <&intc>; + ti,pmic-shutdown-controller; + charger { interrupts = <0>, <1>; interrupt-names = "USB", "AC"; @@ -487,22 +656,23 @@ }; }; -&tscadc { - status = "okay"; - adc { - ti,adc-channels = <0 1 2 3 4 5 6 7>; - }; -}; +//&tscadc { +// status = "okay"; +// adc { +// ti,adc-channels = <0 1 2 3 4 5 6 7>; +// }; +//}; &uart3 { pinctrl-names = "default"; - pinctrl-0 = <&uart3_pins &bt_pins>; + //pinctrl-0 = <&uart3_pins &bt_pins>; + pinctrl-0 = <&uart3_pins>; status = "okay"; - bluetooth { - compatible = "ti,wl1835-st"; - enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; - }; +// bluetooth { +// compatible = "ti,wl1835-st"; +// enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; +// }; }; &aes { @@ -515,10 +685,20 @@ &rtc { system-power-controller; - clocks = <&clk_32768_ck>, <&l4_per_clkctrl AM3_CLKDIV32K_CLKCTRL 0>; - clock-names = "ext-clk", "int-clk"; }; +&wkup_m3_ipc { + ti,scale-data-fw = "am335x-bone-scale-data.bin"; +}; + +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@4a300000 { +// status = "okay"; +// }; +//}; + &dcan1 { pinctrl-names = "default"; pinctrl-0 = <&dcan1_pins>; @@ -533,3 +713,360 @@ line-name = "LS_BUF_EN"; }; }; + +&ocp { + /* activate the static pinmux helper list of pin modes above */ + test_helper: helper { + compatible = "bone-pinmux-helper"; + pinctrl-names = "default"; + pinctrl-0 = <&mux_helper_pins>; + + status = "okay"; + }; + + + /* UART 2 RX GPS */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm"; + pinctrl-0 = <&P9_22_uart_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_spi_pin>; + pinctrl-5 = <&P9_22_uart_pin>; + pinctrl-6 = <&P9_22_i2c_pin>; + pinctrl-7 = <&P9_22_pwm_pin>; + }; + + /* UART 2 TX GPS*/ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm"; + pinctrl-0 = <&P9_21_uart_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_spi_pin>; + pinctrl-5 = <&P9_21_uart_pin>; + pinctrl-6 = <&P9_21_i2c_pin>; + pinctrl-7 = <&P9_21_pwm_pin>; + }; + + /* UART 1 RX */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c", "pruin"; + pinctrl-0 = <&P9_26_uart_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_uart_pin>; + pinctrl-5 = <&P9_26_can_pin>; + pinctrl-6 = <&P9_26_i2c_pin>; + pinctrl-7 = <&P9_26_pruin_pin>; + }; + + /* UART 1 TX */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c", "pruin"; + pinctrl-0 = <&P9_24_uart_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_uart_pin>; + pinctrl-5 = <&P9_24_can_pin>; + pinctrl-6 = <&P9_24_i2c_pin>; + pinctrl-7 = <&P9_24_pruin_pin>; + }; + + /* SPI MOSI */ + P9_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin"; + pinctrl-0 = <&P9_30_spi_pin>; + pinctrl-1 = <&P9_30_gpio_pin>; + pinctrl-2 = <&P9_30_gpio_pu_pin>; + pinctrl-3 = <&P9_30_gpio_pd_pin>; + pinctrl-4 = <&P9_30_pwm_pin>; + pinctrl-5 = <&P9_30_spi_pin>; + pinctrl-6 = <&P9_30_pruout_pin>; + pinctrl-7 = <&P9_30_pruin_pin>; + }; + + /* SPI MISO */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin"; + pinctrl-0 = <&P9_29_spi_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_pwm_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pruout_pin>; + pinctrl-7 = <&P9_29_pruin_pin>; + }; + + /* SPI SCK */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_sclk", "pruout", "pruin"; + pinctrl-0 = <&P9_31_spi_sclk_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_pwm_pin>; + pinctrl-5 = <&P9_31_spi_pin>; + pinctrl-6 = <&P9_31_spi_sclk_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* Encoder 4 (U13) */ + P8_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "pruin_pu", "gpio", "gpio_pu", "gpio_pd", "pruin", "qep", "pruecapin_pu"; + pinctrl-0 = <&P8_15_pruin_pu_pin>; + pinctrl-1 = <&P8_15_pruin_pu_pin>; + pinctrl-2 = <&P8_15_gpio_pin>; + pinctrl-3 = <&P8_15_gpio_pu_pin>; + pinctrl-4 = <&P8_15_gpio_pd_pin>; + pinctrl-5 = <&P8_15_pruin_pin>; + pinctrl-6 = <&P8_15_qep_pin>; + pinctrl-7 = <&P8_15_pruecapin_pu_pin>; + }; + + /* UART4 RX DSM */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "uart", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&P9_11_uart_pin>; + pinctrl-1 = <&P9_11_uart_pin>; + pinctrl-2 = <&P9_11_gpio_pin>; + pinctrl-3 = <&P9_11_gpio_pu_pin>; + pinctrl-4 = <&P9_11_gpio_pd_pin>; + }; + + /* U16 BLUE_GP0_PIN_3 gpio 1_25*/ + U16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&U16_default_pin>; + pinctrl-1 = <&U16_gpio_pin>; + pinctrl-2 = <&U16_gpio_pu_pin>; + pinctrl-3 = <&U16_gpio_pd_pin>; + }; + + + /* BLUE_GP0_PIN_3 gpio1_17*/ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm"; + pinctrl-0 = <&P9_23_default_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_pwm_pin>; + }; + + /* BLUE_GP0_PIN_5 gpio3_20 */ + D13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&D13_default_pin>; + pinctrl-1 = <&D13_gpio_pin>; + pinctrl-2 = <&D13_gpio_pu_pin>; + pinctrl-3 = <&D13_gpio_pd_pin>; + }; + + /* BLUE_GP0_PIN_6 gpio3_17 */ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_cs", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_default_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_pwm_pin>; + pinctrl-5 = <&P9_28_spi_pin>; + pinctrl-6 = <&P9_28_spi_cs_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* BLUE_GP1_PIN_3 gpio3_2 */ + J15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&J15_default_pin>; + pinctrl-1 = <&J15_gpio_pin>; + pinctrl-2 = <&J15_gpio_pu_pin>; + pinctrl-3 = <&J15_gpio_pd_pin>; + }; + + /* BLUE_GP1_PIN_4 gpio3_1 */ + H17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&H17_default_pin>; + pinctrl-1 = <&H17_gpio_pin>; + pinctrl-2 = <&H17_gpio_pu_pin>; + pinctrl-3 = <&H17_gpio_pd_pin>; + }; + + + //////////////////////////////////////////////////////////////////////// + // Pinmux options only for BB BLue + //////////////////////////////////////////////////////////////////////// + + + + /* H18 SPI1_SS1 default SPI*/ + H18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi"; + pinctrl-0 = <&H18_spi_pin>; + pinctrl-1 = <&H18_gpio_pin>; + pinctrl-2 = <&H18_gpio_pu_pin>; + pinctrl-3 = <&H18_gpio_pd_pin>; + pinctrl-4 = <&H18_spi_pin>; + }; + + + /* C18 SPI1_SS2 default spi*/ + C18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi"; + pinctrl-0 = <&C18_spi_pin>; + pinctrl-1 = <&C18_gpio_pin>; + pinctrl-2 = <&C18_gpio_pu_pin>; + pinctrl-3 = <&C18_gpio_pd_pin>; + pinctrl-4 = <&C18_spi_pin>; + }; + +}; + +/******************************************************************************* +* PWMSS +*******************************************************************************/ +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; +}; + +&ehrpwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&ehrpwm1_pins>; + status = "okay"; +}; + +&ehrpwm2 { + pinctrl-names = "default"; + pinctrl-0 = <&ehrpwm2_pins>; + status = "okay"; +}; + +&ecap0 { status="okay"; }; +&ecap1 { status="okay"; }; +&ecap2 { status="okay"; }; + +/******************************************************************************* +* EQEP +*******************************************************************************/ +&eqep0 { + pinctrl-names = "default"; + pinctrl-0 = <&eqep0_pins>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +&eqep1 { + pinctrl-names = "default"; + pinctrl-0 = <&eqep1_pins>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +&eqep2 { + pinctrl-names = "default"; + pinctrl-0 = <&eqep2_pins>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +/******************************************************************************* + SPI +*******************************************************************************/ +&spi1 { + + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.1"; + reg = <1>; + spi-max-frequency = <24000000>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-common.dtsi b/arch/arm/boot/dts/am335x-bonegreen-common.dtsi index 853e6d3a028dc..7a8826633cef7 100644 --- a/arch/arm/boot/dts/am335x-bonegreen-common.dtsi +++ b/arch/arm/boot/dts/am335x-bonegreen-common.dtsi @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ &ldo3_reg { @@ -27,8 +24,8 @@ &am33xx_pinmux { uart2_pins: uart2_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x950, PIN_INPUT | MUX_MODE1) /* spi0_sclk.uart2_rxd */ - AM33XX_IOPAD(0x954, PIN_OUTPUT | MUX_MODE1) /* spi0_d0.uart2_txd */ + AM33XX_PADCONF(AM335X_PIN_SPI0_SCLK, PIN_INPUT, MUX_MODE1) /* spi0_sclk.uart2_rxd */ + AM33XX_PADCONF(AM335X_PIN_SPI0_D0, PIN_OUTPUT, MUX_MODE1) /* spi0_d0.uart2_txd */ >; }; }; diff --git a/arch/arm/boot/dts/am335x-bonegreen-gateway.dts b/arch/arm/boot/dts/am335x-bonegreen-gateway.dts new file mode 100644 index 0000000000000..5b3a5b8095586 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-gateway.dts @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bonegreen-common.dtsi" +#include +/* #include "am335x-bone-jtag.dtsi" */ + +/ { + model = "SeeedStudio BeagleBone Green Gateway"; + compatible = "ti,am335x-bone-green-gateway", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + aliases { + rtc0 = &extrtc; + rtc1 = &rtc; + }; + + chosen { + base_dtb = "am335x-bonegreen-gateway.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + + wlan_en_reg: fixedregulator@2 { + compatible = "regulator-fixed"; + regulator-name = "wlan-en-regulator"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + startup-delay-us= <70000>; + + /* WL_EN */ + gpio = <&gpio3 9 0>; + enable-active-high; + }; + + leds { + led6 { + label = "beaglebone:green:usr4"; + gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "netdev"; + default-state = "off"; + }; + }; +}; + +&cpu0_opp_table { + /* + * Octavo Systems: + * The EFUSE_SMA register is not programmed for any of the AM335x wafers + * we get and we are not programming them during our production test. + * Therefore, from a DEVICE_ID revision point of view, the silicon looks + * like it is Revision 2.1. However, from an EFUSE_SMA point of view for + * the HW OPP table, the silicon looks like it is Revision 1.0 (ie the + * EFUSE_SMA register reads as all zeros). + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + +&sgx { + status = "okay"; +}; + +&am33xx_pinmux { + pinctrl-names = "default"; + pinctrl-0 = <&usbhost_pins>; + + bt_pins: pinmux_bt_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_txd0.gpio0_28 - BT_EN */ + >; + }; + + mmc3_pins: pinmux_mmc3_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (L15) gmii1_rxd1.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLUP, MUX_MODE6 ) /* (J16) gmii1_txen.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J17) gmii1_rxdv.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (J18) gmii1_txd3.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (K15) gmii1_txd2.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLUP, MUX_MODE5 ) /* (H16) gmii1_col.mmc2_dat3 */ + >; + }; + + uart2_grove_pins: pinmux_uart2_grove_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x90c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) + AM33XX_IOPAD(0x910, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) + >; + }; + + uart3_pins: pinmux_uart3_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* mdio_clk.uart3_rtsn */ + >; + }; + + wl18xx_pins: pinmux_wl18xx_pins { + pinctrl-single,pins = < + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gmii1_txclk.gpio3_9 WL_EN */ + AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) /* rmii1_refclk.gpio0_29 WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gmii1_rxclk.gpio3_10 LS_BUF_EN */ + >; + }; + + usbhost_pins: pinmux_usbhost_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gmii1_txd1.gpio0[21] */ + >; + }; +}; + +&mac { + status = "disabled"; +}; + +&mmc3 { + dmas = <&edma_xbar 12 0 1 + &edma_xbar 13 0 2>; + dma-names = "tx", "rx"; + status = "okay"; + vmmc-supply = <&wlan_en_reg>; + bus-width = <4>; + non-removable; + cap-power-off-card; + ti,needs-special-hs-handling; + keep-power-in-suspend; + pinctrl-names = "default"; + pinctrl-0 = <&mmc3_pins &wl18xx_pins>; + + #address-cells = <1>; + #size-cells = <0>; + wlcore: wlcore@2 { + compatible = "ti,wl1835"; + reg = <2>; + interrupt-parent = <&gpio0>; + interrupts = <29 IRQ_TYPE_EDGE_RISING>; + }; +}; + +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <&uart2_grove_pins>; + status = "okay"; +}; + +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&uart3_pins &bt_pins>; + status = "okay"; + + bluetooth { + compatible = "ti,wl1835-st"; + enable-gpios = <&gpio0 28 GPIO_ACTIVE_HIGH>; + }; +}; + +&i2c0 { + extrtc: rtc@68 { + compatible = "dallas,ds1340"; + reg = <0x68>; + }; +}; + +// (K16) gmii1_txd1.gpio0[21] +&gpio0 { + usb_reset { + gpio-hog; + gpios = <21 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "usb_reset"; + }; +}; + +&gpio3 { + ls_buf_en { + gpio-hog; + gpios = <10 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "LS_BUF_EN"; + }; +}; + +&usb1 { + #address-cells = <1>; + #size-cells = <0>; + + hub@1 { + compatible = "usb424,9512"; + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + ethernet: ethernet@1 { + compatible = "usb424,ec00"; + reg = <1>; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi b/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi new file mode 100644 index 0000000000000..019f90e02e3f1 --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless-common-univ.dtsi @@ -0,0 +1,2835 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ + +&am33xx_pinmux { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0818, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad6.gpio1_6 */ + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x081c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad7.gpio1_7 */ + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0808, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad2.gpio1_2 */ + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x080c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad3.gpio1_3 */ + + /* P8_07 (ZCZ ball R7) gpio2_2 */ + P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_advn_ale.gpio2_2 */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0890, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_advn_ale.timer4 */ + + /* P8_08 (ZCZ ball T7) gpio2_3 */ + P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_oen_ren.gpio2_3 */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0894, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_oen_ren.timer7 */ + + /* P8_09 (ZCZ ball T6) gpio2_5 */ + P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be0n_cle.gpio2_5 */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x089c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_be0n_cle.timer5 */ + + /* P8_10 (ZCZ ball U6) gpio2_4 */ + P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wen.gpio2_4 */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0898, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* gpmc_wen.timer6 */ + + /* P8_11 (ZCZ ball R12) gpio1_13 */ + P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P8_12 (ZCZ ball T12) gpio1_12 */ + P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P8_13 (ZCZ ball T10) gpio0_23 */ + P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P8_14 (ZCZ ball T11) wl1835: wl_en */ + + /* P8_15 (ZCZ ball U13) gpio1_15 */ + P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P8_15_qep_pin: pinmux_P8_15_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P8_16 (ZCZ ball V13) gpio1_14 */ + P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P8_16_qep_pin: pinmux_P8_16_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P8_17 (ZCZ ball U12) wl1835: wl_irq */ + + /* P8_18 (ZCZ ball V12) gpio2_1 */ + P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P8_19 (ZCZ ball U10) gpio0_22 */ + P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad8.gpio0_22 */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0820, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad8.ehrpwm2a */ + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn2.gpio1_31 */ + P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn2.pru1_out13 */ + P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0884, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn2.pru1_in13 */ + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn1.gpio1_30 */ + P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_csn1.pru1_out12 */ + P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0880, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_csn1.pru1_in12 */ + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0814, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad5.gpio1_5 */ + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0810, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad4.gpio1_4 */ + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0804, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad1.gpio1_1 */ + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0800, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad0.gpio1_0 */ + + /* P8_26 (ZCZ ball V6) gpio-hog wl1835 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data14.gpio0_10 */ + P8_31_qep_pin: pinmux_P8_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data14.eqep1_index */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data14.uart5_rxd */ + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data15.gpio0_11 */ + P8_32_qep_pin: pinmux_P8_32_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08dc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data15.eqep1_strobe */ + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data13.gpio0_9 */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data13.eqep1b_in */ + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data11.gpio2_17 */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08cc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data11.ehrpwm1b */ + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data12.gpio0_8 */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08d0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* lcd_data12.eqep1a_in */ + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data10.gpio2_16 */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data10.ehrpwm1a */ + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data8.gpio2_14 */ + P8_37_pwm_pin: pinmux_P8_37_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data8.ehrpwm1_tripzone_input */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data8.uart5_txd */ + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data9.gpio2_15 */ + P8_38_pwm_pin: pinmux_P8_38_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE2) >; }; /* lcd_data9.ehrpwm0_synco */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08c4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* lcd_data9.uart5_rxd */ + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data6.gpio2_12 */ + P8_39_qep_pin: pinmux_P8_39_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data6.eqep2_index */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data6.pru1_out6 */ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data6.pru1_in6 */ + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data7.gpio2_13 */ + P8_40_qep_pin: pinmux_P8_40_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data7.eqep2_strobe */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data7.pru1_out7 */ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08bc, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data7.pru1_in7 */ + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data4.gpio2_10 */ + P8_41_qep_pin: pinmux_P8_41_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data4.eqep2a_in */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data4.pru1_out4 */ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data4.pru1_in4 */ + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data5.gpio2_11 */ + P8_42_qep_pin: pinmux_P8_42_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* lcd_data5.eqep2b_in */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data5.pru1_out5 */ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08b4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data5.pru1_in5 */ + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data2.gpio2_8 */ + P8_43_pwm_pin: pinmux_P8_43_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data2.ehrpwm2_tripzone_input */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data2.pru1_out2 */ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data2.pru1_in2 */ + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data3.gpio2_9 */ + P8_44_pwm_pin: pinmux_P8_44_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data3.ehrpwm0_synco */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data3.pru1_out3 */ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ac, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data3.pru1_in3 */ + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data0.gpio2_6 */ + P8_45_pwm_pin: pinmux_P8_45_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data0.ehrpwm2a */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data0.pru1_out0 */ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data0.pru1_in0 */ + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_data1.gpio2_7 */ + P8_46_pwm_pin: pinmux_P8_46_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* lcd_data1.ehrpwm2b */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_data1.pru1_out1 */ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08a4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_data1.pru1_in1 */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) gpio0_30 */ + P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P9_12 (ZCZ ball U18) gpio1_28 */ + P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P9_13 (ZCZ ball U17) gpio0_31 */ + P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P9_14 (ZCZ ball U14) gpio1_18 */ + P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P9_15 (ZCZ ball R13) gpio1_16 */ + P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a0.gpio1_16 */ + P9_15_pwm_pin: pinmux_P9_15_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0840, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a0.ehrpwm1_tripzone_input */ + + /* P9_16 (ZCZ ball T14) gpio1_19 */ + P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a3.gpio1_19 */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x084c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a3.ehrpwm1b */ + + /* P9_17 (ZCZ ball A16) gpio0_5 */ + P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P9_17_pwm_pin: pinmux_P9_17_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P9_17_pru_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P9_18 (ZCZ ball B16) gpio0_4 */ + P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P9_18_pwm_pin: pinmux_P9_18_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P9_18_pru_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P9_19 (ZCZ ball D17) i2c2_scl */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P9_19_timer_pin: pinmux_P9_19_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_rtsn.timer5 */ + P9_19_can_pin: pinmux_P9_19_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P9_19_spi_cs_pin: pinmux_P9_19_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P9_19_pru_uart_pin: pinmux_P9_19_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P9_20 (ZCZ ball D18) i2c2_sda */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P9_20_timer_pin: pinmux_P9_20_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart1_ctsn.timer6 */ + P9_20_can_pin: pinmux_P9_20_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P9_20_spi_cs_pin: pinmux_P9_20_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P9_20_pru_uart_pin: pinmux_P9_20_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P9_21 (ZCZ ball B17) gpio0_3 */ + P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P9_21_i2c_pin: pinmux_P9_21_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P9_21_pwm_pin: pinmux_P9_21_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P9_21_pru_uart_pin: pinmux_P9_21_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P9_22 (ZCZ ball A17) gpio0_2 */ + P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P9_22_i2c_pin: pinmux_P9_22_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P9_22_pwm_pin: pinmux_P9_22_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P9_22_pru_uart_pin: pinmux_P9_22_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P9_23 (ZCZ ball V14) gpio1_17 */ + P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a1.gpio1_17 */ + P9_23_pwm_pin: pinmux_P9_23_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0844, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a1.ehrpwm0_synco */ + + /* P9_24 (ZCZ ball D15) gpio0_15 */ + P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P9_24_pru_uart_pin: pinmux_P9_24_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P9_24_pruin_pin: pinmux_P9_24_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P9_25_qep_pin: pinmux_P9_25_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P9_26 (ZCZ ball D16) gpio0_14 */ + P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P9_26_pru_uart_pin: pinmux_P9_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P9_27 (ZCZ ball C13) gpio3_19 */ + P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P9_28_pwm_pin: pinmux_P9_28_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P9_28_pwm2_pin: pinmux_P9_28_pwm2_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* mcasp0_ahclkr.ecap2_in_pwm2_out */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P9_29_pwm_pin: pinmux_P9_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P9_30 (ZCZ ball D12) gpio-hog wl1835 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P9_31_pwm_pin: pinmux_P9_31_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) gpio0_20 */ + P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P9_41_timer_pin: pinmux_P9_41_timer_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr1.timer7 */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) gpio3_20 */ + P9_91_default_pin: pinmux_P9_91_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pin: pinmux_P9_91_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pu_pin: pinmux_P9_91_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_pd_pin: pinmux_P9_91_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_gpio_input_pin: pinmux_P9_91_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P9_91_qep_pin: pinmux_P9_91_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P9_91_pruout_pin: pinmux_P9_91_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P9_91_pruin_pin: pinmux_P9_91_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P9_42 (ZCZ ball C18) gpio0_7 */ + P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P9_42_pwm_pin: pinmux_P9_42_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P9_42_uart_pin: pinmux_P9_42_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P9_42_pru_ecap_pin: pinmux_P9_42_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P9_42_spi_sclk_pin: pinmux_P9_42_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) gpio3_18 */ + P9_92_default_pin: pinmux_P9_92_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pin: pinmux_P9_92_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pu_pin: pinmux_P9_92_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_pd_pin: pinmux_P9_92_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_gpio_input_pin: pinmux_P9_92_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P9_92_qep_pin: pinmux_P9_92_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P9_92_pruout_pin: pinmux_P9_92_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P9_92_pruin_pin: pinmux_P9_92_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_default_pin: pinmux_A15_default_pin { + pinctrl-single,pins = <0x1b0 (PIN_OUTPUT_PULLUP | MUX_MODE7)>; }; /* Mode 7, Pull-Up */ + A15_clkout_pin: pinmux_A15_clkout_pin { + pinctrl-single,pins = <0x1b0 0x0b>; }; /* Mode 3 */ + A15_gpio_pin: pinmux_A15_gpio_pin { + pinctrl-single,pins = <0x1b0 0x2f>; }; /* Mode 7, RxActive */ + A15_gpio_pu_pin: pinmux_A15_gpio_pu_pin { + pinctrl-single,pins = <0x1b0 0x37>; }; /* Mode 7, Pull-Up, RxActive */ + A15_gpio_pd_pin: pinmux_A15_gpio_pd_pin { + pinctrl-single,pins = <0x1b0 0x27>; }; /* Mode 7, Pull-Down, RxActive */ +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/1"; +}; + +&i2c2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + clock-frequency = <100000>; + symlink = "bone/i2c/2"; +}; + +&uart1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/1"; +}; + +&uart2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/2"; +}; + +&uart3 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/3"; +}; + +&uart4 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/4"; +}; + +&uart5 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/uart/5"; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/0"; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + symlink = "bone/can/1"; +}; + +&eqep0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/0"; +}; + +&eqep1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/1"; +}; + +&eqep2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + symlink = "bone/eqep/2"; +}; + +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ecap2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&spi0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/0.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +&spi1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +/**********************************************************************/ +/* Pin Multiplex Helpers */ +/* */ +/* These provide userspace runtime pin configuration for the */ +/* BeagleBone cape expansion headers */ +/**********************************************************************/ + +&ocp { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ZCZ ball R9) emmc */ + P8_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_03_default_pin>; + pinctrl-1 = <&P8_03_gpio_pin>; + pinctrl-2 = <&P8_03_gpio_pu_pin>; + pinctrl-3 = <&P8_03_gpio_pd_pin>; + pinctrl-4 = <&P8_03_gpio_input_pin>; + }; + + /* P8_04 (ZCZ ball T9) emmc */ + P8_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_04_default_pin>; + pinctrl-1 = <&P8_04_gpio_pin>; + pinctrl-2 = <&P8_04_gpio_pu_pin>; + pinctrl-3 = <&P8_04_gpio_pd_pin>; + pinctrl-4 = <&P8_04_gpio_input_pin>; + }; + + /* P8_05 (ZCZ ball R8) emmc */ + P8_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_05_default_pin>; + pinctrl-1 = <&P8_05_gpio_pin>; + pinctrl-2 = <&P8_05_gpio_pu_pin>; + pinctrl-3 = <&P8_05_gpio_pd_pin>; + pinctrl-4 = <&P8_05_gpio_input_pin>; + }; + + /* P8_06 (ZCZ ball T8) emmc */ + P8_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_06_default_pin>; + pinctrl-1 = <&P8_06_gpio_pin>; + pinctrl-2 = <&P8_06_gpio_pu_pin>; + pinctrl-3 = <&P8_06_gpio_pd_pin>; + pinctrl-4 = <&P8_06_gpio_input_pin>; + }; + + /* P8_07 (ZCZ ball R7) */ + P8_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_07_default_pin>; + pinctrl-1 = <&P8_07_gpio_pin>; + pinctrl-2 = <&P8_07_gpio_pu_pin>; + pinctrl-3 = <&P8_07_gpio_pd_pin>; + pinctrl-4 = <&P8_07_gpio_input_pin>; + pinctrl-5 = <&P8_07_timer_pin>; + }; + + /* P8_08 (ZCZ ball T7) */ + P8_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_08_default_pin>; + pinctrl-1 = <&P8_08_gpio_pin>; + pinctrl-2 = <&P8_08_gpio_pu_pin>; + pinctrl-3 = <&P8_08_gpio_pd_pin>; + pinctrl-4 = <&P8_08_gpio_input_pin>; + pinctrl-5 = <&P8_08_timer_pin>; + }; + + /* P8_09 (ZCZ ball T6) */ + P8_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_09_default_pin>; + pinctrl-1 = <&P8_09_gpio_pin>; + pinctrl-2 = <&P8_09_gpio_pu_pin>; + pinctrl-3 = <&P8_09_gpio_pd_pin>; + pinctrl-4 = <&P8_09_gpio_input_pin>; + pinctrl-5 = <&P8_09_timer_pin>; + }; + + /* P8_10 (ZCZ ball U6) */ + P8_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer"; + pinctrl-0 = <&P8_10_default_pin>; + pinctrl-1 = <&P8_10_gpio_pin>; + pinctrl-2 = <&P8_10_gpio_pu_pin>; + pinctrl-3 = <&P8_10_gpio_pd_pin>; + pinctrl-4 = <&P8_10_gpio_input_pin>; + pinctrl-5 = <&P8_10_timer_pin>; + }; + + /* P8_11 (ZCZ ball R12) */ + P8_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_11_default_pin>; + pinctrl-1 = <&P8_11_gpio_pin>; + pinctrl-2 = <&P8_11_gpio_pu_pin>; + pinctrl-3 = <&P8_11_gpio_pd_pin>; + pinctrl-4 = <&P8_11_gpio_input_pin>; + pinctrl-5 = <&P8_11_qep_pin>; + pinctrl-6 = <&P8_11_pruout_pin>; + }; + + /* P8_12 (ZCZ ball T12) */ + P8_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P8_12_default_pin>; + pinctrl-1 = <&P8_12_gpio_pin>; + pinctrl-2 = <&P8_12_gpio_pu_pin>; + pinctrl-3 = <&P8_12_gpio_pd_pin>; + pinctrl-4 = <&P8_12_gpio_input_pin>; + pinctrl-5 = <&P8_12_qep_pin>; + pinctrl-6 = <&P8_12_pruout_pin>; + }; + + /* P8_13 (ZCZ ball T10) */ + P8_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_13_default_pin>; + pinctrl-1 = <&P8_13_gpio_pin>; + pinctrl-2 = <&P8_13_gpio_pu_pin>; + pinctrl-3 = <&P8_13_gpio_pd_pin>; + pinctrl-4 = <&P8_13_gpio_input_pin>; + pinctrl-5 = <&P8_13_pwm_pin>; + }; + + /* P8_14 (ZCZ ball T11) wl1835: wl_en */ + + /* P8_15 (ZCZ ball U13) */ + P8_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P8_15_default_pin>; + pinctrl-1 = <&P8_15_gpio_pin>; + pinctrl-2 = <&P8_15_gpio_pu_pin>; + pinctrl-3 = <&P8_15_gpio_pd_pin>; + pinctrl-4 = <&P8_15_gpio_input_pin>; + pinctrl-5 = <&P8_15_qep_pin>; + pinctrl-6 = <&P8_15_pru_ecap_pin>; + pinctrl-7 = <&P8_15_pruin_pin>; + }; + + /* P8_16 (ZCZ ball V13) */ + P8_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P8_16_default_pin>; + pinctrl-1 = <&P8_16_gpio_pin>; + pinctrl-2 = <&P8_16_gpio_pu_pin>; + pinctrl-3 = <&P8_16_gpio_pd_pin>; + pinctrl-4 = <&P8_16_gpio_input_pin>; + pinctrl-5 = <&P8_16_qep_pin>; + pinctrl-6 = <&P8_16_pruin_pin>; + }; + + /* P8_17 (ZCZ ball U12) wl1835: wl_irq */ + + /* P8_18 (ZCZ ball V12) */ + P8_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_18_default_pin>; + pinctrl-1 = <&P8_18_gpio_pin>; + pinctrl-2 = <&P8_18_gpio_pu_pin>; + pinctrl-3 = <&P8_18_gpio_pd_pin>; + pinctrl-4 = <&P8_18_gpio_input_pin>; + }; + + /* P8_19 (ZCZ ball U10) */ + P8_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_19_default_pin>; + pinctrl-1 = <&P8_19_gpio_pin>; + pinctrl-2 = <&P8_19_gpio_pu_pin>; + pinctrl-3 = <&P8_19_gpio_pd_pin>; + pinctrl-4 = <&P8_19_gpio_input_pin>; + pinctrl-5 = <&P8_19_pwm_pin>; + }; + + /* P8_20 (ZCZ ball V9) emmc */ + P8_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_20_default_pin>; + pinctrl-1 = <&P8_20_gpio_pin>; + pinctrl-2 = <&P8_20_gpio_pu_pin>; + pinctrl-3 = <&P8_20_gpio_pd_pin>; + pinctrl-4 = <&P8_20_gpio_input_pin>; + pinctrl-5 = <&P8_20_pruout_pin>; + pinctrl-6 = <&P8_20_pruin_pin>; + }; + + /* P8_21 (ZCZ ball U9) emmc */ + P8_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_21_default_pin>; + pinctrl-1 = <&P8_21_gpio_pin>; + pinctrl-2 = <&P8_21_gpio_pu_pin>; + pinctrl-3 = <&P8_21_gpio_pd_pin>; + pinctrl-4 = <&P8_21_gpio_input_pin>; + pinctrl-5 = <&P8_21_pruout_pin>; + pinctrl-6 = <&P8_21_pruin_pin>; + }; + + /* P8_22 (ZCZ ball V8) emmc */ + P8_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_22_default_pin>; + pinctrl-1 = <&P8_22_gpio_pin>; + pinctrl-2 = <&P8_22_gpio_pu_pin>; + pinctrl-3 = <&P8_22_gpio_pd_pin>; + pinctrl-4 = <&P8_22_gpio_input_pin>; + }; + + /* P8_23 (ZCZ ball U8) emmc */ + P8_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_23_default_pin>; + pinctrl-1 = <&P8_23_gpio_pin>; + pinctrl-2 = <&P8_23_gpio_pu_pin>; + pinctrl-3 = <&P8_23_gpio_pd_pin>; + pinctrl-4 = <&P8_23_gpio_input_pin>; + }; + + /* P8_24 (ZCZ ball V7) emmc */ + P8_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_24_default_pin>; + pinctrl-1 = <&P8_24_gpio_pin>; + pinctrl-2 = <&P8_24_gpio_pu_pin>; + pinctrl-3 = <&P8_24_gpio_pd_pin>; + pinctrl-4 = <&P8_24_gpio_input_pin>; + }; + + /* P8_25 (ZCZ ball U7) emmc */ + P8_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P8_25_default_pin>; + pinctrl-1 = <&P8_25_gpio_pin>; + pinctrl-2 = <&P8_25_gpio_pu_pin>; + pinctrl-3 = <&P8_25_gpio_pd_pin>; + pinctrl-4 = <&P8_25_gpio_input_pin>; + }; + + /* P8_26 (ZCZ ball V6) gpio-hog wl1835 */ + + /* P8_27 (ZCZ ball U5) hdmi */ + P8_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_27_default_pin>; + pinctrl-1 = <&P8_27_gpio_pin>; + pinctrl-2 = <&P8_27_gpio_pu_pin>; + pinctrl-3 = <&P8_27_gpio_pd_pin>; + pinctrl-4 = <&P8_27_gpio_input_pin>; + pinctrl-5 = <&P8_27_pruout_pin>; + pinctrl-6 = <&P8_27_pruin_pin>; + }; + + /* P8_28 (ZCZ ball V5) hdmi */ + P8_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_28_default_pin>; + pinctrl-1 = <&P8_28_gpio_pin>; + pinctrl-2 = <&P8_28_gpio_pu_pin>; + pinctrl-3 = <&P8_28_gpio_pd_pin>; + pinctrl-4 = <&P8_28_gpio_input_pin>; + pinctrl-5 = <&P8_28_pruout_pin>; + pinctrl-6 = <&P8_28_pruin_pin>; + }; + + /* P8_29 (ZCZ ball R5) hdmi */ + P8_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_29_default_pin>; + pinctrl-1 = <&P8_29_gpio_pin>; + pinctrl-2 = <&P8_29_gpio_pu_pin>; + pinctrl-3 = <&P8_29_gpio_pd_pin>; + pinctrl-4 = <&P8_29_gpio_input_pin>; + pinctrl-5 = <&P8_29_pruout_pin>; + pinctrl-6 = <&P8_29_pruin_pin>; + }; + + /* P8_30 (ZCZ ball R6) hdmi */ + P8_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P8_30_default_pin>; + pinctrl-1 = <&P8_30_gpio_pin>; + pinctrl-2 = <&P8_30_gpio_pu_pin>; + pinctrl-3 = <&P8_30_gpio_pd_pin>; + pinctrl-4 = <&P8_30_gpio_input_pin>; + pinctrl-5 = <&P8_30_pruout_pin>; + pinctrl-6 = <&P8_30_pruin_pin>; + }; + + /* P8_31 (ZCZ ball V4) hdmi */ + P8_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "qep"; + pinctrl-0 = <&P8_31_default_pin>; + pinctrl-1 = <&P8_31_gpio_pin>; + pinctrl-2 = <&P8_31_gpio_pu_pin>; + pinctrl-3 = <&P8_31_gpio_pd_pin>; + pinctrl-4 = <&P8_31_gpio_input_pin>; + pinctrl-5 = <&P8_31_uart_pin>; + pinctrl-6 = <&P8_31_qep_pin>; + }; + + /* P8_32 (ZCZ ball T5) hdmi */ + P8_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_32_default_pin>; + pinctrl-1 = <&P8_32_gpio_pin>; + pinctrl-2 = <&P8_32_gpio_pu_pin>; + pinctrl-3 = <&P8_32_gpio_pd_pin>; + pinctrl-4 = <&P8_32_gpio_input_pin>; + pinctrl-5 = <&P8_32_qep_pin>; + }; + + /* P8_33 (ZCZ ball V3) hdmi */ + P8_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_33_default_pin>; + pinctrl-1 = <&P8_33_gpio_pin>; + pinctrl-2 = <&P8_33_gpio_pu_pin>; + pinctrl-3 = <&P8_33_gpio_pd_pin>; + pinctrl-4 = <&P8_33_gpio_input_pin>; + pinctrl-5 = <&P8_33_qep_pin>; + }; + + /* P8_34 (ZCZ ball U4) hdmi */ + P8_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_34_default_pin>; + pinctrl-1 = <&P8_34_gpio_pin>; + pinctrl-2 = <&P8_34_gpio_pu_pin>; + pinctrl-3 = <&P8_34_gpio_pd_pin>; + pinctrl-4 = <&P8_34_gpio_input_pin>; + pinctrl-5 = <&P8_34_pwm_pin>; + }; + + /* P8_35 (ZCZ ball V2) hdmi */ + P8_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P8_35_default_pin>; + pinctrl-1 = <&P8_35_gpio_pin>; + pinctrl-2 = <&P8_35_gpio_pu_pin>; + pinctrl-3 = <&P8_35_gpio_pd_pin>; + pinctrl-4 = <&P8_35_gpio_input_pin>; + pinctrl-5 = <&P8_35_qep_pin>; + }; + + /* P8_36 (ZCZ ball U3) hdmi */ + P8_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P8_36_default_pin>; + pinctrl-1 = <&P8_36_gpio_pin>; + pinctrl-2 = <&P8_36_gpio_pu_pin>; + pinctrl-3 = <&P8_36_gpio_pd_pin>; + pinctrl-4 = <&P8_36_gpio_input_pin>; + pinctrl-5 = <&P8_36_pwm_pin>; + }; + + /* P8_37 (ZCZ ball U1) hdmi */ + P8_37_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_37_default_pin>; + pinctrl-1 = <&P8_37_gpio_pin>; + pinctrl-2 = <&P8_37_gpio_pu_pin>; + pinctrl-3 = <&P8_37_gpio_pd_pin>; + pinctrl-4 = <&P8_37_gpio_input_pin>; + pinctrl-5 = <&P8_37_uart_pin>; + pinctrl-6 = <&P8_37_pwm_pin>; + }; + + /* P8_38 (ZCZ ball U2) hdmi */ + P8_38_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "pwm"; + pinctrl-0 = <&P8_38_default_pin>; + pinctrl-1 = <&P8_38_gpio_pin>; + pinctrl-2 = <&P8_38_gpio_pu_pin>; + pinctrl-3 = <&P8_38_gpio_pd_pin>; + pinctrl-4 = <&P8_38_gpio_input_pin>; + pinctrl-5 = <&P8_38_uart_pin>; + pinctrl-6 = <&P8_38_pwm_pin>; + }; + + /* P8_39 (ZCZ ball T3) hdmi */ + P8_39_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_39_default_pin>; + pinctrl-1 = <&P8_39_gpio_pin>; + pinctrl-2 = <&P8_39_gpio_pu_pin>; + pinctrl-3 = <&P8_39_gpio_pd_pin>; + pinctrl-4 = <&P8_39_gpio_input_pin>; + pinctrl-5 = <&P8_39_qep_pin>; + pinctrl-6 = <&P8_39_pruout_pin>; + pinctrl-7 = <&P8_39_pruin_pin>; + }; + + /* P8_40 (ZCZ ball T4) hdmi */ + P8_40_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_40_default_pin>; + pinctrl-1 = <&P8_40_gpio_pin>; + pinctrl-2 = <&P8_40_gpio_pu_pin>; + pinctrl-3 = <&P8_40_gpio_pd_pin>; + pinctrl-4 = <&P8_40_gpio_input_pin>; + pinctrl-5 = <&P8_40_qep_pin>; + pinctrl-6 = <&P8_40_pruout_pin>; + pinctrl-7 = <&P8_40_pruin_pin>; + }; + + /* P8_41 (ZCZ ball T1) hdmi */ + P8_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_41_default_pin>; + pinctrl-1 = <&P8_41_gpio_pin>; + pinctrl-2 = <&P8_41_gpio_pu_pin>; + pinctrl-3 = <&P8_41_gpio_pd_pin>; + pinctrl-4 = <&P8_41_gpio_input_pin>; + pinctrl-5 = <&P8_41_qep_pin>; + pinctrl-6 = <&P8_41_pruout_pin>; + pinctrl-7 = <&P8_41_pruin_pin>; + }; + + /* P8_42 (ZCZ ball T2) hdmi */ + P8_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P8_42_default_pin>; + pinctrl-1 = <&P8_42_gpio_pin>; + pinctrl-2 = <&P8_42_gpio_pu_pin>; + pinctrl-3 = <&P8_42_gpio_pd_pin>; + pinctrl-4 = <&P8_42_gpio_input_pin>; + pinctrl-5 = <&P8_42_qep_pin>; + pinctrl-6 = <&P8_42_pruout_pin>; + pinctrl-7 = <&P8_42_pruin_pin>; + }; + + /* P8_43 (ZCZ ball R3) hdmi */ + P8_43_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_43_default_pin>; + pinctrl-1 = <&P8_43_gpio_pin>; + pinctrl-2 = <&P8_43_gpio_pu_pin>; + pinctrl-3 = <&P8_43_gpio_pd_pin>; + pinctrl-4 = <&P8_43_gpio_input_pin>; + pinctrl-5 = <&P8_43_pwm_pin>; + pinctrl-6 = <&P8_43_pruout_pin>; + pinctrl-7 = <&P8_43_pruin_pin>; + }; + + /* P8_44 (ZCZ ball R4) hdmi */ + P8_44_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_44_default_pin>; + pinctrl-1 = <&P8_44_gpio_pin>; + pinctrl-2 = <&P8_44_gpio_pu_pin>; + pinctrl-3 = <&P8_44_gpio_pd_pin>; + pinctrl-4 = <&P8_44_gpio_input_pin>; + pinctrl-5 = <&P8_44_pwm_pin>; + pinctrl-6 = <&P8_44_pruout_pin>; + pinctrl-7 = <&P8_44_pruin_pin>; + }; + + /* P8_45 (ZCZ ball R1) hdmi */ + P8_45_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_45_default_pin>; + pinctrl-1 = <&P8_45_gpio_pin>; + pinctrl-2 = <&P8_45_gpio_pu_pin>; + pinctrl-3 = <&P8_45_gpio_pd_pin>; + pinctrl-4 = <&P8_45_gpio_input_pin>; + pinctrl-5 = <&P8_45_pwm_pin>; + pinctrl-6 = <&P8_45_pruout_pin>; + pinctrl-7 = <&P8_45_pruin_pin>; + }; + + /* P8_46 (ZCZ ball R2) hdmi */ + P8_46_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P8_46_default_pin>; + pinctrl-1 = <&P8_46_gpio_pin>; + pinctrl-2 = <&P8_46_gpio_pu_pin>; + pinctrl-3 = <&P8_46_gpio_pd_pin>; + pinctrl-4 = <&P8_46_gpio_input_pin>; + pinctrl-5 = <&P8_46_pwm_pin>; + pinctrl-6 = <&P8_46_pruout_pin>; + pinctrl-7 = <&P8_46_pruin_pin>; + }; + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11 (ZCZ ball T17) */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_11_default_pin>; + pinctrl-1 = <&P9_11_gpio_pin>; + pinctrl-2 = <&P9_11_gpio_pu_pin>; + pinctrl-3 = <&P9_11_gpio_pd_pin>; + pinctrl-4 = <&P9_11_gpio_input_pin>; + pinctrl-5 = <&P9_11_uart_pin>; + }; + + /* P9_12 (ZCZ ball U18) */ + P9_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P9_12_default_pin>; + pinctrl-1 = <&P9_12_gpio_pin>; + pinctrl-2 = <&P9_12_gpio_pu_pin>; + pinctrl-3 = <&P9_12_gpio_pd_pin>; + pinctrl-4 = <&P9_12_gpio_input_pin>; + }; + + /* P9_13 (ZCZ ball U17) */ + P9_13_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P9_13_default_pin>; + pinctrl-1 = <&P9_13_gpio_pin>; + pinctrl-2 = <&P9_13_gpio_pu_pin>; + pinctrl-3 = <&P9_13_gpio_pd_pin>; + pinctrl-4 = <&P9_13_gpio_input_pin>; + pinctrl-5 = <&P9_13_uart_pin>; + }; + + /* P9_14 (ZCZ ball U14) */ + P9_14_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_14_default_pin>; + pinctrl-1 = <&P9_14_gpio_pin>; + pinctrl-2 = <&P9_14_gpio_pu_pin>; + pinctrl-3 = <&P9_14_gpio_pd_pin>; + pinctrl-4 = <&P9_14_gpio_input_pin>; + pinctrl-5 = <&P9_14_pwm_pin>; + }; + + /* P9_15 (ZCZ ball R13) */ + P9_15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_15_default_pin>; + pinctrl-1 = <&P9_15_gpio_pin>; + pinctrl-2 = <&P9_15_gpio_pu_pin>; + pinctrl-3 = <&P9_15_gpio_pd_pin>; + pinctrl-4 = <&P9_15_gpio_input_pin>; + pinctrl-5 = <&P9_15_pwm_pin>; + }; + + /* P9_16 (ZCZ ball T14) */ + P9_16_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_16_default_pin>; + pinctrl-1 = <&P9_16_gpio_pin>; + pinctrl-2 = <&P9_16_gpio_pu_pin>; + pinctrl-3 = <&P9_16_gpio_pd_pin>; + pinctrl-4 = <&P9_16_gpio_input_pin>; + pinctrl-5 = <&P9_16_pwm_pin>; + }; + + /* P9_17 (ZCZ ball A16) */ + P9_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_17_default_pin>; + pinctrl-1 = <&P9_17_gpio_pin>; + pinctrl-2 = <&P9_17_gpio_pu_pin>; + pinctrl-3 = <&P9_17_gpio_pd_pin>; + pinctrl-4 = <&P9_17_gpio_input_pin>; + pinctrl-5 = <&P9_17_spi_cs_pin>; + pinctrl-6 = <&P9_17_i2c_pin>; + pinctrl-7 = <&P9_17_pwm_pin>; + pinctrl-8 = <&P9_17_pru_uart_pin>; + }; + + /* P9_18 (ZCZ ball B16) */ + P9_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_18_default_pin>; + pinctrl-1 = <&P9_18_gpio_pin>; + pinctrl-2 = <&P9_18_gpio_pu_pin>; + pinctrl-3 = <&P9_18_gpio_pd_pin>; + pinctrl-4 = <&P9_18_gpio_input_pin>; + pinctrl-5 = <&P9_18_spi_pin>; + pinctrl-6 = <&P9_18_i2c_pin>; + pinctrl-7 = <&P9_18_pwm_pin>; + pinctrl-8 = <&P9_18_pru_uart_pin>; + }; + + /* P9_19 (ZCZ ball D17) i2c */ + P9_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_19_default_pin>; + pinctrl-1 = <&P9_19_gpio_pin>; + pinctrl-2 = <&P9_19_gpio_pu_pin>; + pinctrl-3 = <&P9_19_gpio_pd_pin>; + pinctrl-4 = <&P9_19_gpio_input_pin>; + pinctrl-5 = <&P9_19_spi_cs_pin>; + pinctrl-6 = <&P9_19_can_pin>; + pinctrl-7 = <&P9_19_i2c_pin>; + pinctrl-8 = <&P9_19_pru_uart_pin>; + pinctrl-9 = <&P9_19_timer_pin>; + }; + + /* P9_20 (ZCZ ball D18) i2c */ + P9_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart", "timer"; + pinctrl-0 = <&P9_20_default_pin>; + pinctrl-1 = <&P9_20_gpio_pin>; + pinctrl-2 = <&P9_20_gpio_pu_pin>; + pinctrl-3 = <&P9_20_gpio_pd_pin>; + pinctrl-4 = <&P9_20_gpio_input_pin>; + pinctrl-5 = <&P9_20_spi_cs_pin>; + pinctrl-6 = <&P9_20_can_pin>; + pinctrl-7 = <&P9_20_i2c_pin>; + pinctrl-8 = <&P9_20_pru_uart_pin>; + pinctrl-9 = <&P9_20_timer_pin>; + }; + + /* P9_21 (ZCZ ball B17) */ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_21_default_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_gpio_input_pin>; + pinctrl-5 = <&P9_21_spi_pin>; + pinctrl-6 = <&P9_21_uart_pin>; + pinctrl-7 = <&P9_21_i2c_pin>; + pinctrl-8 = <&P9_21_pwm_pin>; + pinctrl-9 = <&P9_21_pru_uart_pin>; + }; + + /* P9_22 (ZCZ ball A17) */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P9_22_default_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_gpio_input_pin>; + pinctrl-5 = <&P9_22_spi_sclk_pin>; + pinctrl-6 = <&P9_22_uart_pin>; + pinctrl-7 = <&P9_22_i2c_pin>; + pinctrl-8 = <&P9_22_pwm_pin>; + pinctrl-9 = <&P9_22_pru_uart_pin>; + }; + + /* P9_23 (ZCZ ball V14) */ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P9_23_default_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_gpio_input_pin>; + pinctrl-5 = <&P9_23_pwm_pin>; + }; + + /* P9_24 (ZCZ ball D15) */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_24_default_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_gpio_input_pin>; + pinctrl-5 = <&P9_24_uart_pin>; + pinctrl-6 = <&P9_24_can_pin>; + pinctrl-7 = <&P9_24_i2c_pin>; + pinctrl-8 = <&P9_24_pru_uart_pin>; + pinctrl-9 = <&P9_24_pruin_pin>; + }; + + /* P9_25 (ZCZ ball A14) audio */ + P9_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_25_default_pin>; + pinctrl-1 = <&P9_25_gpio_pin>; + pinctrl-2 = <&P9_25_gpio_pu_pin>; + pinctrl-3 = <&P9_25_gpio_pd_pin>; + pinctrl-4 = <&P9_25_gpio_input_pin>; + pinctrl-5 = <&P9_25_qep_pin>; + pinctrl-6 = <&P9_25_pruout_pin>; + pinctrl-7 = <&P9_25_pruin_pin>; + }; + + /* P9_26 (ZCZ ball D16) */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P9_26_default_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_gpio_input_pin>; + pinctrl-5 = <&P9_26_uart_pin>; + pinctrl-6 = <&P9_26_can_pin>; + pinctrl-7 = <&P9_26_i2c_pin>; + pinctrl-8 = <&P9_26_pru_uart_pin>; + pinctrl-9 = <&P9_26_pruin_pin>; + }; + + /* P9_27 (ZCZ ball C13) */ + P9_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_27_default_pin>; + pinctrl-1 = <&P9_27_gpio_pin>; + pinctrl-2 = <&P9_27_gpio_pu_pin>; + pinctrl-3 = <&P9_27_gpio_pd_pin>; + pinctrl-4 = <&P9_27_gpio_input_pin>; + pinctrl-5 = <&P9_27_qep_pin>; + pinctrl-6 = <&P9_27_pruout_pin>; + pinctrl-7 = <&P9_27_pruin_pin>; + }; + + /* P9_28 (ZCZ ball C12) audio */ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_default_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_gpio_input_pin>; + pinctrl-5 = <&P9_28_spi_cs_pin>; + pinctrl-6 = <&P9_28_pwm_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* P9_29 (ZCZ ball B13) audio */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_29_default_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_gpio_input_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pwm_pin>; + pinctrl-7 = <&P9_29_pruout_pin>; + pinctrl-8 = <&P9_29_pruin_pin>; + }; + + /* P9_30 (ZCZ ball D12) gpio-hog wl1835 */ + + /* P9_31 (ZCZ ball A13) audio */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P9_31_default_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_gpio_input_pin>; + pinctrl-5 = <&P9_31_spi_sclk_pin>; + pinctrl-6 = <&P9_31_pwm_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* P9_32 VADC */ + + /* P9_33 (ZCZ ball C8) AIN4 */ + + /* P9_34 AGND */ + + /* P9_35 (ZCZ ball A8) AIN6 */ + + /* P9_36 (ZCZ ball B8) AIN5 */ + + /* P9_37 (ZCZ ball B7) AIN2 */ + + /* P9_38 (ZCZ ball A7) AIN3 */ + + /* P9_39 (ZCZ ball B6) AIN0 */ + + /* P9_40 (ZCZ ball C7) AIN1 */ + + /* P9_41 (ZCZ ball D14) */ + P9_41_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "timer", "pruin"; + pinctrl-0 = <&P9_41_default_pin>; + pinctrl-1 = <&P9_41_gpio_pin>; + pinctrl-2 = <&P9_41_gpio_pu_pin>; + pinctrl-3 = <&P9_41_gpio_pd_pin>; + pinctrl-4 = <&P9_41_gpio_input_pin>; + pinctrl-5 = <&P9_41_timer_pin>; + pinctrl-6 = <&P9_41_pruin_pin>; + }; + + /* P9_41.1 */ + /* P9_91 (ZCZ ball D13) */ + P9_91_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_91_default_pin>; + pinctrl-1 = <&P9_91_gpio_pin>; + pinctrl-2 = <&P9_91_gpio_pu_pin>; + pinctrl-3 = <&P9_91_gpio_pd_pin>; + pinctrl-4 = <&P9_91_gpio_input_pin>; + pinctrl-5 = <&P9_91_qep_pin>; + pinctrl-6 = <&P9_91_pruout_pin>; + pinctrl-7 = <&P9_91_pruin_pin>; + }; + + /* P9_42 (ZCZ ball C18) */ + P9_42_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P9_42_default_pin>; + pinctrl-1 = <&P9_42_gpio_pin>; + pinctrl-2 = <&P9_42_gpio_pu_pin>; + pinctrl-3 = <&P9_42_gpio_pd_pin>; + pinctrl-4 = <&P9_42_gpio_input_pin>; + pinctrl-5 = <&P9_42_spi_cs_pin>; + pinctrl-6 = <&P9_42_spi_sclk_pin>; + pinctrl-7 = <&P9_42_uart_pin>; + pinctrl-8 = <&P9_42_pwm_pin>; + pinctrl-9 = <&P9_42_pru_ecap_pin>; + }; + + /* P9_42.1 */ + /* P9_92 (ZCZ ball B12) */ + P9_92_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P9_92_default_pin>; + pinctrl-1 = <&P9_92_gpio_pin>; + pinctrl-2 = <&P9_92_gpio_pu_pin>; + pinctrl-3 = <&P9_92_gpio_pd_pin>; + pinctrl-4 = <&P9_92_gpio_input_pin>; + pinctrl-5 = <&P9_92_qep_pin>; + pinctrl-6 = <&P9_92_pruout_pin>; + pinctrl-7 = <&P9_92_pruin_pin>; + }; + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + /* (ZCZ ball A15) */ + A15_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "clkout", "gpio", "gpio_pu", "gpio_pd"; + pinctrl-0 = <&A15_default_pin>; + pinctrl-1 = <&A15_clkout_pin>; + pinctrl-2 = <&A15_gpio_pin>; + pinctrl-3 = <&A15_gpio_pu_pin>; + pinctrl-4 = <&A15_gpio_pd_pin>; + }; + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P8_03 { + gpio-name = "P8_03"; + gpio = <&gpio1 6 0>; + input; + dir-changeable; + }; + + P8_04 { + gpio-name = "P8_04"; + gpio = <&gpio1 7 0>; + input; + dir-changeable; + }; + + P8_05 { + gpio-name = "P8_05"; + gpio = <&gpio1 2 0>; + input; + dir-changeable; + }; + + P8_06 { + gpio-name = "P8_06"; + gpio = <&gpio1 3 0>; + input; + dir-changeable; + }; + + P8_07 { + gpio-name = "P8_07"; + gpio = <&gpio2 2 0>; + input; + dir-changeable; + }; + + P8_08 { + gpio-name = "P8_08"; + gpio = <&gpio2 3 0>; + input; + dir-changeable; + }; + + P8_09 { + gpio-name = "P8_09"; + gpio = <&gpio2 5 0>; + input; + dir-changeable; + }; + + P8_10 { + gpio-name = "P8_10"; + gpio = <&gpio2 4 0>; + input; + dir-changeable; + }; + + P8_11 { + gpio-name = "P8_11"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P8_12 { + gpio-name = "P8_12"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P8_13 { + gpio-name = "P8_13"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P8_15 { + gpio-name = "P8_15"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P8_16 { + gpio-name = "P8_16"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P8_18 { + gpio-name = "P8_18"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P8_19 { + gpio-name = "P8_19"; + gpio = <&gpio0 22 0>; + input; + dir-changeable; + }; + + P8_20 { + gpio-name = "P8_20"; + gpio = <&gpio1 31 0>; + input; + dir-changeable; + }; + + P8_21 { + gpio-name = "P8_21"; + gpio = <&gpio1 30 0>; + input; + dir-changeable; + }; + + P8_22 { + gpio-name = "P8_22"; + gpio = <&gpio1 5 0>; + input; + dir-changeable; + }; + + P8_23 { + gpio-name = "P8_23"; + gpio = <&gpio1 4 0>; + input; + dir-changeable; + }; + + P8_24 { + gpio-name = "P8_24"; + gpio = <&gpio1 1 0>; + input; + dir-changeable; + }; + + P8_25 { + gpio-name = "P8_25"; + gpio = <&gpio1 0 0>; + input; + dir-changeable; + }; + + P8_27 { + gpio-name = "P8_27"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + P8_28 { + gpio-name = "P8_28"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P8_29 { + gpio-name = "P8_29"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P8_30 { + gpio-name = "P8_30"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P8_31 { + gpio-name = "P8_31"; + gpio = <&gpio0 10 0>; + input; + dir-changeable; + }; + + P8_32 { + gpio-name = "P8_32"; + gpio = <&gpio0 11 0>; + input; + dir-changeable; + }; + + P8_33 { + gpio-name = "P8_33"; + gpio = <&gpio0 9 0>; + input; + dir-changeable; + }; + + P8_34 { + gpio-name = "P8_34"; + gpio = <&gpio2 17 0>; + input; + dir-changeable; + }; + + P8_35 { + gpio-name = "P8_35"; + gpio = <&gpio0 8 0>; + input; + dir-changeable; + }; + + P8_36 { + gpio-name = "P8_36"; + gpio = <&gpio2 16 0>; + input; + dir-changeable; + }; + + P8_37 { + gpio-name = "P8_37"; + gpio = <&gpio2 14 0>; + input; + dir-changeable; + }; + + P8_38 { + gpio-name = "P8_38"; + gpio = <&gpio2 15 0>; + input; + dir-changeable; + }; + + P8_39 { + gpio-name = "P8_39"; + gpio = <&gpio2 12 0>; + input; + dir-changeable; + }; + + P8_40 { + gpio-name = "P8_40"; + gpio = <&gpio2 13 0>; + input; + dir-changeable; + }; + + P8_41 { + gpio-name = "P8_41"; + gpio = <&gpio2 10 0>; + input; + dir-changeable; + }; + + P8_42 { + gpio-name = "P8_42"; + gpio = <&gpio2 11 0>; + input; + dir-changeable; + }; + + P8_43 { + gpio-name = "P8_43"; + gpio = <&gpio2 8 0>; + input; + dir-changeable; + }; + + P8_44 { + gpio-name = "P8_44"; + gpio = <&gpio2 9 0>; + input; + dir-changeable; + }; + + P8_45 { + gpio-name = "P8_45"; + gpio = <&gpio2 6 0>; + input; + dir-changeable; + }; + + P8_46 { + gpio-name = "P8_46"; + gpio = <&gpio2 7 0>; + input; + dir-changeable; + }; + + P9_11 { + gpio-name = "P9_11"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P9_12 { + gpio-name = "P9_12"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P9_13 { + gpio-name = "P9_13"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P9_14 { + gpio-name = "P9_14"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P9_15 { + gpio-name = "P9_15"; + gpio = <&gpio1 16 0>; + input; + dir-changeable; + }; + + P9_16 { + gpio-name = "P9_16"; + gpio = <&gpio1 19 0>; + input; + dir-changeable; + }; + + P9_17 { + gpio-name = "P9_17"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P9_18 { + gpio-name = "P9_18"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P9_21 { + gpio-name = "P9_21"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P9_22 { + gpio-name = "P9_22"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P9_23 { + gpio-name = "P9_23"; + gpio = <&gpio1 17 0>; + input; + dir-changeable; + }; + + P9_24 { + gpio-name = "P9_24"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P9_25 { + gpio-name = "P9_25"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P9_26 { + gpio-name = "P9_26"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P9_27 { + gpio-name = "P9_27"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P9_28 { + gpio-name = "P9_28"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P9_29 { + gpio-name = "P9_29"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P9_31 { + gpio-name = "P9_31"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P9_41 { + gpio-name = "P9_41"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P9_91 { + gpio-name = "P9_91"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P9_42 { + gpio-name = "P9_42"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P9_92 { + gpio-name = "P9_92"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + A15 { + gpio-name = "A15"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts b/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts new file mode 100644 index 0000000000000..3adb9721f9f7b --- /dev/null +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless-uboot-univ.dts @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + */ +/dts-v1/; + +#include "am33xx.dtsi" +#include "am335x-bone-common.dtsi" +#include "am335x-bonegreen-wireless-common-univ.dtsi" + +/ { + model = "TI AM335x BeagleBone Green Wireless"; + compatible = "ti,am335x-bone-green-wireless", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bonegreen-wireless-uboot-univ.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; +}; + +&ldo3_reg { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; +}; + +&mmc1 { + vmmc-supply = <&vmmcsd_fixed>; +}; + +&gpio1 { + ls_buf_en { + gpio-hog; + gpios = <29 GPIO_ACTIVE_HIGH>; + output-high; + line-name = "LS_BUF_EN"; + }; +}; + +/* BT_AUD_OUT from wl1835 has to be pulled low when WL_EN is activated.*/ +/* in case it isn't, wilink8 ends up in one of the test modes that */ +/* intruces various issues (elp wkaeup timeouts etc.) */ +/* On the BBGW this pin is routed through the level shifter (U21) that */ +/* introduces a pullup on the line and wilink8 ends up in a bad state. */ +/* use a gpio hog to force this pin low. An alternative may be adding */ +/* an external pulldown on U21 pin 4. */ + +&gpio3 { + bt_aud_in { + gpio-hog; + gpios = <16 GPIO_ACTIVE_HIGH>; + output-low; + line-name = "MCASP0_AHCLKR"; + }; +}; diff --git a/arch/arm/boot/dts/am335x-bonegreen-wireless.dts b/arch/arm/boot/dts/am335x-bonegreen-wireless.dts index 57731f0daf103..8372b8e496622 100644 --- a/arch/arm/boot/dts/am335x-bonegreen-wireless.dts +++ b/arch/arm/boot/dts/am335x-bonegreen-wireless.dts @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; @@ -11,11 +8,17 @@ #include "am335x-bone-common.dtsi" #include "am335x-bonegreen-common.dtsi" #include +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Green Wireless"; compatible = "ti,am335x-bone-green-wireless", "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + chosen { + base_dtb = "am335x-bonegreen-wireless.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + wlan_en_reg: fixedregulator@2 { compatible = "regulator-fixed"; regulator-name = "wlan-en-regulator"; @@ -29,38 +32,42 @@ }; }; +&sgx { + status = "okay"; +}; + &am33xx_pinmux { bt_pins: pinmux_bt_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x878, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_ad12.gpio1_28 BT_EN */ + AM33XX_PADCONF(AM335X_PIN_GPMC_BEN1, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_ad12.gpio1_28 BT_EN */ >; }; mmc3_pins: pinmux_mmc3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x830, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_ad12.mmc2_dat0 */ - AM33XX_IOPAD(0x834, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_ad13.mmc2_dat1 */ - AM33XX_IOPAD(0x838, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_ad14.mmc2_dat2 */ - AM33XX_IOPAD(0x83c, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_ad15.mmc2_dat3 */ - AM33XX_IOPAD(0x888, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_csn3.mmc2_cmd */ - AM33XX_IOPAD(0x88c, PIN_INPUT_PULLUP | MUX_MODE3) /* gpmc_clk.mmc2_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD12, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_ad12.mmc2_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD13, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_ad13.mmc2_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD14, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_ad14.mmc2_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD15, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_ad15.mmc2_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN3, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_csn3.mmc2_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CLK, PIN_INPUT_PULLUP, MUX_MODE3) /* gpmc_clk.mmc2_clk */ >; }; uart3_pins: pinmux_uart3_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ - AM33XX_IOPAD(0x938, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* gmii1_rxd2.uart3_txd */ - AM33XX_IOPAD(0x948, PIN_INPUT | MUX_MODE3) /* mdio_data.uart3_ctsn */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* mdio_clk.uart3_rtsn */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gmii1_rxd3.uart3_rxd */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* gmii1_rxd2.uart3_txd */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT, MUX_MODE3) /* mdio_data.uart3_ctsn */ + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLDOWN, MUX_MODE3) /* mdio_clk.uart3_rtsn */ >; }; wl18xx_pins: pinmux_wl18xx_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x828, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad10.gpio0_26 WL_EN */ - AM33XX_IOPAD(0x82C, PIN_INPUT_PULLDOWN | MUX_MODE7) /* gpmc_ad11.gpio0_27 WL_IRQ */ - AM33XX_IOPAD(0x87C, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_csn0.gpio1_29 LS_BUF_EN */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD10, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_ad10.gpio0_26 WL_EN */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD11, PIN_INPUT_PULLDOWN, MUX_MODE7) /* gpmc_ad11.gpio0_27 WL_IRQ */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN0, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_csn0.gpio1_29 LS_BUF_EN */ >; }; }; diff --git a/arch/arm/boot/dts/am335x-bonegreen.dts b/arch/arm/boot/dts/am335x-bonegreen.dts index a8b4d969ce2a0..dc6f69a85bf08 100644 --- a/arch/arm/boot/dts/am335x-bonegreen.dts +++ b/arch/arm/boot/dts/am335x-bonegreen.dts @@ -1,17 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; #include "am33xx.dtsi" #include "am335x-bone-common.dtsi" #include "am335x-bonegreen-common.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ / { model = "TI AM335x BeagleBone Green"; compatible = "ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-bonegreen.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; }; diff --git a/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi b/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi new file mode 100644 index 0000000000000..d2704c6921083 --- /dev/null +++ b/arch/arm/boot/dts/am335x-cape-bbb-exp-c.dtsi @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +#include "am335x-peripheral-can0.dtsi" +#include "am335x-bone-pinmux-can0.dtsi" + +#include "am335x-peripheral-ttyS1.dtsi" +#include "am335x-bone-pinmux-ttyS1.dtsi" + +#include "am335x-peripheral-ttyS2.dtsi" +#include "am335x-bone-pinmux-ttyS2.dtsi" + +#include "am335x-peripheral-ttyS4.dtsi" +#include "am335x-bone-pinmux-ttyS4.dtsi" + +&am33xx_pinmux { + user_leds_s1: user_leds_s1 { + pinctrl-single,pins = < + 0x98 0x7 /* gpmc_wen.gpio2_4, OUTPUT | MODE7 */ + 0x9c 0x7 /* gpmc_ben0_cle.gpio2_5, OUTPUT | MODE7 */ + >; + }; + + bb_lcd_pwm_backlight_pins: pinmux_bb_lcd_pwm_backlight_pins { + pinctrl-single,pins = < + BONE_P9_14 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* gpmc_a2.ehrpwm1a */ + >; + }; + + keymap3_pins: pinmux_keymap3_pins { + pinctrl-single,pins = < + 0x040 0x2f /* KEY_UP gpmc_a0.gpio1_16, INPUT | PULLDIS | MODE7 */ + 0x04c 0x2f /* KEY_DOWN gpmc_a3.gpio1_19, INPUT | PULLDIS | MODE7 */ + 0x078 0x2f /* KEY_RIGHT gpmc_ben1.gpio1_28, INPUT | PULLDIS | MODE7 */ + 0x164 0x2f /* KEY_LEFT ecap0_in_pwm0_out.gpio0_7, INPUT | PULLDIS | MODE7 */ + 0x1a4 0x2f /* KEY_ENTER mcasp0_fxr.gpio3_19, INPUT | PULLDIS | MODE7 */ + >; + }; + + edt_ft5306_ts_pins: pinmux_edt_ft5306_ts_pins { + pinctrl-single,pins = < + /* CAP_TSC gpmc_a1.gpio1_17, INPUT | MODE7 */ + BONE_P9_23 (PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + i2c1_pins: pinmux_i2c1_pins { + pinctrl-single,pins = < + /* spi0_d1.i2c1_sda, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */ + BONE_P9_18 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2) + /* spi0_cs0.i2c1_scl, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */ + BONE_P9_17 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2) + >; + }; + + mcasp0_pins: pinmux_mcasp0_pins { + pinctrl-single,pins = < + 0x190 0x20 /* mcasp0_aclkx.mcasp0_aclkx, INPUT | MODE0 */ + 0x194 0x20 /* mcasp0_fsx.mcasp0_fsx, INPUT | MODE0 */ + 0x198 0x20 /* mcasp0_axr0.mcasp0_axr0, INPUT | MODE0 */ + 0x19c 0x22 /* mcasp0_ahclkr.mcasp0_axr2, INPUT | MODE2 */ + >; + }; +}; + +&epwmss1 { + status = "okay"; +}; + + +&ehrpwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&bb_lcd_pwm_backlight_pins>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + clock-frequency = <400000>; + + edt-ft5306@38 { + status = "okay"; + compatible = "edt,edt-ft5306", "edt,edt-ft5x06"; + pinctrl-names = "default"; + pinctrl-0 = <&edt_ft5306_ts_pins>; + + reg = <0x38>; + interrupt-parent = <&gpio1>; + interrupts = <17 0>; + + touchscreen-size-x = <600>; + touchscreen-size-y = <1024>; + touchscreen-inverted-x; + touchscreen-inverted-y; + touchscreen-swapped-x-y; + }; + + tlv320aic3x: tlv320aic3x@1b { + compatible = "ti,tlv320aic3x"; + reg = <0x1b>; + status = "okay"; + }; +}; + +&mcasp0 { + pinctrl-names = "default"; + pinctrl-0 = <&mcasp0_pins>; + + status = "okay"; + + op-mode = <0>; /* MCASP_IIS_MODE */ + tdm-slots = <2>; + num-serializer = <16>; + serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */ + 1 0 2 0 + 0 0 0 0 + 0 0 0 0 + 0 0 0 0 + >; + tx-num-evt = <1>; + rx-num-evt = <1>; +}; + +/ { + backlight { + status = "okay"; + compatible = "pwm-backlight"; + pwms = <&ehrpwm1 0 50000 0>; + brightness-levels = <0 51 53 56 62 75 101 152 255>; + default-brightness-level = <8>; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-names = "default"; + pinctrl-0 = <&keymap3_pins>; + + #address-cells = <1>; + #size-cells = <0>; + + button@1 { + debounce_interval = <50>; + linux,code = <105>; + label = "left"; + gpios = <&gpio0 7 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@2 { + debounce_interval = <50>; + linux,code = <106>; + label = "right"; + gpios = <&gpio1 28 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@3 { + debounce_interval = <50>; + linux,code = <103>; + label = "up"; + gpios = <&gpio1 16 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@4 { + debounce_interval = <50>; + linux,code = <108>; + label = "down"; + gpios = <&gpio1 19 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@5 { + debounce_interval = <50>; + linux,code = <28>; + label = "enter"; + gpios = <&gpio3 19 0x1>; + gpio-key,wakeup; + }; + }; + + gpio-leds-cape-lcd { + compatible = "gpio-leds"; + pinctrl-names = "default"; + + pinctrl-0 = <&user_leds_s1>; + + lcd-led0 { + label = "lcd:green:usr0"; + gpios = <&gpio2 4 0>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + lcd-led1 { + label = "lcd:green:usr1"; + gpios = <&gpio2 5 0>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + }; + + sound { + compatible = "ti,da830-evm-audio"; + ti,model = "DA830 EVM"; + ti,audio-codec = <&tlv320aic3x>; + ti,mcasp-controller = <&mcasp0>; + ti,codec-clock-rate = <12000000>; + ti,audio-routing = + "Headphone Jack", "HPLOUT", + "Headphone Jack", "HPROUT", + "MIC3L", "Mic Jack", + "MIC3R", "Mic Jack"; + }; +}; + +#include "am335x-peripheral-panel-1024x600-24bit.dtsi" +#include "am335x-bone-pinmux-panel-1024x600-24bit.dtsi" diff --git a/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi b/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi new file mode 100644 index 0000000000000..539409c9fcd81 --- /dev/null +++ b/arch/arm/boot/dts/am335x-cape-bbb-exp-r.dtsi @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include + +#include "am335x-peripheral-can0.dtsi" +#include "am335x-bone-pinmux-can0.dtsi" + +#include "am335x-peripheral-ttyS1.dtsi" +#include "am335x-bone-pinmux-ttyS1.dtsi" + +#include "am335x-peripheral-ttyS2.dtsi" +#include "am335x-bone-pinmux-ttyS2.dtsi" + +#include "am335x-peripheral-ttyS4.dtsi" +#include "am335x-bone-pinmux-ttyS4.dtsi" + +&am33xx_pinmux { + user_leds_s1: user_leds_s1 { + pinctrl-single,pins = < + 0x98 0x7 /* gpmc_wen.gpio2_4, OUTPUT | MODE7 */ + 0x9c 0x7 /* gpmc_ben0_cle.gpio2_5, OUTPUT | MODE7 */ + >; + }; + + bb_lcd_pwm_backlight_pins: pinmux_bb_lcd_pwm_backlight_pins { + pinctrl-single,pins = < + BONE_P9_14 (PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* gpmc_a2.ehrpwm1a */ + >; + }; + + keymap3_pins: pinmux_keymap3_pins { + pinctrl-single,pins = < + 0x040 0x2f /* KEY_UP gpmc_a0.gpio1_16, INPUT | PULLDIS | MODE7 */ + 0x04c 0x2f /* KEY_DOWN gpmc_a3.gpio1_19, INPUT | PULLDIS | MODE7 */ + 0x078 0x2f /* KEY_RIGHT gpmc_ben1.gpio1_28, INPUT | PULLDIS | MODE7 */ + 0x164 0x2f /* KEY_LEFT ecap0_in_pwm0_out.gpio0_7, INPUT | PULLDIS | MODE7 */ + 0x1a4 0x2f /* KEY_ENTER mcasp0_fxr.gpio3_19, INPUT | PULLDIS | MODE7 */ + >; + }; + + i2c1_pins: pinmux_i2c1_pins { + pinctrl-single,pins = < + /* spi0_d1.i2c1_sda, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */ + BONE_P9_18 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2) + /* spi0_cs0.i2c1_scl, SLEWCTRL_SLOW | INPUT_PULLUP | MODE2 */ + BONE_P9_17 (PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE2) + >; + }; + + mcasp0_pins: pinmux_mcasp0_pins { + pinctrl-single,pins = < + 0x190 0x20 /* mcasp0_aclkx.mcasp0_aclkx, INPUT | MODE0 */ + 0x194 0x20 /* mcasp0_fsx.mcasp0_fsx, INPUT | MODE0 */ + 0x198 0x20 /* mcasp0_axr0.mcasp0_axr0, INPUT | MODE0 */ + 0x19c 0x22 /* mcasp0_ahclkr.mcasp0_axr2, INPUT | MODE2 */ + >; + }; +}; + +&epwmss1 { + status = "okay"; +}; + + +&ehrpwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&bb_lcd_pwm_backlight_pins>; + status = "okay"; +}; + +&i2c1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + clock-frequency = <400000>; + + tlv320aic3x: tlv320aic3x@1b { + compatible = "ti,tlv320aic3x"; + reg = <0x1b>; + status = "okay"; + }; +}; + +&mcasp0 { + pinctrl-names = "default"; + pinctrl-0 = <&mcasp0_pins>; + + status = "okay"; + + op-mode = <0>; /* MCASP_IIS_MODE */ + tdm-slots = <2>; + num-serializer = <16>; + serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */ + 1 0 2 0 + 0 0 0 0 + 0 0 0 0 + 0 0 0 0 + >; + tx-num-evt = <1>; + rx-num-evt = <1>; +}; + +&tscadc { + status = "okay"; + tsc { + ti,wires = <4>; + ti,x-plate-resistance = <200>; + ti,coordinate-readouts = <5>; + ti,wire-config = <0x00 0x11 0x22 0x33>; + }; + + adc { + ti,adc-channels = <4 5 6 7>; + }; +}; + +/ { + backlight { + status = "okay"; + compatible = "pwm-backlight"; + pwms = <&ehrpwm1 0 50000 0>; + brightness-levels = <0 51 53 56 62 75 101 152 255>; + default-brightness-level = <8>; + }; + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-names = "default"; + pinctrl-0 = <&keymap3_pins>; + + #address-cells = <1>; + #size-cells = <0>; + + button@1 { + debounce_interval = <50>; + linux,code = <105>; + label = "left"; + gpios = <&gpio0 7 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@2 { + debounce_interval = <50>; + linux,code = <106>; + label = "right"; + gpios = <&gpio1 28 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@3 { + debounce_interval = <50>; + linux,code = <103>; + label = "up"; + gpios = <&gpio1 16 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@4 { + debounce_interval = <50>; + linux,code = <108>; + label = "down"; + gpios = <&gpio1 19 0x1>; + gpio-key,wakeup; + autorepeat; + }; + button@5 { + debounce_interval = <50>; + linux,code = <28>; + label = "enter"; + gpios = <&gpio3 19 0x1>; + gpio-key,wakeup; + }; + }; + + gpio-leds-cape-lcd { + compatible = "gpio-leds"; + pinctrl-names = "default"; + + pinctrl-0 = <&user_leds_s1>; + + lcd-led0 { + label = "lcd:green:usr0"; + gpios = <&gpio2 4 0>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + lcd-led1 { + label = "lcd:green:usr1"; + gpios = <&gpio2 5 0>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + }; + + sound { + compatible = "ti,da830-evm-audio"; + ti,model = "DA830 EVM"; + ti,audio-codec = <&tlv320aic3x>; + ti,mcasp-controller = <&mcasp0>; + ti,codec-clock-rate = <12000000>; + ti,audio-routing = + "Headphone Jack", "HPLOUT", + "Headphone Jack", "HPROUT", + "MIC3L", "Mic Jack", + "MIC3R", "Mic Jack"; + }; +}; + +#include "am335x-peripheral-panel-1024x600-24bit.dtsi" +#include "am335x-bone-pinmux-panel-1024x600-24bit.dtsi" diff --git a/arch/arm/boot/dts/am335x-osd3358-sm-red.dts b/arch/arm/boot/dts/am335x-osd3358-sm-red.dts index d9e92671055bd..3d5f3c46a24c9 100644 --- a/arch/arm/boot/dts/am335x-osd3358-sm-red.dts +++ b/arch/arm/boot/dts/am335x-osd3358-sm-red.dts @@ -13,10 +13,20 @@ #include #include +/* #include "am335x-bone-jtag.dtsi" */ + +&sgx { + status = "okay"; +}; / { model = "Octavo Systems OSD3358-SM-RED"; compatible = "oct,osd3358-sm-refdesign", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-osd3358-sm-red.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &ldo3_reg { @@ -40,61 +50,54 @@ &am33xx_pinmux { nxp_hdmi_bonelt_pins: nxp-hdmi-bonelt-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr0 */ - AM33XX_IOPAD(0x8a0, PIN_OUTPUT | MUX_MODE0) /* lcd_data0.lcd_data0 */ - AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE0) /* lcd_data1.lcd_data1 */ - AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE0) /* lcd_data2.lcd_data2 */ - AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE0) /* lcd_data3.lcd_data3 */ - AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE0) /* lcd_data4.lcd_data4 */ - AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE0) /* lcd_data5.lcd_data5 */ - AM33XX_IOPAD(0x8b8, PIN_OUTPUT | MUX_MODE0) /* lcd_data6.lcd_data6 */ - AM33XX_IOPAD(0x8bc, PIN_OUTPUT | MUX_MODE0) /* lcd_data7.lcd_data7 */ - AM33XX_IOPAD(0x8c0, PIN_OUTPUT | MUX_MODE0) /* lcd_data8.lcd_data8 */ - AM33XX_IOPAD(0x8c4, PIN_OUTPUT | MUX_MODE0) /* lcd_data9.lcd_data9 */ - AM33XX_IOPAD(0x8c8, PIN_OUTPUT | MUX_MODE0) /* lcd_data10.lcd_data10 */ - AM33XX_IOPAD(0x8cc, PIN_OUTPUT | MUX_MODE0) /* lcd_data11.lcd_data11 */ - AM33XX_IOPAD(0x8d0, PIN_OUTPUT | MUX_MODE0) /* lcd_data12.lcd_data12 */ - AM33XX_IOPAD(0x8d4, PIN_OUTPUT | MUX_MODE0) /* lcd_data13.lcd_data13 */ - AM33XX_IOPAD(0x8d8, PIN_OUTPUT | MUX_MODE0) /* lcd_data14.lcd_data14 */ - AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE0) /* lcd_data15.lcd_data15 */ - AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_vsync.lcd_vsync */ - AM33XX_IOPAD(0x8e4, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_hsync.lcd_hsync */ - AM33XX_IOPAD(0x8e8, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_pclk.lcd_pclk */ - AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_ac_bias_en.lcd_ac_bias_en */ - >; - }; - - nxp_hdmi_bonelt_off_pins: nxp-hdmi-bonelt-off-pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x9b0, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr0 */ + AM33XX_PADCONF(AM335X_PIN_LCD_DATA0, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA1, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA2, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA3, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA4, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA5, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA6, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA7, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA8, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA9, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA10, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA11, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA12, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA13, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA14, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_DATA15, PIN_OUTPUT, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_VSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_HSYNC, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_PCLK, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_LCD_AC_BIAS_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE0) >; }; mcasp0_pins: mcasp0-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x9ac, PIN_INPUT_PULLUP | MUX_MODE0) /* mcasp0_ahcklx.mcasp0_ahclkx */ - AM33XX_IOPAD(0x99c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ - AM33XX_IOPAD(0x994, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mcasp0_fsx.mcasp0_fsx */ - AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mcasp0_aclkx.mcasp0_aclkx */ - AM33XX_IOPAD(0x86c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a11.GPIO1_27 */ + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKX, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MCASP0_AHCLKR, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mcasp0_ahclkr.mcasp0_axr2*/ + AM33XX_PADCONF(AM335X_PIN_MCASP0_FSX, PIN_OUTPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_GPMC_A11, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a11.GPIO1_27 */ >; }; flash_enable: flash-enable { pinctrl-single,pins = < - AM33XX_IOPAD(0x944, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* rmii1_ref_clk.gpio0_29 */ + AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* rmii1_ref_clk.gpio0_29 */ >; }; imu_interrupt: imu-interrupt { pinctrl-single,pins = < - AM33XX_IOPAD(0x910, PIN_INPUT_PULLDOWN | MUX_MODE7) /* mii1_rx_er.gpio3_2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_ER, PIN_INPUT_PULLDOWN, MUX_MODE7) /* mii1_rx_er.gpio3_2 */ >; }; ethernet_interrupt: ethernet-interrupt{ pinctrl-single,pins = < - AM33XX_IOPAD(0x908, PIN_INPUT_PULLDOWN | MUX_MODE7) /* mii1_col.gpio3_0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_COL, PIN_INPUT_PULLDOWN, MUX_MODE7) /* mii1_col.gpio3_0 */ >; }; }; @@ -124,9 +127,8 @@ compatible = "nxp,tda998x"; reg = <0x70>; - pinctrl-names = "default", "off"; + pinctrl-names = "default"; pinctrl-0 = <&nxp_hdmi_bonelt_pins>; - pinctrl-1 = <&nxp_hdmi_bonelt_off_pins>; /* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */ /* video-ports = <0x234501>; */ @@ -264,114 +266,105 @@ }; &am33xx_pinmux { - pinctrl-names = "default"; - pinctrl-0 = <&clkout2_pin>; - user_leds_s0: user-leds-s0 { pinctrl-single,pins = < - AM33XX_IOPAD(0x854, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a5.gpio1_21 */ - AM33XX_IOPAD(0x858, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a6.gpio1_22 */ - AM33XX_IOPAD(0x85c, PIN_OUTPUT_PULLDOWN | MUX_MODE7) /* gpmc_a7.gpio1_23 */ - AM33XX_IOPAD(0x860, PIN_OUTPUT_PULLUP | MUX_MODE7) /* gpmc_a8.gpio1_24 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a5.gpio1_21 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT_PULLDOWN, MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT_PULLUP, MUX_MODE7) /* gpmc_a8.gpio1_24 */ >; }; i2c2_pins: pinmux-i2c2-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_ctsn.i2c2_sda */ - AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3) /* uart1_rtsn.i2c2_scl */ + AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_ctsn.i2c2_sda */ + AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* uart1_rtsn.i2c2_scl */ >; }; uart0_pins: pinmux-uart0-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0) /* uart0_rxd.uart0_rxd */ - AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* uart0_txd.uart0_txd */ - >; - }; - - clkout2_pin: pinmux-clkout2-pin { - pinctrl-single,pins = < - AM33XX_IOPAD(0x9b4, PIN_OUTPUT_PULLDOWN | MUX_MODE3) /* xdma_event_intr1.clkout2 */ + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) >; }; cpsw_default: cpsw-default { pinctrl-single,pins = < /* Slave 1 */ - AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txen.rgmii1_tctl */ - AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxdv.rgmii1_rctl */ - AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd3.rgmii1_txd3 */ - AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd2.rgmii1_txd2 */ - AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd1.rgmii1_txd1 */ - AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd0.rgmii1_txd0 */ - AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txclk.rgmii1_txclk */ - AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxclk.rgmii1_rxclk */ - AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd3.rgmii1_rxd3 */ - AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd2.rgmii1_rxd2 */ - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd1.rgmii1_rxd1 */ - AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd0.rgmii1_rxd0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txen.rgmii1_tctl */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxdv.rgmii1_rctl */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE2) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE2) >; }; cpsw_sleep: cpsw-sleep { pinctrl-single,pins = < /* Slave 1 reset value */ - AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) >; }; davinci_mdio_default: davinci-mdio-default { pinctrl-single,pins = < /* MDIO */ - AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0) >; }; davinci_mdio_sleep: davinci-mdio-sleep { pinctrl-single,pins = < /* MDIO reset value */ - AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7) >; }; mmc1_pins: pinmux-mmc1-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */ - AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* (G16) mmc0_dat0.mmc0_dat0 */ - AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0) /* (G15) mmc0_dat1.mmc0_dat1 */ - AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0) /* (F18) mmc0_dat2.mmc0_dat2 */ - AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* (F17) mmc0_dat3.mmc0_dat3 */ - AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* (G18) mmc0_cmd.mmc0_cmd */ - AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* (G17) mmc0_clk.mmc0_clk */ + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */ + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0) >; }; emmc_pins: pinmux-emmc-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */ - AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ - AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ - AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ - AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ - AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ - AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ - AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ - AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ - AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN1, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + AM33XX_PADCONF(AM335X_PIN_GPMC_CSN2, PIN_INPUT_PULLUP, MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD0, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD1, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD2, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD3, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD4, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD5, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD6, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_AD7, PIN_INPUT_PULLUP, MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ >; }; }; diff --git a/arch/arm/boot/dts/am335x-osd335x-common.dtsi b/arch/arm/boot/dts/am335x-osd335x-common.dtsi index f8ff473f94f01..a8b6842489f74 100644 --- a/arch/arm/boot/dts/am335x-osd335x-common.dtsi +++ b/arch/arm/boot/dts/am335x-osd335x-common.dtsi @@ -36,8 +36,8 @@ &am33xx_pinmux { i2c0_pins: pinmux-i2c0-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0) /* (C17) I2C0_SDA.I2C0_SDA */ - AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0) /* (C16) I2C0_SCL.I2C0_SCL */ + AM33XX_PADCONF(AM335X_PIN_I2C0_SDA, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_I2C0_SCL, PIN_INPUT_PULLUP, MUX_MODE0) >; }; }; diff --git a/arch/arm/boot/dts/am335x-peripheral-can0.dtsi b/arch/arm/boot/dts/am335x-peripheral-can0.dtsi new file mode 100644 index 0000000000000..4335e39bf1999 --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-can0.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&dcan0 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi b/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi new file mode 100644 index 0000000000000..f7c8f8af499fa --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-panel-1024x600-24bit.dtsi @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&lcdc { + status = "okay"; + blue-and-red-wiring = "crossed"; +}; + +/ { + panel { + status = "okay"; + compatible = "ti,tilcdc,panel"; + pinctrl-names = "default"; + + panel-info { + ac-bias = <255>; + ac-bias-intrpt = <0>; + dma-burst-sz = <16>; + bpp = <32>; + fdd = <0x80>; + sync-edge = <0>; + sync-ctrl = <0>; + raster-order = <1>; + fifo-th = <0>; + }; + display-timings { + native-mode = <&timing0>; + timing0: 1024x600 { + clock-frequency = <36000000>; + hactive = <1024>; + vactive = <600>; + hfront-porch = <1>; + hback-porch = <45>; + hsync-len = <30>; + vback-porch = <22>; + vfront-porch = <12>; + vsync-len = <2>; + hsync-active = <1>; + vsync-active = <1>; + de-active = <1>; + pixelclk-active = <0>; + }; + }; + }; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi new file mode 100644 index 0000000000000..f59fa4c5025fa --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-ttyS1.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&uart1 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi new file mode 100644 index 0000000000000..a25d6cfcb1d6a --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-ttyS2.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&uart2 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi b/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi new file mode 100644 index 0000000000000..adc89f046fbb5 --- /dev/null +++ b/arch/arm/boot/dts/am335x-peripheral-ttyS4.dtsi @@ -0,0 +1,13 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&uart4 { + pinctrl-names = "default"; + + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am335x-pocketbeagle.dts b/arch/arm/boot/dts/am335x-pocketbeagle.dts index 62fe5cab9fae5..120a2cac41c50 100644 --- a/arch/arm/boot/dts/am335x-pocketbeagle.dts +++ b/arch/arm/boot/dts/am335x-pocketbeagle.dts @@ -9,12 +9,18 @@ #include "am33xx.dtsi" #include "am335x-osd335x-common.dtsi" +&sgx { + status = "okay"; +}; + / { model = "TI AM335x PocketBeagle"; compatible = "ti,am335x-pocketbeagle", "ti,am335x-bone", "ti,am33xx"; chosen { stdout-path = &uart0; + base_dtb = "am335x-pocketbeagle.dts"; + base_dtb_timestamp = __TIMESTAMP__; }; leds { @@ -60,78 +66,904 @@ }; &am33xx_pinmux { - i2c2_pins: pinmux-i2c2-pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x97c, PIN_INPUT_PULLUP | MUX_MODE3) /* (D17) uart1_rtsn.I2C2_SCL */ - AM33XX_IOPAD(0x978, PIN_INPUT_PULLUP | MUX_MODE3) /* (D18) uart1_ctsn.I2C2_SDA */ - >; - }; +// i2c2_pins: pinmux-i2c2-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_UART1_RTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D17) uart1_rtsn.I2C2_SCL */ +// AM33XX_PADCONF(AM335X_PIN_UART1_CTSN, PIN_INPUT_PULLUP, MUX_MODE3) /* (D18) uart1_ctsn.I2C2_SDA */ +// >; +// }; - ehrpwm0_pins: pinmux-ehrpwm0-pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x990, PIN_OUTPUT_PULLDOWN | MUX_MODE1) /* (A13) mcasp0_aclkx.ehrpwm0A */ - >; - }; +// ehrpwm0_pins: pinmux-ehrpwm0-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_MCASP0_ACLKX, PIN_OUTPUT_PULLDOWN, MUX_MODE1) /* (A13) mcasp0_aclkx.ehrpwm0A */ +// >; +// }; - ehrpwm1_pins: pinmux-ehrpwm1-pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x848, PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* (U14) gpmc_a2.ehrpwm1A */ - >; - }; +// ehrpwm1_pins: pinmux-ehrpwm1-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_GPMC_A2, PIN_OUTPUT_PULLDOWN, MUX_MODE6) /* (U14) gpmc_a2.ehrpwm1A */ +// >; +// }; mmc0_pins: pinmux-mmc0-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x960, PIN_INPUT | MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */ - AM33XX_IOPAD(0x8fc, PIN_INPUT_PULLUP | MUX_MODE0) /* (G16) mmc0_dat0.mmc0_dat0 */ - AM33XX_IOPAD(0x8f8, PIN_INPUT_PULLUP | MUX_MODE0) /* (G15) mmc0_dat1.mmc0_dat1 */ - AM33XX_IOPAD(0x8f4, PIN_INPUT_PULLUP | MUX_MODE0) /* (F18) mmc0_dat2.mmc0_dat2 */ - AM33XX_IOPAD(0x8f0, PIN_INPUT_PULLUP | MUX_MODE0) /* (F17) mmc0_dat3.mmc0_dat3 */ - AM33XX_IOPAD(0x904, PIN_INPUT_PULLUP | MUX_MODE0) /* (G18) mmc0_cmd.mmc0_cmd */ - AM33XX_IOPAD(0x900, PIN_INPUT_PULLUP | MUX_MODE0) /* (G17) mmc0_clk.mmc0_clk */ - AM33XX_IOPAD(0x9a0, PIN_INPUT | MUX_MODE4) /* (B12) mcasp0_aclkr.mmc0_sdwp */ + AM33XX_PADCONF(AM335X_PIN_SPI0_CS1, PIN_INPUT, MUX_MODE7) /* (C15) spi0_cs1.gpio0[6] */ + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT0, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT1, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT2, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_DAT3, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CMD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MMC0_CLK, PIN_INPUT_PULLUP, MUX_MODE0) >; }; - spi0_pins: pinmux-spi0-pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x950, PIN_INPUT_PULLUP | MUX_MODE0) /* (A17) spi0_sclk.spi0_sclk */ - AM33XX_IOPAD(0x954, PIN_INPUT_PULLUP | MUX_MODE0) /* (B17) spi0_d0.spi0_d0 */ - AM33XX_IOPAD(0x958, PIN_INPUT_PULLUP | MUX_MODE0) /* (B16) spi0_d1.spi0_d1 */ - AM33XX_IOPAD(0x95c, PIN_INPUT_PULLUP | MUX_MODE0) /* (A16) spi0_cs0.spi0_cs0 */ - >; - }; +// spi0_pins: pinmux-spi0-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_SPI0_SCLK, PIN_INPUT_PULLUP, MUX_MODE0) /* (A17) spi0_sclk.spi0_sclk */ +// AM33XX_PADCONF(AM335X_PIN_SPI0_D0, PIN_INPUT_PULLUP, MUX_MODE0) /* (B17) spi0_d0.spi0_d0 */ +// AM33XX_PADCONF(AM335X_PIN_SPI0_D1, PIN_INPUT_PULLUP, MUX_MODE0) /* (B16) spi0_d1.spi0_d1 */ +// AM33XX_PADCONF(AM335X_PIN_SPI0_CS0, PIN_INPUT_PULLUP, MUX_MODE0) /* (A16) spi0_cs0.spi0_cs0 */ +// >; +// }; - spi1_pins: pinmux-spi1-pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x964, PIN_INPUT_PULLUP | MUX_MODE4) /* (C18) eCAP0_in_PWM0_out.spi1_sclk */ - AM33XX_IOPAD(0x968, PIN_INPUT_PULLUP | MUX_MODE4) /* (E18) uart0_ctsn.spi1_d0 */ - AM33XX_IOPAD(0x96c, PIN_INPUT_PULLUP | MUX_MODE4) /* (E17) uart0_rtsn.spi1_d1 */ - AM33XX_IOPAD(0x9b0, PIN_INPUT_PULLUP | MUX_MODE4) /* (A15) xdma_event_intr0.spi1_cs1 */ - >; - }; +// spi1_pins: pinmux-spi1-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_ECAP0_IN_PWM0_OUT, PIN_INPUT_PULLUP, MUX_MODE4) /* (C18) eCAP0_in_PWM0_out.spi1_sclk */ +// AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_INPUT_PULLUP, MUX_MODE4) /* (E18) uart0_ctsn.spi1_d0 */ +// AM33XX_PADCONF(AM335X_PIN_UART0_RTSN, PIN_INPUT_PULLUP, MUX_MODE4) /* (E17) uart0_rtsn.spi1_d1 */ +// AM33XX_PADCONF(AM335X_PIN_XDMA_EVENT_INTR0, PIN_INPUT_PULLUP, MUX_MODE4) /* (A15) xdma_event_intr0.spi1_cs1 */ +// >; +// }; usr_leds_pins: pinmux-usr-leds-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x854, PIN_OUTPUT | MUX_MODE7) /* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */ - AM33XX_IOPAD(0x858, PIN_OUTPUT | MUX_MODE7) /* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */ - AM33XX_IOPAD(0x85c, PIN_OUTPUT | MUX_MODE7) /* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */ - AM33XX_IOPAD(0x860, PIN_OUTPUT | MUX_MODE7) /* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A5, PIN_OUTPUT, MUX_MODE7) /* (V15) gpmc_a5.gpio1[21] - USR_LED_0 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A6, PIN_OUTPUT, MUX_MODE7) /* (U15) gpmc_a6.gpio1[22] - USR_LED_1 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A7, PIN_OUTPUT, MUX_MODE7) /* (T15) gpmc_a7.gpio1[23] - USR_LED_2 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A8, PIN_OUTPUT, MUX_MODE7) /* (V16) gpmc_a8.gpio1[24] - USR_LED_3 */ >; }; uart0_pins: pinmux-uart0-pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0) /* (E15) uart0_rxd.uart0_rxd */ - AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* (E16) uart0_txd.uart0_txd */ + AM33XX_PADCONF(AM335X_PIN_UART0_RXD, PIN_INPUT_PULLUP, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_UART0_TXD, PIN_OUTPUT_PULLDOWN, MUX_MODE0) >; }; - uart4_pins: pinmux-uart4-pins { - pinctrl-single,pins = < - AM33XX_IOPAD(0x870, PIN_INPUT_PULLUP | MUX_MODE6) /* (T17) gpmc_wait0.uart4_rxd */ - AM33XX_IOPAD(0x874, PIN_OUTPUT_PULLDOWN | MUX_MODE6) /* (U17) gpmc_wpn.uart4_txd */ - >; - }; +// uart4_pins: pinmux-uart4-pins { +// pinctrl-single,pins = < +// AM33XX_PADCONF(AM335X_PIN_GPMC_WAIT0, PIN_INPUT_PULLUP, MUX_MODE6) /* (T17) gpmc_wait0.uart4_rxd */ +// AM33XX_PADCONF(AM335X_PIN_GPMC_WPN, PIN_OUTPUT_PULLDOWN, MUX_MODE6) /* (U17) gpmc_wpn.uart4_txd */ +// >; +// }; + + /************************/ + /* P1 Header */ + /************************/ + + /* P1_01 VIN-AC */ + + /* P1_02 (ZCZ ball R5) gpio2_23 */ + P1_02_default_pin: pinmux_P1_02_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pin: pinmux_P1_02_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pu_pin: pinmux_P1_02_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_pd_pin: pinmux_P1_02_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_gpio_input_pin: pinmux_P1_02_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE7) >; }; /* lcd_hsync.gpio2_23 */ + P1_02_pruout_pin: pinmux_P1_02_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_hsync.pru1_out9 */ + P1_02_pruin_pin: pinmux_P1_02_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e4, PIN_INPUT | MUX_MODE6) >; }; /* lcd_hsync.pru1_in9 */ + + /* P1_03 (ZCZ ball F15) usb1_vbus_out */ + + /* P1_04 (ZCZ ball R6) gpio2_25 */ + P1_04_default_pin: pinmux_P1_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pin: pinmux_P1_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pu_pin: pinmux_P1_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_pd_pin: pinmux_P1_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_gpio_input_pin: pinmux_P1_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE7) >; }; /* lcd_ac_bias_en.gpio2_25 */ + P1_04_pruout_pin: pinmux_P1_04_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_ac_bias_en.pru1_out11 */ + P1_04_pruin_pin: pinmux_P1_04_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08ec, PIN_INPUT | MUX_MODE6) >; }; /* lcd_ac_bias_en.pru1_in11 */ + + /* P1_05 (ZCZ ball T18) usb1_vbus_in */ + + /* P1_06 (ZCZ ball A16) spi0_cs0 */ + P1_06_default_pin: pinmux_P1_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P1_06_gpio_pin: pinmux_P1_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_pu_pin: pinmux_P1_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_pd_pin: pinmux_P1_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_gpio_input_pin: pinmux_P1_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_INPUT | MUX_MODE7) >; }; /* spi0_cs0.gpio0_5 */ + P1_06_spi_cs_pin: pinmux_P1_06_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_cs0.spi0_cs0 */ + P1_06_i2c_pin: pinmux_P1_06_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_cs0.i2c1_scl */ + P1_06_pwm_pin: pinmux_P1_06_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_cs0.ehrpwm0_synci */ + P1_06_pru_uart_pin: pinmux_P1_06_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x095c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_cs0.pr1_uart0_txd */ + + /* P1_07 VIN-USB */ + + /* P1_08 (ZCZ ball A17) spi0_sclk */ + P1_08_default_pin: pinmux_P1_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P1_08_gpio_pin: pinmux_P1_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_pu_pin: pinmux_P1_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_pd_pin: pinmux_P1_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_gpio_input_pin: pinmux_P1_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_INPUT | MUX_MODE7) >; }; /* spi0_sclk.gpio0_2 */ + P1_08_spi_sclk_pin: pinmux_P1_08_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_sclk.spi0_sclk */ + P1_08_uart_pin: pinmux_P1_08_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_sclk.uart2_rxd */ + P1_08_i2c_pin: pinmux_P1_08_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_sclk.i2c2_sda */ + P1_08_pwm_pin: pinmux_P1_08_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_sclk.ehrpwm0a */ + P1_08_pru_uart_pin: pinmux_P1_08_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0950, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_sclk.pr1_uart0_cts_n */ + + /* P1_09 (ZCZ ball R18) USB1-DN */ + + /* P1_10 (ZCZ ball B17) spi0_d0 */ + P1_10_default_pin: pinmux_P1_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P1_10_gpio_pin: pinmux_P1_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_pu_pin: pinmux_P1_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_pd_pin: pinmux_P1_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_gpio_input_pin: pinmux_P1_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d0.gpio0_3 */ + P1_10_spi_pin: pinmux_P1_10_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d0.spi0_d0 */ + P1_10_uart_pin: pinmux_P1_10_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi0_d0.uart2_txd */ + P1_10_i2c_pin: pinmux_P1_10_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d0.i2c2_scl */ + P1_10_pwm_pin: pinmux_P1_10_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d0.ehrpwm0b */ + P1_10_pru_uart_pin: pinmux_P1_10_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0954, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d0.pr1_uart0_rts_n */ + + /* P1_11 (ZCZ ball R17) USB1-DP */ + + /* P1_12 (ZCZ ball B16) spi0_d1 */ + P1_12_default_pin: pinmux_P1_12_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P1_12_gpio_pin: pinmux_P1_12_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_pu_pin: pinmux_P1_12_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_pd_pin: pinmux_P1_12_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_gpio_input_pin: pinmux_P1_12_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_INPUT | MUX_MODE7) >; }; /* spi0_d1.gpio0_4 */ + P1_12_spi_pin: pinmux_P1_12_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi0_d1.spi0_d1 */ + P1_12_i2c_pin: pinmux_P1_12_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* spi0_d1.i2c1_sda */ + P1_12_pwm_pin: pinmux_P1_12_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* spi0_d1.ehrpwm0_tripzone_input */ + P1_12_pru_uart_pin: pinmux_P1_12_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0958, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* spi0_d1.pr1_uart0_rxd */ + + /* P1_13 (ZCZ ball P17) USB1-ID */ + + /* P1_14 VOUT-3.3V */ + + /* P1_15 GND */ + + /* P1_16 GND */ + + /* P1_17 (ZCZ ball A9) VREFN */ + + /* P1_18 (ZCZ ball B9) VREFP */ + + /* P1_19 (ZCZ ball B6) AIN0 */ + + /* P1_20 (ZCZ ball D14) gpio0_20 */ + P1_20_default_pin: pinmux_P1_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pin: pinmux_P1_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pu_pin: pinmux_P1_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_pd_pin: pinmux_P1_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_gpio_input_pin: pinmux_P1_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr1.gpio0_20 */ + P1_20_pruin_pin: pinmux_P1_20_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b4, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr1.pru0_in16 */ + + /* P1_21 (ZCZ ball C7) AIN1 */ + + /* P1_22 GND */ + + /* P1_23 (ZCZ ball B7) AIN2 */ + + /* P1_24 VOUT-5V */ + + /* P1_25 (ZCZ ball A7) AIN3 */ + + /* P1_26 (ZCZ ball D18) i2c2_sda */ + P1_26_default_pin: pinmux_P1_26_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P1_26_gpio_pin: pinmux_P1_26_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_pu_pin: pinmux_P1_26_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_pd_pin: pinmux_P1_26_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_gpio_input_pin: pinmux_P1_26_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_INPUT | MUX_MODE7) >; }; /* uart1_ctsn.gpio0_12 */ + P1_26_can_pin: pinmux_P1_26_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_ctsn.dcan0_tx */ + P1_26_i2c_pin: pinmux_P1_26_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_ctsn.i2c2_sda */ + P1_26_spi_cs_pin: pinmux_P1_26_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_ctsn.spi1_cs0 */ + P1_26_pru_uart_pin: pinmux_P1_26_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0978, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_ctsn.pr1_uart0_cts_n */ + + /* P1_27 (ZCZ ball C8) AIN4 */ + + /* P1_28 (ZCZ ball D17) i2c2_scl */ + P1_28_default_pin: pinmux_P1_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P1_28_gpio_pin: pinmux_P1_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_pu_pin: pinmux_P1_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_pd_pin: pinmux_P1_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_gpio_input_pin: pinmux_P1_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rtsn.gpio0_13 */ + P1_28_can_pin: pinmux_P1_28_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rtsn.dcan0_rx */ + P1_28_i2c_pin: pinmux_P1_28_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rtsn.i2c2_scl */ + P1_28_spi_cs_pin: pinmux_P1_28_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart1_rtsn.spi1_cs1 */ + P1_28_pru_uart_pin: pinmux_P1_28_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x097c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rtsn.pr1_uart0_rts_n */ + + /* P1_29 (ZCZ ball A14) pru0_in7 */ + P1_29_default_pin: pinmux_P1_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + P1_29_gpio_pin: pinmux_P1_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_pu_pin: pinmux_P1_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_pd_pin: pinmux_P1_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_gpio_input_pin: pinmux_P1_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkx.gpio3_21 */ + P1_29_qep_pin: pinmux_P1_29_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkx.eqep0_strobe */ + P1_29_pruout_pin: pinmux_P1_29_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkx.pru0_out7 */ + P1_29_pruin_pin: pinmux_P1_29_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09ac, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkx.pru0_in7 */ + + /* P1_30 (ZCZ ball E16) uart0_txd */ + P1_30_default_pin: pinmux_P1_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_txd.uart0_txd */ + P1_30_gpio_pin: pinmux_P1_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_pu_pin: pinmux_P1_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_pd_pin: pinmux_P1_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_gpio_input_pin: pinmux_P1_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT | MUX_MODE7) >; }; /* uart0_txd.gpio1_11 */ + P1_30_uart_pin: pinmux_P1_30_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_txd.uart0_txd */ + P1_30_spi_cs_pin: pinmux_P1_30_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_txd.spi1_cs1 */ + P1_30_can_pin: pinmux_P1_30_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart0_txd.dcan0_rx */ + P1_30_i2c_pin: pinmux_P1_30_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_txd.i2c2_scl */ + P1_30_pruout_pin: pinmux_P1_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* uart0_txd.pru1_out15 */ + P1_30_pruin_pin: pinmux_P1_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0974, PIN_INPUT | MUX_MODE6) >; }; /* uart0_txd.pru1_in15 */ + + /* P1_31 (ZCZ ball B12) pru0_in4 */ + P1_31_default_pin: pinmux_P1_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + P1_31_gpio_pin: pinmux_P1_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_pu_pin: pinmux_P1_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_pd_pin: pinmux_P1_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_gpio_input_pin: pinmux_P1_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkr.gpio3_18 */ + P1_31_qep_pin: pinmux_P1_31_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkr.eqep0a_in */ + P1_31_pruout_pin: pinmux_P1_31_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkr.pru0_out4 */ + P1_31_pruin_pin: pinmux_P1_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a0, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkr.pru0_in4 */ + + /* P1_32 (ZCZ ball E15) uart0_rxd */ + P1_32_default_pin: pinmux_P1_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_rxd.uart0_rxd */ + P1_32_gpio_pin: pinmux_P1_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_pu_pin: pinmux_P1_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_pd_pin: pinmux_P1_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_gpio_input_pin: pinmux_P1_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_INPUT | MUX_MODE7) >; }; /* uart0_rxd.gpio1_10 */ + P1_32_uart_pin: pinmux_P1_32_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart0_rxd.uart0_rxd */ + P1_32_spi_cs_pin: pinmux_P1_32_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_rxd.spi1_cs0 */ + P1_32_can_pin: pinmux_P1_32_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart0_rxd.dcan0_tx */ + P1_32_i2c_pin: pinmux_P1_32_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_rxd.i2c2_sda */ + P1_32_pruout_pin: pinmux_P1_32_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* uart0_rxd.pru1_out14 */ + P1_32_pruin_pin: pinmux_P1_32_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0970, PIN_INPUT | MUX_MODE6) >; }; /* uart0_rxd.pru1_in14 */ + + /* P1_33 (ZCZ ball B13) pru0_in1 */ + P1_33_default_pin: pinmux_P1_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + P1_33_gpio_pin: pinmux_P1_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_pu_pin: pinmux_P1_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_pd_pin: pinmux_P1_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_gpio_input_pin: pinmux_P1_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsx.gpio3_15 */ + P1_33_pwm_pin: pinmux_P1_33_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsx.ehrpwm0b */ + P1_33_spi_pin: pinmux_P1_33_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_fsx.spi1_d0 */ + P1_33_pruout_pin: pinmux_P1_33_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsx.pru0_out1 */ + P1_33_pruin_pin: pinmux_P1_33_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0994, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsx.pru0_in1 */ + + /* P1_34 (ZCZ ball T11) gpio0_26 */ + P1_34_default_pin: pinmux_P1_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pin: pinmux_P1_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pu_pin: pinmux_P1_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_pd_pin: pinmux_P1_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_gpio_input_pin: pinmux_P1_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad10.gpio0_26 */ + P1_34_pwm_pin: pinmux_P1_34_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0828, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad10.ehrpwm2_tripzone_input */ + + /* P1_35 (ZCZ ball V5) pru1_in10 */ + P1_35_default_pin: pinmux_P1_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + P1_35_gpio_pin: pinmux_P1_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_pu_pin: pinmux_P1_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_pd_pin: pinmux_P1_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_gpio_input_pin: pinmux_P1_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE7) >; }; /* lcd_pclk.gpio2_24 */ + P1_35_pruout_pin: pinmux_P1_35_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_pclk.pru1_out10 */ + P1_35_pruin_pin: pinmux_P1_35_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e8, PIN_INPUT | MUX_MODE6) >; }; /* lcd_pclk.pru1_in10 */ + + /* P1_36 (ZCZ ball A13) ehrpwm0a */ + P1_36_default_pin: pinmux_P1_36_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P1_36_gpio_pin: pinmux_P1_36_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_pu_pin: pinmux_P1_36_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_pd_pin: pinmux_P1_36_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_gpio_input_pin: pinmux_P1_36_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_aclkx.gpio3_14 */ + P1_36_pwm_pin: pinmux_P1_36_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_aclkx.ehrpwm0a */ + P1_36_spi_sclk_pin: pinmux_P1_36_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_aclkx.spi1_sclk */ + P1_36_pruout_pin: pinmux_P1_36_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_aclkx.pru0_out0 */ + P1_36_pruin_pin: pinmux_P1_36_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0990, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_aclkx.pru0_in0 */ + + + /************************/ + /* P2 Header */ + /************************/ + + /* P2_01 (ZCZ ball U14) ehrpwm1a */ + P2_01_default_pin: pinmux_P2_01_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + P2_01_gpio_pin: pinmux_P2_01_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_pu_pin: pinmux_P2_01_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_pd_pin: pinmux_P2_01_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_gpio_input_pin: pinmux_P2_01_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a2.gpio1_18 */ + P2_01_pwm_pin: pinmux_P2_01_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0848, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a2.ehrpwm1a */ + + /* P2_02 (ZCZ ball V17) gpio1_27 */ + P2_02_default_pin: pinmux_P2_02_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pin: pinmux_P2_02_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pu_pin: pinmux_P2_02_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_pd_pin: pinmux_P2_02_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + P2_02_gpio_input_pin: pinmux_P2_02_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x086c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a11.gpio1_27 */ + + /* P2_03 (ZCZ ball T10) gpio0_23 */ + P2_03_default_pin: pinmux_P2_03_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pin: pinmux_P2_03_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pu_pin: pinmux_P2_03_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_pd_pin: pinmux_P2_03_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_gpio_input_pin: pinmux_P2_03_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad9.gpio0_23 */ + P2_03_pwm_pin: pinmux_P2_03_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0824, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad9.ehrpwm2b */ + + /* P2_04 (ZCZ ball T16) gpio1_26 */ + P2_04_default_pin: pinmux_P2_04_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pin: pinmux_P2_04_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pu_pin: pinmux_P2_04_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_pd_pin: pinmux_P2_04_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + P2_04_gpio_input_pin: pinmux_P2_04_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0868, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a10.gpio1_26 */ + + /* P2_05 (ZCZ ball T17) uart4_rxd */ + P2_05_default_pin: pinmux_P2_05_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + P2_05_gpio_pin: pinmux_P2_05_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_pu_pin: pinmux_P2_05_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_pd_pin: pinmux_P2_05_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_gpio_input_pin: pinmux_P2_05_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wait0.gpio0_30 */ + P2_05_uart_pin: pinmux_P2_05_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0870, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wait0.uart4_rxd */ + + /* P2_06 (ZCZ ball U16) gpio1_25 */ + P2_06_default_pin: pinmux_P2_06_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pin: pinmux_P2_06_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pu_pin: pinmux_P2_06_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_pd_pin: pinmux_P2_06_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + P2_06_gpio_input_pin: pinmux_P2_06_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0864, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a9.gpio1_25 */ + + /* P2_07 (ZCZ ball U17) uart4_txd */ + P2_07_default_pin: pinmux_P2_07_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + P2_07_gpio_pin: pinmux_P2_07_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_pu_pin: pinmux_P2_07_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_pd_pin: pinmux_P2_07_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_gpio_input_pin: pinmux_P2_07_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_wpn.gpio0_31 */ + P2_07_uart_pin: pinmux_P2_07_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0874, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_wpn.uart4_txd */ + + /* P2_08 (ZCZ ball U18) gpio1_28 */ + P2_08_default_pin: pinmux_P2_08_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pin: pinmux_P2_08_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pu_pin: pinmux_P2_08_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_pd_pin: pinmux_P2_08_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + P2_08_gpio_input_pin: pinmux_P2_08_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0878, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_be1n.gpio1_28 */ + + /* P2_09 (ZCZ ball D15) i2c1_scl */ + P2_09_default_pin: pinmux_P2_09_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P2_09_gpio_pin: pinmux_P2_09_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_pu_pin: pinmux_P2_09_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_pd_pin: pinmux_P2_09_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_gpio_input_pin: pinmux_P2_09_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE7) >; }; /* uart1_txd.gpio0_15 */ + P2_09_uart_pin: pinmux_P2_09_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_txd.uart1_txd */ + P2_09_can_pin: pinmux_P2_09_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart1_txd.dcan1_rx */ + P2_09_i2c_pin: pinmux_P2_09_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_txd.i2c1_scl */ + P2_09_pru_uart_pin: pinmux_P2_09_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_txd.pr1_uart0_txd */ + P2_09_pruin_pin: pinmux_P2_09_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0984, PIN_INPUT | MUX_MODE6) >; }; /* uart1_txd.pru0_in16 */ + + /* P2_10 (ZCZ ball R14) gpio1_20 */ + P2_10_default_pin: pinmux_P2_10_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pin: pinmux_P2_10_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pu_pin: pinmux_P2_10_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_pd_pin: pinmux_P2_10_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_gpio_input_pin: pinmux_P2_10_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_a4.gpio1_20 */ + P2_10_qep_pin: pinmux_P2_10_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0850, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE6) >; }; /* gpmc_a4.eqep1a_in */ + + /* P2_11 (ZCZ ball D16) i2c1_sda */ + P2_11_default_pin: pinmux_P2_11_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P2_11_gpio_pin: pinmux_P2_11_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_pu_pin: pinmux_P2_11_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_pd_pin: pinmux_P2_11_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_gpio_input_pin: pinmux_P2_11_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE7) >; }; /* uart1_rxd.gpio0_14 */ + P2_11_uart_pin: pinmux_P2_11_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* uart1_rxd.uart1_rxd */ + P2_11_can_pin: pinmux_P2_11_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart1_rxd.dcan1_tx */ + P2_11_i2c_pin: pinmux_P2_11_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart1_rxd.i2c1_sda */ + P2_11_pru_uart_pin: pinmux_P2_11_pru_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart1_rxd.pr1_uart0_rxd */ + P2_11_pruin_pin: pinmux_P2_11_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0980, PIN_INPUT | MUX_MODE6) >; }; /* uart1_rxd.pru1_in16 */ + + /* P2_12 POWER_BUTTON */ + + /* P2_13 VOUT-5V */ + + /* P2_14 BAT-VIN */ + + /* P2_15 GND */ + + /* P2_16 BAT-TEMP */ + + /* P2_17 (ZCZ ball V12) gpio2_1 */ + P2_17_default_pin: pinmux_P2_17_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pin: pinmux_P2_17_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pu_pin: pinmux_P2_17_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_pd_pin: pinmux_P2_17_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + P2_17_gpio_input_pin: pinmux_P2_17_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x088c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_clk.gpio2_1 */ + + /* P2_18 (ZCZ ball U13) gpio1_15 */ + P2_18_default_pin: pinmux_P2_18_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pin: pinmux_P2_18_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pu_pin: pinmux_P2_18_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_pd_pin: pinmux_P2_18_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_gpio_input_pin: pinmux_P2_18_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad15.gpio1_15 */ + P2_18_qep_pin: pinmux_P2_18_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad15.eqep2_strobe */ + P2_18_pru_ecap_pin: pinmux_P2_18_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* gpmc_ad15.pr1_ecap0_ecap_capin_apwm_o */ + P2_18_pruin_pin: pinmux_P2_18_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x083c, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad15.pru0_in15 */ + + /* P2_19 (ZCZ ball U12) gpio0_27 */ + P2_19_default_pin: pinmux_P2_19_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pin: pinmux_P2_19_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pu_pin: pinmux_P2_19_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_pd_pin: pinmux_P2_19_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_gpio_input_pin: pinmux_P2_19_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad11.gpio0_27 */ + P2_19_pwm_pin: pinmux_P2_19_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x082c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad11.ehrpwm0_synco */ + + /* P2_20 (ZCZ ball T13) gpio2_0 */ + P2_20_default_pin: pinmux_P2_20_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pin: pinmux_P2_20_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pu_pin: pinmux_P2_20_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_pd_pin: pinmux_P2_20_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + P2_20_gpio_input_pin: pinmux_P2_20_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0888, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_csn3.gpio2_0 */ + + /* P2_21 GND */ + + /* P2_22 (ZCZ ball V13) gpio1_14 */ + P2_22_default_pin: pinmux_P2_22_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pin: pinmux_P2_22_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pu_pin: pinmux_P2_22_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_pd_pin: pinmux_P2_22_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_gpio_input_pin: pinmux_P2_22_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad14.gpio1_14 */ + P2_22_qep_pin: pinmux_P2_22_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad14.eqep2_index */ + P2_22_pruin_pin: pinmux_P2_22_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0838, PIN_INPUT | MUX_MODE6) >; }; /* gpmc_ad14.pru0_in14 */ + + /* P2_23 VOUT-3.3V */ + + /* P2_24 (ZCZ ball T12) gpio1_12 */ + P2_24_default_pin: pinmux_P2_24_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pin: pinmux_P2_24_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pu_pin: pinmux_P2_24_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_pd_pin: pinmux_P2_24_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_gpio_input_pin: pinmux_P2_24_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad12.gpio1_12 */ + P2_24_qep_pin: pinmux_P2_24_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad12.eqep2a_in */ + P2_24_pruout_pin: pinmux_P2_24_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0830, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad12.pru0_out14 */ + + /* P2_25 (ZCZ ball E17) spi1_d1 */ + P2_25_default_pin: pinmux_P2_25_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_rtsn.spi1_d1 */ + P2_25_gpio_pin: pinmux_P2_25_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_pu_pin: pinmux_P2_25_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_pd_pin: pinmux_P2_25_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_gpio_input_pin: pinmux_P2_25_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_INPUT | MUX_MODE7) >; }; /* uart0_rtsn.gpio1_9 */ + P2_25_uart_pin: pinmux_P2_25_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_rtsn.uart4_txd */ + P2_25_can_pin: pinmux_P2_25_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* uart0_rtsn.dcan1_rx */ + P2_25_i2c_pin: pinmux_P2_25_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_rtsn.i2c1_scl */ + P2_25_spi_pin: pinmux_P2_25_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_rtsn.spi1_d1 */ + P2_25_spi_cs_pin: pinmux_P2_25_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x096c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE5) >; }; /* uart0_rtsn.spi1_cs0 */ + + /* P2_26 RESET# */ + + /* P2_27 (ZCZ ball E18) spi1_d0 */ + P2_27_default_pin: pinmux_P2_27_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_ctsn.spi1_d0 */ + P2_27_gpio_pin: pinmux_P2_27_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_pu_pin: pinmux_P2_27_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_pd_pin: pinmux_P2_27_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_gpio_input_pin: pinmux_P2_27_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_INPUT | MUX_MODE7) >; }; /* uart0_ctsn.gpio1_8 */ + P2_27_uart_pin: pinmux_P2_27_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* uart0_ctsn.uart4_rxd */ + P2_27_can_pin: pinmux_P2_27_can_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* uart0_ctsn.dcan1_tx */ + P2_27_i2c_pin: pinmux_P2_27_i2c_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* uart0_ctsn.i2c1_sda */ + P2_27_spi_pin: pinmux_P2_27_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0968, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* uart0_ctsn.spi1_d0 */ + + /* P2_28 (ZCZ ball D13) pru0_in6 */ + P2_28_default_pin: pinmux_P2_28_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + P2_28_gpio_pin: pinmux_P2_28_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_pu_pin: pinmux_P2_28_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_pd_pin: pinmux_P2_28_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_gpio_input_pin: pinmux_P2_28_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr1.gpio3_20 */ + P2_28_qep_pin: pinmux_P2_28_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr1.eqep0_index */ + P2_28_pruout_pin: pinmux_P2_28_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr1.pru0_out6 */ + P2_28_pruin_pin: pinmux_P2_28_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a8, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr1.pru0_in6 */ + + /* P2_29 (ZCZ ball C18) spi1_sclk */ + P2_29_default_pin: pinmux_P2_29_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + P2_29_gpio_pin: pinmux_P2_29_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_pu_pin: pinmux_P2_29_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_pd_pin: pinmux_P2_29_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_gpio_input_pin: pinmux_P2_29_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_INPUT | MUX_MODE7) >; }; /* eCAP0_in_PWM0_out.gpio0_7 */ + P2_29_pwm_pin: pinmux_P2_29_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE0) >; }; /* eCAP0_in_PWM0_out.ecap0_in_pwm0_out */ + P2_29_uart_pin: pinmux_P2_29_uart_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* eCAP0_in_PWM0_out.uart3_txd */ + P2_29_spi_cs_pin: pinmux_P2_29_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE2) >; }; /* eCAP0_in_PWM0_out.spi1_cs1 */ + P2_29_pru_ecap_pin: pinmux_P2_29_pru_ecap_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE3) >; }; /* eCAP0_in_PWM0_out.pr1_ecap0_ecap_capin_apwm_o */ + P2_29_spi_sclk_pin: pinmux_P2_29_spi_sclk_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0964, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* eCAP0_in_PWM0_out.spi1_sclk */ + + /* P2_30 (ZCZ ball C12) pru0_in3 */ + P2_30_default_pin: pinmux_P2_30_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + P2_30_gpio_pin: pinmux_P2_30_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_pu_pin: pinmux_P2_30_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_pd_pin: pinmux_P2_30_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_gpio_input_pin: pinmux_P2_30_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_ahclkr.gpio3_17 */ + P2_30_pwm_pin: pinmux_P2_30_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_ahclkr.ehrpwm0_synci */ + P2_30_spi_cs_pin: pinmux_P2_30_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_ahclkr.spi1_cs0 */ + P2_30_pruout_pin: pinmux_P2_30_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_ahclkr.pru0_out3 */ + P2_30_pruin_pin: pinmux_P2_30_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x099c, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_ahclkr.pru0_in3 */ + + /* P2_31 (ZCZ ball A15) spi1_cs1 */ + P2_31_default_pin: pinmux_P2_31_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr0.spi1_cs1 */ + P2_31_gpio_pin: pinmux_P2_31_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_pu_pin: pinmux_P2_31_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_pd_pin: pinmux_P2_31_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_gpio_input_pin: pinmux_P2_31_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_INPUT | MUX_MODE7) >; }; /* xdma_event_intr0.gpio0_19 */ + P2_31_spi_cs_pin: pinmux_P2_31_spi_cs_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* xdma_event_intr0.spi1_cs1 */ + P2_31_pruin_pin: pinmux_P2_31_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09b0, PIN_INPUT | MUX_MODE5) >; }; /* xdma_event_intr0.pru1_in16 */ + + /* P2_32 (ZCZ ball D12) pru0_in2 */ + P2_32_default_pin: pinmux_P2_32_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + P2_32_gpio_pin: pinmux_P2_32_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_pu_pin: pinmux_P2_32_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_pd_pin: pinmux_P2_32_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_gpio_input_pin: pinmux_P2_32_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_axr0.gpio3_16 */ + P2_32_pwm_pin: pinmux_P2_32_pwm_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_axr0.ehrpwm0_tripzone_input */ + P2_32_spi_pin: pinmux_P2_32_spi_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp0_axr0.spi1_d1 */ + P2_32_pruout_pin: pinmux_P2_32_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_axr0.pru0_out2 */ + P2_32_pruin_pin: pinmux_P2_32_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0998, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_axr0.pru0_in2 */ + + /* P2_33 (ZCZ ball R12) gpio1_13 */ + P2_33_default_pin: pinmux_P2_33_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pin: pinmux_P2_33_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pu_pin: pinmux_P2_33_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_pd_pin: pinmux_P2_33_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_gpio_input_pin: pinmux_P2_33_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_INPUT | MUX_MODE7) >; }; /* gpmc_ad13.gpio1_13 */ + P2_33_qep_pin: pinmux_P2_33_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* gpmc_ad13.eqep2b_in */ + P2_33_pruout_pin: pinmux_P2_33_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x0834, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE6) >; }; /* gpmc_ad13.pru0_out15 */ + + /* P2_34 (ZCZ ball C13) pru0_in5 */ + P2_34_default_pin: pinmux_P2_34_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + P2_34_gpio_pin: pinmux_P2_34_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_pu_pin: pinmux_P2_34_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_pd_pin: pinmux_P2_34_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_gpio_input_pin: pinmux_P2_34_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE7) >; }; /* mcasp0_fsr.gpio3_19 */ + P2_34_qep_pin: pinmux_P2_34_qep_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* mcasp0_fsr.eqep0b_in */ + P2_34_pruout_pin: pinmux_P2_34_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* mcasp0_fsr.pru0_out5 */ + P2_34_pruin_pin: pinmux_P2_34_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x09a4, PIN_INPUT | MUX_MODE6) >; }; /* mcasp0_fsr.pru0_in5 */ + + /* P2_35 (ZCZ ball U5) gpio2_22 */ + P2_35_default_pin: pinmux_P2_35_default_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pin: pinmux_P2_35_gpio_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pu_pin: pinmux_P2_35_gpio_pu_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_pd_pin: pinmux_P2_35_gpio_pd_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_gpio_input_pin: pinmux_P2_35_gpio_input_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE7) >; }; /* lcd_vsync.gpio2_22 */ + P2_35_pruout_pin: pinmux_P2_35_pruout_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE5) >; }; /* lcd_vsync.pru1_out8 */ + P2_35_pruin_pin: pinmux_P2_35_pruin_pin { pinctrl-single,pins = < + AM33XX_IOPAD(0x08e0, PIN_INPUT | MUX_MODE6) >; }; /* lcd_vsync.pru1_in8 */ + + /* P2_36 (ZCZ ball C9) AIN7 */ }; &epwmss0 { @@ -141,7 +973,29 @@ &ehrpwm0 { status = "okay"; pinctrl-names = "default"; - pinctrl-0 = <&ehrpwm0_pins>; + //pinctrl-0 = <&ehrpwm0_pins>; + pinctrl-0 = <>; +}; + +// EQEP0 is available on: +// P1.31 - A +// P2.34 - B +// P1.29 - STRB +// P2.28 - IDX +&eqep0 { + status = "okay"; + count_mode = <0>; // count_mode is not userspace op_mode + // 0 -> Quadrature mode, normal 90 phase offset cha & chb. + // 1 -> Direction mode. cha input = clock, chb input = direction + // 2 -> UP count mode for frequency measurement QDIR=1, ignore direction input + // 3 -> DOWN count mode for frequency measurement QDIR=0, ignore direction input + + swap_inputs = <0>; // swap channel A and B? (0 - no, 1 - yes) + invert_qa = <1>; // invert channel A input? + invert_qb = <1>; // invert channel B input? + invert_qi = <0>; // invert index input? + invert_qs = <0>; // invert strobe input? + omit_interrupt = <0>; // 1 -> do not install interrupt handler, 0 -> do install }; &epwmss1 { @@ -151,7 +1005,39 @@ &ehrpwm1 { status = "okay"; pinctrl-names = "default"; - pinctrl-0 = <&ehrpwm1_pins>; + //pinctrl-0 = <&ehrpwm1_pins>; + pinctrl-0 = <>; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm2 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +// EQEP2 is available on: +// P2.24 - A +// P2.33 - B +// P2.18 - STRB +// P2.22 - IDX +&eqep2 { + status = "okay"; + count_mode = <0>; // count_mode is not userspace op_mode + // 0 -> Quadrature mode, normal 90 phase offset cha & chb. + // 1 -> Direction mode. cha input = clock, chb input = direction + // 2 -> UP count mode for frequency measurement QDIR=1, ignore direction input + // 3 -> DOWN count mode for frequency measurement QDIR=0, ignore direction input + + swap_inputs = <0>; // swap channel A and B? (0 - no, 1 - yes) + invert_qa = <1>; // invert channel A input? + invert_qb = <1>; // invert channel B input? + invert_qi = <0>; // invert index input? + invert_qs = <0>; // invert strobe input? + omit_interrupt = <0>; // 1 -> do not install interrupt handler, 0 -> do install }; &i2c0 { @@ -161,9 +1047,18 @@ }; }; +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <>; + + status = "okay"; + clock-frequency = <400000>; +}; + &i2c2 { pinctrl-names = "default"; - pinctrl-0 = <&i2c2_pins>; +// pinctrl-0 = <&i2c2_pins>; + pinctrl-0 = <>; status = "okay"; clock-frequency = <400000>; @@ -182,6 +1077,14 @@ system-power-controller; }; +//&pruss_soc_bus { +// status = "okay"; +// +// pruss: pruss@4a300000 { +// status = "okay"; +// }; +//}; + &tscadc { status = "okay"; adc { @@ -194,14 +1097,30 @@ &uart0 { pinctrl-names = "default"; - pinctrl-0 = <&uart0_pins>; + //pinctrl-0 = <&uart0_pins>; + pinctrl-0 = <>; + + status = "okay"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <>; + + status = "okay"; +}; + +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <>; status = "okay"; }; &uart4 { pinctrl-names = "default"; - pinctrl-0 = <&uart4_pins>; + //pinctrl-0 = <&uart4_pins>; + pinctrl-0 = <>; status = "okay"; }; @@ -235,3 +1154,1092 @@ &cppi41dma { status = "okay"; }; + +&spi0 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/0.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/0.1"; + reg = <1>; + spi-max-frequency = <24000000>; + status = "disabled"; + }; +}; + +&spi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.0"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + symlink = "spi/1.1"; + reg = <1>; + spi-max-frequency = <24000000>; + }; +}; + +&dcan0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&dcan1 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; +}; + +&ocp { + /************************/ + /* P1 Header */ + /************************/ + + /* P1_01 VIN-AC */ + + /* P1_02 (ZCZ ball R5) gpio_input */ + P1_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_02_default_pin>; + pinctrl-1 = <&P1_02_gpio_pin>; + pinctrl-2 = <&P1_02_gpio_pu_pin>; + pinctrl-3 = <&P1_02_gpio_pd_pin>; + pinctrl-4 = <&P1_02_gpio_input_pin>; + pinctrl-5 = <&P1_02_pruout_pin>; + pinctrl-6 = <&P1_02_pruin_pin>; + }; + + /* P1_03 (ZCZ ball F15) usb1_vbus_out */ + + /* P1_04 (ZCZ ball R6) */ + P1_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_04_default_pin>; + pinctrl-1 = <&P1_04_gpio_pin>; + pinctrl-2 = <&P1_04_gpio_pu_pin>; + pinctrl-3 = <&P1_04_gpio_pd_pin>; + pinctrl-4 = <&P1_04_gpio_input_pin>; + pinctrl-5 = <&P1_04_pruout_pin>; + pinctrl-6 = <&P1_04_pruin_pin>; + }; + + /* P1_05 (ZCZ ball T18) usb1_vbus_in */ + + /* P1_06 (ZCZ ball A16) spi_cs */ + P1_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_06_default_pin>; + pinctrl-1 = <&P1_06_gpio_pin>; + pinctrl-2 = <&P1_06_gpio_pu_pin>; + pinctrl-3 = <&P1_06_gpio_pd_pin>; + pinctrl-4 = <&P1_06_gpio_input_pin>; + pinctrl-5 = <&P1_06_spi_cs_pin>; + pinctrl-6 = <&P1_06_i2c_pin>; + pinctrl-7 = <&P1_06_pwm_pin>; + pinctrl-8 = <&P1_06_pru_uart_pin>; + }; + + /* P1_07 VIN-USB */ + + /* P1_08 (ZCZ ball A17) spi_sclk */ + P1_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_08_default_pin>; + pinctrl-1 = <&P1_08_gpio_pin>; + pinctrl-2 = <&P1_08_gpio_pu_pin>; + pinctrl-3 = <&P1_08_gpio_pd_pin>; + pinctrl-4 = <&P1_08_gpio_input_pin>; + pinctrl-5 = <&P1_08_spi_sclk_pin>; + pinctrl-6 = <&P1_08_uart_pin>; + pinctrl-7 = <&P1_08_i2c_pin>; + pinctrl-8 = <&P1_08_pwm_pin>; + pinctrl-9 = <&P1_08_pru_uart_pin>; + }; + + /* P1_09 (ZCZ ball R18) USB1-DN */ + + /* P1_10 (ZCZ ball B17) spi */ + P1_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_10_default_pin>; + pinctrl-1 = <&P1_10_gpio_pin>; + pinctrl-2 = <&P1_10_gpio_pu_pin>; + pinctrl-3 = <&P1_10_gpio_pd_pin>; + pinctrl-4 = <&P1_10_gpio_input_pin>; + pinctrl-5 = <&P1_10_spi_pin>; + pinctrl-6 = <&P1_10_uart_pin>; + pinctrl-7 = <&P1_10_i2c_pin>; + pinctrl-8 = <&P1_10_pwm_pin>; + pinctrl-9 = <&P1_10_pru_uart_pin>; + }; + + /* P1_11 (ZCZ ball R17) USB1-DP */ + + /* P1_12 (ZCZ ball B16) spi */ + P1_12_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "i2c", "pwm", "pru_uart"; + pinctrl-0 = <&P1_12_default_pin>; + pinctrl-1 = <&P1_12_gpio_pin>; + pinctrl-2 = <&P1_12_gpio_pu_pin>; + pinctrl-3 = <&P1_12_gpio_pd_pin>; + pinctrl-4 = <&P1_12_gpio_input_pin>; + pinctrl-5 = <&P1_12_spi_pin>; + pinctrl-6 = <&P1_12_i2c_pin>; + pinctrl-7 = <&P1_12_pwm_pin>; + pinctrl-8 = <&P1_12_pru_uart_pin>; + }; + + /* P1_13 (ZCZ ball P17) USB1-ID */ + + /* P1_14 VOUT-3.3V */ + + /* P1_15 GND */ + + /* P1_16 GND */ + + /* P1_17 (ZCZ ball A9) VREFN */ + + /* P1_18 (ZCZ ball B9) VREFP */ + + /* P1_19 (ZCZ ball B6) AIN0 */ + + /* P1_20 (ZCZ ball D14) */ + P1_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruin"; + pinctrl-0 = <&P1_20_default_pin>; + pinctrl-1 = <&P1_20_gpio_pin>; + pinctrl-2 = <&P1_20_gpio_pu_pin>; + pinctrl-3 = <&P1_20_gpio_pd_pin>; + pinctrl-4 = <&P1_20_gpio_input_pin>; + pinctrl-5 = <&P1_20_pruin_pin>; + }; + + /* P1_21 (ZCZ ball C7) AIN1 */ + + /* P1_22 GND */ + + /* P1_23 (ZCZ ball B7) AIN2 */ + + /* P1_24 VOUT-5V */ + + /* P1_25 (ZCZ ball A7) AIN3 */ + + /* P1_26 (ZCZ ball D18) i2c */ + P1_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_26_default_pin>; + pinctrl-1 = <&P1_26_gpio_pin>; + pinctrl-2 = <&P1_26_gpio_pu_pin>; + pinctrl-3 = <&P1_26_gpio_pd_pin>; + pinctrl-4 = <&P1_26_gpio_input_pin>; + pinctrl-5 = <&P1_26_spi_cs_pin>; + pinctrl-6 = <&P1_26_can_pin>; + pinctrl-7 = <&P1_26_i2c_pin>; + pinctrl-8 = <&P1_26_pru_uart_pin>; + }; + + /* P1_27 (ZCZ ball C8) AIN4 */ + + /* P1_28 (ZCZ ball D17) i2c */ + P1_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "can", "i2c", "pru_uart"; + pinctrl-0 = <&P1_28_default_pin>; + pinctrl-1 = <&P1_28_gpio_pin>; + pinctrl-2 = <&P1_28_gpio_pu_pin>; + pinctrl-3 = <&P1_28_gpio_pd_pin>; + pinctrl-4 = <&P1_28_gpio_input_pin>; + pinctrl-5 = <&P1_28_spi_cs_pin>; + pinctrl-6 = <&P1_28_can_pin>; + pinctrl-7 = <&P1_28_i2c_pin>; + pinctrl-8 = <&P1_28_pru_uart_pin>; + }; + + /* P1_29 (ZCZ ball A14) pruin */ + P1_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_29_default_pin>; + pinctrl-1 = <&P1_29_gpio_pin>; + pinctrl-2 = <&P1_29_gpio_pu_pin>; + pinctrl-3 = <&P1_29_gpio_pd_pin>; + pinctrl-4 = <&P1_29_gpio_input_pin>; + pinctrl-5 = <&P1_29_qep_pin>; + pinctrl-6 = <&P1_29_pruout_pin>; + pinctrl-7 = <&P1_29_pruin_pin>; + }; + + /* P1_30 (ZCZ ball E16) uart */ + P1_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_30_default_pin>; + pinctrl-1 = <&P1_30_gpio_pin>; + pinctrl-2 = <&P1_30_gpio_pu_pin>; + pinctrl-3 = <&P1_30_gpio_pd_pin>; + pinctrl-4 = <&P1_30_gpio_input_pin>; + pinctrl-5 = <&P1_30_spi_cs_pin>; + pinctrl-6 = <&P1_30_uart_pin>; + pinctrl-7 = <&P1_30_can_pin>; + pinctrl-8 = <&P1_30_i2c_pin>; + pinctrl-9 = <&P1_30_pruout_pin>; + pinctrl-10 = <&P1_30_pruin_pin>; + }; + + /* P1_31 (ZCZ ball B12) pruin */ + P1_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P1_31_default_pin>; + pinctrl-1 = <&P1_31_gpio_pin>; + pinctrl-2 = <&P1_31_gpio_pu_pin>; + pinctrl-3 = <&P1_31_gpio_pd_pin>; + pinctrl-4 = <&P1_31_gpio_input_pin>; + pinctrl-5 = <&P1_31_qep_pin>; + pinctrl-6 = <&P1_31_pruout_pin>; + pinctrl-7 = <&P1_31_pruin_pin>; + }; + + /* P1_32 (ZCZ ball E15) uart */ + P1_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "uart", "can", "i2c", "pruout", "pruin"; + pinctrl-0 = <&P1_32_default_pin>; + pinctrl-1 = <&P1_32_gpio_pin>; + pinctrl-2 = <&P1_32_gpio_pu_pin>; + pinctrl-3 = <&P1_32_gpio_pd_pin>; + pinctrl-4 = <&P1_32_gpio_input_pin>; + pinctrl-5 = <&P1_32_spi_cs_pin>; + pinctrl-6 = <&P1_32_uart_pin>; + pinctrl-7 = <&P1_32_can_pin>; + pinctrl-8 = <&P1_32_i2c_pin>; + pinctrl-9 = <&P1_32_pruout_pin>; + pinctrl-10 = <&P1_32_pruin_pin>; + }; + + /* P1_33 (ZCZ ball B13) pruin */ + P1_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_33_default_pin>; + pinctrl-1 = <&P1_33_gpio_pin>; + pinctrl-2 = <&P1_33_gpio_pu_pin>; + pinctrl-3 = <&P1_33_gpio_pd_pin>; + pinctrl-4 = <&P1_33_gpio_input_pin>; + pinctrl-5 = <&P1_33_spi_pin>; + pinctrl-6 = <&P1_33_pwm_pin>; + pinctrl-7 = <&P1_33_pruout_pin>; + pinctrl-8 = <&P1_33_pruin_pin>; + }; + + /* P1_34 (ZCZ ball T11) */ + P1_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P1_34_default_pin>; + pinctrl-1 = <&P1_34_gpio_pin>; + pinctrl-2 = <&P1_34_gpio_pu_pin>; + pinctrl-3 = <&P1_34_gpio_pd_pin>; + pinctrl-4 = <&P1_34_gpio_input_pin>; + pinctrl-5 = <&P1_34_pwm_pin>; + }; + + /* P1_35 (ZCZ ball V5) pruin */ + P1_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P1_35_default_pin>; + pinctrl-1 = <&P1_35_gpio_pin>; + pinctrl-2 = <&P1_35_gpio_pu_pin>; + pinctrl-3 = <&P1_35_gpio_pd_pin>; + pinctrl-4 = <&P1_35_gpio_input_pin>; + pinctrl-5 = <&P1_35_pruout_pin>; + pinctrl-6 = <&P1_35_pruin_pin>; + }; + + /* P1_36 (ZCZ ball A13) pwm */ + P1_36_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_sclk", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P1_36_default_pin>; + pinctrl-1 = <&P1_36_gpio_pin>; + pinctrl-2 = <&P1_36_gpio_pu_pin>; + pinctrl-3 = <&P1_36_gpio_pd_pin>; + pinctrl-4 = <&P1_36_gpio_input_pin>; + pinctrl-5 = <&P1_36_spi_sclk_pin>; + pinctrl-6 = <&P1_36_pwm_pin>; + pinctrl-7 = <&P1_36_pruout_pin>; + pinctrl-8 = <&P1_36_pruin_pin>; + }; + + + /************************/ + /* P2 Header */ + /************************/ + + /* P2_01 (ZCZ ball U14) pwm */ + P2_01_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_01_default_pin>; + pinctrl-1 = <&P2_01_gpio_pin>; + pinctrl-2 = <&P2_01_gpio_pu_pin>; + pinctrl-3 = <&P2_01_gpio_pd_pin>; + pinctrl-4 = <&P2_01_gpio_input_pin>; + pinctrl-5 = <&P2_01_pwm_pin>; + }; + + /* P2_02 (ZCZ ball V17) */ + P2_02_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_02_default_pin>; + pinctrl-1 = <&P2_02_gpio_pin>; + pinctrl-2 = <&P2_02_gpio_pu_pin>; + pinctrl-3 = <&P2_02_gpio_pd_pin>; + pinctrl-4 = <&P2_02_gpio_input_pin>; + }; + + /* P2_03 (ZCZ ball T10) */ + P2_03_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_03_default_pin>; + pinctrl-1 = <&P2_03_gpio_pin>; + pinctrl-2 = <&P2_03_gpio_pu_pin>; + pinctrl-3 = <&P2_03_gpio_pd_pin>; + pinctrl-4 = <&P2_03_gpio_input_pin>; + pinctrl-5 = <&P2_03_pwm_pin>; + }; + + /* P2_04 (ZCZ ball T16) */ + P2_04_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_04_default_pin>; + pinctrl-1 = <&P2_04_gpio_pin>; + pinctrl-2 = <&P2_04_gpio_pu_pin>; + pinctrl-3 = <&P2_04_gpio_pd_pin>; + pinctrl-4 = <&P2_04_gpio_input_pin>; + }; + + /* P2_05 (ZCZ ball T17) uart */ + P2_05_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_05_default_pin>; + pinctrl-1 = <&P2_05_gpio_pin>; + pinctrl-2 = <&P2_05_gpio_pu_pin>; + pinctrl-3 = <&P2_05_gpio_pd_pin>; + pinctrl-4 = <&P2_05_gpio_input_pin>; + pinctrl-5 = <&P2_05_uart_pin>; + }; + + /* P2_06 (ZCZ ball U16) */ + P2_06_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_06_default_pin>; + pinctrl-1 = <&P2_06_gpio_pin>; + pinctrl-2 = <&P2_06_gpio_pu_pin>; + pinctrl-3 = <&P2_06_gpio_pd_pin>; + pinctrl-4 = <&P2_06_gpio_input_pin>; + }; + + /* P2_07 (ZCZ ball U17) uart */ + P2_07_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart"; + pinctrl-0 = <&P2_07_default_pin>; + pinctrl-1 = <&P2_07_gpio_pin>; + pinctrl-2 = <&P2_07_gpio_pu_pin>; + pinctrl-3 = <&P2_07_gpio_pd_pin>; + pinctrl-4 = <&P2_07_gpio_input_pin>; + pinctrl-5 = <&P2_07_uart_pin>; + }; + + /* P2_08 (ZCZ ball U18) */ + P2_08_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_08_default_pin>; + pinctrl-1 = <&P2_08_gpio_pin>; + pinctrl-2 = <&P2_08_gpio_pu_pin>; + pinctrl-3 = <&P2_08_gpio_pd_pin>; + pinctrl-4 = <&P2_08_gpio_input_pin>; + }; + + /* P2_09 (ZCZ ball D15) i2c */ + P2_09_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_09_default_pin>; + pinctrl-1 = <&P2_09_gpio_pin>; + pinctrl-2 = <&P2_09_gpio_pu_pin>; + pinctrl-3 = <&P2_09_gpio_pd_pin>; + pinctrl-4 = <&P2_09_gpio_input_pin>; + pinctrl-5 = <&P2_09_uart_pin>; + pinctrl-6 = <&P2_09_can_pin>; + pinctrl-7 = <&P2_09_i2c_pin>; + pinctrl-8 = <&P2_09_pru_uart_pin>; + pinctrl-9 = <&P2_09_pruin_pin>; + }; + + /* P2_10 (ZCZ ball R14) */ + P2_10_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep"; + pinctrl-0 = <&P2_10_default_pin>; + pinctrl-1 = <&P2_10_gpio_pin>; + pinctrl-2 = <&P2_10_gpio_pu_pin>; + pinctrl-3 = <&P2_10_gpio_pd_pin>; + pinctrl-4 = <&P2_10_gpio_input_pin>; + pinctrl-5 = <&P2_10_qep_pin>; + }; + + /* P2_11 (ZCZ ball D16) i2c */ + P2_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "uart", "can", "i2c", "pru_uart", "pruin"; + pinctrl-0 = <&P2_11_default_pin>; + pinctrl-1 = <&P2_11_gpio_pin>; + pinctrl-2 = <&P2_11_gpio_pu_pin>; + pinctrl-3 = <&P2_11_gpio_pd_pin>; + pinctrl-4 = <&P2_11_gpio_input_pin>; + pinctrl-5 = <&P2_11_uart_pin>; + pinctrl-6 = <&P2_11_can_pin>; + pinctrl-7 = <&P2_11_i2c_pin>; + pinctrl-8 = <&P2_11_pru_uart_pin>; + pinctrl-9 = <&P2_11_pruin_pin>; + }; + + /* P2_12 POWER_BUTTON */ + + /* P2_13 VOUT-5V */ + + /* P2_14 BAT-VIN */ + + /* P2_15 GND */ + + /* P2_16 BAT-TEMP */ + + /* P2_17 (ZCZ ball V12) */ + P2_17_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_17_default_pin>; + pinctrl-1 = <&P2_17_gpio_pin>; + pinctrl-2 = <&P2_17_gpio_pu_pin>; + pinctrl-3 = <&P2_17_gpio_pd_pin>; + pinctrl-4 = <&P2_17_gpio_input_pin>; + }; + + /* P2_18 (ZCZ ball U13) */ + P2_18_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pru_ecap", "pruin"; + pinctrl-0 = <&P2_18_default_pin>; + pinctrl-1 = <&P2_18_gpio_pin>; + pinctrl-2 = <&P2_18_gpio_pu_pin>; + pinctrl-3 = <&P2_18_gpio_pd_pin>; + pinctrl-4 = <&P2_18_gpio_input_pin>; + pinctrl-5 = <&P2_18_qep_pin>; + pinctrl-6 = <&P2_18_pru_ecap_pin>; + pinctrl-7 = <&P2_18_pruin_pin>; + }; + + /* P2_19 (ZCZ ball U12) */ + P2_19_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pwm"; + pinctrl-0 = <&P2_19_default_pin>; + pinctrl-1 = <&P2_19_gpio_pin>; + pinctrl-2 = <&P2_19_gpio_pu_pin>; + pinctrl-3 = <&P2_19_gpio_pd_pin>; + pinctrl-4 = <&P2_19_gpio_input_pin>; + pinctrl-5 = <&P2_19_pwm_pin>; + }; + + /* P2_20 (ZCZ ball T13) */ + P2_20_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input"; + pinctrl-0 = <&P2_20_default_pin>; + pinctrl-1 = <&P2_20_gpio_pin>; + pinctrl-2 = <&P2_20_gpio_pu_pin>; + pinctrl-3 = <&P2_20_gpio_pd_pin>; + pinctrl-4 = <&P2_20_gpio_input_pin>; + }; + + /* P2_21 GND */ + + /* P2_22 (ZCZ ball V13) */ + P2_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruin"; + pinctrl-0 = <&P2_22_default_pin>; + pinctrl-1 = <&P2_22_gpio_pin>; + pinctrl-2 = <&P2_22_gpio_pu_pin>; + pinctrl-3 = <&P2_22_gpio_pd_pin>; + pinctrl-4 = <&P2_22_gpio_input_pin>; + pinctrl-5 = <&P2_22_qep_pin>; + pinctrl-6 = <&P2_22_pruin_pin>; + }; + + /* P2_23 VOUT-3.3V */ + + /* P2_24 (ZCZ ball T12) */ + P2_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_24_default_pin>; + pinctrl-1 = <&P2_24_gpio_pin>; + pinctrl-2 = <&P2_24_gpio_pu_pin>; + pinctrl-3 = <&P2_24_gpio_pd_pin>; + pinctrl-4 = <&P2_24_gpio_input_pin>; + pinctrl-5 = <&P2_24_qep_pin>; + pinctrl-6 = <&P2_24_pruout_pin>; + }; + + /* P2_25 (ZCZ ball E17) spi */ + P2_25_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "spi_cs", "uart", "can", "i2c"; + pinctrl-0 = <&P2_25_default_pin>; + pinctrl-1 = <&P2_25_gpio_pin>; + pinctrl-2 = <&P2_25_gpio_pu_pin>; + pinctrl-3 = <&P2_25_gpio_pd_pin>; + pinctrl-4 = <&P2_25_gpio_input_pin>; + pinctrl-5 = <&P2_25_spi_pin>; + pinctrl-6 = <&P2_25_spi_cs_pin>; + pinctrl-7 = <&P2_25_uart_pin>; + pinctrl-8 = <&P2_25_can_pin>; + pinctrl-9 = <&P2_25_i2c_pin>; + }; + + /* P2_26 RESET# */ + + /* P2_27 (ZCZ ball E18) spi */ + P2_27_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "uart", "can", "i2c"; + pinctrl-0 = <&P2_27_default_pin>; + pinctrl-1 = <&P2_27_gpio_pin>; + pinctrl-2 = <&P2_27_gpio_pu_pin>; + pinctrl-3 = <&P2_27_gpio_pd_pin>; + pinctrl-4 = <&P2_27_gpio_input_pin>; + pinctrl-5 = <&P2_27_spi_pin>; + pinctrl-6 = <&P2_27_uart_pin>; + pinctrl-7 = <&P2_27_can_pin>; + pinctrl-8 = <&P2_27_i2c_pin>; + }; + + /* P2_28 (ZCZ ball D13) pruin */ + P2_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_28_default_pin>; + pinctrl-1 = <&P2_28_gpio_pin>; + pinctrl-2 = <&P2_28_gpio_pu_pin>; + pinctrl-3 = <&P2_28_gpio_pd_pin>; + pinctrl-4 = <&P2_28_gpio_input_pin>; + pinctrl-5 = <&P2_28_qep_pin>; + pinctrl-6 = <&P2_28_pruout_pin>; + pinctrl-7 = <&P2_28_pruin_pin>; + }; + + /* P2_29 (ZCZ ball C18) spi_sclk */ + P2_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "spi_sclk", "uart", "pwm", "pru_ecap"; + pinctrl-0 = <&P2_29_default_pin>; + pinctrl-1 = <&P2_29_gpio_pin>; + pinctrl-2 = <&P2_29_gpio_pu_pin>; + pinctrl-3 = <&P2_29_gpio_pd_pin>; + pinctrl-4 = <&P2_29_gpio_input_pin>; + pinctrl-5 = <&P2_29_spi_cs_pin>; + pinctrl-6 = <&P2_29_spi_sclk_pin>; + pinctrl-7 = <&P2_29_uart_pin>; + pinctrl-8 = <&P2_29_pwm_pin>; + pinctrl-9 = <&P2_29_pru_ecap_pin>; + }; + + /* P2_30 (ZCZ ball C12) pruin */ + P2_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_30_default_pin>; + pinctrl-1 = <&P2_30_gpio_pin>; + pinctrl-2 = <&P2_30_gpio_pu_pin>; + pinctrl-3 = <&P2_30_gpio_pd_pin>; + pinctrl-4 = <&P2_30_gpio_input_pin>; + pinctrl-5 = <&P2_30_spi_cs_pin>; + pinctrl-6 = <&P2_30_pwm_pin>; + pinctrl-7 = <&P2_30_pruout_pin>; + pinctrl-8 = <&P2_30_pruin_pin>; + }; + + /* P2_31 (ZCZ ball A15) spi_cs */ + P2_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi_cs", "pruin"; + pinctrl-0 = <&P2_31_default_pin>; + pinctrl-1 = <&P2_31_gpio_pin>; + pinctrl-2 = <&P2_31_gpio_pu_pin>; + pinctrl-3 = <&P2_31_gpio_pd_pin>; + pinctrl-4 = <&P2_31_gpio_input_pin>; + pinctrl-5 = <&P2_31_spi_cs_pin>; + pinctrl-6 = <&P2_31_pruin_pin>; + }; + + /* P2_32 (ZCZ ball D12) pruin */ + P2_32_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "spi", "pwm", "pruout", "pruin"; + pinctrl-0 = <&P2_32_default_pin>; + pinctrl-1 = <&P2_32_gpio_pin>; + pinctrl-2 = <&P2_32_gpio_pu_pin>; + pinctrl-3 = <&P2_32_gpio_pd_pin>; + pinctrl-4 = <&P2_32_gpio_input_pin>; + pinctrl-5 = <&P2_32_spi_pin>; + pinctrl-6 = <&P2_32_pwm_pin>; + pinctrl-7 = <&P2_32_pruout_pin>; + pinctrl-8 = <&P2_32_pruin_pin>; + }; + + /* P2_33 (ZCZ ball R12) */ + P2_33_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout"; + pinctrl-0 = <&P2_33_default_pin>; + pinctrl-1 = <&P2_33_gpio_pin>; + pinctrl-2 = <&P2_33_gpio_pu_pin>; + pinctrl-3 = <&P2_33_gpio_pd_pin>; + pinctrl-4 = <&P2_33_gpio_input_pin>; + pinctrl-5 = <&P2_33_qep_pin>; + pinctrl-6 = <&P2_33_pruout_pin>; + }; + + /* P2_34 (ZCZ ball C13) pruin */ + P2_34_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "qep", "pruout", "pruin"; + pinctrl-0 = <&P2_34_default_pin>; + pinctrl-1 = <&P2_34_gpio_pin>; + pinctrl-2 = <&P2_34_gpio_pu_pin>; + pinctrl-3 = <&P2_34_gpio_pd_pin>; + pinctrl-4 = <&P2_34_gpio_input_pin>; + pinctrl-5 = <&P2_34_qep_pin>; + pinctrl-6 = <&P2_34_pruout_pin>; + pinctrl-7 = <&P2_34_pruin_pin>; + }; + + /* P2_35 (ZCZ ball U5) gpio_input */ + P2_35_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "gpio_input", "pruout", "pruin"; + pinctrl-0 = <&P2_35_default_pin>; + pinctrl-1 = <&P2_35_gpio_pin>; + pinctrl-2 = <&P2_35_gpio_pu_pin>; + pinctrl-3 = <&P2_35_gpio_pd_pin>; + pinctrl-4 = <&P2_35_gpio_input_pin>; + pinctrl-5 = <&P2_35_pruout_pin>; + pinctrl-6 = <&P2_35_pruin_pin>; + }; + + /* P2_36 (ZCZ ball C9) AIN7 */ + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P1_02 { + gpio-name = "P1_02"; + gpio = <&gpio2 23 0>; + input; + dir-changeable; + }; + + P1_04 { + gpio-name = "P1_04"; + gpio = <&gpio2 25 0>; + input; + dir-changeable; + }; + + P1_06 { + gpio-name = "P1_06"; + gpio = <&gpio0 5 0>; + input; + dir-changeable; + }; + + P1_08 { + gpio-name = "P1_08"; + gpio = <&gpio0 2 0>; + input; + dir-changeable; + }; + + P1_10 { + gpio-name = "P1_10"; + gpio = <&gpio0 3 0>; + input; + dir-changeable; + }; + + P1_12 { + gpio-name = "P1_12"; + gpio = <&gpio0 4 0>; + input; + dir-changeable; + }; + + P1_20 { + gpio-name = "P1_20"; + gpio = <&gpio0 20 0>; + input; + dir-changeable; + }; + + P1_26 { + gpio-name = "P1_26"; + gpio = <&gpio0 12 0>; + input; + dir-changeable; + }; + + P1_28 { + gpio-name = "P1_28"; + gpio = <&gpio0 13 0>; + input; + dir-changeable; + }; + + P1_29 { + gpio-name = "P1_29"; + gpio = <&gpio3 21 0>; + input; + dir-changeable; + }; + + P1_30 { + gpio-name = "P1_30"; + gpio = <&gpio1 11 0>; + input; + dir-changeable; + }; + + P1_31 { + gpio-name = "P1_31"; + gpio = <&gpio3 18 0>; + input; + dir-changeable; + }; + + P1_32 { + gpio-name = "P1_32"; + gpio = <&gpio1 10 0>; + input; + dir-changeable; + }; + + P1_33 { + gpio-name = "P1_33"; + gpio = <&gpio3 15 0>; + input; + dir-changeable; + }; + + P1_34 { + gpio-name = "P1_34"; + gpio = <&gpio0 26 0>; + input; + dir-changeable; + }; + + P1_35 { + gpio-name = "P1_35"; + gpio = <&gpio2 24 0>; + input; + dir-changeable; + }; + + P1_36 { + gpio-name = "P1_36"; + gpio = <&gpio3 14 0>; + input; + dir-changeable; + }; + + P2_01 { + gpio-name = "P2_01"; + gpio = <&gpio1 18 0>; + input; + dir-changeable; + }; + + P2_02 { + gpio-name = "P2_02"; + gpio = <&gpio1 27 0>; + input; + dir-changeable; + }; + + P2_03 { + gpio-name = "P2_03"; + gpio = <&gpio0 23 0>; + input; + dir-changeable; + }; + + P2_04 { + gpio-name = "P2_04"; + gpio = <&gpio1 26 0>; + input; + dir-changeable; + }; + + P2_05 { + gpio-name = "P2_05"; + gpio = <&gpio0 30 0>; + input; + dir-changeable; + }; + + P2_06 { + gpio-name = "P2_06"; + gpio = <&gpio1 25 0>; + input; + dir-changeable; + }; + + P2_07 { + gpio-name = "P2_07"; + gpio = <&gpio0 31 0>; + input; + dir-changeable; + }; + + P2_08 { + gpio-name = "P2_08"; + gpio = <&gpio1 28 0>; + input; + dir-changeable; + }; + + P2_09 { + gpio-name = "P2_09"; + gpio = <&gpio0 15 0>; + input; + dir-changeable; + }; + + P2_10 { + gpio-name = "P2_10"; + gpio = <&gpio1 20 0>; + input; + dir-changeable; + }; + + P2_11 { + gpio-name = "P2_11"; + gpio = <&gpio0 14 0>; + input; + dir-changeable; + }; + + P2_17 { + gpio-name = "P2_17"; + gpio = <&gpio2 1 0>; + input; + dir-changeable; + }; + + P2_18 { + gpio-name = "P2_18"; + gpio = <&gpio1 15 0>; + input; + dir-changeable; + }; + + P2_19 { + gpio-name = "P2_19"; + gpio = <&gpio0 27 0>; + input; + dir-changeable; + }; + + P2_20 { + gpio-name = "P2_20"; + gpio = <&gpio2 0 0>; + input; + dir-changeable; + }; + + P2_22 { + gpio-name = "P2_22"; + gpio = <&gpio1 14 0>; + input; + dir-changeable; + }; + + P2_24 { + gpio-name = "P2_24"; + gpio = <&gpio1 12 0>; + input; + dir-changeable; + }; + + P2_25 { + gpio-name = "P2_25"; + gpio = <&gpio1 9 0>; + input; + dir-changeable; + }; + + P2_27 { + gpio-name = "P2_27"; + gpio = <&gpio1 8 0>; + input; + dir-changeable; + }; + + P2_28 { + gpio-name = "P2_28"; + gpio = <&gpio3 20 0>; + input; + dir-changeable; + }; + + P2_29 { + gpio-name = "P2_29"; + gpio = <&gpio0 7 0>; + input; + dir-changeable; + }; + + P2_30 { + gpio-name = "P2_30"; + gpio = <&gpio3 17 0>; + input; + dir-changeable; + }; + + P2_31 { + gpio-name = "P2_31"; + gpio = <&gpio0 19 0>; + input; + dir-changeable; + }; + + P2_32 { + gpio-name = "P2_32"; + gpio = <&gpio3 16 0>; + input; + dir-changeable; + }; + + P2_33 { + gpio-name = "P2_33"; + gpio = <&gpio1 13 0>; + input; + dir-changeable; + }; + + P2_34 { + gpio-name = "P2_34"; + gpio = <&gpio3 19 0>; + input; + dir-changeable; + }; + + P2_35 { + gpio-name = "P2_35"; + gpio = <&gpio2 22 0>; + input; + dir-changeable; + }; + + }; +}; diff --git a/arch/arm/boot/dts/am335x-revolve.dts b/arch/arm/boot/dts/am335x-revolve.dts new file mode 100644 index 0000000000000..a3caaf075e283 --- /dev/null +++ b/arch/arm/boot/dts/am335x-revolve.dts @@ -0,0 +1,984 @@ +/* + * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "am33xx.dtsi" +/* #include "am335x-bone-jtag.dtsi" */ + +/ { + + model = "Revolve:00A0"; + compatible = "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + cpus { + cpu@0 { + cpu0-supply = <&dcdc2_reg>; + }; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x80000000 0x10000000>; /* 256 MB */ + }; + + chosen { + stdout-path = &uart0; + }; + + vmmcsd_fixed: fixedregulator0 { + compatible = "regulator-fixed"; + regulator-name = "vmmcsd_fixed"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&user_leds_s0>; + + compatible = "gpio-leds"; + + led1 { + label = "beaglebone:green:usr1"; + gpios = <&gpio1 22 GPIO_ACTIVE_HIGH>; + trigger-sources = <&usb1>; + /*linux,default-trigger = "mmc0";*/ + default-state = "off"; + }; + + led2 { + label = "beaglebone:green:usr2"; + gpios = <&gpio1 23 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "cpu0"; + default-state = "off"; + }; + + led3 { + label = "beaglebone:green:usr3"; + gpios = <&gpio1 24 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + led4 { + label = "beaglebone:green:usr0"; + gpios = <&gpio1 25 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + }; + + + gpio_keys { + compatible = "gpio-keys"; + pinctrl-names = "default"; + pinctrl-0 = <&end_stop_pins>; + pinctrl-1 = <&stepper_fault_pins>; + #address-cells = <1>; + #size-cells = <0>; + + switch_x1 { + label = "Replic:End-stop-X1"; + debounce_interval = <50>; + linux,code = <112>; + gpios = <&gpio1 28 0x5>; + gpio-key,wakeup; + }; + switch_x2 { + label = "Replic:End-stop-X2"; + debounce_interval = <50>; + linux,code = <113>; + gpios = <&gpio1 17 0x5>; + gpio-key,wakeup; + }; + switch_y1 { + label = "Replic:End-stop-Y1"; + debounce_interval = <50>; + linux,code = <114>; + gpios = <&gpio1 27 0x5>; + gpio-key,wakeup; + }; + switch_y2 { + label = "Replic:End-stop-Y2"; + debounce_interval = <50>; + linux,code = <115>; + gpios = <&gpio1 29 0x5>; + gpio-key,wakeup; + }; + switch_z1 { + label = "Replic:End-stop-Z1"; + debounce_interval = <50>; + linux,code = <116>; + gpios = <&gpio1 14 0x5>; + gpio-key,wakeup; + }; + switch_z2 { + label = "Replic:End-stop-Z2"; + debounce_interval = <50>; + linux,code = <117>; + gpios = <&gpio1 15 0x5>; + gpio-key,wakeup; + }; + fault_x { + label = "Replic:Fault-X"; + debounce_interval = <50>; + linux,code = <90>; + gpios = <&gpio0 27 0x5>; + gpio-key,wakeup; + }; + fault_y { + label = "Replic:Fault-Y"; + debounce_interval = <50>; + linux,code = <91>; + gpios = <&gpio2 0 0x5>; + gpio-key,wakeup; + }; + fault_z { + label = "Replic:Fault-Z"; + debounce_interval = <50>; + linux,code = <92>; + gpios = <&gpio1 16 0x5>; + gpio-key,wakeup; + }; + fault_e { + label = "Replic:Fault-E"; + debounce_interval = <50>; + linux,code = <93>; + gpios = <&gpio2 1 0x5>; + gpio-key,wakeup; + }; + fault_h { + label = "Replic:Fault-H"; + debounce_interval = <50>; + linux,code = <94>; + gpios = <&gpio0 29 0x5>; + gpio-key,wakeup; + }; + fault_a { + label = "Replic:Fault-A"; + debounce_interval = <50>; + linux,code = <94>; + gpios = <&gpio0 26 0x5>; + gpio-key,wakeup; + }; + }; + + + pwm4: pwm-heater-e { + compatible = "ti,omap-dmtimer-pwm"; + ti,timers = <&timer4>; + #pwm-cells = <3>; + pinctrl-names = "default"; + pinctrl-0 = <&timer4_pins>; + + }; + + pwm5: pwm-heater-h { + compatible = "ti,omap-dmtimer-pwm"; + ti,timers = <&timer5>; + #pwm-cells = <3>; + pinctrl-names = "default"; + pinctrl-0 = <&timer5_pins>; + + }; + + pwm6: pwm-heater-hbp { + compatible = "ti,omap-dmtimer-pwm"; + ti,timers = <&timer6>; + #pwm-cells = <3>; + pinctrl-names = "default"; + pinctrl-0 = <&timer6_pins>; + + }; + + pwm7: pwm-heater-a { + compatible = "ti,omap-dmtimer-pwm"; + ti,timers = <&timer7>; + #pwm-cells = <3>; + pinctrl-names = "default"; + pinctrl-0 = <&timer7_pins>; + + }; +}; + + +&cpu0_opp_table { + /* + * All PG 2.0 silicon may not support 1GHz but some of the early + * BeagleBone Blacks have PG 2.0 silicon which is guaranteed + * to support 1GHz OPP so enable it for PG 2.0 on this board. + */ + oppnitro-1000000000 { + opp-supported-hw = <0x06 0x0100>; + }; +}; + + +&am33xx_pinmux { + nxp_hdmi_bonelt_pins: nxp_hdmi_bonelt_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x8a0, PIN_OUTPUT | MUX_MODE0) /* lcd_data0.lcd_data0 */ + AM33XX_IOPAD(0x8a4, PIN_OUTPUT | MUX_MODE0) /* lcd_data1.lcd_data1 */ + AM33XX_IOPAD(0x8a8, PIN_OUTPUT | MUX_MODE0) /* lcd_data2.lcd_data2 */ + AM33XX_IOPAD(0x8ac, PIN_OUTPUT | MUX_MODE0) /* lcd_data3.lcd_data3 */ + AM33XX_IOPAD(0x8b0, PIN_OUTPUT | MUX_MODE0) /* lcd_data4.lcd_data4 */ + AM33XX_IOPAD(0x8b4, PIN_OUTPUT | MUX_MODE0) /* lcd_data5.lcd_data5 */ + AM33XX_IOPAD(0x8b8, PIN_OUTPUT | MUX_MODE0) /* lcd_data6.lcd_data6 */ + AM33XX_IOPAD(0x8bc, PIN_OUTPUT | MUX_MODE0) /* lcd_data7.lcd_data7 */ + AM33XX_IOPAD(0x8c0, PIN_OUTPUT | MUX_MODE0) /* lcd_data8.lcd_data8 */ + AM33XX_IOPAD(0x8c4, PIN_OUTPUT | MUX_MODE0) /* lcd_data9.lcd_data9 */ + AM33XX_IOPAD(0x8c8, PIN_OUTPUT | MUX_MODE0) /* lcd_data10.lcd_data10 */ + AM33XX_IOPAD(0x8cc, PIN_OUTPUT | MUX_MODE0) /* lcd_data11.lcd_data11 */ + AM33XX_IOPAD(0x8d0, PIN_OUTPUT | MUX_MODE0) /* lcd_data12.lcd_data12 */ + AM33XX_IOPAD(0x8d4, PIN_OUTPUT | MUX_MODE0) /* lcd_data13.lcd_data13 */ + AM33XX_IOPAD(0x8d8, PIN_OUTPUT | MUX_MODE0) /* lcd_data14.lcd_data14 */ + AM33XX_IOPAD(0x8dc, PIN_OUTPUT | MUX_MODE0) /* lcd_data15.lcd_data15 */ + AM33XX_IOPAD(0x8e0, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_vsync.lcd_vsync */ + AM33XX_IOPAD(0x8e4, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_hsync.lcd_hsync */ + AM33XX_IOPAD(0x8e8, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_pclk.lcd_pclk */ + AM33XX_IOPAD(0x8ec, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* lcd_ac_bias_en.lcd_ac_bias_en */ + >; + }; + + ehrpwm1_pins: ehrpwm1_pins_default { + pinctrl-single,pins = < + 0x20 ( PIN_OUTPUT | MUX_MODE4 ) /* (U10) gpmc_ad8.ehrpwm2A */ + 0x24 ( PIN_OUTPUT | MUX_MODE4 ) /* (T10) gpmc_ad9.ehrpwm2B */ + >; + }; + + ehrpwm2_pins: ehrpwm2_pins_default { + pinctrl-single,pins = < + 0x48 ( PIN_OUTPUT | MUX_MODE6 ) /* (U14) gpmc_a2.ehrpwm1A */ + 0x4c ( PIN_OUTPUT | MUX_MODE6 ) /* (T14) gpmc_a3.ehrpwm1B */ + >; + }; + + spi0_pins_default: spi0_pins_deafult { + pinctrl-single,pins = < + 0x150 ( PIN_INPUT | MUX_MODE0 ) /* (A17) spi0_sclk.spi0_sclk */ + 0x154 ( PIN_INPUT | MUX_MODE0 ) /* (B17) spi0_d0.spi0_d0 */ + 0x158 ( PIN_INPUT | MUX_MODE0 ) /* (B16) spi0_d1.spi0_d1 */ + 0x15c ( PIN_OUTPUT | MUX_MODE0 ) /* (A16) spi0_cs0.spi0_cs0 */ + 0x160 ( PIN_OUTPUT | MUX_MODE0 ) /* (C15) spi0_cs1.spi0_cs1 */ + >; + }; + + spi0_pins_reset: spi0_pins_reset { + pinctrl-single,pins = < + 0x150 ( PIN_INPUT | MUX_MODE7 ) /* (A17) spi0_sclk.spi0_sclk */ + 0x154 ( PIN_INPUT | MUX_MODE7 ) /* (B17) spi0_d0.spi0_d0 */ + 0x158 ( PIN_INPUT | MUX_MODE7 ) /* (B16) spi0_d1.spi0_d1 */ + 0x15c ( PIN_INPUT | MUX_MODE7 ) /* (A16) spi0_cs0.spi0_cs0 */ + 0x160 ( PIN_INPUT | MUX_MODE7 ) /* (C15) spi0_cs1.spi0_cs1 */ + >; + }; + + spi1_pins: spi1_pins { + pinctrl-single,pins = < + 0x164 ( PIN_INPUT | MUX_MODE4 ) /* (C18) eCAP0_in_PWM0_out.spi1_sclk */ + 0x168 ( PIN_INPUT | MUX_MODE4 ) /* (E18) uart0_ctsn.spi1_d0 */ + 0x16c ( PIN_INPUT | MUX_MODE4 ) /* (E17) uart0_rtsn.spi1_d1 */ + 0x178 ( PIN_INPUT | MUX_MODE4 ) /* (D18) uart1_ctsn.spi1_cs0 */ + 0x17c ( PIN_INPUT | MUX_MODE4 ) /* (D17) uart1_rtsn.spi1_cs1 */ + >; + }; + + pru_stepper_pins: pru_pruss11_pins { + pinctrl-single,pins = < + 0x190 ( PIN_OUTPUT | MUX_MODE5 ) /* (A13) mcasp0_aclkx.pr1_pru0_pru_r30[0] */ + 0x194 ( PIN_OUTPUT | MUX_MODE5 ) /* (B13) mcasp0_fsx.pr1_pru0_pru_r30[1] */ + 0x198 ( PIN_OUTPUT | MUX_MODE5 ) /* (D12) mcasp0_axr0.pr1_pru0_pru_r30[2] */ + 0x19c ( PIN_OUTPUT | MUX_MODE5 ) /* (C12) mcasp0_ahclkr.pr1_pru0_pru_r30[3] */ + 0x1a0 ( PIN_OUTPUT | MUX_MODE5 ) /* (B12) mcasp0_aclkr.pr1_pru0_pru_r30[4] */ + 0x1a4 ( PIN_OUTPUT | MUX_MODE5 ) /* (C13) mcasp0_fsr.pr1_pru0_pru_r30[5] */ + 0x1a8 ( PIN_OUTPUT | MUX_MODE5 ) /* (D13) mcasp0_axr1.pr1_pru0_pru_r30[6] */ + 0x1ac ( PIN_OUTPUT | MUX_MODE5 ) /* (A14) mcasp0_ahclkx.pr1_pru0_pru_r30[7] */ + 0xf0 ( PIN_OUTPUT | MUX_MODE5 ) /* (F17) mmc0_dat3.pr1_pru0_pru_r30[8] */ + 0xf4 ( PIN_OUTPUT | MUX_MODE5 ) /* (F18) mmc0_dat2.pr1_pru0_pru_r30[9] */ + 0xf8 ( PIN_OUTPUT | MUX_MODE5 ) /* (G15) mmc0_dat1.pr1_pru0_pru_r30[10] */ + 0xfc ( PIN_OUTPUT | MUX_MODE5 ) /* (G16) mmc0_dat0.pr1_pru0_pru_r30[11] */ + 0x100 ( PIN_OUTPUT | MUX_MODE5 ) /* (G17) mmc0_clk.pr1_pru0_pru_r30[12] */ + 0x104 ( PIN_OUTPUT | MUX_MODE5 ) /* (G18) mmc0_cmd.pr1_pru0_pru_r30[13] */ + 0x30 ( PIN_OUTPUT | MUX_MODE6 ) /* (T12) gpmc_ad12.pr1_pru0_pru_r30[14] */ + 0x34 ( PIN_OUTPUT | MUX_MODE6 ) /* (R12) gpmc_ad13.pr1_pru0_pru_r30[15] */ + /*0x38 ( PIN_INPUT | MUX_MODE6 )*/ /* (V13) gpmc_ad14.pr1_pru0_pru_r31[14] */ + /*0x3c ( PIN_INPUT | MUX_MODE6 )*/ /* (U13) gpmc_ad15.pr1_pru0_pru_r31[15] */ + 0x1b4 ( PIN_INPUT | MUX_MODE5 ) /* (D14) xdma_event_intr1.pr1_pru0_pru_r31[16] */ + >; + }; + + pru_stepper_pins_gpio: pru_pruss11_pins_gpio { + pinctrl-single,pins = < + 0x190 ( PIN_OUTPUT | MUX_MODE7 ) /* (A13) mcasp0_aclkx.pr1_pru0_pru_r30[0] */ + 0x194 ( PIN_OUTPUT | MUX_MODE7 ) /* (B13) mcasp0_fsx.pr1_pru0_pru_r30[1] */ + 0x198 ( PIN_OUTPUT | MUX_MODE7 ) /* (D12) mcasp0_axr0.pr1_pru0_pru_r30[2] */ + 0x19c ( PIN_OUTPUT | MUX_MODE7 ) /* (C12) mcasp0_ahclkr.pr1_pru0_pru_r30[3] */ + 0x1a0 ( PIN_OUTPUT | MUX_MODE7 ) /* (B12) mcasp0_aclkr.pr1_pru0_pru_r30[4] */ + 0x1a4 ( PIN_OUTPUT | MUX_MODE7 ) /* (C13) mcasp0_fsr.pr1_pru0_pru_r30[5] */ + 0x1a8 ( PIN_OUTPUT | MUX_MODE7 ) /* (D13) mcasp0_axr1.pr1_pru0_pru_r30[6] */ + 0x1ac ( PIN_OUTPUT | MUX_MODE7 ) /* (A14) mcasp0_ahclkx.pr1_pru0_pru_r30[7] */ + 0xf0 ( PIN_OUTPUT | MUX_MODE7 ) /* (F17) mmc0_dat3.pr1_pru0_pru_r30[8] */ + 0xf4 ( PIN_OUTPUT | MUX_MODE7 ) /* (F18) mmc0_dat2.pr1_pru0_pru_r30[9] */ + 0xf8 ( PIN_OUTPUT | MUX_MODE7 ) /* (G15) mmc0_dat1.pr1_pru0_pru_r30[10] */ + 0xfc ( PIN_OUTPUT | MUX_MODE7 ) /* (G16) mmc0_dat0.pr1_pru0_pru_r30[11] */ + 0x100 ( PIN_OUTPUT | MUX_MODE7 ) /* (G17) mmc0_clk.pr1_pru0_pru_r30[12] */ + 0x104 ( PIN_OUTPUT | MUX_MODE7 ) /* (G18) mmc0_cmd.pr1_pru0_pru_r30[13] */ + 0x30 ( PIN_OUTPUT | MUX_MODE7 ) /* (T12) gpmc_ad12.pr1_pru0_pru_r30[14] */ + 0x34 ( PIN_OUTPUT | MUX_MODE7 ) /* (R12) gpmc_ad13.pr1_pru0_pru_r30[15] */ + >; + }; + + timer4_pins: timer4_pins { + pinctrl-single,pins = < + 0x90 ( PIN_OUTPUT | MUX_MODE2 ) /* (R7) gpmc_advn_ale.timer4 */ + >; + }; + + timer5_pins: timer5_pins { + pinctrl-single,pins = < + 0x9c ( PIN_OUTPUT | MUX_MODE2 ) /* (T6) gpmc_be0n_cle.timer5 */ + >; + }; + timer6_pins: timer6_pins { + pinctrl-single,pins = < + 0x98 ( PIN_OUTPUT | MUX_MODE2 ) /* (U6) gpmc_wen.timer6 */ + >; + }; + timer7_pins: timer7_pins { + pinctrl-single,pins = < + 0x94 ( PIN_OUTPUT | MUX_MODE2 ) /* (T7) gpmc_oen_ren.timer7 */ + >; + }; + + user_leds_s0: user_leds_s0 { + pinctrl-single,pins = < + 0x64 ( PIN_OUTPUT | MUX_MODE7 ) /* (U16) gpmc_a9.gpio1[25] */ + AM33XX_IOPAD(0x858, PIN_OUTPUT | MUX_MODE7) /* gpmc_a6.gpio1_22 */ + AM33XX_IOPAD(0x85c, PIN_OUTPUT | MUX_MODE7) /* gpmc_a7.gpio1_23 */ + AM33XX_IOPAD(0x860, PIN_OUTPUT | MUX_MODE7) /* gpmc_a8.gpio1_24 */ + >; + }; + + i2c0_pins: pinmux_i2c0_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x988, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_sda.i2c0_sda */ + AM33XX_IOPAD(0x98c, PIN_INPUT_PULLUP | MUX_MODE0) /* i2c0_scl.i2c0_scl */ + >; + }; + + i2c1_pins: i2c1_pins { + pinctrl-single,pins = < + 0x184 ( PIN_INPUT | MUX_MODE3 ) /* (D15) uart1_txd.I2C1_SCL */ + 0x180 ( PIN_INPUT | MUX_MODE3 ) /* (D16) uart1_rxd.I2C1_SDA */ + >; + }; + + + uart0_pins: pinmux_uart0_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x970, PIN_INPUT_PULLUP | MUX_MODE0) /* uart0_rxd.uart0_rxd */ + AM33XX_IOPAD(0x974, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* uart0_txd.uart0_txd */ + >; + }; + + uart4_pins: pinmux_uart4_pins { + pinctrl-single,pins = < + 0x70 ( PIN_INPUT | MUX_MODE6 ) /* (T17) gpmc_wait0.uart4_rxd */ + 0x74 ( PIN_OUTPUT | MUX_MODE6 ) /* (U17) gpmc_wpn.uart4_txd */ + >; + }; + + cpsw_default: cpsw_default { + pinctrl-single,pins = < + /* Slave 1 */ + 0x108 (PIN_INPUT | MUX_MODE0) /* mii1_col.mii1_col */ + 0x10c (PIN_INPUT | MUX_MODE0) /* mii1_crs.mii1_crs */ + AM33XX_IOPAD(0x910, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxerr.mii1_rxerr */ + AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txen.mii1_txen */ + AM33XX_IOPAD(0x918, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxdv.mii1_rxdv */ + AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd3.mii1_txd3 */ + AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd2.mii1_txd2 */ + AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd1.mii1_txd1 */ + AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE0) /* mii1_txd0.mii1_txd0 */ + AM33XX_IOPAD(0x92c, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_txclk.mii1_txclk */ + AM33XX_IOPAD(0x930, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxclk.mii1_rxclk */ + AM33XX_IOPAD(0x934, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd3.mii1_rxd3 */ + AM33XX_IOPAD(0x938, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd2.mii1_rxd2 */ + AM33XX_IOPAD(0x93c, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd1.mii1_rxd1 */ + AM33XX_IOPAD(0x940, PIN_INPUT_PULLUP | MUX_MODE0) /* mii1_rxd0.mii1_rxd0 */ + >; + }; + + cpsw_sleep: cpsw_sleep { + pinctrl-single,pins = < + /* Slave 1 reset value */ + 0x108 (PIN_INPUT_PULLDOWN | MUX_MODE7) + 0x10c (PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x910, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + davinci_mdio_default: davinci_mdio_default { + pinctrl-single,pins = < + /* MDIO */ + AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */ + AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */ + >; + }; + + davinci_mdio_sleep: davinci_mdio_sleep { + pinctrl-single,pins = < + /* MDIO reset value */ + AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7) + >; + }; + + emmc_pins: pinmux_emmc_pins { + pinctrl-single,pins = < + AM33XX_IOPAD(0x880, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn1.mmc1_clk */ + AM33XX_IOPAD(0x884, PIN_INPUT_PULLUP | MUX_MODE2) /* gpmc_csn2.mmc1_cmd */ + AM33XX_IOPAD(0x800, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad0.mmc1_dat0 */ + AM33XX_IOPAD(0x804, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad1.mmc1_dat1 */ + AM33XX_IOPAD(0x808, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad2.mmc1_dat2 */ + AM33XX_IOPAD(0x80c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad3.mmc1_dat3 */ + AM33XX_IOPAD(0x810, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad4.mmc1_dat4 */ + AM33XX_IOPAD(0x814, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad5.mmc1_dat5 */ + AM33XX_IOPAD(0x818, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad6.mmc1_dat6 */ + AM33XX_IOPAD(0x81c, PIN_INPUT_PULLUP | MUX_MODE1) /* gpmc_ad7.mmc1_dat7 */ + >; + }; + + stepper_fault_pins: pinmux_stepper_fault_pins{ + pinctrl-single,pins = < + 0x2c ( PIN_INPUT | MUX_MODE7 ) // FAULT X + 0x88 ( PIN_INPUT | MUX_MODE7 ) // FAULT Y + 0x40 ( PIN_INPUT | MUX_MODE7 ) // FAULT Z + 0x8c ( PIN_INPUT | MUX_MODE7 ) // FAULT E + 0x144 ( PIN_INPUT | MUX_MODE7 ) // FAULT H + 0x28 ( PIN_INPUT | MUX_MODE7 ) // FAULT A + >; + }; + + end_stop_pins: pinmux_end_stop_pins{ + pinctrl-single,pins = < + 0x78 ( PIN_INPUT | MUX_MODE7 ) // End stop X1 = GPIO3_21 + 0x44 ( PIN_INPUT | MUX_MODE7 ) // End stop X2 = GPIO0_30 + 0x6c ( PIN_INPUT | MUX_MODE7 ) // End stop Y1 = GPIO1_27 + 0x7c ( PIN_INPUT | MUX_MODE7 ) // End stop Z1 = GPIO1_14 + 0x38 ( PIN_INPUT | MUX_MODE7 ) // End stop Z2 = GPIO1_15 + 0x3c ( PIN_INPUT | MUX_MODE7 ) // End stop Y2 = GPIO3_17 + >; + }; + + enable_stuff_pins: enable_stuff_pins{ + pinctrl-single,pins = < + 0x21c ( PIN_INPUT | MUX_MODE7 ) /* (F16) USB0_DRVVBUS.gpio0[18] */ + >; + }; + + eqep1_pins_default: myeqep1_pins_default { + pinctrl-single,pins = < + 0x50 ( PIN_INPUT | MUX_MODE6 ) /* (R14) gpmc_a4.eQEP1A_in */ + 0x54 ( PIN_INPUT | MUX_MODE6 ) /* (V15) gpmc_a5.eQEP1B_in */ + >; + }; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_pins>; + status = "okay"; +}; + +&uart4 { + pinctrl-names = "default"; + pinctrl-0 = <&uart4_pins>; + status = "okay"; +}; + +&usb { + status = "okay"; +}; + +&usb_ctrl_mod { + status = "okay"; +}; + +&usb0_phy { + status = "okay"; + #phy-cells = <0>; +}; + +&usb1_phy { + status = "okay"; + #phy-cells = <0>; +}; + +&usb0 { + status = "okay"; + dr_mode = "peripheral"; + interrupts-extended = <&intc 18 &tps 0>; + interrupt-names = "mc", "vbus"; +}; + +&usb1 { + status = "okay"; + dr_mode = "host"; +}; + +&cppi41dma { + status = "okay"; +}; + +&gpio0{ + status = "okay"; +}; + +&i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + + status = "okay"; + clock-frequency = <400000>; + + tps: tps@24 { + reg = <0x24>; + }; + + baseboard_eeprom: baseboard_eeprom@50 { + compatible = "atmel,24c256"; + reg = <0x50>; + + #address-cells = <1>; + #size-cells = <1>; + baseboard_data: baseboard_data@0 { + reg = <0 0x100>; + }; + }; + + tda19988: tda19988@70 { + compatible = "nxp,tda998x"; + reg = <0x70>; + + pinctrl-names = "default"; + pinctrl-0 = <&nxp_hdmi_bonelt_pins>; + + /* Convert 24bit BGR to RGB, e.g. cross red and blue wiring */ + /* video-ports = <0x234501>; */ + ports { + port@0 { + hdmi_0: endpoint@0 { + remote-endpoint = <&lcdc_0>; + }; + }; + }; + }; +}; + +&i2c1{ + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + status = "okay"; + clock-frequency = <400000>; +}; + +&epwmss1{ + status = "okay"; +}; + +&ehrpwm1{ + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&ehrpwm1_pins>; +}; + +&eqep1{ + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&eqep1_pins_default>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ +}; + +&epwmss2{ + status = "okay"; +}; + +&ehrpwm2{ + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&ehrpwm2_pins>; +}; + + +&timer4{ + status = "okay"; + ti,timer-pwm; +}; +&timer5{ + status = "okay"; + ti,timer-pwm; +}; +&timer6{ + status = "okay"; + ti,timer-pwm; +}; +&timer7{ + status = "okay"; + ti,timer-pwm; +}; + +&spi0{ + status = "okay"; + pinctrl-names = "default"; + + stepper_control@0{ + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + reg = <0>; + spi-max-frequency = <2000000>; + }; + aux_control@1{ + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +&spi1{ + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&spi1_pins>; + + ext_stepper_B@0{ + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + reg = <0>; + spi-max-frequency = <16000000>; + }; + + ext_stepper_C@1{ + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +&pruss{ + status = "okay"; +}; + +&tscadc{ + status = "okay"; + adc { + ti,adc-channels = <0 1 2 3 4 5 6 7>; + }; +}; + +/include/ "tps65217.dtsi" + +&tps { + /* + * Configure pmic to enter OFF-state instead of SLEEP-state ("RTC-only + * mode") at poweroff. Most BeagleBone versions do not support RTC-only + * mode and risk hardware damage if this mode is entered. + * + * For details, see linux-omap mailing list May 2015 thread + * [PATCH] ARM: dts: am335x-bone* enable pmic-shutdown-controller + * In particular, messages: + * http://www.spinics.net/lists/linux-omap/msg118585.html + * http://www.spinics.net/lists/linux-omap/msg118615.html + * + * You can override this later with + * &tps { /delete-property/ ti,pmic-shutdown-controller; } + * if you want to use RTC-only mode and made sure you are not affected + * by the hardware problems. (Tip: double-check by performing a current + * measurement after shutdown: it should be less than 1 mA.) + */ + + interrupts = <7>; /* NMI */ + interrupt-parent = <&intc>; + + ti,pmic-shutdown-controller; + + charger { + status = "okay"; + }; + + pwrbutton { + status = "okay"; + }; + + regulators { + dcdc1_reg: regulator@0 { + regulator-name = "vdds_dpr"; + regulator-always-on; + }; + + dcdc2_reg: regulator@1 { + /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */ + regulator-name = "vdd_mpu"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1351500>; + regulator-boot-on; + regulator-always-on; + }; + + dcdc3_reg: regulator@2 { + /* VDD_CORE voltage limits 0.95V - 1.1V with +/-4% tolerance */ + regulator-name = "vdd_core"; + regulator-min-microvolt = <925000>; + regulator-max-microvolt = <1150000>; + regulator-boot-on; + regulator-always-on; + }; + + ldo1_reg: regulator@3 { + regulator-name = "vio,vrtc,vdds"; + regulator-always-on; + }; + + ldo2_reg: regulator@4 { + regulator-name = "vdd_3v3aux"; + regulator-always-on; + }; + + ldo3_reg: regulator@5 { + regulator-name = "vdd_1v8"; + regulator-always-on; + }; + + ldo4_reg: regulator@6 { + regulator-name = "vdd_3v3a"; + regulator-always-on; + }; + }; +}; + +&cpsw_emac0 { + phy_id = <&davinci_mdio>, <0>; + phy-mode = "mii"; +}; + +&mac { + slaves = <1>; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&cpsw_default>; + pinctrl-1 = <&cpsw_sleep>; + status = "okay"; +}; + +&davinci_mdio { + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&davinci_mdio_default>; + pinctrl-1 = <&davinci_mdio_sleep>; + status = "okay"; +}; + +&mmc2 { + vmmc-supply = <&vmmcsd_fixed>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pins>; + bus-width = <8>; + status = "okay"; +}; + + +&lcdc { + status = "okay"; + + /* If you want to get 24 bit RGB and 16 BGR mode instead of + * current 16 bit RGB and 24 BGR modes, set the propety + * below to "crossed" and uncomment the video-ports -property + * in tda19988 node. + */ + blue-and-red-wiring = "straight"; + + port { + lcdc_0: endpoint@0 { + remote-endpoint = <&hdmi_0>; + }; + }; +}; + +&aes { + status = "okay"; +}; + +&sham { + status = "okay"; +}; + +&rtc { + system-power-controller; +}; + + +/* the cape manager */ +/ { + bone_capemgr { + compatible = "ti,bone-capemgr"; + status = "okay"; + + nvmem-cells = <&baseboard_data>; + nvmem-cell-names = "baseboard"; + #slots = <0>; + + /* map board revisions to compatible definitions */ + baseboardmaps { + baseboard_revolve: board@0 { + board-name = "A335RVLV"; + compatible-name = "ti,beaglebone"; + }; + }; + }; +}; + + +&ocp { + enable_stuff_pinmux { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&enable_stuff_pins>; + + enable_stuff{ + gpio-name = "enable_stuff"; + gpio = <&gpio0 18 0>; + output; + }; + }; + + sgx@0x56000000 { + compatible = "ti,sgx"; + ti,hwmods = "gfx"; + reg = <0x56000000 0x1000000>; + interrupts = <0x25>; + status = "okay"; + }; + + pruss_pins { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&pru_stepper_pins_gpio>; + + stepper_x_step{ + gpio-name = "step_x"; + gpio = <&gpio3 14 0>; + output; + }; + stepper_x_dir{ + gpio-name = "dir_x"; + gpio = <&gpio3 15 0>; + output; + }; + stepper_y_step{ + gpio-name = "step_y"; + gpio = <&gpio3 16 0>; + output; + }; + stepper_y_dir{ + gpio-name = "dir_y"; + gpio = <&gpio3 17 0>; + output; + }; + stepper_z_step{ + gpio-name = "step_z"; + gpio = <&gpio3 18 0>; + output; + }; + stepper_z_dir{ + gpio-name = "dir_z"; + gpio = <&gpio3 19 0>; + output; + }; + stepper_e_step{ + gpio-name = "step_e"; + gpio = <&gpio3 20 0>; + output; + }; + stepper_e_dir{ + gpio-name = "dir_e"; + gpio = <&gpio3 21 0>; + output; + }; + stepper_h_step{ + gpio-name = "step_h"; + gpio = <&gpio2 26 0>; + output; + }; + stepper_h_dir{ + gpio-name = "dir_h"; + gpio = <&gpio2 27 0>; + output; + }; + stepper_a_step{ + gpio-name = "step_a"; + gpio = <&gpio2 28 0>; + output; + }; + stepper_a_dir{ + gpio-name = "dir_a"; + gpio = <&gpio2 29 0>; + output; + }; + stepper_b_step{ + gpio-name = "step_b"; + gpio = <&gpio2 30 0>; + output; + }; + stepper_b_dir{ + gpio-name = "dir_b"; + gpio = <&gpio2 31 0>; + output; + }; + stepper_c_step{ + gpio-name = "step_c"; + gpio = <&gpio1 12 0>; + output; + }; + stepper_c_dir{ + gpio-name = "dir_c"; + gpio = <&gpio1 13 0>; + output; + }; + }; + + spi0_pins { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "reset"; + pinctrl-0 = <&spi0_pins_default>; + pinctrl-1 = <&spi0_pins_reset>; + }; + +}; diff --git a/arch/arm/boot/dts/am335x-roboticscape.dtsi b/arch/arm/boot/dts/am335x-roboticscape.dtsi new file mode 100644 index 0000000000000..2f7180da76e98 --- /dev/null +++ b/arch/arm/boot/dts/am335x-roboticscape.dtsi @@ -0,0 +1,489 @@ +/* + * @file am335x-roboticscape.dtsi + * + * used by: + * am335x-boneblack-roboticscape.dts + * am335x-boneblack-wireless-roboticscape.dts + * + * + * @file am335x-boneblack-roboticscape.dtb + * + * based on am335x-boneblack.dtb + * + * This device tree serves to replace the need for an overlay when using the + * RoboticsCape. It is similar to the boneblue tree but preserves pin config for + * the black. + * + * This file was updated in April 2018 to support LED driver like in BB Blue + * This goes in sync with V0.4.0 of the Robotics Cape library + * + * @author James Strawson + * @date April 19, 2018 + */ + +&aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + serial4 = &uart4; + serial5 = &uart5; + d-can0 = &dcan0; + d-can1 = &dcan1; + usb0 = &usb0; + usb1 = &usb1; + phy0 = &usb0_phy; + phy1 = &usb1_phy; + ethernet0 = &cpsw_emac0; + ethernet1 = &cpsw_emac1; + /* librobotcontrol expects /dev/spidev1 */ + spi0 = &spi0; + spi1 = &spi1; + /delete-property/ spi2; +}; + +/******************************************************************************* +* Pin Muxing +*******************************************************************************/ +&am33xx_pinmux { + +/******************************************************************************* +* Static Pinmux +*******************************************************************************/ + mux_helper_pins: pins { + pinctrl-single,pins = < + + /* GPIO Input Pullup */ + 0x09c 0x37 /*P8.9 T6 PAUSE_BTN */ + 0x098 0x37 /*P8.10 U6 MODE_BTN */ + 0x1AC 0x37 /*P9.25 A14 IMU_INT */ + + /* Motor Control GPIO Out*/ + 0x0cc 0x0F /*P8.34 MDIR_2B different from blue!!*/ + 0x0a8 0x0F /*P8.43 MDIR_3B*/ + 0x0ac 0x0F /*P8.44 MDIR_3A*/ + 0x0a0 0x0F /*P8.45 MDIR_4A*/ + 0x0a4 0x0F /*P8.46 MDIR_4B*/ + 0x078 0x0F /*P9.12 MDIR_1A different from blue!!*/ + 0x074 0x0F /*P9.13 MDIR_1B*/ + 0x040 0x0F /*P9.15 MDIR_2A*/ + 0x1b4 0x0F /*P9.41 MOT_STBY*/ + + /* PRU encoder input */ + 0x03c 0x36 /* P8_15,PRU0_r31_15,MODE6 */ + 0x038 0x36 /* P8_16,PRU0_r31_16,MODE6 */ + + /* PRU Servo output */ + 0x0e0 0x05 /*pru1_pru_r30_8, MODE5*/ + 0x0e8 0x05 /*pru1_pru_r30_10, MODE5 */ + 0x0e4 0x05 /*pr1_pru1_pru_r30_9, MODE5 */ + 0x0ec 0x05 /*pru1_pru_r30_11, MODE5 */ + 0x0b8 0x05 /*pru1_pru_r30_6, MODE5 */ + 0x0bc 0x05 /*pru1_pru_r30_7, MODE5 */ + 0x0b0 0x05 /*pru1_pru_r30_4, MODE5 */ + 0x0b4 0x05 /*pru1_pru_r30_5, MODE5 */ + 0x0C8 0x0F /*P8.36, SERVO_PWR GPIO OUT*/ + + /* I2C1 */ + 0x15C 0x32 /* P9.17,i2c1_scl,INPUT_PULLUP,MODE2 */ + 0x158 0x32 /* P9.18,i2c1_sda,INPUT_PULLUP,MODE2 */ + + /* I2C2 */ + //0x17c 0x73 /* P9.19, i2c2_sda, mode 3 */ + //0x178 0x73 /* P9.20, i2c2_sda, mode 3 */ + + /* UART5 */ + 0x0C4 0x34 /* P8.38,uart5_rxd,MODE4 */ + 0x0C0 0x14 /* P8.37,uart5_txd,MODE4 */ + + >; + }; + + + led_pins: pinmux_led_pins { + pinctrl-single,pins = < + 0x090 (PIN_OUTPUT | MUX_MODE7) /* (R7) gpmc_advn_ale.gpio2[2] - P8.7, LED_RED, GP1_PIN_5 */ + 0x094 (PIN_OUTPUT | MUX_MODE7) /* (T7) gpmc_oen_ren.gpio2[3] - P8.8, LED_GREEN, GP1_PIN_6 */ + 0x02C (PIN_OUTPUT | MUX_MODE7) /* (U12) gpmc_ad11.gpio0[27] - P8.17, BATT_LED_1 */ + 0x08c (PIN_OUTPUT | MUX_MODE7) /* (V12) gpio2.1 - P8.18 BATT_LED_2 diff from BLUE! */ + 0x07C (PIN_OUTPUT | MUX_MODE7) /* (V6) gpmc_csn0.gpio1[29] - P8.26, BATT_LED_3 */ + 0x028 (PIN_OUTPUT | MUX_MODE7) /* (T11) gpmc_ad10.gpio0[26] - P8.14, BATT_LED_4 */ + >; + }; + +}; + + +/******************************************************************************* +* apply static and dynamic pinmux modes listed above. Configurable pins get the +* modes from am335x-boneblack-common-universal-pins.dtsi +*******************************************************************************/ +&ocp { + /* activate the static pinmux helper list of pin modes above */ + test_helper: helper { + compatible = "bone-pinmux-helper"; + pinctrl-names = "default"; + pinctrl-0 = <&mux_helper_pins>; + + status = "okay"; + }; + + /* UART4 RX DSM */ + P9_11_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart"; + pinctrl-0 = <&P9_11_default_pin>; + pinctrl-1 = <&P9_11_gpio_pin>; + pinctrl-2 = <&P9_11_gpio_pu_pin>; + pinctrl-3 = <&P9_11_gpio_pd_pin>; + pinctrl-4 = <&P9_11_uart_pin>; + }; + + /* UART 2 TX GPS*/ + P9_21_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm"; + pinctrl-0 = <&P9_21_uart_pin>; + pinctrl-1 = <&P9_21_gpio_pin>; + pinctrl-2 = <&P9_21_gpio_pu_pin>; + pinctrl-3 = <&P9_21_gpio_pd_pin>; + pinctrl-4 = <&P9_21_spi_pin>; + pinctrl-5 = <&P9_21_uart_pin>; + pinctrl-6 = <&P9_21_i2c_pin>; + pinctrl-7 = <&P9_21_pwm_pin>; + }; + + /* UART 2 RX GPS */ + P9_22_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "spi", "uart", "i2c", "pwm"; + pinctrl-0 = <&P9_22_uart_pin>; + pinctrl-1 = <&P9_22_gpio_pin>; + pinctrl-2 = <&P9_22_gpio_pu_pin>; + pinctrl-3 = <&P9_22_gpio_pd_pin>; + pinctrl-4 = <&P9_22_spi_pin>; + pinctrl-5 = <&P9_22_uart_pin>; + pinctrl-6 = <&P9_22_i2c_pin>; + pinctrl-7 = <&P9_22_pwm_pin>; + }; + + /* SPI MISO */ + P9_29_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin"; + pinctrl-0 = <&P9_29_spi_pin>; + pinctrl-1 = <&P9_29_gpio_pin>; + pinctrl-2 = <&P9_29_gpio_pu_pin>; + pinctrl-3 = <&P9_29_gpio_pd_pin>; + pinctrl-4 = <&P9_29_pwm_pin>; + pinctrl-5 = <&P9_29_spi_pin>; + pinctrl-6 = <&P9_29_pruout_pin>; + pinctrl-7 = <&P9_29_pruin_pin>; + }; + + /* SPI MOSI */ + P9_30_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "pruout", "pruin"; + pinctrl-0 = <&P9_30_spi_pin>; + pinctrl-1 = <&P9_30_gpio_pin>; + pinctrl-2 = <&P9_30_gpio_pu_pin>; + pinctrl-3 = <&P9_30_gpio_pd_pin>; + pinctrl-4 = <&P9_30_pwm_pin>; + pinctrl-5 = <&P9_30_spi_pin>; + pinctrl-6 = <&P9_30_pruout_pin>; + pinctrl-7 = <&P9_30_pruin_pin>; + }; + + /* SPI SCK */ + P9_31_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_sclk", "pruout", "pruin"; + pinctrl-0 = <&P9_31_spi_sclk_pin>; + pinctrl-1 = <&P9_31_gpio_pin>; + pinctrl-2 = <&P9_31_gpio_pu_pin>; + pinctrl-3 = <&P9_31_gpio_pd_pin>; + pinctrl-4 = <&P9_31_pwm_pin>; + pinctrl-5 = <&P9_31_spi_pin>; + pinctrl-6 = <&P9_31_spi_sclk_pin>; + pinctrl-7 = <&P9_31_pruout_pin>; + pinctrl-8 = <&P9_31_pruin_pin>; + }; + + /* SPI SS1 GPIO3_17*/ + P9_28_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm", "spi", "spi_cs", "pwm2", "pruout", "pruin"; + pinctrl-0 = <&P9_28_spi_cs_pin>; + pinctrl-1 = <&P9_28_gpio_pin>; + pinctrl-2 = <&P9_28_gpio_pu_pin>; + pinctrl-3 = <&P9_28_gpio_pd_pin>; + pinctrl-4 = <&P9_28_pwm_pin>; + pinctrl-5 = <&P9_28_spi_pin>; + pinctrl-6 = <&P9_28_spi_cs_pin>; + pinctrl-7 = <&P9_28_pwm2_pin>; + pinctrl-8 = <&P9_28_pruout_pin>; + pinctrl-9 = <&P9_28_pruin_pin>; + }; + + /* SPI SS1 GPIO1_17*/ + P9_23_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "pwm"; + pinctrl-0 = <&P9_23_gpio_pin>; + pinctrl-1 = <&P9_23_gpio_pin>; + pinctrl-2 = <&P9_23_gpio_pu_pin>; + pinctrl-3 = <&P9_23_gpio_pd_pin>; + pinctrl-4 = <&P9_23_pwm_pin>; + }; + + /* UART 1 TX */ + P9_24_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c", "pruin"; + pinctrl-0 = <&P9_24_uart_pin>; + pinctrl-1 = <&P9_24_gpio_pin>; + pinctrl-2 = <&P9_24_gpio_pu_pin>; + pinctrl-3 = <&P9_24_gpio_pd_pin>; + pinctrl-4 = <&P9_24_uart_pin>; + pinctrl-5 = <&P9_24_can_pin>; + pinctrl-6 = <&P9_24_i2c_pin>; + pinctrl-7 = <&P9_24_pruin_pin>; + }; + + /* UART 1 RX */ + P9_26_pinmux { + compatible = "bone-pinmux-helper"; + status = "okay"; + pinctrl-names = "default", "gpio", "gpio_pu", "gpio_pd", "uart", "can", "i2c", "pruin"; + pinctrl-0 = <&P9_26_uart_pin>; + pinctrl-1 = <&P9_26_gpio_pin>; + pinctrl-2 = <&P9_26_gpio_pu_pin>; + pinctrl-3 = <&P9_26_gpio_pd_pin>; + pinctrl-4 = <&P9_26_uart_pin>; + pinctrl-5 = <&P9_26_can_pin>; + pinctrl-6 = <&P9_26_i2c_pin>; + pinctrl-7 = <&P9_26_pruin_pin>; + }; + + +}; + + +/******************************************************************************* +* PWMSS +*******************************************************************************/ +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm0 { + status = "okay"; +}; + +&ehrpwm1 { + pinctrl-names = "default"; + pinctrl-0 = <&P8_19_pwm_pin &P8_13_pwm_pin>; + status = "okay"; +}; + +&ehrpwm2 { + pinctrl-names = "default"; + pinctrl-0 = <&P9_14_pwm_pin &P9_16_pwm_pin>; + status = "okay"; +}; + +/******************************************************************************* +* EQEP +*******************************************************************************/ +&eqep0 { + pinctrl-names = "default"; + pinctrl-0 = <&P9_92_qep_pin &P9_27_qep_pin>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +&eqep1 { + pinctrl-names = "default"; + pinctrl-0 = <&P8_33_qep_pin &P8_35_qep_pin>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + +&eqep2 { + pinctrl-names = "default"; + pinctrl-0 = <&P8_12_qep_pin &P8_11_qep_pin>; + + count_mode = <0>; /* 0 - Quadrature mode, normal 90 phase offset cha & chb. 1 - Direction mode. cha input = clock, chb input = direction */ + swap_inputs = <0>; /* Are channel A and channel B swapped? (0 - no, 1 - yes) */ + invert_qa = <1>; /* Should we invert the channel A input? */ + invert_qb = <1>; /* Should we invert the channel B input? I invert these because my encoder outputs drive transistors that pull down the pins */ + invert_qi = <0>; /* Should we invert the index input? */ + invert_qs = <0>; /* Should we invert the strobe input? */ + status = "okay"; +}; + + +/******************************************************************************* +* UART +*******************************************************************************/ +&uart1 { + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&uart4 { + status = "okay"; +}; + +&uart5 { + status = "okay"; +}; + + +/******************************************************************************* +* PRU Encoder and Servos +*******************************************************************************/ +&pruss_soc_bus { + status = "okay"; + + pruss: pruss@4a300000 { + status = "okay"; + }; +}; + + +/******************************************************************************* +* I2C +*******************************************************************************/ +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + #address-cells = <1>; + #size-cells = <0>; +}; + +/******************************************************************************* +* SPI +*******************************************************************************/ +&spi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.0"; + + reg = <0>; + spi-max-frequency = <16000000>; + spi-cpha; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + + compatible = "spidev"; + symlink = "spi/1.1"; + + reg = <1>; + spi-max-frequency = <16000000>; + }; +}; + +/******************************************************************************* +* LEDs +*******************************************************************************/ +/ { + leds { + pinctrl-names = "default"; + pinctrl-0 = <&led_pins>; + compatible = "gpio-leds"; + + red_led { + label = "red"; + gpios = <&gpio2 2 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + green_led { + label = "green"; + gpios = <&gpio2 3 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_1_led { + label = "bat25"; + gpios = <&gpio0 27 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_2_led { + label = "bat50"; + gpios = <&gpio2 1 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_3_led { + label = "bat75"; + gpios = <&gpio1 29 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_4_led { + label = "bat100"; + gpios = <&gpio0 26 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; + +}; + +&tscadc { + status = "okay"; + adc { + ti,adc-channels = <0 1 2 3 4 5 6 7>; + ti,chan-step-avg = <0x16 0x16 0x16 0x16 0x16 0x16 0x16 0x16>; + ti,chan-step-opendelay = <0x98 0x98 0x98 0x98 0x98 0x98 0x98 0x98>; + ti,chan-step-sampledelay = <0x0 0x0 0x0 0x0 0x0 0x0 0x0 0x0>; + }; +}; diff --git a/arch/arm/boot/dts/am335x-sancloud-bbe.dts b/arch/arm/boot/dts/am335x-sancloud-bbe.dts index 7b8e7417a11e6..caeaf768ac46f 100644 --- a/arch/arm/boot/dts/am335x-sancloud-bbe.dts +++ b/arch/arm/boot/dts/am335x-sancloud-bbe.dts @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2012 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; @@ -15,6 +12,15 @@ / { model = "SanCloud BeagleBone Enhanced"; compatible = "sancloud,am335x-boneenhanced", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"; + + chosen { + base_dtb = "am335x-sancloud-bbe.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; +}; + +&sgx { + status = "okay"; }; &am33xx_pinmux { @@ -23,70 +29,70 @@ cpsw_default: cpsw_default { pinctrl-single,pins = < /* Slave 1 */ - AM33XX_IOPAD(0x914, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txen.rgmii1_tctl */ - AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxdv.rgmii1_rctl */ - AM33XX_IOPAD(0x91c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd3.rgmii1_td3 */ - AM33XX_IOPAD(0x920, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd2.rgmii1_td2 */ - AM33XX_IOPAD(0x924, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd1.rgmii1_td1 */ - AM33XX_IOPAD(0x928, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txd0.rgmii1_td0 */ - AM33XX_IOPAD(0x92c, PIN_OUTPUT_PULLDOWN | MUX_MODE2) /* mii1_txclk.rgmii1_tclk */ - AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxclk.rgmii1_rclk */ - AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd3.rgmii1_rd3 */ - AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd2.rgmii1_rd2 */ - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd1.rgmii1_rd1 */ - AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE2) /* mii1_rxd0.rgmii1_rd0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txen.rgmii1_tctl */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxdv.rgmii1_rctl */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txd3.rgmii1_td3 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txd2.rgmii1_td2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txd1.rgmii1_td1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txd0.rgmii1_td0 */ + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_OUTPUT_PULLDOWN, MUX_MODE2) /* mii1_txclk.rgmii1_tclk */ + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxclk.rgmii1_rclk */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxd3.rgmii1_rd3 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxd2.rgmii1_rd2 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxd1.rgmii1_rd1 */ + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE2) /* mii1_rxd0.rgmii1_rd0 */ >; }; cpsw_sleep: cpsw_sleep { pinctrl-single,pins = < /* Slave 1 reset value */ - AM33XX_IOPAD(0x914, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x918, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x91c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x920, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x924, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x928, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x92c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x930, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x934, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x938, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x93c, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x940, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_EN, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_DV, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_TX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RX_CLK, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD3, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD2, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD1, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MII1_RXD0, PIN_INPUT_PULLDOWN, MUX_MODE7) >; }; davinci_mdio_default: davinci_mdio_default { pinctrl-single,pins = < /* MDIO */ - AM33XX_IOPAD(0x948, PIN_INPUT_PULLUP | SLEWCTRL_FAST | MUX_MODE0) /* mdio_data.mdio_data */ - AM33XX_IOPAD(0x94c, PIN_OUTPUT_PULLUP | MUX_MODE0) /* mdio_clk.mdio_clk */ + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLUP | SLEWCTRL_FAST, MUX_MODE0) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_OUTPUT_PULLUP, MUX_MODE0) >; }; davinci_mdio_sleep: davinci_mdio_sleep { pinctrl-single,pins = < /* MDIO reset value */ - AM33XX_IOPAD(0x948, PIN_INPUT_PULLDOWN | MUX_MODE7) - AM33XX_IOPAD(0x94c, PIN_INPUT_PULLDOWN | MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDIO, PIN_INPUT_PULLDOWN, MUX_MODE7) + AM33XX_PADCONF(AM335X_PIN_MDC, PIN_INPUT_PULLDOWN, MUX_MODE7) >; }; usb_hub_ctrl: usb_hub_ctrl { pinctrl-single,pins = < - AM33XX_IOPAD(0x944, PIN_OUTPUT_PULLUP | MUX_MODE7) /* rmii1_refclk.gpio0_29 */ + AM33XX_PADCONF(AM335X_PIN_RMII1_REF_CLK, PIN_OUTPUT_PULLUP, MUX_MODE7) /* rmii1_refclk.gpio0_29 */ >; }; mpu6050_pins: pinmux_mpu6050_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x968, PIN_INPUT | MUX_MODE7) /* uart0_ctsn.gpio1_8 */ + AM33XX_PADCONF(AM335X_PIN_UART0_CTSN, PIN_INPUT, MUX_MODE7) /* uart0_ctsn.gpio1_8 */ >; }; lps3331ap_pins: pinmux_lps3331ap_pins { pinctrl-single,pins = < - AM33XX_IOPAD(0x868, PIN_INPUT | MUX_MODE7) /* gpmc_a10.gpio1_26 */ + AM33XX_PADCONF(AM335X_PIN_GPMC_A10, PIN_INPUT, MUX_MODE7) /* gpmc_a10.gpio1_26 */ >; }; }; diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi index 68acb39141f61..3c75e40f4371f 100644 --- a/arch/arm/boot/dts/am33xx.dtsi +++ b/arch/arm/boot/dts/am33xx.dtsi @@ -19,7 +19,7 @@ #size-cells = <1>; chosen { }; - aliases { + aliases: aliases { i2c0 = &i2c0; i2c1 = &i2c1; i2c2 = &i2c2; @@ -171,7 +171,7 @@ * for the moment, just use a fake OCP bus entry to represent * the whole bus hierarchy. */ - ocp { + ocp: ocp { compatible = "simple-bus"; #address-cells = <1>; #size-cells = <1>; @@ -318,6 +318,39 @@ #interrupt-cells = <2>; reg = <0x44e07000 0x1000>; interrupts = <96>; + gpio-line-names = + "MDIO_DATA", // 0 + "MDIO_CLK", // 1 + "SPI0_SCLK", // 2 + "SPI0_D0", // 3 + "SPI0_D1", // 4 + "SPI0_CS0", // 5 + "SPI0_CS1", // 6 + "ECAP0_IN_PWM0_OUT", // 7 + "LCD_DATA12", // 8 + "LCD_DATA13", // 9 + "LCD_DATA14", // 10 + "LCD_DATA15", // 11 + "UART1_CTSN", // 12 + "UART1_RTSN", // 13 + "UART1_RXD", // 14 + "UART1_TXD", // 15 + "GMII1_TXD3", // 16 + "GMII1_TXD2", // 17 + "USB0_DRVVBUS", // 18 + "XDMA_EVENT_INTR0", // 19 + "XDMA_EVENT_INTR1", // 20 + "GMII1_TXD1", // 21 + "GPMC_AD8", // 22 + "GPMC_AD9", // 23 + "NC", // 24 + "NC", // 25 + "GPMC_AD10", // 26 + "GPMC_AD11", // 27 + "GMII1_TXD0", // 28 + "RMII1_REFCLK", // 29 + "GPMC_WAIT0", // 30 + "GPMC_WPN"; // 31 }; gpio1: gpio@4804c000 { @@ -329,6 +362,39 @@ #interrupt-cells = <2>; reg = <0x4804c000 0x1000>; interrupts = <98>; + gpio-line-names = + "GPMC_AD0", // 0 + "GPMC_AD1", // 1 + "GPMC_AD2", // 2 + "GPMC_AD3", // 3 + "GPMC_AD4", // 4 + "GPMC_AD5", // 5 + "GPMC_AD6", // 6 + "GPMC_AD7", // 7 + "UART0_CTSN", // 8 + "UART0_RTSN", // 9 + "UART0_RXD", // 10 + "UART0_TXD", // 11 + "GPMC_AD12", // 12 + "GPMC_AD13", // 13 + "GPMC_AD14", // 14 + "GPMC_AD15", // 15 + "GPMC_A0", // 16 + "GPMC_A1", // 17 + "GPMC_A2", // 18 + "GPMC_A3", // 19 + "GPMC_A4", // 20 + "GPMC_A5", // 21 + "GPMC_A6", // 22 + "GPMC_A7", // 23 + "GPMC_A8", // 24 + "GPMC_A9", // 25 + "GPMC_A10", // 26 + "GPMC_A11", // 27 + "GPMC_BE1N", // 28 + "GPMC_CSN0", // 29 + "GPMC_CSN1", // 30 + "GPMC_CSN2"; // 31 }; gpio2: gpio@481ac000 { @@ -340,6 +406,39 @@ #interrupt-cells = <2>; reg = <0x481ac000 0x1000>; interrupts = <32>; + gpio-line-names = + "GPMC_CSN3", // 0 + "GPMC_CLK", // 1 + "GPMC_ADVN_ALE", // 2 + "GPMC_OEN_REN", // 3 + "GPMC_WEN", // 4 + "GPMC_BE0N_CLE", // 5 + "LCD_DATA0", // 6 + "LCD_DATA1", // 7 + "LCD_DATA2", // 8 + "LCD_DATA3", // 9 + "LCD_DATA4", // 10 + "LCD_DATA5", // 11 + "LCD_DATA6", // 12 + "LCD_DATA7", // 13 + "LCD_DATA8", // 14 + "LCD_DATA9", // 15 + "LCD_DATA10", // 16 + "LCD_DATA11", // 17 + "GMII1_RXD3", // 18 + "GMII1_RXD2", // 19 + "GMII1_RXD1", // 20 + "GMII1_RXD0", // 21 + "LCD_VSYNC", // 22 + "LCD_HSYNC", // 23 + "LCD_PCLK", // 24 + "LCD_AC_BIAS_EN", // 25 + "MMC0_DAT3", // 26 + "MMC0_DAT2", // 27 + "MMC0_DAT1", // 28 + "MMC0_DAT0", // 29 + "MMC0_CLK", // 30 + "MMC0_CMD"; // 31 }; gpio3: gpio@481ae000 { @@ -351,6 +450,39 @@ #interrupt-cells = <2>; reg = <0x481ae000 0x1000>; interrupts = <62>; + gpio-line-names = + "GMII1_COL", // 0 + "GMII1_CRS", // 1 + "GMII1_RXER", // 2 + "GMII1_TXEN", // 3 + "GMII1_RXDV", // 4 + "I2C0_SDA", // 5 + "I2C0_SCL", // 6 + "EMU0", // 7 + "EMU1", // 8 + "GMII1_TXCLK", // 9 + "GMII1_RXCLK", // 10 + "NC", // 11 + "NC", // 12 + "USB1_DRVVBUS", // 13 + "MCASP0_ACLKX", // 14 + "MCASP0_FSX", // 15 + "MCASP0_AXR0", // 16 + "MCASP0_AHCLKR", // 17 + "MCASP0_ACLKR", // 18 + "MCASP0_FSR", // 19 + "MCASP0_AXR1", // 20 + "MCASP0_AHCLKX", // 21 + "NC", // 22 + "NC", // 23 + "NC", // 24 + "NC", // 25 + "NC", // 26 + "NC", // 27 + "NC", // 28 + "NC", // 29 + "NC", // 30 + "NC"; // 31 }; uart0: serial@44e09000 { @@ -782,6 +914,16 @@ status = "disabled"; }; + eqep0: eqep@0x48300180 { + compatible = "ti,am33xx-eqep"; + reg = <0x48300180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupt-parent = <&intc>; + interrupts = <79>; + status = "disabled"; + }; + ehrpwm0: pwm@48300200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; @@ -816,6 +958,16 @@ status = "disabled"; }; + eqep1: eqep@0x48302180 { + compatible = "ti,am33xx-eqep"; + reg = <0x48302180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupt-parent = <&intc>; + interrupts = <88>; + status = "disabled"; + }; + ehrpwm1: pwm@48302200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; @@ -850,6 +1002,16 @@ status = "disabled"; }; + eqep2: eqep@0x48304180 { + compatible = "ti,am33xx-eqep"; + reg = <0x48304180 0x80>; + clocks = <&l4ls_gclk>; + clock-names = "fck"; + interrupt-parent = <&intc>; + interrupts = <89>; + status = "disabled"; + }; + ehrpwm2: pwm@48304200 { compatible = "ti,am3352-ehrpwm", "ti,am33xx-ehrpwm"; diff --git a/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts b/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts new file mode 100644 index 0000000000000..023e16c8aa3f8 --- /dev/null +++ b/arch/arm/boot/dts/am5729-beagleboneai-roboticscape.dts @@ -0,0 +1,242 @@ +/* + * Copyright (C) 2014-2019 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include "am5729-beagleboneai.dts" + +/ { + chosen { + base_dtb = "am5729-beagleboneai-roboticscape.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; + + rcleds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&cape_pins_rc>; + + red_led { + label = "red"; + gpios = <&gpio6 5 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + green_led { + label = "green"; + gpios = <&gpio6 6 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_1_led { + label = "bat25"; + gpios = <&gpio8 18 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_2_led { + label = "bat50"; + gpios = <&gpio4 9 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_3_led { + label = "bat75"; + gpios = <&gpio4 28 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + batt_4_led { + label = "bat100"; + gpios = <&gpio4 13 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + }; +}; + +&ocp { + pruss1_shmem: pruss_shmem@4b200000 { + status = "okay"; + compatible = "ti,pruss-shmem"; + reg = <0x4b200000 0x020000>; + }; + + pruss2_shmem: pruss_shmem@4b280000 { + status = "okay"; + compatible = "ti,pruss-shmem"; + reg = <0x4b280000 0x020000>; + }; +}; + +&cape_pins { + status = "disabled"; +}; + +&dra7_pmx_core { + cape_pins_rc: cape_pins_rc { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, MUX_MODE15) /* AB8: P8.3: mmc3_dat6.off */ + DRA7XX_CORE_IOPAD(0x37A0, MUX_MODE15) /* AB5: P8.4: mmc3_dat7.off */ + DRA7XX_CORE_IOPAD(0x378C, MUX_MODE15) /* AC9: P8.5: mmc3_dat2.off */ + DRA7XX_CORE_IOPAD(0x3790, MUX_MODE15) /* AC3: P8.6: mmc3_dat3.off */ + DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT | MUX_MODE14) /* G14: mcasp1_axr14.gpio6_5 - P8_7 - RED */ + DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT | MUX_MODE14) /* F14: mcasp1_axr15.gpio6_6 - P8_8 - GREEN */ + DRA7XX_CORE_IOPAD(0x3698, PIN_INPUT_PULLUP | MUX_MODE14) /* E17: P8.9: xref_clk1.gpio6_18 - PAUSE */ + DRA7XX_CORE_IOPAD(0x36E8, PIN_INPUT_PULLUP | MUX_MODE14) /* A13: P8.10: mcasp1_axr13.gpio6_4 - MODE */ + DRA7XX_CORE_IOPAD(0x3510, PIN_INPUT | MUX_MODE10) /* AH4: P8.11: vin1a_d7.eQEP2B_in - QEP2B */ + DRA7XX_CORE_IOPAD(0x350C, PIN_INPUT | MUX_MODE10) /* AG6: P8.12: vin1a_d6.eQEP2A_in - QEP2A */ + DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT | MUX_MODE10) /* D3: P8.13: vin2a_d10.ehrpwm2B - PWM2B */ + DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT | MUX_MODE14) /* D5: vin2a_d12.gpio4_13 - P8_14 - BAT100 */ + DRA7XX_CORE_IOPAD(0x3570, MUX_MODE15) /* D1: P8.15a: vin2a_d2.off */ + DRA7XX_CORE_IOPAD(0x35B4, PIN_INPUT | MUX_MODE12) /* A3: P8.15b: vin2a_d19.pr1_pru1_gpi16 - QEP4B */ + DRA7XX_CORE_IOPAD(0x35BC, PIN_INPUT | MUX_MODE12) /* B4: P8.16: vin2a_d21.pr1_pru1_gpi18 - QEP4A */ + DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT | MUX_MODE14) /* A7: vout1_d18.gpio8_18 - P8_17 - BAT25 */ + DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT | MUX_MODE14) /* F5: vin2a_d8.gpio4_9 - P8_18 - BAT50 */ + DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT | MUX_MODE10) /* E6: P8.19: vin2a_d9.ehrpwm2A - PWM2A */ + DRA7XX_CORE_IOPAD(0x3780, MUX_MODE15) /* AC4: P8.20: mmc3_cmd.off */ + DRA7XX_CORE_IOPAD(0x377C, MUX_MODE15) /* AD4: P8.21: mmc3_clk.off */ + DRA7XX_CORE_IOPAD(0x3798, MUX_MODE15) /* AD6: P8.22: mmc3_dat5.off */ + DRA7XX_CORE_IOPAD(0x3794, MUX_MODE15) /* AC8: P8.23: mmc3_dat4.off */ + DRA7XX_CORE_IOPAD(0x3788, MUX_MODE15) /* AC6: P8.24: mmc3_dat1.off */ + DRA7XX_CORE_IOPAD(0x3784, MUX_MODE15) /* AC7: P8.25: mmc3_dat0.off */ + DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT | MUX_MODE14) /* B3: vin2a_d20.gpio4_28 - P8_26 - BAT75 */ + DRA7XX_CORE_IOPAD(0x35D8, MUX_MODE15) /* E11: P8.27a: vout1_vsync.off */ + DRA7XX_CORE_IOPAD(0x3628, PIN_OUTPUT | MUX_MODE13) /* A8: P8.27b: vout1_d19.pr2_pru0_gpo16 - SVO1 */ + DRA7XX_CORE_IOPAD(0x35C8, MUX_MODE15) /* D11: P8.28a: vout1_clk.off */ + DRA7XX_CORE_IOPAD(0x362C, PIN_OUTPUT | MUX_MODE13) /* C9: P8.28b: vout1_d20.pr2_pru0_gpo17 - SVO2 */ + DRA7XX_CORE_IOPAD(0x35D4, MUX_MODE15) /* C11: P8.29a: vout1_hsync.off */ + DRA7XX_CORE_IOPAD(0x3630, PIN_OUTPUT | MUX_MODE13) /* A9: P8.29b: vout1_d21.pr2_pru0_gpo18 - SVO3 */ + DRA7XX_CORE_IOPAD(0x35CC, MUX_MODE15) /* B10: P8.30a: vout1_de.off */ + DRA7XX_CORE_IOPAD(0x3634, PIN_OUTPUT | MUX_MODE13) /* B9: P8.30b: vout1_d22.pr2_pru0_gpo19 - SVO4 */ + DRA7XX_CORE_IOPAD(0x3614, MUX_MODE15) /* C8: P8.31a: vout1_d14.off */ + DRA7XX_CORE_IOPAD(0x373C, MUX_MODE15) /* G16: P8.31b: mcasp4_axr0.off */ + DRA7XX_CORE_IOPAD(0x3618, MUX_MODE15) /* C7: P8.32a: vout1_d15.off */ + DRA7XX_CORE_IOPAD(0x3740, MUX_MODE15) /* D17: P8.32b: mcasp4_axr1.off */ + DRA7XX_CORE_IOPAD(0x3610, MUX_MODE15) /* C6: P8.33a: vout1_d13.off */ + DRA7XX_CORE_IOPAD(0x34E8, PIN_INPUT | MUX_MODE10) /* AF9: P8.33b: vin1a_fld0.eQEP1B_in - QEP1B */ + DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT | MUX_MODE14) /* D8: P8.34a: vout1_d11.gpio8_11 - MDIR2B */ + DRA7XX_CORE_IOPAD(0x3564, MUX_MODE15) /* G6: P8.34b: vin2a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x360C, MUX_MODE15) /* A5: P8.35a: vout1_d12.off */ + DRA7XX_CORE_IOPAD(0x34E4, PIN_INPUT | MUX_MODE10) /* AD9: P8.35b: vin1a_de0.eQEP1A_in - QEP1A */ + DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT | MUX_MODE14) /* D7: P8.36a: vout1_d10.gpio8_10 - SERVO_EN */ + DRA7XX_CORE_IOPAD(0x3568, MUX_MODE15) /* F2: P8.36b: vin2a_d0.off */ + DRA7XX_CORE_IOPAD(0x35FC, MUX_MODE15) /* E8: P8.37a: vout1_d8.off */ + DRA7XX_CORE_IOPAD(0x3738, PIN_OUTPUT | MUX_MODE3) /* A21: P8.37b: mcasp4_fsx.uart8_txd - UART5_TX */ + DRA7XX_CORE_IOPAD(0x3600, MUX_MODE15) /* D9: P8.38a: vout1_d9.off */ + DRA7XX_CORE_IOPAD(0x3734, PIN_INPUT | MUX_MODE3) /* C18: P8.38b: mcasp4_aclkx.uart8_rxd - UART5_RX */ + DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT | MUX_MODE13) /* F8: P8.39: vout1_d6.pr2_pru0_gpo3 - SVO5 */ + DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT | MUX_MODE13) /* E7: P8.40: vout1_d7.pr2_pru0_gpo4 - SVO6 */ + DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT | MUX_MODE13) /* E9: P8.41: vout1_d4.pr2_pru0_gpo1 - SVO7 */ + DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT | MUX_MODE13) /* F9: P8.42: vout1_d5.pr2_pru0_gpo2 - SVO8 */ + DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT | MUX_MODE14 ) /* F10: P8.43: vout1_d2.gpio8_2 - MDIR3B */ + DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT | MUX_MODE14 ) /* G11: P8.44: vout1_d3.gpio8_3 - MDIR3A */ + DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT | MUX_MODE14 ) /* F11: P8.45a: vout1_d0.gpio8_0 - MDIR4A */ + DRA7XX_CORE_IOPAD(0x361C, MUX_MODE15) /* B7: P8.45b: vout1_d16.off */ + DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT | MUX_MODE14 ) /* G10: P8.46a: vout1_d1.gpio8_1 - MDIR4B */ + DRA7XX_CORE_IOPAD(0x3638, MUX_MODE15) /* A10: P8.46b: vout1_d23.off */ + DRA7XX_CORE_IOPAD(0x372C, PIN_INPUT | MUX_MODE4) /* B19: P9.11a: mcasp3_axr0.uart5_rxd - UART4_RX */ + DRA7XX_CORE_IOPAD(0x3620, MUX_MODE15) /* B8: P9.11b: vout1_d17.off */ + DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT | MUX_MODE14) /* B14: P9.12: mcasp1_aclkr.gpio5_0 - MDIR1A */ + DRA7XX_CORE_IOPAD(0x3730, PIN_OUTPUT | MUX_MODE14) /* C17: P9.13: mcasp3_axr1.off - MDIR1B */ + DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT | MUX_MODE10) /* D6: P9.14: vin2a_d17.ehrpwm3A - PWM1A */ + DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT | MUX_MODE14) /* AG4: P9.15: vin1a_d8.gpio3_12 - MDIR2A */ + DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT | MUX_MODE10) /* C5: P9.16: vin2a_d18.ehrpwm3B - PWM1B */ + DRA7XX_CORE_IOPAD(0x37CC, PIN_INPUT | MUX_MODE14) /* B24: P9.17a: spi2_cs0.gpio7_17 */ + DRA7XX_CORE_IOPAD(0x36B8, PIN_INPUT_PULLUP | SLEWCONTROL | MUX_MODE10) /* F12: P9.17b: mcasp1_axr1.i2c5_scl - I2C1_SCL */ + DRA7XX_CORE_IOPAD(0x37C8, PIN_INPUT | MUX_MODE14) /* G17: P9.18a: spi2_d0.gpio7_16 */ + DRA7XX_CORE_IOPAD(0x36B4, PIN_INPUT_PULLUP | SLEWCONTROL | MUX_MODE10) /* G12: P9.18b: mcasp1_axr0.i2c5_sda - I2C1_SDA */ + DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT | MUX_MODE7) /* R6: P9.19a: gpmc_a0.i2c4_scl - I2C2_SCL */ + DRA7XX_CORE_IOPAD(0x357C, PIN_INPUT | MUX_MODE12 ) /* F4: P9.19b: vin2a_d5.pr1_pru1_gpi2 */ + DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT | MUX_MODE7) /* T9: P9.20a: gpmc_a1.i2c4_sda - I2C2_SDA */ + DRA7XX_CORE_IOPAD(0x3578, PIN_INPUT | MUX_MODE12) /* D2: P9.20b: vin2a_d4.pr1_pru1_gpi1 */ + DRA7XX_CORE_IOPAD(0x34F0, MUX_MODE15) /* AF8: P9.21a: vin1a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT | MUX_MODE1) /* B22: P9.21b: spi2_d1.uart3_txd - UART2_TX */ + DRA7XX_CORE_IOPAD(0x369C, MUX_MODE15) /* B26: P9.22a: xref_clk2.off */ + DRA7XX_CORE_IOPAD(0x37C0, PIN_INPUT | MUX_MODE1) /* A26: P9.22b: spi2_sclk.uart3_rxd - UART2_RX */ + DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT | MUX_MODE14) /* A22: P9.23: spi1_cs1.gpio7_11 - SPI1_SS2 */ + DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT | MUX_MODE3) /* F20: P9.24: gpio6_15.uart10_txd - UART1_TX */ + DRA7XX_CORE_IOPAD(0x3694, PIN_INPUT | MUX_MODE14) /* D18: P9.25: xref_clk0.gpio6_17 - IMU_INT */ + DRA7XX_CORE_IOPAD(0x3688, PIN_INPUT | MUX_MODE3) /* E21: P9.26a: gpio6_14.uart10_rxd - UART1_RX */ + DRA7XX_CORE_IOPAD(0x3544, MUX_MODE15) /* AE2: P9.26b: vin1a_d20.off */ + DRA7XX_CORE_IOPAD(0x35A0, PIN_INPUT | MUX_MODE10) /* C3: P9.27a: vin2a_d14.eQEP3B_in - QEP0B */ + DRA7XX_CORE_IOPAD(0x36B0, MUX_MODE15) /* J14: P9.27b: mcasp1_fsr.off */ + DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT | MUX_MODE3) /* A12: P9.28: mcasp1_axr11.spi3_cs0 - SPI1_SS1 */ + DRA7XX_CORE_IOPAD(0x36D8, PIN_INPUT | MUX_MODE3) /* A11: P9.29a: mcasp1_axr9.spi3_d1 - SPI1_MISO */ + DRA7XX_CORE_IOPAD(0x36A8, MUX_MODE15) /* D14: P9.29b: mcasp1_fsx.off */ + DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT | MUX_MODE3) /* B13: P9.30: mcasp1_axr10.spi3_d0 - SPI1_MOSI */ + DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT | MUX_MODE3) /* B12: P9.31a: mcasp1_axr8.spi3_sclk - SPI1_SCK */ + DRA7XX_CORE_IOPAD(0x36A4, MUX_MODE15) /* C14: P9.31b: mcasp1_aclkx.off */ + DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT | MUX_MODE14) /* C23: P9.41a: xref_clk3.gpio6_20 - MOT_STBY */ + DRA7XX_CORE_IOPAD(0x3580, MUX_MODE15) /* C1: P9.41b: vin2a_d6.off */ + DRA7XX_CORE_IOPAD(0x36E4, MUX_MODE15) /* E14: P9.42a: mcasp1_axr12.off */ + DRA7XX_CORE_IOPAD(0x359C, PIN_INPUT | MUX_MODE10) /* C2: P9.42b: vin2a_d13.eQEP3A_in - QEP0A */ + >; + }; +}; + +&i2c5 { + status = "okay"; + clock-frequency = <100000>; +}; + +&epwmss0 { + status = "okay"; +}; + +&epwmss1 { + status = "okay"; +}; + +&epwmss2 { + status = "okay"; +}; + +&ehrpwm1 { + status = "okay"; +}; + +&ehrpwm2 { + status = "okay"; +}; + +&uart8 { + status = "okay"; +}; + +&uart5 { + status = "okay"; +}; + +&uart3 { + status = "okay"; +}; + +&uart10 { + status = "okay"; +}; + +&mcspi1 { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + cs-gpios = <0>, <&gpio7 11 0>; + + channel@0 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + reg = <0>; + spi-max-frequency = <24000000>; + }; + + channel@1 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "spidev"; + reg = <1>; + spi-max-frequency = <24000000>; + }; +}; diff --git a/arch/arm/boot/dts/am5729-beagleboneai.dts b/arch/arm/boot/dts/am5729-beagleboneai.dts index d0b377d2a507d..457c52f73e734 100644 --- a/arch/arm/boot/dts/am5729-beagleboneai.dts +++ b/arch/arm/boot/dts/am5729-beagleboneai.dts @@ -6,6 +6,7 @@ /dts-v1/; #include "dra74x.dtsi" +#include "am57xx-commercial-grade.dtsi" #include "dra74x-mmc-iodelay.dtsi" #include "dra74-ipu-dsp-common.dtsi" #include @@ -25,6 +26,8 @@ chosen { stdout-path = &uart1; + base_dtb = "am5729-beagleboneai.dts"; + base_dtb_timestamp = __TIMESTAMP__; }; memory@0 { @@ -37,6 +40,12 @@ #size-cells = <2>; ranges; + cmem_block_mem_1_ocmc3: cmem_block_mem@40500000 { + reg = <0x0 0x40500000 0x0 0x100000>; + no-map; + status = "okay"; + }; + ipu2_memory_region: ipu2-memory@95800000 { compatible = "shared-dma-pool"; reg = <0x0 0x95800000 0x0 0x3800000>; @@ -65,8 +74,32 @@ status = "okay"; }; + cmem_block_mem_0: cmem_block_mem@a0000000 { + reg = <0x0 0xa0000000 0x0 0x18000000>; + no-map; + status = "okay"; + }; }; + cmem { + compatible = "ti,cmem"; + #address-cells = <1>; + #size-cells = <0>; + #pool-size-cells = <2>; + status = "okay"; + + cmem_block_0: cmem_block@0 { + reg = <0>; + memory-region = <&cmem_block_mem_0>; + cmem-buf-pools = <1 0x0 0x18000000>; + }; + + cmem_block_1: cmem_block@1 { + reg = <1>; + memory-region = <&cmem_block_mem_1_ocmc3>; + }; + }; + vdd_adc: gpioregulator-vdd_adc { compatible = "regulator-gpio"; regulator-name = "vdd_adc"; @@ -102,6 +135,8 @@ leds { compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&led_pins_default>; led0 { label = "beaglebone:green:usr0"; @@ -185,18 +220,1966 @@ emmc_pwrseq: emmc_pwrseq { compatible = "mmc-pwrseq-emmc"; reset-gpios = <&gpio5 7 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_pwrseq_pins_default>; }; brcmf_pwrseq: brcmf_pwrseq { compatible = "mmc-pwrseq-simple"; + pinctrl-names = "default"; + pinctrl-0 = <&brcmf_pwrseq_pins_default>; reset-gpios = <&gpio3 22 GPIO_ACTIVE_LOW>, /* BT-REG-ON */ <&gpio3 18 GPIO_ACTIVE_LOW>; /* WL-REG-ON */ }; + unused_pins: unused_pins { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&unused_pins_default>; + }; + + cape_pins: cape_pins { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&cape_pins_default>; + }; + extcon_usb1: extcon_usb1 { compatible = "linux,extcon-usb-gpio"; ti,enable-id-detection; id-gpio = <&gpio3 13 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&extcon_usb1_pins_default>; + }; +}; + +&ocp { + pruss1_shmem: pruss_shmem@4b200000 { + status = "okay"; + compatible = "ti,pruss-shmem"; + reg = <0x4b200000 0x020000>; + }; + + pruss2_shmem: pruss_shmem@4b280000 { + status = "okay"; + compatible = "ti,pruss-shmem"; + reg = <0x4b280000 0x020000>; + }; + + cape-universal { + compatible = "gpio-of-helper"; + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <>; + + P8_03 { + gpio-name = "P8_03"; + gpio = <&gpio1 24 0>; + input; + dir-changeable; + }; + + P8_04 { + gpio-name = "P8_04"; + gpio = <&gpio1 25 0>; + input; + dir-changeable; + }; + + P8_05 { + gpio-name = "P8_05"; + gpio = <&gpio7 1 0>; + input; + dir-changeable; + }; + + P8_06 { + gpio-name = "P8_06"; + gpio = <&gpio7 2 0>; + input; + dir-changeable; + }; + + P8_07 { + gpio-name = "P8_07"; + gpio = <&gpio6 5 0>; + input; + dir-changeable; + }; + + P8_08 { + gpio-name = "P8_08"; + gpio = <&gpio6 6 0>; + input; + dir-changeable; + }; + + P8_09 { + gpio-name = "P8_09"; + gpio = <&gpio6 18 0>; + input; + dir-changeable; + }; + + P8_10 { + gpio-name = "P8_10"; + gpio = <&gpio6 4 0>; + input; + dir-changeable; + }; + + P8_11 { + gpio-name = "P8_11"; + gpio = <&gpio3 11 0>; + input; + dir-changeable; + }; + + P8_12 { + gpio-name = "P8_12"; + gpio = <&gpio3 10 0>; + input; + dir-changeable; + }; + + P8_13 { + gpio-name = "P8_13"; + gpio = <&gpio4 11 0>; + input; + dir-changeable; + }; + + P8_14 { + gpio-name = "P8_14"; + gpio = <&gpio4 13 0>; + input; + dir-changeable; + }; + + P8_15 { + gpio-name = "P8_15"; + gpio = <&gpio4 3 0>; + input; + dir-changeable; + }; + + P8_16 { + gpio-name = "P8_16"; + gpio = <&gpio4 29 0>; + input; + dir-changeable; + }; + + P8_17 { + gpio-name = "P8_17"; + gpio = <&gpio8 18 0>; + input; + dir-changeable; + }; + + P8_18 { + gpio-name = "P8_18"; + gpio = <&gpio4 9 0>; + input; + dir-changeable; + }; + + P8_19 { + gpio-name = "P8_19"; + gpio = <&gpio4 10 0>; + input; + dir-changeable; + }; + + P8_20 { + gpio-name = "P8_20"; + gpio = <&gpio6 30 0>; + input; + dir-changeable; + }; + + P8_21 { + gpio-name = "P8_21"; + gpio = <&gpio6 29 0>; + input; + dir-changeable; + }; + + P8_22 { + gpio-name = "P8_22"; + gpio = <&gpio1 23 0>; + input; + dir-changeable; + }; + + P8_23 { + gpio-name = "P8_23"; + gpio = <&gpio1 22 0>; + input; + dir-changeable; + }; + + P8_24 { + gpio-name = "P8_24"; + gpio = <&gpio7 0 0>; + input; + dir-changeable; + }; + + P8_25 { + gpio-name = "P8_25"; + gpio = <&gpio6 31 0>; + input; + dir-changeable; + }; + + P8_26 { + gpio-name = "P8_26"; + gpio = <&gpio4 28 0>; + input; + dir-changeable; + }; + + P8_27 { + gpio-name = "P8_27"; + gpio = <&gpio4 23 0>; + input; + dir-changeable; + }; + + P8_28 { + gpio-name = "P8_28"; + gpio = <&gpio4 19 0>; + input; + dir-changeable; + }; + + P8_29 { + gpio-name = "P8_29"; + gpio = <&gpio4 22 0>; + input; + dir-changeable; + }; + + P8_30 { + gpio-name = "P8_30"; + gpio = <&gpio4 20 0>; + input; + dir-changeable; + }; + + P8_31 { + gpio-name = "P8_31"; + gpio = <&gpio8 14 0>; + input; + dir-changeable; + }; + + P8_32 { + gpio-name = "P8_32"; + gpio = <&gpio8 15 0>; + input; + dir-changeable; + }; + + P8_33 { + gpio-name = "P8_33"; + gpio = <&gpio8 13 0>; + input; + dir-changeable; + }; + + P8_34 { + gpio-name = "P8_34"; + gpio = <&gpio8 11 0>; + input; + dir-changeable; + }; + + P8_35 { + gpio-name = "P8_35"; + gpio = <&gpio8 12 0>; + input; + dir-changeable; + }; + + P8_36 { + gpio-name = "P8_36"; + gpio = <&gpio8 10 0>; + input; + dir-changeable; + }; + + P8_37 { + gpio-name = "P8_37"; + gpio = <&gpio8 8 0>; + input; + dir-changeable; + }; + + P8_38 { + gpio-name = "P8_38"; + gpio = <&gpio8 9 0>; + input; + dir-changeable; + }; + + P8_39 { + gpio-name = "P8_39"; + gpio = <&gpio8 6 0>; + input; + dir-changeable; + }; + + P8_40 { + gpio-name = "P8_40"; + gpio = <&gpio8 7 0>; + input; + dir-changeable; + }; + + P8_41 { + gpio-name = "P8_41"; + gpio = <&gpio8 4 0>; + input; + dir-changeable; + }; + + P8_42 { + gpio-name = "P8_42"; + gpio = <&gpio8 5 0>; + input; + dir-changeable; + }; + + P8_43 { + gpio-name = "P8_43"; + gpio = <&gpio8 2 0>; + input; + dir-changeable; + }; + + P8_44 { + gpio-name = "P8_44"; + gpio = <&gpio8 3 0>; + input; + dir-changeable; + }; + + P8_45 { + gpio-name = "P8_45"; + gpio = <&gpio8 0 0>; + input; + dir-changeable; + }; + + P8_46 { + gpio-name = "P8_46"; + gpio = <&gpio8 1 0>; + input; + dir-changeable; + }; + + P9_11 { + gpio-name = "P9_11"; + gpio = <&gpio8 17 0>; + input; + dir-changeable; + }; + + P9_12 { + gpio-name = "P9_12"; + gpio = <&gpio5 0 0>; + input; + dir-changeable; + }; + + P9_13 { + gpio-name = "P9_13"; + gpio = <&gpio6 12 0>; + input; + dir-changeable; + }; + + P9_14 { + gpio-name = "P9_14"; + gpio = <&gpio4 25 0>; + input; + dir-changeable; + }; + + P9_15 { + gpio-name = "P9_15"; + gpio = <&gpio3 12 0>; + input; + dir-changeable; + }; + + P9_16 { + gpio-name = "P9_16"; + gpio = <&gpio4 26 0>; + input; + dir-changeable; + }; + + P9_17 { + gpio-name = "P9_17"; + gpio = <&gpio7 17 0>; + input; + dir-changeable; + }; + + P9_18 { + gpio-name = "P9_18"; + gpio = <&gpio7 16 0>; + input; + dir-changeable; + }; + + P9_19 { + gpio-name = "P9_19"; + gpio = <&gpio7 3 0>; + input; + dir-changeable; + }; + + P9_20 { + gpio-name = "P9_20"; + gpio = <&gpio7 4 0>; + input; + dir-changeable; + }; + + P9_21 { + gpio-name = "P9_21"; + gpio = <&gpio3 3 0>; + input; + dir-changeable; + }; + + P9_22 { + gpio-name = "P9_22"; + gpio = <&gpio6 19 0>; + input; + dir-changeable; + }; + + P9_23 { + gpio-name = "P9_23"; + gpio = <&gpio7 11 0>; + input; + dir-changeable; + }; + + P9_24 { + gpio-name = "P9_24"; + gpio = <&gpio6 15 0>; + input; + dir-changeable; + }; + + P9_25 { + gpio-name = "P9_25"; + gpio = <&gpio6 17 0>; + input; + dir-changeable; + }; + + P9_26 { + gpio-name = "P9_26"; + gpio = <&gpio6 14 0>; + input; + dir-changeable; + }; + + P9_27 { + gpio-name = "P9_27"; + gpio = <&gpio6 15 0>; + input; + dir-changeable; + }; + + P9_28 { + gpio-name = "P9_28"; + gpio = <&gpio4 17 0>; + input; + dir-changeable; + }; + + P9_29 { + gpio-name = "P9_29"; + gpio = <&gpio5 11 0>; + input; + dir-changeable; + }; + + P9_30 { + gpio-name = "P9_30"; + gpio = <&gpio5 12 0>; + input; + dir-changeable; + }; + + P9_31 { + gpio-name = "P9_31"; + gpio = <&gpio5 10 0>; + input; + dir-changeable; + }; + + P9_41 { + gpio-name = "P9_41"; + gpio = <&gpio6 20 0>; + input; + dir-changeable; + }; + + P9_42 { + gpio-name = "P9_42"; + gpio = <&gpio4 18 0>; + input; + dir-changeable; + }; + }; +}; + +&dra7_pmx_core { + /************************/ + /* P8 Header */ + /************************/ + + /* P8_01 GND */ + + /* P8_02 GND */ + + + /* P8_03 (ball AB8) gpio1_24 */ + P8_03_default_pin: pinmux_P8_03_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat6.gpio1_24 */ + P8_03_gpio_pin: pinmux_P8_03_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat6.gpio1_24 */ + P8_03_gpio_pu_pin: pinmux_P8_03_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat6.gpio1_24 */ + P8_03_gpio_pd_pin: pinmux_P8_03_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat6.gpio1_24 */ + P8_03_gpio_input_pin: pinmux_P8_03_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_dat6.gpio1_24 */ + + /* P8_04 (ball AB5) gpio1_25 */ + P8_04_default_pin: pinmux_P8_04_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat7.gpio1_25 */ + P8_04_gpio_pin: pinmux_P8_04_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37A0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat7.gpio1_25 */ + P8_04_gpio_pu_pin: pinmux_P8_04_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat7.gpio1_25 */ + P8_04_gpio_pd_pin: pinmux_P8_04_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat7.gpio1_25 */ + P8_04_gpio_input_pin: pinmux_P8_04_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37A0, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_dat7.gpio1_25 */ + + /* P8_05 (ball AC9) gpio7_1 */ + P8_05_default_pin: pinmux_P8_05_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x378C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat2.gpio7_1 */ + P8_05_gpio_pin: pinmux_P8_05_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x378C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat2.gpio7_1 */ + P8_05_gpio_pu_pin: pinmux_P8_05_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x378C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat2.gpio7_1 */ + P8_05_gpio_pd_pin: pinmux_P8_05_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x378C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat2.gpio7_1 */ + P8_05_gpio_input_pin: pinmux_P8_05_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x378C, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_dat2.gpio7_1 */ + + /* P8_06 (ball AC3) gpio7_2 */ + P8_06_default_pin: pinmux_P8_06_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3790, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat3.gpio7_2 */ + P8_06_gpio_pin: pinmux_P8_06_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3790, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat3.gpio7_2 */ + P8_06_gpio_pu_pin: pinmux_P8_06_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3790, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat3.gpio7_2 */ + P8_06_gpio_pd_pin: pinmux_P8_06_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3790, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat3.gpio7_2 */ + P8_06_gpio_input_pin: pinmux_P8_06_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3790, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_dat3.gpio7_2 */ + + /* P8_07 (ball G14) gpio6_5*/ + P8_07_default_pin: pinmux_P8_07_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr14.gpio6_5 */ + P8_07_gpio_pin: pinmux_P8_07_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr14.gpio6_5 */ + P8_07_gpio_pu_pin: pinmux_P8_07_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr14.gpio6_5 */ + P8_07_gpio_pd_pin: pinmux_P8_07_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr14.gpio6_5 */ + P8_07_gpio_input_pin: pinmux_P8_07_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36EC, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_axr14.gpio6_5 */ + P8_07_timer_pin: pinmux_P8_07_timer_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* mcasp1_axr14.timer11 */ + + /* P8_08 (ball F14) gpio6_6 */ + P8_08_default_pin: pinmux_P8_08_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr15.gpio6_6 */ + P8_08_gpio_pin: pinmux_P8_08_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr15.gpio6_6 */ + P8_08_gpio_pu_pin: pinmux_P8_08_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr15.gpio6_6 */ + P8_08_gpio_pd_pin: pinmux_P8_08_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr15.gpio6_6 */ + P8_08_gpio_input_pin: pinmux_P8_08_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36F0, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_axr15.gpio6_6 */ + P8_08_timer_pin: pinmux_P8_08_timer_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* mcasp1_axr15.timer12 */ + + /* P8_09 (ball E17) gpio6_18 */ + P8_09_default_pin: pinmux_P8_09_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* xref_clk1.gpio6_18 */ + P8_09_gpio_pin: pinmux_P8_09_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* xref_clk1.gpio6_18 */ + P8_09_gpio_pu_pin: pinmux_P8_09_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* xref_clk1.gpio6_18 */ + P8_09_gpio_pd_pin: pinmux_P8_09_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* xref_clk1.gpio6_18 */ + P8_09_gpio_input_pin: pinmux_P8_09_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3698, PIN_INPUT | MUX_MODE14) >; }; /* xref_clk1.gpio6_18 */ + P8_09_timer_pin: pinmux_P8_09_timer_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3698, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* xref_clk1.timer14 */ + + /* P8_10 (ball A13) gpio6_4 */ + P8_10_default_pin: pinmux_P8_10_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr13.gpio6_4 */ + P8_10_gpio_pin: pinmux_P8_10_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr13.gpio6_4 */ + P8_10_gpio_pu_pin: pinmux_P8_10_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr13.gpio6_4 */ + P8_10_gpio_pd_pin: pinmux_P8_10_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr13.gpio6_4 */ + P8_10_gpio_input_pin: pinmux_P8_10_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E8, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_axr13.gpio6_4 */ + P8_10_timer_pin: pinmux_P8_10_timer_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* mcasp1_axr13.timer10 */ + + /* P8_11 (ball AH4) gpio3_11 */ + P8_11_default_pin: pinmux_P8_11_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d7.gpio3_11 */ + P8_11_gpio_pin: pinmux_P8_11_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d7.gpio3_11 */ + P8_11_gpio_pu_pin: pinmux_P8_11_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d7.gpio3_11 */ + P8_11_gpio_pd_pin: pinmux_P8_11_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d7.gpio3_11 */ + P8_11_gpio_input_pin: pinmux_P8_11_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3510, PIN_INPUT | MUX_MODE14) >; }; /* vin1a_d7.gpio3_11 */ + P8_11_qep_pin: pinmux_P8_11_qep_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* vin1a_d7.eQEP2B_in */ + P8_11_pruout_pin: pinmux_P8_11_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3510, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vin1a_d7.pr1_pru0_gpo4 */ + + /* P8_12 (ball AG6) gpio3_10 */ + P8_12_default_pin: pinmux_P8_12_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d6.gpio3_10 */ + P8_12_gpio_pin: pinmux_P8_12_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d6.gpio3_10 */ + P8_12_gpio_pu_pin: pinmux_P8_12_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d6.gpio3_10 */ + P8_12_gpio_pd_pin: pinmux_P8_12_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d6.gpio3_10 */ + P8_12_gpio_input_pin: pinmux_P8_12_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x350C, PIN_INPUT | MUX_MODE14) >; }; /* vin1a_d6.gpio3_10 */ + P8_12_qep_pin: pinmux_P8_12_qep_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* vin1a_d6.eQEP2A_in */ + P8_12_pruout_pin: pinmux_P8_12_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x350C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vin1a_d6.pr1_pru0_gpo3 */ + + /* P8_13 (ball D3) gpio4_11 */ + P8_13_default_pin: pinmux_P8_13_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d10.gpio4_11 */ + P8_13_gpio_pin: pinmux_P8_13_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d10.gpio4_11 */ + P8_13_gpio_pu_pin: pinmux_P8_13_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d10.gpio4_11 */ + P8_13_gpio_pd_pin: pinmux_P8_13_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d10.gpio4_11 */ + P8_13_gpio_input_pin: pinmux_P8_13_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3590, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d10.gpio4_11 */ + P8_13_pwm_pin: pinmux_P8_13_pwm_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3590, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d10.ehrpwm2B */ + + /* P8_14 (ball D5) gpio4_13*/ + P8_14_default_pin: pinmux_P8_14_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d12.gpio4_13 */ + P8_14_gpio_pin: pinmux_P8_14_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d12.gpio4_13 */ + P8_14_gpio_pu_pin: pinmux_P8_14_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d12.gpio4_13 */ + P8_14_gpio_pd_pin: pinmux_P8_14_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d12.gpio4_13 */ + P8_14_gpio_input_pin: pinmux_P8_14_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3598, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d12.gpio4_13 */ + P8_14_pwm_pin: pinmux_P8_14_pwm_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3598, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d12.eCAP2_in_PWM2_out */ + + /* P8_15a (ball D1) gpio4_3*/ + P8_15_default_pin: pinmux_P8_15_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d2.gpio4_3 */ + P8_15_gpio_pin: pinmux_P8_15_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d2.gpio4_3 */ + P8_15_gpio_pu_pin: pinmux_P8_15_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d2.gpio4_3 */ + P8_15_gpio_pd_pin: pinmux_P8_15_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d2.gpio4_3 */ + P8_15_gpio_input_pin: pinmux_P8_15_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3570, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d2.gpio4_3 */ + P8_15_pru_ecap_pin: pinmux_P8_15_pru_ecap_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3570, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE11) >; }; /* vin2a_d2.pr1_ecap0_ecap_capin_apwm_o */ + + /* P8_15b (ball A3) gpio4_27 */ + P8_15_pruin_pin: pinmux_P8_15_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B4, PIN_INPUT | MUX_MODE12) >; }; /* vin2a_d19.pr1_pru1_gpi16*/ + + /* P8_16 (ball B4) gpio4_29 */ + P8_16_default_pin: pinmux_P8_16_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d21.gpio4_29 */ + P8_16_gpio_pin: pinmux_P8_16_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d21.gpio4_29 */ + P8_16_gpio_pu_pin: pinmux_P8_16_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d21.gpio4_29 */ + P8_16_gpio_pd_pin: pinmux_P8_16_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35BC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d21.gpio4_29 */ + P8_16_gpio_input_pin: pinmux_P8_16_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35BC, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d21.gpio4_29 */ + P8_16_pruin_pin: pinmux_P8_16_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35BC, PIN_INPUT | MUX_MODE12) >; }; /* vin2a_d21.pr1_pru1_gpi18 */ + + /* P8_17 (ball A7) gpio8_18 */ + P8_17_default_pin: pinmux_P8_17_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d18.gpio8_18 */ + P8_17_gpio_pin: pinmux_P8_17_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d18.gpio8_18 */ + P8_17_gpio_pu_pin: pinmux_P8_17_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d18.gpio8_18 */ + P8_17_gpio_pd_pin: pinmux_P8_17_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3624, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d18.gpio8_18 */ + P8_17_gpio_input_pin: pinmux_P8_17_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3624, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d18.gpio8_18 */ + + /* P8_18 (ball F5) gpio4_9 */ + P8_18_default_pin: pinmux_P8_18_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d8.gpio4_9 */ + P8_18_gpio_pin: pinmux_P8_18_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d8.gpio4_9 */ + P8_18_gpio_pu_pin: pinmux_P8_18_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d8.gpio4_9 */ + P8_18_gpio_pd_pin: pinmux_P8_18_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d8.gpio4_9 */ + P8_18_gpio_input_pin: pinmux_P8_18_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3588, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d8.gpio4_9 */ + P8_18_qep_pin: pinmux_P8_18_qep_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3588, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d8.eQEP2_strobe */ + + /* P8_19 (ball E6) gpio4_10 */ + P8_19_default_pin: pinmux_P8_19_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d9.gpio4_10 */ + P8_19_gpio_pin: pinmux_P8_19_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d9.gpio4_10 */ + P8_19_gpio_pu_pin: pinmux_P8_19_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d9.gpio4_10 */ + P8_19_gpio_pd_pin: pinmux_P8_19_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d9.gpio4_10 */ + P8_19_gpio_input_pin: pinmux_P8_19_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x358C, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d9.gpio4_10 */ + P8_19_pwm_pin: pinmux_P8_19_pwm_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x358C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d9.ehrpwm2A */ + + /* P8_20 (ball AC4) gpio6_30 */ + P8_20_default_pin: pinmux_P8_20_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_cmd.gpio6_30 */ + P8_20_gpio_pin: pinmux_P8_20_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_cmd.gpio6_30 */ + P8_20_gpio_pu_pin: pinmux_P8_20_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_cmd.gpio6_30 */ + P8_20_gpio_pd_pin: pinmux_P8_20_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_cmd.gpio6_30 */ + P8_20_gpio_input_pin: pinmux_P8_20_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3780, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_cmd.gpio6_30 */ + P8_20_pruout_pin: pinmux_P8_20_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3780, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* mmc3_cmd.pr2_pru0_gpo3 */ + P8_20_pruin_pin: pinmux_P8_20_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3780, PIN_INPUT | MUX_MODE12) >; }; /* mmc3_cmd.pr2_pru0_gpi3 */ + + /* P8_21 (ball AD4) gpio6_29 */ + P8_21_default_pin: pinmux_P8_21_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_clk.gpio6_29 */ + P8_21_gpio_pin: pinmux_P8_21_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_clk.gpio6_29 */ + P8_21_gpio_pu_pin: pinmux_P8_21_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_clk.gpio6_29 */ + P8_21_gpio_pd_pin: pinmux_P8_21_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_clk.gpio6_29 */ + P8_21_gpio_input_pin: pinmux_P8_21_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x377C, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_clk.gpio6_29 */ + P8_21_pruout_pin: pinmux_P8_21_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x377C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* mmc3_clk.pr2_pru0_gpo2 */ + P8_21_pruin_pin: pinmux_P8_21_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x377C, PIN_INPUT | MUX_MODE12) >; }; /* mmc3_clk.pr2_pru0_gpi2 */ + + /* P8_22 (ball AD6) gpio1_23 */ + P8_22_default_pin: pinmux_P8_22_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3798, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat5.gpio1_23 */ + P8_22_gpio_pin: pinmux_P8_22_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3798, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat5.gpio1_23 */ + P8_22_gpio_pu_pin: pinmux_P8_22_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3798, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat5.gpio1_23 */ + P8_22_gpio_pd_pin: pinmux_P8_22_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3798, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat5.gpio1_23 */ + P8_22_gpio_input_pin: pinmux_P8_22_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3798, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_dat5.gpio1_23 */ + + /* P8_23 (ball AC8) gpio1_22 */ + P8_23_default_pin: pinmux_P8_23_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3794, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat4.gpio1_22 */ + P8_23_gpio_pin: pinmux_P8_23_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3794, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat4.gpio1_22 */ + P8_23_gpio_pu_pin: pinmux_P8_23_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3794, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat4.gpio1_22 */ + P8_23_gpio_pd_pin: pinmux_P8_23_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3794, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat4.gpio1_22 */ + P8_23_gpio_input_pin: pinmux_P8_23_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3794, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_dat4.gpio1_22 */ + + /* P8_24 (ball AC6) gpio7_0 */ + P8_24_default_pin: pinmux_P8_24_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3788, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat1.gpio7_0 */ + P8_24_gpio_pin: pinmux_P8_24_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3788, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat1.gpio7_0 */ + P8_24_gpio_pu_pin: pinmux_P8_24_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3788, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat1.gpio7_0 */ + P8_24_gpio_pd_pin: pinmux_P8_24_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3788, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat1.gpio7_0 */ + P8_24_gpio_input_pin: pinmux_P8_24_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3788, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_dat1.gpio7_0 */ + + /* P8_25 (ball AC7) gpio6_31 */ + P8_25_default_pin: pinmux_P8_25_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3784, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat0.gpio6_31 */ + P8_25_gpio_pin: pinmux_P8_25_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3784, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat0.gpio6_31 */ + P8_25_gpio_pu_pin: pinmux_P8_25_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3784, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat0.gpio6_31 */ + P8_25_gpio_pd_pin: pinmux_P8_25_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3784, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mmc3_dat0.gpio6_31 */ + P8_25_gpio_input_pin: pinmux_P8_25_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3784, PIN_INPUT | MUX_MODE14) >; }; /* mmc3_dat0.gpio6_31 */ + + /* P8_26 (ball B3) gpio4_28 */ + P8_26_default_pin: pinmux_P8_26_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d20.gpio4_28 */ + P8_26_gpio_pin: pinmux_P8_26_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d20.gpio4_28 */ + P8_26_gpio_pu_pin: pinmux_P8_26_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d20.gpio4_28 */ + P8_26_gpio_pd_pin: pinmux_P8_26_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d20.gpio4_28 */ + P8_26_gpio_input_pin: pinmux_P8_26_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B8, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d20.gpio4_28 */ + + /* P8_27a (ball E11) gpio4_23 */ + P8_27_default_pin: pinmux_P8_27_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_vsync.gpio4_23 */ + P8_27_gpio_pin: pinmux_P8_27_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_vsync.gpio4_23 */ + P8_27_gpio_pu_pin: pinmux_P8_27_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_vsync.gpio4_23 */ + P8_27_gpio_pd_pin: pinmux_P8_27_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_vsync.gpio4_23 */ + P8_27_gpio_input_pin: pinmux_P8_27_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D8, PIN_INPUT | MUX_MODE14) >; }; /* vout1_vsync.gpio4_23 */ + + /* P8_27b (ball A8) gpio8_19 */ + P8_27_pruout_pin: pinmux_P8_27_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3628, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d19.pr2_pru0_gpo16 */ + P8_27_pruin_pin: pinmux_P8_27_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3628, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d19.pr2_pru0_gpi16 */ + + /* P8_28a (ball D11) gpio4_19 */ + P8_28_default_pin: pinmux_P8_28_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35C8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_clk.gpio4_19 */ + P8_28_gpio_pin: pinmux_P8_28_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35C8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_clk.gpio4_19 */ + P8_28_gpio_pu_pin: pinmux_P8_28_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35C8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_clk.gpio4_19 */ + P8_28_gpio_pd_pin: pinmux_P8_28_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35C8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_clk.gpio4_19 */ + P8_28_gpio_input_pin: pinmux_P8_28_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35C8, PIN_INPUT | MUX_MODE14) >; }; /* vout1_clk.gpio4_19 */ + + /* P8_28b (ball C9) gpio8_20 */ + P8_28_pruout_pin: pinmux_P8_28_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x362C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d20.pr2_pru0_gpo17 */ + P8_28_pruin_pin: pinmux_P8_28_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x362C, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d20.pr2_pru0_gpi17 */ + + /* P8_29a (ball C11) gpio4_22 */ + P8_29_default_pin: pinmux_P8_29_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_hsync.gpio4_22 */ + P8_29_gpio_pin: pinmux_P8_29_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_hsync.gpio4_22 */ + P8_29_gpio_pu_pin: pinmux_P8_29_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_hsync.gpio4_22 */ + P8_29_gpio_pd_pin: pinmux_P8_29_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_hsync.gpio4_22 */ + P8_29_gpio_input_pin: pinmux_P8_29_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35D4, PIN_INPUT | MUX_MODE14) >; }; /* vout1_hsync.gpio4_22 */ + + /* P8_29b (ball A9) gpio8_21 */ + P8_29_pruout_pin: pinmux_P8_29_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3630, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d21.pr2_pru0_gpo18 */ + P8_29_pruin_pin: pinmux_P8_29_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3630, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d21.pr2_pru0_gpi18 */ + + /* P8_30a (ball B10) gpio4_20 */ + P8_30_default_pin: pinmux_P8_30_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35CC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_de.gpio4_20 */ + P8_30_gpio_pin: pinmux_P8_30_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35CC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_de.gpio4_20 */ + P8_30_gpio_pu_pin: pinmux_P8_30_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35CC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_de.gpio4_20 */ + P8_30_gpio_pd_pin: pinmux_P8_30_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35CC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_de.gpio4_20 */ + P8_30_gpio_input_pin: pinmux_P8_30_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35CC, PIN_INPUT | MUX_MODE14) >; }; /* vout1_de.gpio4_20 */ + + /* P8_30b (ball B9) gpio8_22 */ + P8_30_pruout_pin: pinmux_P8_30_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3634, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d22.pr2_pru0_gpo19 */ + P8_30_pruin_pin: pinmux_P8_30_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3634, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d22.pr2_pru0_gpi19 */ + + + /* P8_31a (ball C8) gpio8_14 */ + P8_31_default_pin: pinmux_P8_31_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3614, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d14.gpio8_14 */ + P8_31_gpio_pin: pinmux_P8_31_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3614, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d14.gpio8_14 */ + P8_31_gpio_pu_pin: pinmux_P8_31_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3614, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d14.gpio8_14 */ + P8_31_gpio_pd_pin: pinmux_P8_31_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3614, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d14.gpio8_14 */ + P8_31_gpio_input_pin: pinmux_P8_31_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3614, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d14.gpio8_14 */ + + /* P8_31b (ball G16) */ + P8_31_uart_pin: pinmux_P8_31_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x373C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* mcasp4_axr0.uart4_rxd */ + + /* P8_32a (ball C7) gpio8_15 */ + P8_32_default_pin: pinmux_P8_32_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3618, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d15.gpio8_15 */ + P8_32_gpio_pin: pinmux_P8_32_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3618, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d15.gpio8_15 */ + P8_32_gpio_pu_pin: pinmux_P8_32_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3618, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d15.gpio8_15 */ + P8_32_gpio_pd_pin: pinmux_P8_32_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3618, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d15.gpio8_15 */ + P8_32_gpio_input_pin: pinmux_P8_32_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3618, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d15.gpio8_15 */ + + /* P8_32b (ball D17) */ + P8_32_uart_pin: pinmux_P8_32_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3740, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* mcasp4_axr1.uart4_txd */ + + /* P8_33a (ball C6) gpio8_13 */ + P8_33_default_pin: pinmux_P8_33_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3610, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d13.gpio8_13 */ + P8_33_gpio_pin: pinmux_P8_33_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3610, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d13.gpio8_13 */ + P8_33_gpio_pu_pin: pinmux_P8_33_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3610, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d13.gpio8_13 */ + P8_33_gpio_pd_pin: pinmux_P8_33_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3610, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d13.gpio8_13 */ + P8_33_gpio_input_pin: pinmux_P8_33_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3610, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d13.gpio8_13 */ + + /* P8_33b (ball AF9) gpio3_1 */ + P8_33_qep_pin: pinmux_P8_33_qep_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* vin1a_fld0.eQEP1B_in */ + + + /* P8_34a (ball D8) gpio8_11 */ + P8_34_default_pin: pinmux_P8_34_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d11.gpio8_11 */ + P8_34_gpio_pin: pinmux_P8_34_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d11.gpio8_11 */ + P8_34_gpio_pu_pin: pinmux_P8_34_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d11.gpio8_11 */ + P8_34_gpio_pd_pin: pinmux_P8_34_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3608, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d11.gpio8_11 */ + P8_34_gpio_input_pin: pinmux_P8_34_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3608, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d11.gpio8_11 */ + + /* P8_34b (ball G6) gpio4_0 */ + P8_34_pwm_pin: pinmux_P8_34_pwm_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3564, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; }; /* vin2a_vsync0.ehrpwm1A */ + + /* P8_35a (ball A5) gpio8_12 */ + P8_35_default_pin: pinmux_P8_35_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x360C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d12.gpio8_12 */ + P8_35_gpio_pin: pinmux_P8_35_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x360C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d12.gpio8_12 */ + P8_35_gpio_pu_pin: pinmux_P8_35_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x360C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d12.gpio8_12 */ + P8_35_gpio_pd_pin: pinmux_P8_35_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x360C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d12.gpio8_12 */ + P8_35_gpio_input_pin: pinmux_P8_35_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x360C, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d12.gpio8_12 */ + + /* P8_35b (ball AD9) gpio3_0 */ + P8_35_qep_pin: pinmux_P8_35_qep_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* vin1a_de0.eQEP1A_in */ + + /* P8_36a (ball D7) gpio8_10 */ + P8_36_default_pin: pinmux_P8_36_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d10.gpio8_10 */ + P8_36_gpio_pin: pinmux_P8_36_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d10.gpio8_10 */ + P8_36_gpio_pu_pin: pinmux_P8_36_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d10.gpio8_10 */ + P8_36_gpio_pd_pin: pinmux_P8_36_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3604, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d10.gpio8_10 */ + P8_36_gpio_input_pin: pinmux_P8_36_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3604, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d10.gpio8_10 */ + + /* P8_36b (ball F2) gpio4_1 */ + P8_36_pwm_pin: pinmux_P8_36_pwm_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3568, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d0.ehrpwm1B */ + + /* P8_37a (ball E8) gpio8_8 */ + P8_37_default_pin: pinmux_P8_37_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d8.gpio8_8 */ + P8_37_gpio_pin: pinmux_P8_37_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d8.gpio8_8 */ + P8_37_gpio_pu_pin: pinmux_P8_37_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d8.gpio8_8 */ + P8_37_gpio_pd_pin: pinmux_P8_37_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35FC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d8.gpio8_8 */ + P8_37_gpio_input_pin: pinmux_P8_37_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35FC, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d8.gpio8_8 */ + + /* P8_37b (ball A21) */ + P8_37_uart_pin: pinmux_P8_37_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3738, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp4_fsx.uart8_txd */ + + /* P8_38a (ball D9) gpio8_9 */ + P8_38_default_pin: pinmux_P8_38_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d9.gpio8_9 */ + P8_38_gpio_pin: pinmux_P8_38_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d9.gpio8_9 */ + P8_38_gpio_pu_pin: pinmux_P8_38_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d9.gpio8_9 */ + P8_38_gpio_pd_pin: pinmux_P8_38_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3600, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d9.gpio8_9 */ + P8_38_gpio_input_pin: pinmux_P8_38_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3600, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d9.gpio8_9 */ + + /* P8_38b (ball C18) */ + P8_38_uart_pin: pinmux_P8_38_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3734, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp4_aclkx.uart8_rxd */ + + /* P8_39 (ball F8) gpio8_6 */ + P8_39_default_pin: pinmux_P8_39_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d6.gpio8_6 */ + P8_39_gpio_pin: pinmux_P8_39_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d6.gpio8_6 */ + P8_39_gpio_pu_pin: pinmux_P8_39_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d6.gpio8_6 */ + P8_39_gpio_pd_pin: pinmux_P8_39_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d6.gpio8_6 */ + P8_39_gpio_input_pin: pinmux_P8_39_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F4, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d6.gpio8_6 */ + P8_39_pruout_pin: pinmux_P8_39_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d6.pr2_pru0_gpo3 */ + P8_39_pruin_pin: pinmux_P8_39_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F4, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d6.pr2_pru0_gpi3 */ + + /* P8_40 (ball E7) gpio8_7 */ + P8_40_default_pin: pinmux_P8_40_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d7.gpio8_7 */ + P8_40_gpio_pin: pinmux_P8_40_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d7.gpio8_7 */ + P8_40_gpio_pu_pin: pinmux_P8_40_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d7.gpio8_7 */ + P8_40_gpio_pd_pin: pinmux_P8_40_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d7.gpio8_7 */ + P8_40_gpio_input_pin: pinmux_P8_40_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F8, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d7.gpio8_7 */ + P8_40_pruout_pin: pinmux_P8_40_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d7.pr2_pru0_gpo4 */ + P8_40_pruin_pin: pinmux_P8_40_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F8, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d7.pr2_pru0_gpi4 */ + + /* P8_41 (ball E9) gpio8_4 */ + P8_41_default_pin: pinmux_P8_41_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d4.gpio8_4 */ + P8_41_gpio_pin: pinmux_P8_41_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d4.gpio8_4 */ + P8_41_gpio_pu_pin: pinmux_P8_41_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d4.gpio8_4 */ + P8_41_gpio_pd_pin: pinmux_P8_41_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d4.gpio8_4 */ + P8_41_gpio_input_pin: pinmux_P8_41_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35EC, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d4.gpio8_4 */ + P8_41_pruout_pin: pinmux_P8_41_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35EC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d4.pr2_pru0_gpo1 */ + P8_41_pruin_pin: pinmux_P8_41_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35EC, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d4.pr2_pru0_gpi1 */ + + /* P8_42 (ball F9) gpio8_5 */ + P8_42_default_pin: pinmux_P8_42_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d5.gpio8_5 */ + P8_42_gpio_pin: pinmux_P8_42_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d5.gpio8_5 */ + P8_42_gpio_pu_pin: pinmux_P8_42_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d5.gpio8_5 */ + P8_42_gpio_pd_pin: pinmux_P8_42_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d5.gpio8_5 */ + P8_42_gpio_input_pin: pinmux_P8_42_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F0, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d5.gpio8_5 */ + P8_42_pruout_pin: pinmux_P8_42_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d5.pr2_pru0_gpo2 */ + P8_42_pruin_pin: pinmux_P8_42_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35F0, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d5.pr2_pru0_gpi2 */ + + /* P8_43 (ball F10) gpio8_2 */ + P8_43_default_pin: pinmux_P8_43_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d2.gpio8_2 */ + P8_43_gpio_pin: pinmux_P8_43_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d2.gpio8_2 */ + P8_43_gpio_pu_pin: pinmux_P8_43_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d2.gpio8_2 */ + P8_43_gpio_pd_pin: pinmux_P8_43_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d2.gpio8_2 */ + P8_43_gpio_input_pin: pinmux_P8_43_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E4, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d2.gpio8_2 */ + P8_43_pruout_pin: pinmux_P8_43_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d2.pr2_pru1_gpo20 */ + P8_43_pruin_pin: pinmux_P8_43_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E4, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d2.pr2_pru1_gpi20 */ + + /* P8_44 (ball G11) gpio8_3 */ + P8_44_default_pin: pinmux_P8_44_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d3.gpio8_3 */ + P8_44_gpio_pin: pinmux_P8_44_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d3.gpio8_3 */ + P8_44_gpio_pu_pin: pinmux_P8_44_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d3.gpio8_3 */ + P8_44_gpio_pd_pin: pinmux_P8_44_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d3.gpio8_3 */ + P8_44_gpio_input_pin: pinmux_P8_44_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E8, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d3.gpio8_3 */ + P8_44_pruout_pin: pinmux_P8_44_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d3.pr2_pru0_gpo0 */ + P8_44_pruin_pin: pinmux_P8_44_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E8, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d3.pr2_pru0_gpi0 */ + + /* P8_45a (ball F11) gpio8_0 */ + P8_45_default_pin: pinmux_P8_45_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d0.gpio8_0 */ + P8_45_gpio_pin: pinmux_P8_45_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d0.gpio8_0 */ + P8_45_gpio_pu_pin: pinmux_P8_45_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d0.gpio8_0 */ + P8_45_gpio_pd_pin: pinmux_P8_45_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d0.gpio8_0 */ + P8_45_gpio_input_pin: pinmux_P8_45_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35DC, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d0.gpio8_0 */ + + /* P8_45b (ball B7) gpio8_16 */ + P8_45_pruout_pin: pinmux_P8_45_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x361C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d16.pr2_pru0_gpo13 */ + P8_45_pruin_pin: pinmux_P8_45_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x361C, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d16.pr2_pru0_gpi13 */ + + /* P8_46a (ball G10) gpio8_1 */ + P8_46_default_pin: pinmux_P8_46_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d1.gpio8_1 */ + P8_46_gpio_pin: pinmux_P8_46_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d1.gpio8_1 */ + P8_46_gpio_pu_pin: pinmux_P8_46_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d1.gpio8_1 */ + P8_46_gpio_pd_pin: pinmux_P8_46_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d1.gpio8_1 */ + P8_46_gpio_input_pin: pinmux_P8_46_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35E0, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d1.gpio8_1 */ + + /* P8_46b (ball A10) gpio8_23 */ + P8_46_pruout_pin: pinmux_P8_46_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3638, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vout1_d23.pr2_pru0_gpo20 */ + P8_46_pruin_pin: pinmux_P8_46_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3638, PIN_INPUT | MUX_MODE12) >; }; /* vout1_d23.pr2_pru0_gpi20 */ + + /************************/ + /* P9 Header */ + /************************/ + + /* P9_01 GND */ + + /* P9_02 GND */ + + /* P9_03 3V3 */ + + /* P9_04 3V3 */ + + /* P9_05 VDD_5V */ + + /* P9_06 VDD_5V */ + + /* P9_07 SYS_5V */ + + /* P9_08 SYS_5V */ + + /* P9_09 PWR_BUT */ + + /* P9_10 RSTn */ + + /* P9_11a (ball B19) */ + P9_11_uart_pin: pinmux_P9_11_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x372C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* mcasp3_axr0.uart5_rxd */ + + /* P9_11b (ball B8) gpio8_17 */ + P9_11_default_pin: pinmux_P9_11_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3620, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d17.gpio8_17 */ + P9_11_gpio_pin: pinmux_P9_11_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3620, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vout1_d17.gpio8_17 */ + P9_11_gpio_pu_pin: pinmux_P9_11_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3620, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vout1_d17.gpio8_17 */ + P9_11_gpio_pd_pin: pinmux_P9_11_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3620, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vout1_d17.gpio8_17 */ + P9_11_gpio_input_pin: pinmux_P9_11_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3620, PIN_INPUT | MUX_MODE14) >; }; /* vout1_d17.gpio8_17 */ + + /* P9_12 (ball B14) gpio5_0 */ + P9_12_default_pin: pinmux_P9_12_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_aclkr.gpio5_0 */ + P9_12_gpio_pin: pinmux_P9_12_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_aclkr.gpio5_0 */ + P9_12_gpio_pu_pin: pinmux_P9_12_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_aclkr.gpio5_0 */ + P9_12_gpio_pd_pin: pinmux_P9_12_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36AC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_aclkr.gpio5_0 */ + P9_12_gpio_input_pin: pinmux_P9_12_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36AC, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_aclkr.gpio5_0 */ + + /* P9_13a (ball C17) */ + P9_13_uart_pin: pinmux_P9_13_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3730, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE4) >; }; /* mcasp3_axr1.uart5_txd */ + + /* P9_13b (ball AB10) gpio6_12 */ + P9_13_default_pin: pinmux_P9_13_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3680, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* usb1_drvvbus.gpio6_12 */ + P9_13_gpio_pin: pinmux_P9_13_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3680, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* usb1_drvvbus.gpio6_12 */ + P9_13_gpio_pu_pin: pinmux_P9_13_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3680, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* usb1_drvvbus.gpio6_12 */ + P9_13_gpio_pd_pin: pinmux_P9_13_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3680, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* usb1_drvvbus.gpio6_12 */ + P9_13_gpio_input_pin: pinmux_P9_13_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3680, PIN_INPUT | MUX_MODE14) >; }; /* usb1_drvvbus.gpio6_12 */ + + + /* P9_14 (ball D6) gpio4_25 */ + P9_14_default_pin: pinmux_P9_14_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d17.gpio4_25 */ + P9_14_gpio_pin: pinmux_P9_14_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d17.gpio4_25 */ + P9_14_gpio_pu_pin: pinmux_P9_14_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d17.gpio4_25 */ + P9_14_gpio_pd_pin: pinmux_P9_14_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d17.gpio4_25 */ + P9_14_gpio_input_pin: pinmux_P9_14_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35AC, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d17.gpio4_25 */ + P9_14_pwm_pin: pinmux_P9_14_pwm_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35AC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d17.ehrpwm3A */ + + /* P9_15 (ball AG4) gpio3_12 */ + P9_15_default_pin: pinmux_P9_15_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d8.gpio3_12 */ + P9_15_gpio_pin: pinmux_P9_15_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d8.gpio3_12 */ + P9_15_gpio_pu_pin: pinmux_P9_15_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d8.gpio3_12 */ + P9_15_gpio_pd_pin: pinmux_P9_15_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3514, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin1a_d8.gpio3_12 */ + P9_15_gpio_input_pin: pinmux_P9_15_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3514, PIN_INPUT | MUX_MODE14) >; }; /* vin1a_d8.gpio3_12 */ + + /* P9_16 (ball C5) gpio4_26 */ + P9_16_default_pin: pinmux_P9_16_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d18.gpio4_26 */ + P9_16_gpio_pin: pinmux_P9_16_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d18.gpio4_26 */ + P9_16_gpio_pu_pin: pinmux_P9_16_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d18.gpio4_26 */ + P9_16_gpio_pd_pin: pinmux_P9_16_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d18.gpio4_26 */ + P9_16_gpio_input_pin: pinmux_P9_16_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B0, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d18.gpio4_26 */ + P9_16_pwm_pin: pinmux_P9_16_pwm_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35B0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d18.ehrpwm3B */ + + /* P9_17a (ball B24) gpio7_17 */ + P9_17_default_pin: pinmux_P9_17_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* spi2_cs0.gpio7_17 */ + P9_17_gpio_pin: pinmux_P9_17_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* spi2_cs0.gpio7_17 */ + P9_17_gpio_pu_pin: pinmux_P9_17_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* spi2_cs0.gpio7_17 */ + P9_17_gpio_pd_pin: pinmux_P9_17_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* spi2_cs0.gpio7_17 */ + P9_17_gpio_input_pin: pinmux_P9_17_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37CC, PIN_INPUT | MUX_MODE14) >; }; /* spi2_cs0.gpio7_17 */ + P9_17_spi_cs_pin: pinmux_P9_17_spi_cs_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37CC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi2_cs0.spi2_cs0 */ + + /* P9_17b (ball F12) gpio5_3 */ + P9_17_i2c_pin: pinmux_P9_17_i2c_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36B8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* mcasp1_axr1.i2c5_scl */ + P9_17_uart_pin: pinmux_P9_17_pru_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36B8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp1_axr1.uart6_txd */ + + /* P9_18a (ball G17) gpio7_16 */ + P9_18_default_pin: pinmux_P9_18_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* spi2_d0.gpio7_16 */ + P9_18_gpio_pin: pinmux_P9_18_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* spi2_d0.gpio7_16 */ + P9_18_gpio_pu_pin: pinmux_P9_18_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* spi2_d0.gpio7_16 */ + P9_18_gpio_pd_pin: pinmux_P9_18_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* spi2_d0.gpio7_16 */ + P9_18_gpio_input_pin: pinmux_P9_18_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C8, PIN_INPUT | MUX_MODE14) >; }; /* spi2_d0.gpio7_16 */ + P9_18_spi_pin: pinmux_P9_18_spi_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi2_d0.spi2_d0 */ + + /* P9_18b (ball G12) gpio5_2 */ + P9_18_i2c_pin: pinmux_P9_18_i2c_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36B4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* mcasp1_axr0.i2c5_sda */ + P9_18_uart_pin: pinmux_P9_18_pru_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36B4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp1_axr0.uart6_rxd */ + + /* P9_19a (ball R6) gpio7_3 */ + P9_19_default_pin: pinmux_P9_19_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.gpio7_3 */ + P9_19_gpio_pin: pinmux_P9_19_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* gpmc_a0.gpio7_3 */ + P9_19_gpio_pu_pin: pinmux_P9_19_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* gpmc_a0.gpio7_3 */ + P9_19_gpio_pd_pin: pinmux_P9_19_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* gpmc_a0.gpio7_3 */ + P9_19_gpio_input_pin: pinmux_P9_19_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT | MUX_MODE14) >; }; /* gpmc_a0.gpio7_3 */ + P9_19_i2c_pin: pinmux_P9_19_i2c_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3440, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a0.i2c4_scl */ + + /* P9_19b (ball F4) gpio4_6 */ + P9_19_qep_pin: pinmux_P9_19_qep_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x357C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d5.eQEP2A_in */ + + /* P9_20a (ball T9) gpio7_4 */ + P9_20_default_pin: pinmux_P9_20_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.gpio7_4 */ + P9_20_gpio_pin: pinmux_P9_20_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* gpmc_a1.gpio7_4 */ + P9_20_gpio_pu_pin: pinmux_P9_20_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* gpmc_a1.gpio7_4 */ + P9_20_gpio_pd_pin: pinmux_P9_20_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* gpmc_a1.gpio7_4 */ + P9_20_gpio_input_pin: pinmux_P9_20_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT | MUX_MODE14) >; }; /* gpmc_a1.gpio7_4 */ + P9_20_i2c_pin: pinmux_P9_20_i2c_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3444, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE7) >; }; /* gpmc_a1.i2c4_sda */ + + /* P9_20b (ball D2) gpio4_5*/ + P9_20_pruout_pin: pinmux_P9_20_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3578, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vin2a_d4.pr1_pru1_gpo1 */ + P9_20_pruin_pin: pinmux_P9_20_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3578, PIN_INPUT | MUX_MODE12) >; }; /* vin2a_d4.pr1_pru1_gpi1 */ + + /* P9_21a (ball AF8) gpio3_3 */ + P9_21_default_pin: pinmux_P9_21_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin1a_vsync0.gpio3_3 */ + P9_21_gpio_pin: pinmux_P9_21_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin1a_vsync0.gpio3_3 */ + P9_21_gpio_pu_pin: pinmux_P9_21_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin1a_vsync0.gpio3_3 */ + P9_21_gpio_pd_pin: pinmux_P9_21_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin1a_vsync0.gpio3_3 */ + P9_21_gpio_input_pin: pinmux_P9_21_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34F0, PIN_INPUT | MUX_MODE14) >; }; /* vin1a_vsync0.gpio3_3 */ + P9_21_qep_pin: pinmux_P9_21_qep_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34F0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* vin1a_vsync0.eQEP1_strobe */ + + /* P9_21b (ball B22) gpio7_15 */ + P9_21_spi_pin: pinmux_P9_21_spi_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi2_d1.spi2_d1 */ + P9_21_uart_pin: pinmux_P9_21_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi2_d1.uart3_txd */ + + /* P9_22a (ball B26) gpio6_19 */ + P9_22_default_pin: pinmux_P9_22_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x369C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* xref_clk2.gpio6_19 */ + P9_22_gpio_pin: pinmux_P9_22_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x369C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* xref_clk2.gpio6_19 */ + P9_22_gpio_pu_pin: pinmux_P9_22_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x369C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* xref_clk2.gpio6_19 */ + P9_22_gpio_pd_pin: pinmux_P9_22_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x369C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* xref_clk2.gpio6_19 */ + P9_22_gpio_input_pin: pinmux_P9_22_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x369C, PIN_INPUT | MUX_MODE14) >; }; /* xref_clk2.gpio6_19 */ + + /* P9_22b (ball A26) gpio7_14 */ + P9_22_spi_sclk_pin: pinmux_P9_22_spi_sclk_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE0) >; }; /* spi2_sclk.spi2_sclk */ + P9_22_uart_pin: pinmux_P9_22_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37C0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE1) >; }; /* spi2_sclk.uart3_rxd */ + + /* P9_23 (ball A22) gpio7_11 */ + P9_23_default_pin: pinmux_P9_23_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* spi1_cs1.gpio7_11 */ + P9_23_gpio_pin: pinmux_P9_23_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* spi1_cs1.gpio7_11 */ + P9_23_gpio_pu_pin: pinmux_P9_23_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* spi1_cs1.gpio7_11 */ + P9_23_gpio_pd_pin: pinmux_P9_23_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37B4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* spi1_cs1.gpio7_11 */ + P9_23_gpio_input_pin: pinmux_P9_23_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x37B4, PIN_INPUT | MUX_MODE14) >; }; /* spi1_cs1.gpio7_11 */ + + /* P9_24 (ball F20) gpio6_15*/ + P9_24_default_pin: pinmux_P9_24_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* gpio6_15.gpio6_15 */ + P9_24_gpio_pin: pinmux_P9_24_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* gpio6_15.gpio6_15 */ + P9_24_gpio_pu_pin: pinmux_P9_24_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* gpio6_15.gpio6_15 */ + P9_24_gpio_pd_pin: pinmux_P9_24_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* gpio6_15.gpio6_15 */ + P9_24_gpio_input_pin: pinmux_P9_24_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x368C, PIN_INPUT | MUX_MODE14) >; }; /* gpio6_15.gpio6_15 */ + P9_24_uart_pin: pinmux_P9_24_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* gpio6_15.uart10_txd */ + P9_24_can_pin: pinmux_P9_24_can_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x368C, PIN_INPUT_PULLUP | MUX_MODE2) >; }; /* gpio6_15.dcan2_rx */ + P9_24_i2c_pin: pinmux_P9_24_i2c_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x368C, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE9) >; }; /* gpio6_15.i2c3_scl */ + + /* P9_25 (ball D18) gpio6_17 */ + P9_25_default_pin: pinmux_P9_25_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* xref_clk0.gpio6_17 */ + P9_25_gpio_pin: pinmux_P9_25_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* xref_clk0.gpio6_17 */ + P9_25_gpio_pu_pin: pinmux_P9_25_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* xref_clk0.gpio6_17 */ + P9_25_gpio_pd_pin: pinmux_P9_25_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* xref_clk0.gpio6_17 */ + P9_25_gpio_input_pin: pinmux_P9_25_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3694, PIN_INPUT | MUX_MODE14) >; }; /* xref_clk0.gpio6_17 */ + P9_25_pruout_pin: pinmux_P9_25_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3694, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* xref_clk0.pr2_pru1_gpo5 */ + P9_25_pruin_pin: pinmux_P9_25_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3694, PIN_INPUT | MUX_MODE12) >; }; /* xref_clk0.pr2_pru1_gpi5 */ + + /* P9_26a (ball E21) gpio6_14 */ + P9_26_default_pin: pinmux_P9_26_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* gpio6_14.gpio6_14 */ + P9_26_gpio_pin: pinmux_P9_26_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* gpio6_14.gpio6_14 */ + P9_26_gpio_pu_pin: pinmux_P9_26_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* gpio6_14.gpio6_14 */ + P9_26_gpio_pd_pin: pinmux_P9_26_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* gpio6_14.gpio6_14 */ + P9_26_gpio_input_pin: pinmux_P9_26_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3688, PIN_INPUT | MUX_MODE14) >; }; /* gpio6_14.gpio6_14 */ + P9_26_uart_pin: pinmux_P9_26_uart_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* gpio6_14.uart10_rxd */ + P9_26_can_pin: pinmux_P9_26_can_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | MUX_MODE2) >; }; /* gpio6_14.dcan2_tx */ + P9_26_i2c_pin: pinmux_P9_26_i2c_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3688, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE9) >; }; /* gpio6_14.i2c3_sda */ + + /* P9_26b (ball AE2) gpio3_24 */ + P9_26_pruout_pin: pinmux_P9_26_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3544, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vin1a_d20.pr1_pru0_gpo17*/ + P9_26_pruin_pin: pinmux_P9_26_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3544, PIN_INPUT | MUX_MODE12) >; }; /* vin1a_d20.pr1_pru0_gpi17*/ + + /* P9_27a (ball C3) gpio4_15 */ + P9_27_default_pin: pinmux_P9_27_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d14.gpio4_15 */ + P9_27_gpio_pin: pinmux_P9_27_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d14.gpio4_15 */ + P9_27_gpio_pu_pin: pinmux_P9_27_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d14.gpio4_15 */ + P9_27_gpio_pd_pin: pinmux_P9_27_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* vin2a_d14.gpio4_15 */ + P9_27_gpio_input_pin: pinmux_P9_27_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35A0, PIN_INPUT | MUX_MODE14) >; }; /* vin2a_d14.gpio4_15 */ + P9_27_qep_pin: pinmux_P9_27_qep_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE10) >; }; /* vin2a_d14.eQEP3B_in */ + P9_27_pruout_pin: pinmux_P9_27_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vin2a_d14.pr1_pru1_gpo11 */ + P9_27_pruin_pin: pinmux_P9_27_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x35A0, PIN_INPUT | MUX_MODE12) >; }; /* vin2a_d14.pr1_pru1_gpi11 */ + + /* P9_27b (ball J14) gpio5_1 */ + + + /* P9_28 (ball A12) gpio4_17 */ + P9_28_default_pin: pinmux_P9_28_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr11.gpio4_17 */ + P9_28_gpio_pin: pinmux_P9_28_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr11.gpio4_17 */ + P9_28_gpio_pu_pin: pinmux_P9_28_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr11.gpio4_17 */ + P9_28_gpio_pd_pin: pinmux_P9_28_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr11.gpio4_17 */ + P9_28_gpio_input_pin: pinmux_P9_28_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E0, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_axr11.gpio4_17 */ + P9_28_spi_cs_pin: pinmux_P9_28_spi_cs_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp1_axr11.spi3_cs0 */ + P9_28_pruout_pin: pinmux_P9_28_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* mcasp1_axr11.pr2_pru1_gpo13 */ + P9_28_pruin_pin: pinmux_P9_28_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E0, PIN_INPUT | MUX_MODE12) >; }; /* mcasp1_axr11.pr2_pru1_gpi13 */ + + /* P9_29a (ball A11) gpio5_11*/ + P9_29_default_pin: pinmux_P9_29_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr9.gpio5_11 */ + P9_29_gpio_pin: pinmux_P9_29_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr9.gpio5_11 */ + P9_29_gpio_pu_pin: pinmux_P9_29_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr9.gpio5_11 */ + P9_29_gpio_pd_pin: pinmux_P9_29_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr9.gpio5_11 */ + P9_29_gpio_input_pin: pinmux_P9_29_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D8, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_axr9.gpio5_11 */ + P9_29_spi_pin: pinmux_P9_29_spi_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp1_axr9.spi3_d1 */ + P9_29_pruout_pin: pinmux_P9_29_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D8, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* mcasp1_axr9.pr2_pru1_gpo11 */ + P9_29_pruin_pin: pinmux_P9_29_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D8, PIN_INPUT | MUX_MODE12) >; }; /* mcasp1_axr9.pr2_pru1_gpi11 */ + + /* P9_29b (ball D14) gpio7_30 */ + + + /* P9_30 (ball B13) gpio5_12*/ + P9_30_default_pin: pinmux_P9_30_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr10.gpio5_12 */ + P9_30_gpio_pin: pinmux_P9_30_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr10.gpio5_12 */ + P9_30_gpio_pu_pin: pinmux_P9_30_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr10.gpio5_12 */ + P9_30_gpio_pd_pin: pinmux_P9_30_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr10.gpio5_12 */ + P9_30_gpio_input_pin: pinmux_P9_30_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36DC, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_axr10.gpio5_12 */ + P9_30_spi_pin: pinmux_P9_30_spi_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp1_axr10.spi3_d0 */ + P9_30_pruout_pin: pinmux_P9_30_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36DC, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* mcasp1_axr10.pr2_pru1_gpo12 */ + P9_30_pruin_pin: pinmux_P9_30_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36DC, PIN_INPUT | MUX_MODE12) >; }; /* mcasp1_axr10.pr2_pru1_gpi12 */ + + /* P9_31a (ball B12) gpio5_10 */ + P9_31_default_pin: pinmux_P9_31_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr8.gpio5_10 */ + P9_31_gpio_pin: pinmux_P9_31_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr8.gpio5_10 */ + P9_31_gpio_pu_pin: pinmux_P9_31_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr8.gpio5_10 */ + P9_31_gpio_pd_pin: pinmux_P9_31_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr8.gpio5_10 */ + P9_31_gpio_input_pin: pinmux_P9_31_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D4, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_axr8.gpio5_10 */ + P9_31_spi_sclk_pin: pinmux_P9_31_spi_sclk_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp1_axr8.spi3_sclk */ + P9_31_pruout_pin: pinmux_P9_31_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* mcasp1_axr8.pr2_pru1_gpo10 */ + P9_31_pruin_pin: pinmux_P9_31_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36D4, PIN_INPUT | MUX_MODE12) >; }; /* mcasp1_axr8.pr2_pru1_gpi10 */ + + /* P9_31b (ball C14) gpio7_31*/ + + /* P9_32 VADC */ + + /* P9_33 AIN4*/ + + /* P9_34 AGND */ + + /* P9_35 AIN6 */ + + /* P9_36 AIN5 */ + + /* P9_37 AIN2 */ + + /* P9_38 AIN3*/ + + /* P9_39 AIN0*/ + + /* P9_40 AIN1*/ + + /* P9_41a (ball C23) gpio6_20 */ + P9_41_default_pin: pinmux_P9_41_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* xref_clk3.gpio6_20 */ + P9_41_gpio_pin: pinmux_P9_41_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* xref_clk3.gpio6_20 */ + P9_41_gpio_pu_pin: pinmux_P9_41_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* xref_clk3.gpio6_20 */ + P9_41_gpio_pd_pin: pinmux_P9_41_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36A0, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* xref_clk3.gpio6_20 */ + P9_41_gpio_input_pin: pinmux_P9_41_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36A0, PIN_INPUT | MUX_MODE14) >; }; /* xref_clk3.gpio6_20 */ + + /* P9_41b (ball C1) gpio4_7 */ + P9_41_pruout_pin: pinmux_P9_41_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3580, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vin2a_d6.pr1_pru1_gpo3 */ + P9_41_pruin_pin: pinmux_P9_41_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3580, PIN_INPUT | MUX_MODE12) >; }; /* vin2a_d6.pr1_pru1_gpi3 */ + + /* P9_42a (ball E14) gpio4_18 */ + P9_42_default_pin: pinmux_P9_42_default_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr12.gpio4_18 */ + P9_42_gpio_pin: pinmux_P9_42_gpio_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr12.gpio4_18 */ + P9_42_gpio_pu_pin: pinmux_P9_42_gpio_pu_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr12.gpio4_18 */ + P9_42_gpio_pd_pin: pinmux_P9_42_gpio_pd_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE14) >; }; /* mcasp1_axr12.gpio4_18 */ + P9_42_gpio_input_pin: pinmux_P9_42_gpio_input_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E4, PIN_INPUT | MUX_MODE14) >; }; /* mcasp1_axr12.gpio4_18 */ + P9_42_spi_cs_pin: pinmux_P9_42_spi_cs_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36E4, PIN_OUTPUT_PULLUP | INPUT_EN | MUX_MODE3) >; }; /* mcasp1_axr12.spi3_cs1 */ + + /* P9_42b (ball C2) gpio4_14*/ + P9_42_pruout_pin: pinmux_P9_42_pruout_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x359C, PIN_OUTPUT_PULLDOWN | INPUT_EN | MUX_MODE13) >; }; /* vin2a_d13.pr1_pru1_gpo10 */ + P9_42_pruin_pin: pinmux_P9_42_pruin_pin { pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x359C, PIN_INPUT | MUX_MODE12) >; }; /* vin2a_d13.pr1_pru1_gpi10 */ + + /* P9_43 GND */ + + /* P9_44 GND */ + + /* P9_45 GND */ + + /* P9_46 GND */ + + extcon_usb1_pins_default: extcon_usb1_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3518, PIN_INPUT | MUX_MODE14) /* AG2: vin1a_d9.gpio3_13 - USR0 */ + >; + }; + + led_pins_default: led_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3528, PIN_OUTPUT | MUX_MODE14) /* AF6: vin1a_d13.gpio3_17 - USR0 */ + DRA7XX_CORE_IOPAD(0x36c0, PIN_OUTPUT | MUX_MODE14) /* J11: mcasp1_axr3.gpio5_5 - USR1 */ + DRA7XX_CORE_IOPAD(0x3520, PIN_OUTPUT | MUX_MODE14) /* AG5: vin1a_d12.gpio3_15 - USR2 */ + DRA7XX_CORE_IOPAD(0x351c, PIN_OUTPUT | MUX_MODE14) /* AG3: vin1a_d10.gpio3_14 - USR3 */ + DRA7XX_CORE_IOPAD(0x3500, PIN_OUTPUT | MUX_MODE14) /* AH6: vin1a_d3.gpio3_7 - USR4 */ + >; + }; + + microsd_extra_pins_default: microsd_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x376c, PIN_INPUT_PULLUP | MUX_MODE14) /* W7: mmc1_sdcd.gpio6_27 - MMC1_SDCD */ + >; + }; + + ethphy_extra_pins_default: ethphy_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x34c4, PIN_OUTPUT_PULLUP | MUX_MODE14) /* N1: gpmc_advn_ale.gpio2_23 - RGMII_RST */ + DRA7XX_CORE_IOPAD(0x364c, PIN_INPUT | MUX_MODE14) /* Y1: uart3_txd.gpio5_19 - MII0_INT */ + >; + }; + + emmc_pwrseq_pins_default: emmc_pwrseq_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x36c8, PIN_OUTPUT_PULLUP | MUX_MODE14) /* F13: mcasp1_axr5.gpio5_7 - eMMC_RSTn */ + >; + }; + + brcmf_pwrseq_pins_default: brcmf_pwrseq_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x352c, PIN_OUTPUT_PULLUP | MUX_MODE14) /* AF3: vin1a_d14.gpio3_18 - WL_REG_ON */ + DRA7XX_CORE_IOPAD(0x353c, PIN_OUTPUT_PULLUP | MUX_MODE14) /* AE5: vin1a_d18.gpio3_22 - BT_REG_ON */ + >; + }; + + wifibt_extra_pins_default: wifibt_extra_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3540, PIN_INPUT | MUX_MODE14) /* AE1: vin1a_d19.gpio3_23 - WL_HOST_WAKE */ + DRA7XX_CORE_IOPAD(0x3450, PIN_INPUT | MUX_MODE8) /* P6: vin1a_d20.uart6_rxd - UART6_RXD */ + DRA7XX_CORE_IOPAD(0x3454, PIN_INPUT | MUX_MODE8) /* R9: vin1a_d21.uart6_txd - UART6_TXD */ + DRA7XX_CORE_IOPAD(0x3458, PIN_INPUT | MUX_MODE8) /* R5: vin1a_d22.uart6_ctsn - UART6_CTSN */ + DRA7XX_CORE_IOPAD(0x345c, PIN_INPUT | MUX_MODE8) /* P5: vin1a_d23.uart6_rtsn - UART6_RTSN */ + DRA7XX_CORE_IOPAD(0x3534, PIN_INPUT_PULLDOWN | MUX_MODE14) /* AF1: vin1a_d16.gpio3_20 - BT_HOST_WAKE */ + DRA7XX_CORE_IOPAD(0x3538, PIN_OUTPUT_PULLDOWN | MUX_MODE14) /* AE3: vin1a_d6.gpio3_21 - BT_WAKE */ + >; + }; + + adc_pins_default: adc_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3550, PIN_OUTPUT | MUX_MODE14) /* AD3: vin1a_d23.gpio3_27 - VDD_ADC_SEL */ + DRA7XX_CORE_IOPAD(0x34DC, PIN_INPUT_PULLUP | MUX_MODE14) /* AG8: vin1a_clk0.gpio2_30 - INT_ADC */ + >; + }; + + pmic_pins_default: pmic_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3690, PIN_INPUT_PULLUP | MUX_MODE14) /* F21: gpio6_16.gpio6_16 - PMIC_INT */ + >; + }; + + hdmi_pins_default: hdmi_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3808, PIN_INPUT | MUX_MODE1) /* C25: i2c2_sda.hdmi1_ddc_scl - HDMI_DDC_SCL */ + DRA7XX_CORE_IOPAD(0x380C, PIN_INPUT | MUX_MODE1) /* F17: i2c2_scl.hdmi1_ddc_sda - HDMI_DDC_SDA */ + DRA7XX_CORE_IOPAD(0x37BC, PIN_INPUT | MUX_MODE6) /* B20: spi1_cs3.hdmi1_cec - HDMI_DDC_CEC */ +#if 0 + DRA7XX_CORE_IOPAD(0x37B8, PIN_INPUT | MUX_MODE6) /* B21: spi1_cs2.hdmi1_hpd - HDMI_DDC_HPD */ +#else + DRA7XX_CORE_IOPAD(0x37B8, PIN_INPUT | MUX_MODE14) /* B21: spi1_cs2.gpio7_12 - HDMI_DDC_HPD */ +#endif + >; + }; + + unused_pins_default: unused_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x3400, MUX_MODE15) /* M6: sysboot0 */ + DRA7XX_CORE_IOPAD(0x3404, MUX_MODE15) /* M2: sysboot1 */ + DRA7XX_CORE_IOPAD(0x3408, MUX_MODE15) /* L5: sysboot2 */ + DRA7XX_CORE_IOPAD(0x340C, MUX_MODE15) /* M1: sysboot3 */ + DRA7XX_CORE_IOPAD(0x3410, MUX_MODE15) /* L6: sysboot4 */ + DRA7XX_CORE_IOPAD(0x3414, MUX_MODE15) /* L4: sysboot5 */ + DRA7XX_CORE_IOPAD(0x3418, MUX_MODE15) /* L3: sysboot6 */ + DRA7XX_CORE_IOPAD(0x341C, MUX_MODE15) /* L2: sysboot7 */ + DRA7XX_CORE_IOPAD(0x3420, MUX_MODE15) /* L1: sysboot8 */ + DRA7XX_CORE_IOPAD(0x3424, MUX_MODE15) /* K2: sysboot9 */ + DRA7XX_CORE_IOPAD(0x3428, MUX_MODE15) /* J1: sysboot10 */ + DRA7XX_CORE_IOPAD(0x342C, MUX_MODE15) /* J2: sysboot11 */ + DRA7XX_CORE_IOPAD(0x3430, MUX_MODE15) /* H1: sysboot12 */ + DRA7XX_CORE_IOPAD(0x3434, MUX_MODE15) /* J3: sysboot13 */ + DRA7XX_CORE_IOPAD(0x3438, MUX_MODE15) /* H2: sysboot14 */ + DRA7XX_CORE_IOPAD(0x343C, MUX_MODE15) /* H3: sysboot15 */ + + DRA7XX_CORE_IOPAD(0x3448, MUX_MODE15) /* T6: N/C */ + DRA7XX_CORE_IOPAD(0x344C, MUX_MODE15) /* T7: N/C */ + + DRA7XX_CORE_IOPAD(0x3460, MUX_MODE15) /* N7: N/C */ + DRA7XX_CORE_IOPAD(0x3464, MUX_MODE15) /* R4: N/C */ + DRA7XX_CORE_IOPAD(0x3468, MUX_MODE15) /* N9: N/C */ + DRA7XX_CORE_IOPAD(0x346C, MUX_MODE15) /* P9: N/C */ + DRA7XX_CORE_IOPAD(0x3470, MUX_MODE15) /* P4: N/C */ + DRA7XX_CORE_IOPAD(0x3474, MUX_MODE15) /* R3: N/C */ + DRA7XX_CORE_IOPAD(0x3478, MUX_MODE15) /* T2: N/C */ + DRA7XX_CORE_IOPAD(0x347C, MUX_MODE15) /* U2: N/C */ + DRA7XX_CORE_IOPAD(0x3480, MUX_MODE15) /* U1: N/C */ + DRA7XX_CORE_IOPAD(0x3484, MUX_MODE15) /* P3: N/C */ + DRA7XX_CORE_IOPAD(0x3488, MUX_MODE15) /* R2: N/C */ + + DRA7XX_CORE_IOPAD(0x34B4, MUX_MODE15) /* T1: N/C */ + DRA7XX_CORE_IOPAD(0x34B8, MUX_MODE15) /* P2: N/C */ + DRA7XX_CORE_IOPAD(0x34BC, MUX_MODE15) /* P1: N/C */ + DRA7XX_CORE_IOPAD(0x34C0, MUX_MODE15) /* P7: N/C */ + DRA7XX_CORE_IOPAD(0x34C4, MUX_MODE15) /* N1: N/C */ + DRA7XX_CORE_IOPAD(0x34C8, MUX_MODE15) /* M5: N/C */ + DRA7XX_CORE_IOPAD(0x34CC, MUX_MODE15) /* M3: N/C */ + DRA7XX_CORE_IOPAD(0x34D0, MUX_MODE15) /* N6: N/C */ + DRA7XX_CORE_IOPAD(0x34D4, MUX_MODE15) /* M4: N/C */ + DRA7XX_CORE_IOPAD(0x34D8, MUX_MODE15) /* N2: N/C */ + + DRA7XX_CORE_IOPAD(0x34E0, MUX_MODE15) /* AH7: N/C */ + + DRA7XX_CORE_IOPAD(0x34EC, MUX_MODE15) /* AE9: N/C */ + + DRA7XX_CORE_IOPAD(0x34F4, MUX_MODE15) /* AE8: N/C */ + DRA7XX_CORE_IOPAD(0x34F8, MUX_MODE15) /* AD8: N/C */ + DRA7XX_CORE_IOPAD(0x34FC, MUX_MODE15) /* AG7: N/C */ + + DRA7XX_CORE_IOPAD(0x3504, MUX_MODE15) /* AH3: N/C */ + DRA7XX_CORE_IOPAD(0x3508, MUX_MODE15) /* AH5: N/C */ + + DRA7XX_CORE_IOPAD(0x3524, MUX_MODE15) /* AF2: N/C */ + + DRA7XX_CORE_IOPAD(0x3530, MUX_MODE15) /* AF4: N/C */ + + DRA7XX_CORE_IOPAD(0x354C, MUX_MODE15) /* AD2: N/C */ + + DRA7XX_CORE_IOPAD(0x3554, MUX_MODE15) /* E1: N/C */ + DRA7XX_CORE_IOPAD(0x3558, MUX_MODE15) /* G2: N/C */ + DRA7XX_CORE_IOPAD(0x355C, MUX_MODE15) /* H7: N/C */ + DRA7XX_CORE_IOPAD(0x3560, MUX_MODE15) /* G1: N/C */ + + DRA7XX_CORE_IOPAD(0x356C, MUX_MODE15) /* F3: N/C */ + + DRA7XX_CORE_IOPAD(0x3574, MUX_MODE15) /* E2: N/C */ + + DRA7XX_CORE_IOPAD(0x3584, MUX_MODE15) /* E4: N/C */ + + DRA7XX_CORE_IOPAD(0x3594, MUX_MODE15) /* F6: N/C */ + + DRA7XX_CORE_IOPAD(0x35A4, MUX_MODE15) /* C4: N/C */ + DRA7XX_CORE_IOPAD(0x35A8, MUX_MODE15) /* B2: N/C */ + + DRA7XX_CORE_IOPAD(0x35C0, MUX_MODE15) /* B5: N/C */ + DRA7XX_CORE_IOPAD(0x35C4, MUX_MODE15) /* A4: N/C */ + + DRA7XX_CORE_IOPAD(0x35D0, MUX_MODE15) /* B11: N/C */ + + DRA7XX_CORE_IOPAD(0x3644, MUX_MODE15) /* U3: N/C */ + + DRA7XX_CORE_IOPAD(0x3680, MUX_MODE15) /* AB10: N/C */ + + DRA7XX_CORE_IOPAD(0x36BC, MUX_MODE15) /* G13: N/C */ + + DRA7XX_CORE_IOPAD(0x36C4, MUX_MODE15) /* E12: N/C */ + + DRA7XX_CORE_IOPAD(0x36CC, MUX_MODE15) /* C12: N/C */ + DRA7XX_CORE_IOPAD(0x36D0, MUX_MODE15) /* D12: N/C */ + + DRA7XX_CORE_IOPAD(0x36FC, MUX_MODE15) /* E15: N/C */ + DRA7XX_CORE_IOPAD(0x3700, MUX_MODE15) /* A20: N/C */ + + DRA7XX_CORE_IOPAD(0x370C, MUX_MODE15) /* C15: N/C */ + DRA7XX_CORE_IOPAD(0x3710, MUX_MODE15) /* A16: N/C */ + DRA7XX_CORE_IOPAD(0x3714, MUX_MODE15) /* D15: N/C */ + DRA7XX_CORE_IOPAD(0x3718, MUX_MODE15) /* B16: N/C */ + DRA7XX_CORE_IOPAD(0x371C, MUX_MODE15) /* B17: N/C */ + DRA7XX_CORE_IOPAD(0x3720, MUX_MODE15) /* A17: N/C */ + DRA7XX_CORE_IOPAD(0x3724, MUX_MODE15) /* B18: N/C */ + DRA7XX_CORE_IOPAD(0x3728, MUX_MODE15) /* F15: N/C */ + + DRA7XX_CORE_IOPAD(0x3744, MUX_MODE15) /* AA3: N/C */ + DRA7XX_CORE_IOPAD(0x3748, MUX_MODE15) /* AB9: N/C */ + DRA7XX_CORE_IOPAD(0x374C, MUX_MODE15) /* AB3: N/C */ + DRA7XX_CORE_IOPAD(0x3750, MUX_MODE15) /* AA4: N/C */ + + DRA7XX_CORE_IOPAD(0x3770, MUX_MODE15) /* Y9: N/C */ + DRA7XX_CORE_IOPAD(0x3774, MUX_MODE15) /* AC5: N/C */ + DRA7XX_CORE_IOPAD(0x3778, MUX_MODE15) /* AB4: N/C */ + + DRA7XX_CORE_IOPAD(0x37A4, MUX_MODE15) /* A25: N/C */ + DRA7XX_CORE_IOPAD(0x37A8, MUX_MODE15) /* F16: N/C */ + DRA7XX_CORE_IOPAD(0x37AC, MUX_MODE15) /* B25: N/C */ + DRA7XX_CORE_IOPAD(0x37B0, MUX_MODE15) /* A24: N/C */ + + DRA7XX_CORE_IOPAD(0x37D0, MUX_MODE15) /* G20: N/C */ + DRA7XX_CORE_IOPAD(0x37D4, MUX_MODE15) /* G19: N/C */ + + DRA7XX_CORE_IOPAD(0x3818, MUX_MODE15) /* AD17: N/C */ + DRA7XX_CORE_IOPAD(0x381C, MUX_MODE15) /* AC17: N/C */ + DRA7XX_CORE_IOPAD(0x3820, MUX_MODE15) /* AB16: N/C */ + DRA7XX_CORE_IOPAD(0x3824, MUX_MODE15) /* AC16: N/C */ + >; + }; + + cape_pins_default: cape_pins_default { + pinctrl-single,pins = < + DRA7XX_CORE_IOPAD(0x379C, MUX_MODE14) /* AB8: P8.3: mmc3_dat6.off */ + DRA7XX_CORE_IOPAD(0x37A0, MUX_MODE14) /* AB5: P8.4: mmc3_dat7.off */ + DRA7XX_CORE_IOPAD(0x378C, MUX_MODE14) /* AC9: P8.5: mmc3_dat2.off */ + DRA7XX_CORE_IOPAD(0x3790, MUX_MODE14) /* AC3: P8.6: mmc3_dat3.off */ + DRA7XX_CORE_IOPAD(0x36EC, MUX_MODE14) /* G14: P8.7: mcasp1_axr14.off */ + DRA7XX_CORE_IOPAD(0x36F0, MUX_MODE14) /* F14: P8.8: mcasp1_axr15.off */ + DRA7XX_CORE_IOPAD(0x3698, MUX_MODE14) /* E17: P8.9: xref_clk1.off */ + DRA7XX_CORE_IOPAD(0x36E8, MUX_MODE14) /* A13: P8.10: mcasp1_axr13.off */ + DRA7XX_CORE_IOPAD(0x3510, MUX_MODE14) /* AH4: P8.11: vin1a_d7.off */ + DRA7XX_CORE_IOPAD(0x350C, MUX_MODE14) /* AG6: P8.12: vin1a_d6.off */ + DRA7XX_CORE_IOPAD(0x3590, PIN_INPUT | MUX_MODE12) /* D3: P8.13: vin2a_d10.off */ + DRA7XX_CORE_IOPAD(0x3598, MUX_MODE14) /* D5: P8.14: vin2a_d12.off */ + DRA7XX_CORE_IOPAD(0x3570, MUX_MODE14) /* D1: P8.15a: vin2a_d2.off */ + DRA7XX_CORE_IOPAD(0x35B4, MUX_MODE13) /* A3: P8.15b: vin2a_d19.off */ + DRA7XX_CORE_IOPAD(0x35BC, MUX_MODE13) /* B4: P8.16: vin2a_d21.off */ + DRA7XX_CORE_IOPAD(0x3624, MUX_MODE14) /* A7: P8.17: vout1_d18.off */ + DRA7XX_CORE_IOPAD(0x3588, PIN_INPUT | MUX_MODE12) /* F5: P8.18: vin2a_d8.off */ + DRA7XX_CORE_IOPAD(0x358C, PIN_INPUT | MUX_MODE12) /* E6: P8.19: vin2a_d9.off */ + DRA7XX_CORE_IOPAD(0x3780, MUX_MODE14) /* AC4: P8.20: mmc3_cmd.off */ + DRA7XX_CORE_IOPAD(0x377C, MUX_MODE14) /* AD4: P8.21: mmc3_clk.off */ + DRA7XX_CORE_IOPAD(0x3798, MUX_MODE14) /* AD6: P8.22: mmc3_dat5.off */ + DRA7XX_CORE_IOPAD(0x3794, MUX_MODE14) /* AC8: P8.23: mmc3_dat4.off */ + DRA7XX_CORE_IOPAD(0x3788, MUX_MODE14) /* AC6: P8.24: mmc3_dat1.off */ + DRA7XX_CORE_IOPAD(0x3784, MUX_MODE14) /* AC7: P8.25: mmc3_dat0.off */ + DRA7XX_CORE_IOPAD(0x35B8, MUX_MODE13) /* B3: P8.26: vin2a_d20.off */ + DRA7XX_CORE_IOPAD(0x35D8, MUX_MODE14) /* E11: P8.27a: vout1_vsync.off */ + DRA7XX_CORE_IOPAD(0x3628, MUX_MODE14) /* A8: P8.27b: vout1_d19.off */ + DRA7XX_CORE_IOPAD(0x35C8, MUX_MODE14) /* D11: P8.28a: vout1_clk.off */ + DRA7XX_CORE_IOPAD(0x362C, MUX_MODE14) /* C9: P8.28b: vout1_d20.off */ + DRA7XX_CORE_IOPAD(0x35D4, MUX_MODE14) /* C11: P8.29a: vout1_hsync.off */ + DRA7XX_CORE_IOPAD(0x3630, MUX_MODE14) /* A9: P8.29b: vout1_d21.off */ + DRA7XX_CORE_IOPAD(0x35CC, MUX_MODE14) /* B10: P8.30a: vout1_de.off */ + DRA7XX_CORE_IOPAD(0x3634, MUX_MODE14) /* B9: P8.30b: vout1_d22.off */ + DRA7XX_CORE_IOPAD(0x3614, MUX_MODE14) /* C8: P8.31a: vout1_d14.off */ + DRA7XX_CORE_IOPAD(0x373C, MUX_MODE14) /* G16: P8.31b: mcasp4_axr0.off */ + DRA7XX_CORE_IOPAD(0x3618, MUX_MODE14) /* C7: P8.32a: vout1_d15.off */ + DRA7XX_CORE_IOPAD(0x3740, MUX_MODE14) /* D17: P8.32b: mcasp4_axr1.off */ + DRA7XX_CORE_IOPAD(0x3610, MUX_MODE14) /* C6: P8.33a: vout1_d13.off */ + DRA7XX_CORE_IOPAD(0x34E8, MUX_MODE14) /* AF9: P8.33b: vin1a_fld0.off */ + DRA7XX_CORE_IOPAD(0x3608, MUX_MODE14) /* D8: P8.34a: vout1_d11.off */ + DRA7XX_CORE_IOPAD(0x3564, MUX_MODE14) /* G6: P8.34b: vin2a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x360C, MUX_MODE14) /* A5: P8.35a: vout1_d12.off */ + DRA7XX_CORE_IOPAD(0x34E4, MUX_MODE14) /* AD9: P8.35b: vin1a_de0.off */ + DRA7XX_CORE_IOPAD(0x3604, MUX_MODE14) /* D7: P8.36a: vout1_d10.off */ + DRA7XX_CORE_IOPAD(0x3568, MUX_MODE14) /* F2: P8.36b: vin2a_d0.off */ + DRA7XX_CORE_IOPAD(0x35FC, MUX_MODE14) /* E8: P8.37a: vout1_d8.off */ + DRA7XX_CORE_IOPAD(0x3738, MUX_MODE14) /* A21: P8.37b: mcasp4_fsx.off */ + DRA7XX_CORE_IOPAD(0x3600, MUX_MODE14) /* D9: P8.38a: vout1_d9.off */ + DRA7XX_CORE_IOPAD(0x3734, MUX_MODE14) /* C18: P8.38b: mcasp4_aclkx.off */ + DRA7XX_CORE_IOPAD(0x35F4, MUX_MODE14) /* F8: P8.39: vout1_d6.off */ + DRA7XX_CORE_IOPAD(0x35F8, MUX_MODE14) /* E7: P8.40: vout1_d7.off */ + DRA7XX_CORE_IOPAD(0x35EC, MUX_MODE14) /* E9: P8.41: vout1_d4.off */ + DRA7XX_CORE_IOPAD(0x35F0, MUX_MODE14) /* F9: P8.42: vout1_d5.off */ + DRA7XX_CORE_IOPAD(0x35E4, MUX_MODE14) /* F10: P8.43: vout1_d2.off */ + DRA7XX_CORE_IOPAD(0x35E8, MUX_MODE14) /* G11: P8.44: vout1_d3.off */ + DRA7XX_CORE_IOPAD(0x35DC, MUX_MODE14) /* F11: P8.45a: vout1_d0.off */ + DRA7XX_CORE_IOPAD(0x361C, MUX_MODE14) /* B7: P8.45b: vout1_d16.off */ + DRA7XX_CORE_IOPAD(0x35E0, MUX_MODE14) /* G10: P8.46a: vout1_d1.off */ + DRA7XX_CORE_IOPAD(0x3638, MUX_MODE14) /* A10: P8.46b: vout1_d23.off */ + DRA7XX_CORE_IOPAD(0x372C, MUX_MODE14) /* B19: P9.11a: mcasp3_axr0.off */ + DRA7XX_CORE_IOPAD(0x3620, MUX_MODE14) /* B8: P9.11b: vout1_d17.off */ + DRA7XX_CORE_IOPAD(0x36AC, MUX_MODE14) /* B14: P9.12: mcasp1_aclkr.off */ + DRA7XX_CORE_IOPAD(0x3730, MUX_MODE14) /* C17: P9.13: mcasp3_axr1.off */ + DRA7XX_CORE_IOPAD(0x35AC, MUX_MODE10) /* D6: P9.14: vin2a_d17.off */ + DRA7XX_CORE_IOPAD(0x3514, MUX_MODE14) /* AG4: P9.15: vin1a_d8.off */ + DRA7XX_CORE_IOPAD(0x35B0, MUX_MODE13) /* C5: P9.16: vin2a_d18.off */ + DRA7XX_CORE_IOPAD(0x37CC, MUX_MODE14) /* B24: P9.17a: spi2_cs0.off */ + DRA7XX_CORE_IOPAD(0x36B8, MUX_MODE14) /* F12: P9.17b: mcasp1_axr1.off */ + DRA7XX_CORE_IOPAD(0x37C8, MUX_MODE14) /* G17: P9.18a: spi2_d0.off */ + DRA7XX_CORE_IOPAD(0x36B4, MUX_MODE14) /* G12: P9.18b: mcasp1_axr0.off */ + DRA7XX_CORE_IOPAD(0x3440, PIN_INPUT_PULLUP | MUX_MODE7) /* R6: P9.19a: gpmc_a0.i2c4_scl */ + DRA7XX_CORE_IOPAD(0x357C, PIN_INPUT_PULLUP | MUX_MODE12 ) /* F4: P9.19b: vin2a_d5.pr1_pru1_gpi2 */ + DRA7XX_CORE_IOPAD(0x3444, PIN_INPUT_PULLUP | MUX_MODE7) /* T9: P9.20a: gpmc_a1.i2c4_sda */ + DRA7XX_CORE_IOPAD(0x3578, PIN_INPUT_PULLUP | MUX_MODE12) /* D2: P9.20b: vin2a_d4.pr1_pru1_gpi1 */ + DRA7XX_CORE_IOPAD(0x34F0, MUX_MODE14) /* AF8: P9.21a: vin1a_vsync0.off */ + DRA7XX_CORE_IOPAD(0x37C4, MUX_MODE14) /* B22: P9.21b: spi2_d1.off */ + DRA7XX_CORE_IOPAD(0x369C, MUX_MODE14) /* B26: P9.22a: xref_clk2.off */ + DRA7XX_CORE_IOPAD(0x37C0, MUX_MODE14) /* A26: P9.22b: spi2_sclk.off */ + DRA7XX_CORE_IOPAD(0x37B4, MUX_MODE14) /* A22: P9.23: spi1_cs1.off */ + DRA7XX_CORE_IOPAD(0x368C, MUX_MODE14) /* F20: P9.24: gpio6_15.off */ + DRA7XX_CORE_IOPAD(0x3694, MUX_MODE14) /* D18: P9.25: xref_clk0.off */ + DRA7XX_CORE_IOPAD(0x3688, MUX_MODE14) /* E21: P9.26a: gpio6_14.off */ + DRA7XX_CORE_IOPAD(0x3544, MUX_MODE14) /* AE2: P9.26b: vin1a_d20.off */ + DRA7XX_CORE_IOPAD(0x35A0, MUX_MODE14) /* C3: P9.27a: vin2a_d14.off */ + DRA7XX_CORE_IOPAD(0x36B0, MUX_MODE14) /* J14: P9.27b: mcasp1_fsr.off */ + DRA7XX_CORE_IOPAD(0x36E0, MUX_MODE14) /* A12: P9.28: mcasp1_axr11.off */ + DRA7XX_CORE_IOPAD(0x36D8, MUX_MODE14) /* A11: P9.29a: mcasp1_axr9.off */ + DRA7XX_CORE_IOPAD(0x36A8, MUX_MODE14) /* D14: P9.29b: mcasp1_fsx.off */ + DRA7XX_CORE_IOPAD(0x36DC, MUX_MODE14) /* B13: P9.30: mcasp1_axr10.off */ + DRA7XX_CORE_IOPAD(0x36D4, MUX_MODE14) /* B12: P9.31a: mcasp1_axr8.off */ + DRA7XX_CORE_IOPAD(0x36A4, MUX_MODE14) /* C14: P9.31b: mcasp1_aclkx.off */ + DRA7XX_CORE_IOPAD(0x36A0, MUX_MODE14) /* C23: P9.41a: xref_clk3.off */ + DRA7XX_CORE_IOPAD(0x3580, MUX_MODE14) /* C1: P9.41b: vin2a_d6.off */ + DRA7XX_CORE_IOPAD(0x36E4, MUX_MODE14) /* E14: P9.42a: mcasp1_axr12.off */ + DRA7XX_CORE_IOPAD(0x359C, MUX_MODE14) /* C2: P9.42b: vin2a_d13.off */ + >; }; }; @@ -214,6 +2197,9 @@ interrupt-parent = <&gpio6>; interrupts = <16 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&pmic_pins_default>; + #interrupt-cells = <2>; interrupt-controller; @@ -411,6 +2397,63 @@ #gpio-cells = <2>; }; }; + + /* STMPE811 touch screen controller */ + stmpe811@41 { + compatible = "st,stmpe811"; + reg = <0x41>; + interrupts = <30 IRQ_TYPE_LEVEL_LOW>; + interrupt-parent = <&gpio2>; + interrupt-controller; + id = <0>; + blocks = <0x5>; + irq-trigger = <0x1>; + st,mod-12b = <1>; /* 12-bit ADC */ + st,ref-sel = <0>; /* internal ADC reference */ + st,adc-freq = <1>; /* 3.25 MHz ADC clock speed */ + st,sample-time = <4>; /* ADC converstion time: 80 clocks */ + + pinctrl-names = "default"; + pinctrl-0 = <&adc_pins_default>; + + stmpe_adc { + compatible = "st,stmpe-adc"; + st,norequest-mask = <0x00>; /* mask any channels to be used by touchscreen */ + adc0: iio-device@0 { + #io-channel-cells = <1>; + iio-channels = <&adc0 4>, <&adc0 1>, <&adc0 2>, <&adc0 3>, <&adc0 4>, <&adc0 5>, <&adc0 6>; + iio-channel-names = "AIN0_P9_39", "AIN1_P9_40", "AIN2_P9_37", "AIN3_P9_38", + "AIN4_P9_33", "AIN5_P9_36", "AIN6_P9_35"; + }; + }; + + stmpe_touchscreen { + status = "disabled"; + compatible = "st,stmpe-ts"; + /* 8 sample average control */ + st,ave-ctrl = <3>; + /* 7 length fractional part in z */ + st,fraction-z = <7>; + /* + * 50 mA typical 80 mA max touchscreen drivers + * current limit value + */ + st,i-drive = <1>; + /* 1 ms panel driver settling time */ + st,settling = <3>; + /* 5 ms touch detect interrupt delay */ + st,touch-det-delay = <5>; + }; + + stmpe_gpio { + compatible = "st,stmpe-gpio"; + }; + + stmpe_pwm { + compatible = "st,stmpe-pwm"; + #pwm-cells = <2>; + }; + }; }; &mcspi3 { @@ -434,7 +2477,7 @@ }; &gpu { - status = "ok"; + status = "okay"; }; &pruss_soc_bus1 { @@ -463,6 +2506,8 @@ phy0: ethernet-phy@1 { reg = <4>; + compatible = "ethernet-phy-id004d.d072", + "ethernet-phy-ieee802.3-c22"; eee-broken-100tx; eee-broken-1000t; }; @@ -505,8 +2550,27 @@ bus-width = <4>; cd-gpios = <&gpio6 27 GPIO_ACTIVE_LOW>; /* gpio 219 */ - pinctrl-names = "default"; - pinctrl-0 = <&mmc1_pins_default>; + ti,needs-special-reset; + ti,dual-volt; + sd-uhs-sdr12; + sd-uhs-sdr25; + sd-uhs-sdr50; + sd-uhs-ddr50; + sd-uhs-sdr104; + mmc-hs200-1_8v; + cap-sd-highspeed; + cap-mmc-highspeed; + dmas = <&sdma_xbar 61>, <&sdma_xbar 62>; + dma-names = "tx", "rx"; + + pinctrl-names = "default", "hs", "sdr12", "sdr25", "sdr50", "ddr50", "sdr104"; + pinctrl-0 = <&mmc1_pins_default µsd_extra_pins_default>; + pinctrl-1 = <&mmc1_pins_hs µsd_extra_pins_default>; + pinctrl-2 = <&mmc1_pins_sdr12 µsd_extra_pins_default>; + pinctrl-3 = <&mmc1_pins_sdr25 µsd_extra_pins_default>; + pinctrl-4 = <&mmc1_pins_sdr50 µsd_extra_pins_default>; + pinctrl-5 = <&mmc1_pins_ddr50 &mmc1_iodelay_ddr_rev20_conf µsd_extra_pins_default>; + pinctrl-6 = <&mmc1_pins_sdr104 &mmc1_iodelay_sdr104_rev20_conf µsd_extra_pins_default>; }; &mmc2 { @@ -538,6 +2602,10 @@ /* DDR50: DDR up to 50 MHz (1.8 V signaling). */ status = "okay"; + pinctrl-names = "default", "hs"; + pinctrl-0 = <&mmc4_pins_default &wifibt_extra_pins_default>; + pinctrl-1 = <&mmc4_pins_hs &wifibt_extra_pins_default>; + ti,needs-special-reset; vmmc-supply = <&vdd_3v3>; cap-power-off-card; @@ -557,6 +2625,9 @@ reg = <1>; compatible = "brcm,bcm4329-fmac"; + //brcm,drive-strength = <10>; + //brcm,broken-sg-support; + //brcm,broken_sg_support; brcm,sd-head-align = <4>; brcm,sd_head_align = <4>; brcm,sd_sgentry_align = <512>; @@ -589,6 +2660,10 @@ dr_mode = "host"; }; +&bb2d { + status = "okay"; +}; + &dss { status = "okay"; vdda_video-supply = <&vdd_1v8_pll>; @@ -597,6 +2672,8 @@ &hdmi { status = "okay"; vdda-supply = <&vdd_1v8_phy_ldo4>; + pinctrl-names = "default"; + pinctrl-0 = <&hdmi_pins_default>; port { hdmi_out: endpoint { @@ -609,6 +2686,58 @@ status = "okay"; }; +&mailbox1 { + status = "okay"; +}; + +&mailbox2 { + status = "okay"; +}; + +&mailbox3 { + status = "okay"; +}; + +&mailbox4 { + status = "okay"; +}; + +&mailbox5 { + status = "okay"; +}; + +&mailbox6 { + status = "okay"; +}; + +&mailbox7 { + status = "okay"; +}; + +&mailbox8 { + status = "okay"; +}; + +&mailbox9 { + status = "okay"; +}; + +&mailbox10 { + status = "okay"; +}; + +&mailbox11 { + status = "okay"; +}; + +&mailbox12 { + status = "okay"; +}; + +&mailbox13 { + status = "okay"; +}; + &cpu_alert0 { temperature = <55000>; /* milliCelsius */ }; @@ -650,6 +2779,18 @@ &i2c4 { status = "okay"; clock-frequency = <100000>; + symlink = "bone-i2c2"; +}; + +/* thermal hacks */ +&cpu0_opp_table { + opp_slow-400000000 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <860000 850000 1100000>, + <860000 850000 1100000>; + opp-supported-hw = <0xFF 0x01>; + opp-suspend; + }; }; #include "dra7-ipu-common-early-boot.dtsi" diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi index 09f123c939bf9..a3a9844937ec8 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi +++ b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi @@ -1,9 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2014-2016 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ /dts-v1/; diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts b/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts index a3c44875a0941..bb18a51c0e04a 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15-revb1.dts @@ -1,15 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2014-2016 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include "am57xx-beagle-x15-common.dtsi" +#include "am57xx-cmem.dtsi" / { model = "TI AM5728 BeagleBoard-X15 rev B1"; + + chosen { + base_dtb = "am57xx-beagle-x15-revb1.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts b/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts index 63106dce79bf3..a166ff398950e 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15-revc.dts @@ -1,15 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2014-2017 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include "am57xx-beagle-x15-common.dtsi" +#include "am57xx-cmem.dtsi" / { model = "TI AM5728 BeagleBoard-X15 rev C"; + + chosen { + base_dtb = "am57xx-beagle-x15-revc.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/am57xx-beagle-x15.dts b/arch/arm/boot/dts/am57xx-beagle-x15.dts index 70a71c641066b..509df7fd0599d 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15.dts @@ -1,16 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2014-2016 Texas Instruments Incorporated - http://www.ti.com/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #include "am57xx-beagle-x15-common.dtsi" +#include "am57xx-cmem.dtsi" / { /* NOTE: This describes the "original" pre-production A2 revision */ model = "TI AM5728 BeagleBoard-X15"; + + chosen { + base_dtb = "am57xx-beagle-x15.dts"; + base_dtb_timestamp = __TIMESTAMP__; + }; }; &tpd12s015 { diff --git a/arch/arm/boot/dts/am57xx-cmem.dtsi b/arch/arm/boot/dts/am57xx-cmem.dtsi new file mode 100644 index 0000000000000..e66ef87623ee6 --- /dev/null +++ b/arch/arm/boot/dts/am57xx-cmem.dtsi @@ -0,0 +1,43 @@ +/* https://git.yoctoproject.org/cgit/cgit.cgi/meta-ti/tree/recipes-kernel/linux/files/dra7xx/cmem.dtsi?h=thud */ +/* 0x0c000000 -> 0x18000000 for tild */ + +/ { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + cmem_block_mem_0: cmem_block_mem@a0000000 { + reg = <0x0 0xa0000000 0x0 0x18000000>; + no-map; + status = "okay"; + }; + + cmem_block_mem_1_ocmc3: cmem_block_mem@40500000 { + reg = <0x0 0x40500000 0x0 0x100000>; + no-map; + status = "okay"; + }; + }; + + cmem { + compatible = "ti,cmem"; + #address-cells = <1>; + #size-cells = <0>; + + #pool-size-cells = <2>; + + status = "okay"; + + cmem_block_0: cmem_block@0 { + reg = <0>; + memory-region = <&cmem_block_mem_0>; + cmem-buf-pools = <1 0x0 0x18000000>; + }; + + cmem_block_1: cmem_block@1 { + reg = <1>; + memory-region = <&cmem_block_mem_1_ocmc3>; + }; + }; +}; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index f338e8624b5f3..83c1706fe83b0 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -145,7 +145,7 @@ * the moment, just use a fake OCP bus entry to represent the whole bus * hierarchy. */ - ocp { + ocp: ocp { compatible = "ti,dra7-l3-noc", "simple-bus"; #address-cells = <1>; #size-cells = <1>; diff --git a/include/dt-bindings/board/am335x-bbw-bbb-base.h b/include/dt-bindings/board/am335x-bbw-bbb-base.h new file mode 100644 index 0000000000000..35f6d57ef4923 --- /dev/null +++ b/include/dt-bindings/board/am335x-bbw-bbb-base.h @@ -0,0 +1,103 @@ +/* + * This header provides constants for bbw/bbb pinctrl bindings. + * + * Copyright (C) 2014 Robert Nelson + * + * Numbers Based on: https://github.com/derekmolloy/boneDeviceTree/tree/master/docs + */ + +#ifndef _DT_BINDINGS_BOARD_AM335X_BBW_BBB_BASE_H +#define _DT_BINDINGS_BOARD_AM335X_BBW_BBB_BASE_H + +#define BONE_P8_03 0x018 +#define BONE_P8_04 0x01C + +#define BONE_P8_05 0x008 +#define BONE_P8_06 0x00C +#define BONE_P8_07 0x090 +#define BONE_P8_08 0x094 + +#define BONE_P8_09 0x09C +#define BONE_P8_10 0x098 +#define BONE_P8_11 0x034 +#define BONE_P8_12 0x030 + +#define BONE_P8_13 0x024 +#define BONE_P8_14 0x028 +#define BONE_P8_15 0x03C +#define BONE_P8_16 0x038 + +#define BONE_P8_17 0x02C +#define BONE_P8_18 0x08C +#define BONE_P8_19 0x020 +#define BONE_P8_20 0x084 + +#define BONE_P8_21 0x080 +#define BONE_P8_22 0x014 +#define BONE_P8_23 0x010 +#define BONE_P8_24 0x004 + +#define BONE_P8_25 0x000 +#define BONE_P8_26 0x07C +#define BONE_P8_27 0x0E0 +#define BONE_P8_28 0x0E8 + +#define BONE_P8_29 0x0E4 +#define BONE_P8_30 0x0EC +#define BONE_P8_31 0x0D8 +#define BONE_P8_32 0x0DC + +#define BONE_P8_33 0x0D4 +#define BONE_P8_34 0x0CC +#define BONE_P8_35 0x0D0 +#define BONE_P8_36 0x0C8 + +#define BONE_P8_37 0x0C0 +#define BONE_P8_38 0x0C4 +#define BONE_P8_39 0x0B8 +#define BONE_P8_40 0x0BC + +#define BONE_P8_41 0x0B0 +#define BONE_P8_42 0x0B4 +#define BONE_P8_43 0x0A8 +#define BONE_P8_44 0x0AC + +#define BONE_P8_45 0x0A0 +#define BONE_P8_46 0x0A4 + +#define BONE_P9_11 0x070 +#define BONE_P9_12 0x078 + +#define BONE_P9_13 0x074 +#define BONE_P9_14 0x048 +#define BONE_P9_15 0x040 +#define BONE_P9_16 0x04C + +#define BONE_P9_17 0x15C +#define BONE_P9_18 0x158 +#define BONE_P9_19 0x17C +#define BONE_P9_20 0x178 + +#define BONE_P9_21 0x154 +#define BONE_P9_22 0x150 +#define BONE_P9_23 0x044 +#define BONE_P9_24 0x184 + +#define BONE_P9_25 0x1AC +#define BONE_P9_26 0x180 +#define BONE_P9_27 0x1A4 +#define BONE_P9_28 0x19C + +#define BONE_P9_29 0x194 +#define BONE_P9_30 0x198 +#define BONE_P9_31 0x190 + +/* Shared P21 of P11 */ +#define BONE_P9_41A 0x1B4 +#define BONE_P9_41B 0x1A8 + +/* Shared P22 of P11 */ +#define BONE_P9_42A 0x164 +#define BONE_P9_42B 0x1A0 + +#endif diff --git a/include/dt-bindings/bus/ti-sysc.h b/include/dt-bindings/bus/ti-sysc.h index 7138384e2ef9f..babd08a1d2267 100644 --- a/include/dt-bindings/bus/ti-sysc.h +++ b/include/dt-bindings/bus/ti-sysc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* TI sysc interconnect target module defines */ /* Generic sysc found on omap2 and later, also known as type1 */ diff --git a/include/dt-bindings/clock/am3.h b/include/dt-bindings/clock/am3.h index b396f00e481db..2ed5540ca3598 100644 --- a/include/dt-bindings/clock/am3.h +++ b/include/dt-bindings/clock/am3.h @@ -1,14 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright 2017 Texas Instruments, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __DT_BINDINGS_CLK_AM3_H #define __DT_BINDINGS_CLK_AM3_H @@ -16,6 +8,8 @@ #define AM3_CLKCTRL_OFFSET 0x0 #define AM3_CLKCTRL_INDEX(offset) ((offset) - AM3_CLKCTRL_OFFSET) +/* XXX: Compatibility part begin, remove this once compatibility support is no longer needed */ + /* l4_per clocks */ #define AM3_L4_PER_CLKCTRL_OFFSET 0x14 #define AM3_L4_PER_CLKCTRL_INDEX(offset) ((offset) - AM3_L4_PER_CLKCTRL_OFFSET) diff --git a/include/dt-bindings/clock/dra7.h b/include/dt-bindings/clock/dra7.h index 10f01d636361e..a8a0e2f85479f 100644 --- a/include/dt-bindings/clock/dra7.h +++ b/include/dt-bindings/clock/dra7.h @@ -1,14 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright 2017 Texas Instruments, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __DT_BINDINGS_CLK_DRA7_H #define __DT_BINDINGS_CLK_DRA7_H @@ -16,6 +8,8 @@ #define DRA7_CLKCTRL_OFFSET 0x20 #define DRA7_CLKCTRL_INDEX(offset) ((offset) - DRA7_CLKCTRL_OFFSET) +/* XXX: Compatibility part begin, remove this once compatibility support is no longer needed */ + /* mpu clocks */ #define DRA7_MPU_CLKCTRL DRA7_CLKCTRL_INDEX(0x20) diff --git a/include/dt-bindings/clock/omap5.h b/include/dt-bindings/clock/omap5.h index f51821a912169..f3283957f48dd 100644 --- a/include/dt-bindings/clock/omap5.h +++ b/include/dt-bindings/clock/omap5.h @@ -1,14 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright 2017 Texas Instruments, Inc. - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #ifndef __DT_BINDINGS_CLK_OMAP5_H #define __DT_BINDINGS_CLK_OMAP5_H diff --git a/include/dt-bindings/gpio/gpio.h b/include/dt-bindings/gpio/gpio.h index 2cc10ae4bbb7b..c029467e828b2 100644 --- a/include/dt-bindings/gpio/gpio.h +++ b/include/dt-bindings/gpio/gpio.h @@ -33,4 +33,10 @@ #define GPIO_PERSISTENT 0 #define GPIO_TRANSITORY 8 +/* Bit 4 express pull up */ +#define GPIO_PULL_UP 16 + +/* Bit 5 express pull down */ +#define GPIO_PULL_DOWN 32 + #endif diff --git a/include/dt-bindings/interrupt-controller/arm-gic.h b/include/dt-bindings/interrupt-controller/arm-gic.h index 0c85f65c81c7d..35b6f69b7db6a 100644 --- a/include/dt-bindings/interrupt-controller/arm-gic.h +++ b/include/dt-bindings/interrupt-controller/arm-gic.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ /* * This header provides constants for the ARM GIC. */ diff --git a/include/dt-bindings/interrupt-controller/irq.h b/include/dt-bindings/interrupt-controller/irq.h index a8b310555f14a..9e3d183e13819 100644 --- a/include/dt-bindings/interrupt-controller/irq.h +++ b/include/dt-bindings/interrupt-controller/irq.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ /* * This header provides constants for most IRQ bindings. * diff --git a/include/dt-bindings/media/omap3-isp.h b/include/dt-bindings/media/omap3-isp.h index b18c60e468c78..436c71210e656 100644 --- a/include/dt-bindings/media/omap3-isp.h +++ b/include/dt-bindings/media/omap3-isp.h @@ -1,16 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * include/dt-bindings/media/omap3-isp.h * * Copyright (C) 2015 Sakari Ailus - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. */ #ifndef __DT_BINDINGS_OMAP3_ISP_H__ diff --git a/include/dt-bindings/pinctrl/am33xx.h b/include/dt-bindings/pinctrl/am33xx.h index 7d947a5972209..17877e85980b2 100644 --- a/include/dt-bindings/pinctrl/am33xx.h +++ b/include/dt-bindings/pinctrl/am33xx.h @@ -40,5 +40,133 @@ #undef PIN_OFF_INPUT_PULLDOWN #undef PIN_OFF_WAKEUPENABLE -#endif +#define AM335X_PIN_OFFSET_MIN 0x0800U + +#define AM335X_PIN_GPMC_AD0 0x800 +#define AM335X_PIN_GPMC_AD1 0x804 +#define AM335X_PIN_GPMC_AD2 0x808 +#define AM335X_PIN_GPMC_AD3 0x80c +#define AM335X_PIN_GPMC_AD4 0x810 +#define AM335X_PIN_GPMC_AD5 0x814 +#define AM335X_PIN_GPMC_AD6 0x818 +#define AM335X_PIN_GPMC_AD7 0x81c +#define AM335X_PIN_GPMC_AD8 0x820 +#define AM335X_PIN_GPMC_AD9 0x824 +#define AM335X_PIN_GPMC_AD10 0x828 +#define AM335X_PIN_GPMC_AD11 0x82c +#define AM335X_PIN_GPMC_AD12 0x830 +#define AM335X_PIN_GPMC_AD13 0x834 +#define AM335X_PIN_GPMC_AD14 0x838 +#define AM335X_PIN_GPMC_AD15 0x83c +#define AM335X_PIN_GPMC_A0 0x840 +#define AM335X_PIN_GPMC_A1 0x844 +#define AM335X_PIN_GPMC_A2 0x848 +#define AM335X_PIN_GPMC_A3 0x84c +#define AM335X_PIN_GPMC_A4 0x850 +#define AM335X_PIN_GPMC_A5 0x854 +#define AM335X_PIN_GPMC_A6 0x858 +#define AM335X_PIN_GPMC_A7 0x85c +#define AM335X_PIN_GPMC_A8 0x860 +#define AM335X_PIN_GPMC_A9 0x864 +#define AM335X_PIN_GPMC_A10 0x868 +#define AM335X_PIN_GPMC_A11 0x86c +#define AM335X_PIN_GPMC_WAIT0 0x870 +#define AM335X_PIN_GPMC_WPN 0x874 +#define AM335X_PIN_GPMC_BEN1 0x878 +#define AM335X_PIN_GPMC_CSN0 0x87c +#define AM335X_PIN_GPMC_CSN1 0x880 +#define AM335X_PIN_GPMC_CSN2 0x884 +#define AM335X_PIN_GPMC_CSN3 0x888 +#define AM335X_PIN_GPMC_CLK 0x88c +#define AM335X_PIN_GPMC_ADVN_ALE 0x890 +#define AM335X_PIN_GPMC_OEN_REN 0x894 +#define AM335X_PIN_GPMC_WEN 0x898 +#define AM335X_PIN_GPMC_BEN0_CLE 0x89c +#define AM335X_PIN_LCD_DATA0 0x8a0 +#define AM335X_PIN_LCD_DATA1 0x8a4 +#define AM335X_PIN_LCD_DATA2 0x8a8 +#define AM335X_PIN_LCD_DATA3 0x8ac +#define AM335X_PIN_LCD_DATA4 0x8b0 +#define AM335X_PIN_LCD_DATA5 0x8b4 +#define AM335X_PIN_LCD_DATA6 0x8b8 +#define AM335X_PIN_LCD_DATA7 0x8bc +#define AM335X_PIN_LCD_DATA8 0x8c0 +#define AM335X_PIN_LCD_DATA9 0x8c4 +#define AM335X_PIN_LCD_DATA10 0x8c8 +#define AM335X_PIN_LCD_DATA11 0x8cc +#define AM335X_PIN_LCD_DATA12 0x8d0 +#define AM335X_PIN_LCD_DATA13 0x8d4 +#define AM335X_PIN_LCD_DATA14 0x8d8 +#define AM335X_PIN_LCD_DATA15 0x8dc +#define AM335X_PIN_LCD_VSYNC 0x8e0 +#define AM335X_PIN_LCD_HSYNC 0x8e4 +#define AM335X_PIN_LCD_PCLK 0x8e8 +#define AM335X_PIN_LCD_AC_BIAS_EN 0x8ec +#define AM335X_PIN_MMC0_DAT3 0x8f0 +#define AM335X_PIN_MMC0_DAT2 0x8f4 +#define AM335X_PIN_MMC0_DAT1 0x8f8 +#define AM335X_PIN_MMC0_DAT0 0x8fc +#define AM335X_PIN_MMC0_CLK 0x900 +#define AM335X_PIN_MMC0_CMD 0x904 +#define AM335X_PIN_MII1_COL 0x908 +#define AM335X_PIN_MII1_CRS 0x90c +#define AM335X_PIN_MII1_RX_ER 0x910 +#define AM335X_PIN_MII1_TX_EN 0x914 +#define AM335X_PIN_MII1_RX_DV 0x918 +#define AM335X_PIN_MII1_TXD3 0x91c +#define AM335X_PIN_MII1_TXD2 0x920 +#define AM335X_PIN_MII1_TXD1 0x924 +#define AM335X_PIN_MII1_TXD0 0x928 +#define AM335X_PIN_MII1_TX_CLK 0x92c +#define AM335X_PIN_MII1_RX_CLK 0x930 +#define AM335X_PIN_MII1_RXD3 0x934 +#define AM335X_PIN_MII1_RXD2 0x938 +#define AM335X_PIN_MII1_RXD1 0x93c +#define AM335X_PIN_MII1_RXD0 0x940 +#define AM335X_PIN_RMII1_REF_CLK 0x944 +#define AM335X_PIN_MDIO 0x948 +#define AM335X_PIN_MDC 0x94c +#define AM335X_PIN_SPI0_SCLK 0x950 +#define AM335X_PIN_SPI0_D0 0x954 +#define AM335X_PIN_SPI0_D1 0x958 +#define AM335X_PIN_SPI0_CS0 0x95c +#define AM335X_PIN_SPI0_CS1 0x960 +#define AM335X_PIN_ECAP0_IN_PWM0_OUT 0x964 +#define AM335X_PIN_UART0_CTSN 0x968 +#define AM335X_PIN_UART0_RTSN 0x96c +#define AM335X_PIN_UART0_RXD 0x970 +#define AM335X_PIN_UART0_TXD 0x974 +#define AM335X_PIN_UART1_CTSN 0x978 +#define AM335X_PIN_UART1_RTSN 0x97c +#define AM335X_PIN_UART1_RXD 0x980 +#define AM335X_PIN_UART1_TXD 0x984 +#define AM335X_PIN_I2C0_SDA 0x988 +#define AM335X_PIN_I2C0_SCL 0x98c +#define AM335X_PIN_MCASP0_ACLKX 0x990 +#define AM335X_PIN_MCASP0_FSX 0x994 +#define AM335X_PIN_MCASP0_AXR0 0x998 +#define AM335X_PIN_MCASP0_AHCLKR 0x99c +#define AM335X_PIN_MCASP0_ACLKR 0x9a0 +#define AM335X_PIN_MCASP0_FSR 0x9a4 +#define AM335X_PIN_MCASP0_AXR1 0x9a8 +#define AM335X_PIN_MCASP0_AHCLKX 0x9ac +#define AM335X_PIN_XDMA_EVENT_INTR0 0x9b0 +#define AM335X_PIN_XDMA_EVENT_INTR1 0x9b4 +#define AM335X_PIN_WARMRSTN 0x9b8 +#define AM335X_PIN_NNMI 0x9c0 +#define AM335X_PIN_TMS 0x9d0 +#define AM335X_PIN_TDI 0x9d4 +#define AM335X_PIN_TDO 0x9d8 +#define AM335X_PIN_TCK 0x9dc +#define AM335X_PIN_TRSTN 0x9e0 +#define AM335X_PIN_EMU0 0x9e4 +#define AM335X_PIN_EMU1 0x9e8 +#define AM335X_PIN_RTC_PWRONRSTN 0x9f8 +#define AM335X_PIN_PMIC_POWER_EN 0x9fc +#define AM335X_PIN_EXT_WAKEUP 0xa00 +#define AM335X_PIN_USB0_DRVVBUS 0xa1c +#define AM335X_PIN_USB1_DRVVBUS 0xa34 +#define AM335X_PIN_OFFSET_MAX 0x0a34U + +#endif diff --git a/include/dt-bindings/pinctrl/dra.h b/include/dt-bindings/pinctrl/dra.h index 29423ea8fcc76..e6a5e53ba363c 100644 --- a/include/dt-bindings/pinctrl/dra.h +++ b/include/dt-bindings/pinctrl/dra.h @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * This header provides constants for DRA pinctrl bindings. * * Copyright (C) 2013 Texas Instruments Incorporated - http://www.ti.com/ * Author: Rajendra Nayak - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef _DT_BINDINGS_PINCTRL_DRA_H diff --git a/include/dt-bindings/pinctrl/omap.h b/include/dt-bindings/pinctrl/omap.h index 49b5dea2b3880..6257180424132 100644 --- a/include/dt-bindings/pinctrl/omap.h +++ b/include/dt-bindings/pinctrl/omap.h @@ -65,6 +65,7 @@ #define DM814X_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x0800) (val) #define DM816X_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x0800) (val) #define AM33XX_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x0800) (val) +#define AM33XX_PADCONF(pa, dir, mux) OMAP_IOPAD_OFFSET((pa), 0x0800) ((dir) | (mux)) /* * Macros to allow using the offset from the padconf physical address diff --git a/include/dt-bindings/thermal/thermal.h b/include/dt-bindings/thermal/thermal.h index b5e6b0069ac77..bc7babb1a67cd 100644 --- a/include/dt-bindings/thermal/thermal.h +++ b/include/dt-bindings/thermal/thermal.h @@ -1,10 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * This header provides constants for most thermal bindings. * * Copyright (C) 2013 Texas Instruments * Eduardo Valentin - * - * GPLv2 only */ #ifndef _DT_BINDINGS_THERMAL_THERMAL_H diff --git a/include/uapi/linux/input-event-codes.h b/include/uapi/linux/input-event-codes.h index 61a5799b440b9..85387c76c24f3 100644 --- a/include/uapi/linux/input-event-codes.h +++ b/include/uapi/linux/input-event-codes.h @@ -439,10 +439,12 @@ #define KEY_TITLE 0x171 #define KEY_SUBTITLE 0x172 #define KEY_ANGLE 0x173 -#define KEY_ZOOM 0x174 +#define KEY_FULL_SCREEN 0x174 /* AC View Toggle */ +#define KEY_ZOOM KEY_FULL_SCREEN #define KEY_MODE 0x175 #define KEY_KEYBOARD 0x176 -#define KEY_SCREEN 0x177 +#define KEY_ASPECT_RATIO 0x177 /* HUTRR37: Aspect */ +#define KEY_SCREEN KEY_ASPECT_RATIO #define KEY_PC 0x178 /* Media Select Computer */ #define KEY_TV 0x179 /* Media Select TV */ #define KEY_TV2 0x17a /* Media Select Cable */ @@ -604,6 +606,7 @@ #define KEY_SCREENSAVER 0x245 /* AL Screen Saver */ #define KEY_VOICECOMMAND 0x246 /* Listening Voice Command */ #define KEY_ASSISTANT 0x247 /* AL Context-aware desktop assistant */ +#define KEY_KBD_LAYOUT_NEXT 0x248 /* AC Next Keyboard Layout Select */ #define KEY_BRIGHTNESS_MIN 0x250 /* Set Brightness to Minimum */ #define KEY_BRIGHTNESS_MAX 0x251 /* Set Brightness to Maximum */ @@ -708,6 +711,16 @@ #define REL_DIAL 0x07 #define REL_WHEEL 0x08 #define REL_MISC 0x09 +/* + * 0x0a is reserved and should not be used in input drivers. + * It was used by HID as REL_MISC+1 and userspace needs to detect if + * the next REL_* event is correct or is just REL_MISC + n. + * We define here REL_RESERVED so userspace can rely on it and detect + * the situation described above. + */ +#define REL_RESERVED 0x0a +#define REL_WHEEL_HI_RES 0x0b +#define REL_HWHEEL_HI_RES 0x0c #define REL_MAX 0x0f #define REL_CNT (REL_MAX+1) From 1b5f540215da921cdc8a85daaaf4f8c29d0e1910 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 8 Jun 2020 10:59:45 -0500 Subject: [PATCH 011/213] backports: stmpe: from: linux.git Reference: v5.3.18 Signed-off-by: Robert Nelson --- drivers/iio/adc/stmpe-adc.c | 366 ++++++++++++++++++++++++++++++++++++ drivers/mfd/stmpe.c | 82 +++++++- include/linux/mfd/stmpe.h | 23 ++- 3 files changed, 463 insertions(+), 8 deletions(-) create mode 100644 drivers/iio/adc/stmpe-adc.c diff --git a/drivers/iio/adc/stmpe-adc.c b/drivers/iio/adc/stmpe-adc.c new file mode 100644 index 0000000000000..bd72727fc417a --- /dev/null +++ b/drivers/iio/adc/stmpe-adc.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * STMicroelectronics STMPE811 IIO ADC Driver + * + * 4 channel, 10/12-bit ADC + * + * Copyright (C) 2013-2018 Toradex AG + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STMPE_REG_INT_STA 0x0B +#define STMPE_REG_ADC_INT_EN 0x0E +#define STMPE_REG_ADC_INT_STA 0x0F + +#define STMPE_REG_ADC_CTRL1 0x20 +#define STMPE_REG_ADC_CTRL2 0x21 +#define STMPE_REG_ADC_CAPT 0x22 +#define STMPE_REG_ADC_DATA_CH(channel) (0x30 + 2 * (channel)) + +#define STMPE_REG_TEMP_CTRL 0x60 +#define STMPE_TEMP_CTRL_ENABLE BIT(0) +#define STMPE_TEMP_CTRL_ACQ BIT(1) +#define STMPE_TEMP_CTRL_THRES_EN BIT(3) +#define STMPE_START_ONE_TEMP_CONV (STMPE_TEMP_CTRL_ENABLE | \ + STMPE_TEMP_CTRL_ACQ | \ + STMPE_TEMP_CTRL_THRES_EN) +#define STMPE_REG_TEMP_DATA 0x61 +#define STMPE_REG_TEMP_TH 0x63 +#define STMPE_ADC_LAST_NR 7 +#define STMPE_TEMP_CHANNEL (STMPE_ADC_LAST_NR + 1) + +#define STMPE_ADC_CH(channel) ((1 << (channel)) & 0xff) + +#define STMPE_ADC_TIMEOUT msecs_to_jiffies(1000) + +struct stmpe_adc { + struct stmpe *stmpe; + struct clk *clk; + struct device *dev; + struct mutex lock; + + /* We are allocating plus one for the temperature channel */ + struct iio_chan_spec stmpe_adc_iio_channels[STMPE_ADC_LAST_NR + 2]; + + struct completion completion; + + u8 channel; + u32 value; +}; + +static int stmpe_read_voltage(struct stmpe_adc *info, + struct iio_chan_spec const *chan, int *val) +{ + long ret; + + mutex_lock(&info->lock); + + reinit_completion(&info->completion); + + info->channel = (u8)chan->channel; + + if (info->channel > STMPE_ADC_LAST_NR) { + mutex_unlock(&info->lock); + return -EINVAL; + } + + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_CAPT, + STMPE_ADC_CH(info->channel)); + + ret = wait_for_completion_timeout(&info->completion, STMPE_ADC_TIMEOUT); + + if (ret <= 0) { + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA, + STMPE_ADC_CH(info->channel)); + mutex_unlock(&info->lock); + return -ETIMEDOUT; + } + + *val = info->value; + + mutex_unlock(&info->lock); + + return 0; +} + +static int stmpe_read_temp(struct stmpe_adc *info, + struct iio_chan_spec const *chan, int *val) +{ + long ret; + + mutex_lock(&info->lock); + + reinit_completion(&info->completion); + + info->channel = (u8)chan->channel; + + if (info->channel != STMPE_TEMP_CHANNEL) { + mutex_unlock(&info->lock); + return -EINVAL; + } + + stmpe_reg_write(info->stmpe, STMPE_REG_TEMP_CTRL, + STMPE_START_ONE_TEMP_CONV); + + ret = wait_for_completion_timeout(&info->completion, STMPE_ADC_TIMEOUT); + + if (ret <= 0) { + mutex_unlock(&info->lock); + return -ETIMEDOUT; + } + + /* + * absolute temp = +V3.3 * value /7.51 [K] + * scale to [milli °C] + */ + *val = ((449960l * info->value) / 1024l) - 273150; + + mutex_unlock(&info->lock); + + return 0; +} + +static int stmpe_read_raw(struct iio_dev *indio_dev, + struct iio_chan_spec const *chan, + int *val, + int *val2, + long mask) +{ + struct stmpe_adc *info = iio_priv(indio_dev); + long ret; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + case IIO_CHAN_INFO_PROCESSED: + + switch (chan->type) { + case IIO_VOLTAGE: + ret = stmpe_read_voltage(info, chan, val); + break; + + case IIO_TEMP: + ret = stmpe_read_temp(info, chan, val); + break; + default: + return -EINVAL; + } + + if (ret < 0) + return ret; + + return IIO_VAL_INT; + + case IIO_CHAN_INFO_SCALE: + *val = 3300; + *val2 = info->stmpe->mod_12b ? 12 : 10; + return IIO_VAL_FRACTIONAL_LOG2; + + default: + break; + } + + return -EINVAL; +} + +static irqreturn_t stmpe_adc_isr(int irq, void *dev_id) +{ + struct stmpe_adc *info = (struct stmpe_adc *)dev_id; + u16 data; + + if (info->channel <= STMPE_ADC_LAST_NR) { + int int_sta; + + int_sta = stmpe_reg_read(info->stmpe, STMPE_REG_ADC_INT_STA); + + /* Is the interrupt relevant */ + if (!(int_sta & STMPE_ADC_CH(info->channel))) + return IRQ_NONE; + + /* Read value */ + stmpe_block_read(info->stmpe, + STMPE_REG_ADC_DATA_CH(info->channel), 2, (u8 *) &data); + + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA, int_sta); + } else if (info->channel == STMPE_TEMP_CHANNEL) { + /* Read value */ + stmpe_block_read(info->stmpe, STMPE_REG_TEMP_DATA, 2, + (u8 *) &data); + } else { + return IRQ_NONE; + } + + info->value = (u32) be16_to_cpu(data); + complete(&info->completion); + + return IRQ_HANDLED; +} + +static const struct iio_info stmpe_adc_iio_info = { + .read_raw = &stmpe_read_raw, +}; + +static void stmpe_adc_voltage_chan(struct iio_chan_spec *ics, int chan) +{ + ics->type = IIO_VOLTAGE; + ics->info_mask_separate = BIT(IIO_CHAN_INFO_RAW); + ics->info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE); + ics->indexed = 1; + ics->channel = chan; +} + +static void stmpe_adc_temp_chan(struct iio_chan_spec *ics, int chan) +{ + ics->type = IIO_TEMP; + ics->info_mask_separate = BIT(IIO_CHAN_INFO_PROCESSED); + ics->indexed = 1; + ics->channel = chan; +} + +static int stmpe_adc_init_hw(struct stmpe_adc *adc) +{ + int ret; + struct stmpe *stmpe = adc->stmpe; + + ret = stmpe_enable(stmpe, STMPE_BLOCK_ADC); + if (ret) { + dev_err(stmpe->dev, "Could not enable clock for ADC\n"); + return ret; + } + + ret = stmpe811_adc_common_init(stmpe); + if (ret) { + stmpe_disable(stmpe, STMPE_BLOCK_ADC); + return ret; + } + + /* use temp irq for each conversion completion */ + stmpe_reg_write(stmpe, STMPE_REG_TEMP_TH, 0); + stmpe_reg_write(stmpe, STMPE_REG_TEMP_TH + 1, 0); + + return 0; +} + +static int stmpe_adc_probe(struct platform_device *pdev) +{ + struct iio_dev *indio_dev; + struct stmpe_adc *info; + struct device_node *np; + u32 norequest_mask = 0; + int irq_temp, irq_adc; + int num_chan = 0; + int i = 0; + int ret; + + irq_adc = platform_get_irq_byname(pdev, "STMPE_ADC"); + if (irq_adc < 0) + return irq_adc; + + indio_dev = devm_iio_device_alloc(&pdev->dev, sizeof(struct stmpe_adc)); + if (!indio_dev) { + dev_err(&pdev->dev, "failed allocating iio device\n"); + return -ENOMEM; + } + + info = iio_priv(indio_dev); + mutex_init(&info->lock); + + init_completion(&info->completion); + ret = devm_request_threaded_irq(&pdev->dev, irq_adc, NULL, + stmpe_adc_isr, IRQF_ONESHOT, + "stmpe-adc", info); + if (ret < 0) { + dev_err(&pdev->dev, "failed requesting irq, irq = %d\n", + irq_adc); + return ret; + } + + irq_temp = platform_get_irq_byname(pdev, "STMPE_TEMP_SENS"); + if (irq_temp >= 0) { + ret = devm_request_threaded_irq(&pdev->dev, irq_temp, NULL, + stmpe_adc_isr, IRQF_ONESHOT, + "stmpe-adc", info); + if (ret < 0) + dev_warn(&pdev->dev, "failed requesting irq for" + " temp sensor, irq = %d\n", irq_temp); + } + + platform_set_drvdata(pdev, indio_dev); + + indio_dev->name = dev_name(&pdev->dev); + indio_dev->dev.parent = &pdev->dev; + indio_dev->info = &stmpe_adc_iio_info; + indio_dev->modes = INDIO_DIRECT_MODE; + + info->stmpe = dev_get_drvdata(pdev->dev.parent); + + np = pdev->dev.of_node; + + if (!np) + dev_err(&pdev->dev, "no device tree node found\n"); + + of_property_read_u32(np, "st,norequest-mask", &norequest_mask); + + for_each_clear_bit(i, (unsigned long *) &norequest_mask, + (STMPE_ADC_LAST_NR + 1)) { + stmpe_adc_voltage_chan(&info->stmpe_adc_iio_channels[num_chan], i); + num_chan++; + } + stmpe_adc_temp_chan(&info->stmpe_adc_iio_channels[num_chan], i); + num_chan++; + indio_dev->channels = info->stmpe_adc_iio_channels; + indio_dev->num_channels = num_chan; + + ret = stmpe_adc_init_hw(info); + if (ret) + return ret; + + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_EN, + ~(norequest_mask & 0xFF)); + + stmpe_reg_write(info->stmpe, STMPE_REG_ADC_INT_STA, + ~(norequest_mask & 0xFF)); + + return devm_iio_device_register(&pdev->dev, indio_dev); +} + +static int __maybe_unused stmpe_adc_resume(struct device *dev) +{ + struct iio_dev *indio_dev = dev_get_drvdata(dev); + struct stmpe_adc *info = iio_priv(indio_dev); + + stmpe_adc_init_hw(info); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(stmpe_adc_pm_ops, NULL, stmpe_adc_resume); + +static struct platform_driver stmpe_adc_driver = { + .probe = stmpe_adc_probe, + .driver = { + .name = "stmpe-adc", + .pm = &stmpe_adc_pm_ops, + }, +}; +module_platform_driver(stmpe_adc_driver); + +static const struct of_device_id stmpe_adc_ids[] = { + { .compatible = "st,stmpe-adc", }, + { }, +}; +MODULE_DEVICE_TABLE(of, stmpe_adc_ids); + +MODULE_AUTHOR("Stefan Agner "); +MODULE_DESCRIPTION("STMPEXXX ADC driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:stmpe-adc"); diff --git a/drivers/mfd/stmpe.c b/drivers/mfd/stmpe.c index 566caca4efd8e..1aee3b3253fc9 100644 --- a/drivers/mfd/stmpe.c +++ b/drivers/mfd/stmpe.c @@ -1,9 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * ST Microelectronics MFD: stmpe's driver * * Copyright (C) ST-Ericsson SA 2010 * - * License Terms: GNU General Public License, version 2 * Author: Rabin Vincent for ST-Ericsson */ @@ -463,6 +463,28 @@ static const struct mfd_cell stmpe_ts_cell = { .num_resources = ARRAY_SIZE(stmpe_ts_resources), }; +/* + * ADC (STMPE811) + */ + +static struct resource stmpe_adc_resources[] = { + { + .name = "STMPE_TEMP_SENS", + .flags = IORESOURCE_IRQ, + }, + { + .name = "STMPE_ADC", + .flags = IORESOURCE_IRQ, + }, +}; + +static const struct mfd_cell stmpe_adc_cell = { + .name = "stmpe-adc", + .of_compatible = "st,stmpe-adc", + .resources = stmpe_adc_resources, + .num_resources = ARRAY_SIZE(stmpe_adc_resources), +}; + /* * STMPE811 or STMPE610 */ @@ -497,6 +519,11 @@ static struct stmpe_variant_block stmpe811_blocks[] = { .irq = STMPE811_IRQ_TOUCH_DET, .block = STMPE_BLOCK_TOUCHSCREEN, }, + { + .cell = &stmpe_adc_cell, + .irq = STMPE811_IRQ_TEMP_SENS, + .block = STMPE_BLOCK_ADC, + }, }; static int stmpe811_enable(struct stmpe *stmpe, unsigned int blocks, @@ -517,6 +544,35 @@ static int stmpe811_enable(struct stmpe *stmpe, unsigned int blocks, enable ? 0 : mask); } +int stmpe811_adc_common_init(struct stmpe *stmpe) +{ + int ret; + u8 adc_ctrl1, adc_ctrl1_mask; + + adc_ctrl1 = STMPE_SAMPLE_TIME(stmpe->sample_time) | + STMPE_MOD_12B(stmpe->mod_12b) | + STMPE_REF_SEL(stmpe->ref_sel); + adc_ctrl1_mask = STMPE_SAMPLE_TIME(0xff) | STMPE_MOD_12B(0xff) | + STMPE_REF_SEL(0xff); + + ret = stmpe_set_bits(stmpe, STMPE811_REG_ADC_CTRL1, + adc_ctrl1_mask, adc_ctrl1); + if (ret) { + dev_err(stmpe->dev, "Could not setup ADC\n"); + return ret; + } + + ret = stmpe_set_bits(stmpe, STMPE811_REG_ADC_CTRL2, + STMPE_ADC_FREQ(0xff), STMPE_ADC_FREQ(stmpe->adc_freq)); + if (ret) { + dev_err(stmpe->dev, "Could not setup ADC\n"); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(stmpe811_adc_common_init); + static int stmpe811_get_altfunc(struct stmpe *stmpe, enum stmpe_block block) { /* 0 for touchscreen, 1 for GPIO */ @@ -1302,17 +1358,17 @@ static void stmpe_of_probe(struct stmpe_platform_data *pdata, pdata->autosleep = (pdata->autosleep_timeout) ? true : false; for_each_child_of_node(np, child) { - if (!strcmp(child->name, "stmpe_gpio")) { + if (of_node_name_eq(child, "stmpe_gpio")) { pdata->blocks |= STMPE_BLOCK_GPIO; - } else if (!strcmp(child->name, "stmpe_keypad")) { + } else if (of_node_name_eq(child, "stmpe_keypad")) { pdata->blocks |= STMPE_BLOCK_KEYPAD; - } else if (!strcmp(child->name, "stmpe_touchscreen")) { + } else if (of_node_name_eq(child, "stmpe_touchscreen")) { pdata->blocks |= STMPE_BLOCK_TOUCHSCREEN; - } else if (!strcmp(child->name, "stmpe_adc")) { + } else if (of_node_name_eq(child, "stmpe_adc")) { pdata->blocks |= STMPE_BLOCK_ADC; - } else if (!strcmp(child->name, "stmpe_pwm")) { + } else if (of_node_name_eq(child, "stmpe_pwm")) { pdata->blocks |= STMPE_BLOCK_PWM; - } else if (!strcmp(child->name, "stmpe_rotator")) { + } else if (of_node_name_eq(child, "stmpe_rotator")) { pdata->blocks |= STMPE_BLOCK_ROTATOR; } } @@ -1325,6 +1381,7 @@ int stmpe_probe(struct stmpe_client_info *ci, enum stmpe_partnum partnum) struct device_node *np = ci->dev->of_node; struct stmpe *stmpe; int ret; + u32 val; pdata = devm_kzalloc(ci->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) @@ -1342,6 +1399,15 @@ int stmpe_probe(struct stmpe_client_info *ci, enum stmpe_partnum partnum) mutex_init(&stmpe->irq_lock); mutex_init(&stmpe->lock); + if (!of_property_read_u32(np, "st,sample-time", &val)) + stmpe->sample_time = val; + if (!of_property_read_u32(np, "st,mod-12b", &val)) + stmpe->mod_12b = val; + if (!of_property_read_u32(np, "st,ref-sel", &val)) + stmpe->ref_sel = val; + if (!of_property_read_u32(np, "st,adc-freq", &val)) + stmpe->adc_freq = val; + stmpe->dev = ci->dev; stmpe->client = ci->client; stmpe->pdata = pdata; @@ -1433,6 +1499,8 @@ int stmpe_remove(struct stmpe *stmpe) if (!IS_ERR(stmpe->vcc)) regulator_disable(stmpe->vcc); + __stmpe_disable(stmpe, STMPE_BLOCK_ADC); + mfd_remove_devices(stmpe->dev); return 0; diff --git a/include/linux/mfd/stmpe.h b/include/linux/mfd/stmpe.h index 4a827af17e598..87e29d561e223 100644 --- a/include/linux/mfd/stmpe.h +++ b/include/linux/mfd/stmpe.h @@ -1,7 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) ST-Ericsson SA 2010 * - * License Terms: GNU General Public License, version 2 * Author: Rabin Vincent for ST-Ericsson */ @@ -10,6 +10,20 @@ #include +#define STMPE_SAMPLE_TIME(x) ((x & 0xf) << 4) +#define STMPE_MOD_12B(x) ((x & 0x1) << 3) +#define STMPE_REF_SEL(x) ((x & 0x1) << 1) +#define STMPE_ADC_FREQ(x) (x & 0x3) +#define STMPE_AVE_CTRL(x) ((x & 0x3) << 6) +#define STMPE_DET_DELAY(x) ((x & 0x7) << 3) +#define STMPE_SETTLING(x) (x & 0x7) +#define STMPE_FRACTION_Z(x) (x & 0x7) +#define STMPE_I_DRIVE(x) (x & 0x1) +#define STMPE_OP_MODE(x) ((x & 0x7) << 1) + +#define STMPE811_REG_ADC_CTRL1 0x20 +#define STMPE811_REG_ADC_CTRL2 0x21 + struct device; struct regulator; @@ -123,6 +137,12 @@ struct stmpe { u8 ier[2]; u8 oldier[2]; struct stmpe_platform_data *pdata; + + /* For devices that use an ADC */ + u8 sample_time; + u8 mod_12b; + u8 ref_sel; + u8 adc_freq; }; extern int stmpe_reg_write(struct stmpe *stmpe, u8 reg, u8 data); @@ -136,6 +156,7 @@ extern int stmpe_set_altfunc(struct stmpe *stmpe, u32 pins, enum stmpe_block block); extern int stmpe_enable(struct stmpe *stmpe, unsigned int blocks); extern int stmpe_disable(struct stmpe *stmpe, unsigned int blocks); +extern int stmpe811_adc_common_init(struct stmpe *stmpe); #define STMPE_GPIO_NOREQ_811_TOUCH (0xf0) From 3862bf93a0962f7de1885e70c375e9ec7bf43aec Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Fri, 26 Jul 2019 17:36:59 -0500 Subject: [PATCH 012/213] stmpe: wire up adc Kconfig/Makefile Signed-off-by: Robert Nelson --- drivers/iio/adc/Kconfig | 7 +++++++ drivers/iio/adc/Makefile | 1 + 2 files changed, 8 insertions(+) diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index 9421c1ec86f73..2fa99a9c86d01 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -705,6 +705,13 @@ config STM32_DFSDM_ADC This driver can also be built as a module. If so, the module will be called stm32-dfsdm-adc. +config STMPE_ADC + tristate "STMicroelectronics STMPE ADC driver" + depends on OF && MFD_STMPE + help + Say yes here to build support for ST Microelectronics STMPE + built-in ADC block (stmpe811). + config STX104 tristate "Apex Embedded Systems STX104 driver" depends on PC104 && X86 diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile index 03db7b578f9c0..5ce8f8adce874 100644 --- a/drivers/iio/adc/Makefile +++ b/drivers/iio/adc/Makefile @@ -67,6 +67,7 @@ obj-$(CONFIG_STM32_ADC_CORE) += stm32-adc-core.o obj-$(CONFIG_STM32_ADC) += stm32-adc.o obj-$(CONFIG_STM32_DFSDM_CORE) += stm32-dfsdm-core.o obj-$(CONFIG_STM32_DFSDM_ADC) += stm32-dfsdm-adc.o +obj-$(CONFIG_STMPE_ADC) += stmpe-adc.o obj-$(CONFIG_TI_ADC081C) += ti-adc081c.o obj-$(CONFIG_TI_ADC0832) += ti-adc0832.o obj-$(CONFIG_TI_ADC084S021) += ti-adc084s021.o From 19f978f8adf3e2381a717dfc5ada2dbe0d325786 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 8 Jun 2020 11:00:26 -0500 Subject: [PATCH 013/213] backports: vl53l0x: from: linux.git Reference: v5.0.21 Signed-off-by: Robert Nelson --- drivers/iio/proximity/vl53l0x-i2c.c | 164 ++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 drivers/iio/proximity/vl53l0x-i2c.c diff --git a/drivers/iio/proximity/vl53l0x-i2c.c b/drivers/iio/proximity/vl53l0x-i2c.c new file mode 100644 index 0000000000000..b48216cc18589 --- /dev/null +++ b/drivers/iio/proximity/vl53l0x-i2c.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for ST VL53L0X FlightSense ToF Ranging Sensor on a i2c bus. + * + * Copyright (C) 2016 STMicroelectronics Imaging Division. + * Copyright (C) 2018 Song Qiang + * + * Datasheet available at + * + * + * Default 7-bit i2c slave address 0x29. + * + * TODO: FIFO buffer, continuous mode, interrupts, range selection, + * sensor ID check. + */ + +#include +#include +#include + +#include + +#define VL_REG_SYSRANGE_START 0x00 + +#define VL_REG_SYSRANGE_MODE_MASK GENMASK(3, 0) +#define VL_REG_SYSRANGE_MODE_SINGLESHOT 0x00 +#define VL_REG_SYSRANGE_MODE_START_STOP BIT(0) +#define VL_REG_SYSRANGE_MODE_BACKTOBACK BIT(1) +#define VL_REG_SYSRANGE_MODE_TIMED BIT(2) +#define VL_REG_SYSRANGE_MODE_HISTOGRAM BIT(3) + +#define VL_REG_RESULT_INT_STATUS 0x13 +#define VL_REG_RESULT_RANGE_STATUS 0x14 +#define VL_REG_RESULT_RANGE_STATUS_COMPLETE BIT(0) + +struct vl53l0x_data { + struct i2c_client *client; +}; + +static int vl53l0x_read_proximity(struct vl53l0x_data *data, + const struct iio_chan_spec *chan, + int *val) +{ + struct i2c_client *client = data->client; + u16 tries = 20; + u8 buffer[12]; + int ret; + + ret = i2c_smbus_write_byte_data(client, VL_REG_SYSRANGE_START, 1); + if (ret < 0) + return ret; + + do { + ret = i2c_smbus_read_byte_data(client, + VL_REG_RESULT_RANGE_STATUS); + if (ret < 0) + return ret; + + if (ret & VL_REG_RESULT_RANGE_STATUS_COMPLETE) + break; + + usleep_range(1000, 5000); + } while (--tries); + if (!tries) + return -ETIMEDOUT; + + ret = i2c_smbus_read_i2c_block_data(client, VL_REG_RESULT_RANGE_STATUS, + 12, buffer); + if (ret < 0) + return ret; + else if (ret != 12) + return -EREMOTEIO; + + /* Values should be between 30~1200 in millimeters. */ + *val = (buffer[10] << 8) + buffer[11]; + + return 0; +} + +static const struct iio_chan_spec vl53l0x_channels[] = { + { + .type = IIO_DISTANCE, + .info_mask_separate = BIT(IIO_CHAN_INFO_RAW) | + BIT(IIO_CHAN_INFO_SCALE), + }, +}; + +static int vl53l0x_read_raw(struct iio_dev *indio_dev, + const struct iio_chan_spec *chan, + int *val, int *val2, long mask) +{ + struct vl53l0x_data *data = iio_priv(indio_dev); + int ret; + + if (chan->type != IIO_DISTANCE) + return -EINVAL; + + switch (mask) { + case IIO_CHAN_INFO_RAW: + ret = vl53l0x_read_proximity(data, chan, val); + if (ret < 0) + return ret; + + return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + *val = 0; + *val2 = 1000; + + return IIO_VAL_INT_PLUS_MICRO; + default: + return -EINVAL; + } +} + +static const struct iio_info vl53l0x_info = { + .read_raw = vl53l0x_read_raw, +}; + +static int vl53l0x_probe(struct i2c_client *client) +{ + struct vl53l0x_data *data; + struct iio_dev *indio_dev; + + indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*data)); + if (!indio_dev) + return -ENOMEM; + + data = iio_priv(indio_dev); + data->client = client; + i2c_set_clientdata(client, indio_dev); + + if (!i2c_check_functionality(client->adapter, + I2C_FUNC_SMBUS_READ_I2C_BLOCK | + I2C_FUNC_SMBUS_BYTE_DATA)) + return -EOPNOTSUPP; + + indio_dev->dev.parent = &client->dev; + indio_dev->name = "vl53l0x"; + indio_dev->info = &vl53l0x_info; + indio_dev->channels = vl53l0x_channels; + indio_dev->num_channels = ARRAY_SIZE(vl53l0x_channels); + indio_dev->modes = INDIO_DIRECT_MODE; + + return devm_iio_device_register(&client->dev, indio_dev); +} + +static const struct of_device_id st_vl53l0x_dt_match[] = { + { .compatible = "st,vl53l0x", }, + { } +}; +MODULE_DEVICE_TABLE(of, st_vl53l0x_dt_match); + +static struct i2c_driver vl53l0x_driver = { + .driver = { + .name = "vl53l0x-i2c", + .of_match_table = st_vl53l0x_dt_match, + }, + .probe_new = vl53l0x_probe, +}; +module_i2c_driver(vl53l0x_driver); + +MODULE_AUTHOR("Song Qiang "); +MODULE_DESCRIPTION("ST vl53l0x ToF ranging sensor driver"); +MODULE_LICENSE("GPL v2"); From ad9ddec114388a0ead15978a6e31ce0c0117824d Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 29 Jan 2019 09:29:10 -0600 Subject: [PATCH 014/213] wire up VL53L0X_I2C Signed-off-by: Robert Nelson --- drivers/iio/proximity/Kconfig | 11 +++++++++++ drivers/iio/proximity/Makefile | 2 ++ 2 files changed, 13 insertions(+) diff --git a/drivers/iio/proximity/Kconfig b/drivers/iio/proximity/Kconfig index 388ef70c11d24..b99367a89f810 100644 --- a/drivers/iio/proximity/Kconfig +++ b/drivers/iio/proximity/Kconfig @@ -92,4 +92,15 @@ config SRF08 To compile this driver as a module, choose M here: the module will be called srf08. +config VL53L0X_I2C + tristate "STMicroelectronics VL53L0X ToF ranger sensor (I2C)" + depends on I2C + help + Say Y here to build a driver for STMicroelectronics VL53L0X + ToF ranger sensors with i2c interface. + This driver can be used to measure the distance of objects. + + To compile this driver as a module, choose M here: the + module will be called vl53l0x-i2c. + endmenu diff --git a/drivers/iio/proximity/Makefile b/drivers/iio/proximity/Makefile index cac3d7d3325e3..6d031f903c4c2 100644 --- a/drivers/iio/proximity/Makefile +++ b/drivers/iio/proximity/Makefile @@ -11,3 +11,5 @@ obj-$(CONFIG_RFD77402) += rfd77402.o obj-$(CONFIG_SRF04) += srf04.o obj-$(CONFIG_SRF08) += srf08.o obj-$(CONFIG_SX9500) += sx9500.o +obj-$(CONFIG_VL53L0X_I2C) += vl53l0x-i2c.o + From 89cab6ac5335cb29fa3472cc72df81ef8c4faeb6 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 8 Jun 2020 11:01:07 -0500 Subject: [PATCH 015/213] backports: brcm80211: from: linux.git Reference: v4.14.77 Signed-off-by: Robert Nelson --- .../net/wireless/broadcom/brcm80211/Kconfig | 1 + .../broadcom/brcm80211/brcmfmac/bcdc.c | 34 +- .../broadcom/brcm80211/brcmfmac/bcdc.h | 6 +- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 532 ++++-- .../broadcom/brcm80211/brcmfmac/btcoex.c | 8 +- .../broadcom/brcm80211/brcmfmac/bus.h | 14 +- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 378 ++-- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 19 +- .../broadcom/brcm80211/brcmfmac/chip.c | 16 +- .../broadcom/brcm80211/brcmfmac/chip.h | 3 +- .../broadcom/brcm80211/brcmfmac/common.c | 161 +- .../broadcom/brcm80211/brcmfmac/common.h | 3 +- .../broadcom/brcm80211/brcmfmac/core.c | 305 +-- .../broadcom/brcm80211/brcmfmac/core.h | 11 +- .../broadcom/brcm80211/brcmfmac/debug.c | 45 +- .../broadcom/brcm80211/brcmfmac/debug.h | 17 + .../broadcom/brcm80211/brcmfmac/feature.c | 101 +- .../broadcom/brcm80211/brcmfmac/feature.h | 13 +- .../broadcom/brcm80211/brcmfmac/firmware.c | 264 +-- .../broadcom/brcm80211/brcmfmac/firmware.h | 82 +- .../broadcom/brcm80211/brcmfmac/flowring.c | 2 - .../broadcom/brcm80211/brcmfmac/fweh.c | 5 + .../broadcom/brcm80211/brcmfmac/fweh.h | 16 +- .../broadcom/brcm80211/brcmfmac/fwil.c | 24 +- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 74 +- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 69 +- .../broadcom/brcm80211/brcmfmac/fwsignal.h | 4 +- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 43 +- .../broadcom/brcm80211/brcmfmac/msgbuf.h | 6 +- .../broadcom/brcm80211/brcmfmac/p2p.c | 51 +- .../broadcom/brcm80211/brcmfmac/p2p.h | 2 - .../broadcom/brcm80211/brcmfmac/pcie.c | 235 +-- .../broadcom/brcm80211/brcmfmac/proto.c | 13 +- .../broadcom/brcm80211/brcmfmac/proto.h | 24 +- .../broadcom/brcm80211/brcmfmac/sdio.c | 767 ++++---- .../broadcom/brcm80211/brcmfmac/sdio.h | 82 +- .../broadcom/brcm80211/brcmfmac/usb.c | 150 +- .../broadcom/brcm80211/brcmfmac/vendor.c | 5 +- .../broadcom/brcm80211/brcmsmac/channel.c | 1 + .../broadcom/brcm80211/brcmsmac/debug.c | 2 +- .../broadcom/brcm80211/brcmsmac/mac80211_if.c | 40 +- .../broadcom/brcm80211/brcmsmac/main.c | 7 +- .../broadcom/brcm80211/brcmsmac/main.h | 1 - .../broadcom/brcm80211/brcmsmac/phy/phy_cmn.c | 4 +- .../broadcom/brcm80211/brcmsmac/phy/phy_lcn.c | 17 +- .../broadcom/brcm80211/brcmsmac/phy/phy_n.c | 1683 +++++++++-------- .../brcm80211/brcmsmac/phy/phytbl_n.c | 462 ++++- .../broadcom/brcm80211/brcmutil/d11.c | 37 +- .../broadcom/brcm80211/include/brcm_hw_ids.h | 1 - .../broadcom/brcm80211/include/brcmu_wifi.h | 2 - 50 files changed, 2749 insertions(+), 3093 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/Kconfig b/drivers/net/wireless/broadcom/brcm80211/Kconfig index 6acba67bca07a..9d99eb42d9176 100644 --- a/drivers/net/wireless/broadcom/brcm80211/Kconfig +++ b/drivers/net/wireless/broadcom/brcm80211/Kconfig @@ -60,6 +60,7 @@ config BRCMFMAC_PCIE bool "PCIE bus interface support for FullMAC driver" depends on BRCMFMAC depends on PCI + depends on HAS_DMA select BRCMFMAC_PROTO_MSGBUF select FW_LOADER ---help--- diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c index 144e0b83b24b8..9f2d0b0cf6e5c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c @@ -165,7 +165,7 @@ static int brcmf_proto_bcdc_cmplt(struct brcmf_pub *drvr, u32 id, u32 len) static int brcmf_proto_bcdc_query_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd, - void *buf, uint len, int *fwerr) + void *buf, uint len) { struct brcmf_bcdc *bcdc = (struct brcmf_bcdc *)drvr->proto->pd; struct brcmf_proto_bcdc_dcmd *msg = &bcdc->msg; @@ -175,7 +175,6 @@ brcmf_proto_bcdc_query_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd, brcmf_dbg(BCDC, "Enter, cmd %d len %d\n", cmd, len); - *fwerr = 0; ret = brcmf_proto_bcdc_msg(drvr, ifidx, cmd, buf, len, false); if (ret < 0) { brcmf_err("brcmf_proto_bcdc_msg failed w/status %d\n", @@ -212,27 +211,25 @@ brcmf_proto_bcdc_query_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd, memcpy(buf, info, len); } - ret = 0; - /* Check the ERROR flag */ if (flags & BCDC_DCMD_ERROR) - *fwerr = le32_to_cpu(msg->status); + ret = le32_to_cpu(msg->status); + done: return ret; } static int brcmf_proto_bcdc_set_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd, - void *buf, uint len, int *fwerr) + void *buf, uint len) { struct brcmf_bcdc *bcdc = (struct brcmf_bcdc *)drvr->proto->pd; struct brcmf_proto_bcdc_dcmd *msg = &bcdc->msg; - int ret; + int ret = 0; u32 flags, id; brcmf_dbg(BCDC, "Enter, cmd %d len %d\n", cmd, len); - *fwerr = 0; ret = brcmf_proto_bcdc_msg(drvr, ifidx, cmd, buf, len, true); if (ret < 0) goto done; @@ -252,11 +249,9 @@ brcmf_proto_bcdc_set_dcmd(struct brcmf_pub *drvr, int ifidx, uint cmd, goto done; } - ret = 0; - /* Check the ERROR flag */ if (flags & BCDC_DCMD_ERROR) - *fwerr = le32_to_cpu(msg->status); + ret = le32_to_cpu(msg->status); done: return ret; @@ -445,11 +440,6 @@ brcmf_proto_bcdc_init_done(struct brcmf_pub *drvr) return 0; } -static void brcmf_proto_bcdc_debugfs_create(struct brcmf_pub *drvr) -{ - brcmf_fws_debugfs_create(drvr); -} - int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) { struct brcmf_bcdc *bcdc; @@ -477,7 +467,6 @@ int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) drvr->proto->del_if = brcmf_proto_bcdc_del_if; drvr->proto->reset_if = brcmf_proto_bcdc_reset_if; drvr->proto->init_done = brcmf_proto_bcdc_init_done; - drvr->proto->debugfs_create = brcmf_proto_bcdc_debugfs_create; drvr->proto->pd = bcdc; drvr->hdrlen += BCDC_HEADER_LEN + BRCMF_PROT_FW_SIGNAL_MAX_TXBYTES; @@ -490,18 +479,11 @@ int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) return -ENOMEM; } -void brcmf_proto_bcdc_detach_pre_delif(struct brcmf_pub *drvr) -{ - struct brcmf_bcdc *bcdc = drvr->proto->pd; - - brcmf_fws_detach_pre_delif(bcdc->fws); -} - -void brcmf_proto_bcdc_detach_post_delif(struct brcmf_pub *drvr) +void brcmf_proto_bcdc_detach(struct brcmf_pub *drvr) { struct brcmf_bcdc *bcdc = drvr->proto->pd; drvr->proto->pd = NULL; - brcmf_fws_detach_post_delif(bcdc->fws); + brcmf_fws_detach(bcdc->fws); kfree(bcdc); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h index 4bc52240ccea2..3b0e9eff21b58 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.h @@ -18,16 +18,14 @@ #ifdef CONFIG_BRCMFMAC_PROTO_BCDC int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr); -void brcmf_proto_bcdc_detach_pre_delif(struct brcmf_pub *drvr); -void brcmf_proto_bcdc_detach_post_delif(struct brcmf_pub *drvr); +void brcmf_proto_bcdc_detach(struct brcmf_pub *drvr); void brcmf_proto_bcdc_txflowblock(struct device *dev, bool state); void brcmf_proto_bcdc_txcomplete(struct device *dev, struct sk_buff *txp, bool success); struct brcmf_fws_info *drvr_to_fws(struct brcmf_pub *drvr); #else static inline int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) { return 0; } -static void brcmf_proto_bcdc_detach_pre_delif(struct brcmf_pub *drvr) {}; -static inline void brcmf_proto_bcdc_detach_post_delif(struct brcmf_pub *drvr) {} +static inline void brcmf_proto_bcdc_detach(struct brcmf_pub *drvr) {} #endif #endif /* BRCMFMAC_BCDC_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index d2f788d886681..dd6e27513cc1b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -118,7 +118,7 @@ int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev) ret = request_irq(pdata->oob_irq_nr, brcmf_sdiod_oob_irqhandler, pdata->oob_irq_flags, "brcmf_oob_intr", - &sdiodev->func1->dev); + &sdiodev->func[1]->dev); if (ret != 0) { brcmf_err("request_irq failed %d\n", ret); return ret; @@ -132,40 +132,39 @@ int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev) } sdiodev->irq_wake = true; - sdio_claim_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); if (sdiodev->bus_if->chip == BRCM_CC_43362_CHIP_ID) { /* assign GPIO to SDIO core */ addr = CORE_CC_REG(SI_ENUM_BASE, gpiocontrol); - gpiocontrol = brcmf_sdiod_readl(sdiodev, addr, &ret); + gpiocontrol = brcmf_sdiod_regrl(sdiodev, addr, &ret); gpiocontrol |= 0x2; - brcmf_sdiod_writel(sdiodev, addr, gpiocontrol, &ret); + brcmf_sdiod_regwl(sdiodev, addr, gpiocontrol, &ret); - brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_SELECT, - 0xf, &ret); - brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_OUT, 0, &ret); - brcmf_sdiod_writeb(sdiodev, SBSDIO_GPIO_EN, 0x2, &ret); + brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_SELECT, 0xf, + &ret); + brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_OUT, 0, &ret); + brcmf_sdiod_regwb(sdiodev, SBSDIO_GPIO_EN, 0x2, &ret); } /* must configure SDIO_CCCR_IENx to enable irq */ - data = brcmf_sdiod_func0_rb(sdiodev, SDIO_CCCR_IENx, &ret); - data |= SDIO_CCCR_IEN_FUNC1 | SDIO_CCCR_IEN_FUNC2 | - SDIO_CCCR_IEN_FUNC0; - brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_IENx, data, &ret); + data = brcmf_sdiod_regrb(sdiodev, SDIO_CCCR_IENx, &ret); + data |= 1 << SDIO_FUNC_1 | 1 << SDIO_FUNC_2 | 1; + brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_IENx, data, &ret); /* redirect, configure and enable io for interrupt signal */ - data = SDIO_CCCR_BRCM_SEPINT_MASK | SDIO_CCCR_BRCM_SEPINT_OE; + data = SDIO_SEPINT_MASK | SDIO_SEPINT_OE; if (pdata->oob_irq_flags & IRQF_TRIGGER_HIGH) - data |= SDIO_CCCR_BRCM_SEPINT_ACT_HI; - brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_SEPINT, - data, &ret); - sdio_release_host(sdiodev->func1); + data |= SDIO_SEPINT_ACT_HI; + brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_SEPINT, data, &ret); + + sdio_release_host(sdiodev->func[1]); } else { brcmf_dbg(SDIO, "Entering\n"); - sdio_claim_host(sdiodev->func1); - sdio_claim_irq(sdiodev->func1, brcmf_sdiod_ib_irqhandler); - sdio_claim_irq(sdiodev->func2, brcmf_sdiod_dummy_irqhandler); - sdio_release_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); + sdio_claim_irq(sdiodev->func[1], brcmf_sdiod_ib_irqhandler); + sdio_claim_irq(sdiodev->func[2], brcmf_sdiod_dummy_irqhandler); + sdio_release_host(sdiodev->func[1]); sdiodev->sd_irq_requested = true; } @@ -183,26 +182,26 @@ void brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev) struct brcmfmac_sdio_pd *pdata; pdata = &sdiodev->settings->bus.sdio; - sdio_claim_host(sdiodev->func1); - brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_SEPINT, 0, NULL); - brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_IENx, 0, NULL); - sdio_release_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); + brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_SEPINT, 0, NULL); + brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_IENx, 0, NULL); + sdio_release_host(sdiodev->func[1]); sdiodev->oob_irq_requested = false; if (sdiodev->irq_wake) { disable_irq_wake(pdata->oob_irq_nr); sdiodev->irq_wake = false; } - free_irq(pdata->oob_irq_nr, &sdiodev->func1->dev); + free_irq(pdata->oob_irq_nr, &sdiodev->func[1]->dev); sdiodev->irq_en = false; sdiodev->oob_irq_requested = false; } if (sdiodev->sd_irq_requested) { - sdio_claim_host(sdiodev->func1); - sdio_release_irq(sdiodev->func2); - sdio_release_irq(sdiodev->func1); - sdio_release_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); + sdio_release_irq(sdiodev->func[2]); + sdio_release_irq(sdiodev->func[1]); + sdio_release_host(sdiodev->func[1]); sdiodev->sd_irq_requested = false; } } @@ -231,121 +230,271 @@ void brcmf_sdiod_change_state(struct brcmf_sdio_dev *sdiodev, sdiodev->state = state; } -static int brcmf_sdiod_set_backplane_window(struct brcmf_sdio_dev *sdiodev, - u32 addr) +static inline int brcmf_sdiod_f0_writeb(struct sdio_func *func, + uint regaddr, u8 byte) { - u32 v, bar0 = addr & SBSDIO_SBWINDOW_MASK; - int err = 0, i; + int err_ret; - if (bar0 == sdiodev->sbwad) - return 0; + /* + * Can only directly write to some F0 registers. + * Handle CCCR_IENx and CCCR_ABORT command + * as a special case. + */ + if ((regaddr == SDIO_CCCR_ABORT) || + (regaddr == SDIO_CCCR_IENx)) + sdio_writeb(func, byte, regaddr, &err_ret); + else + sdio_f0_writeb(func, byte, regaddr, &err_ret); - v = bar0 >> 8; + return err_ret; +} - for (i = 0 ; i < 3 && !err ; i++, v >>= 8) - brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_SBADDRLOW + i, - v & 0xff, &err); +static int brcmf_sdiod_request_data(struct brcmf_sdio_dev *sdiodev, u8 fn, + u32 addr, u8 regsz, void *data, bool write) +{ + struct sdio_func *func; + int ret = -EINVAL; - if (!err) - sdiodev->sbwad = bar0; + brcmf_dbg(SDIO, "rw=%d, func=%d, addr=0x%05x, nbytes=%d\n", + write, fn, addr, regsz); + + /* only allow byte access on F0 */ + if (WARN_ON(regsz > 1 && !fn)) + return -EINVAL; + func = sdiodev->func[fn]; + + switch (regsz) { + case sizeof(u8): + if (write) { + if (fn) + sdio_writeb(func, *(u8 *)data, addr, &ret); + else + ret = brcmf_sdiod_f0_writeb(func, addr, + *(u8 *)data); + } else { + if (fn) + *(u8 *)data = sdio_readb(func, addr, &ret); + else + *(u8 *)data = sdio_f0_readb(func, addr, &ret); + } + break; + case sizeof(u16): + if (write) + sdio_writew(func, *(u16 *)data, addr, &ret); + else + *(u16 *)data = sdio_readw(func, addr, &ret); + break; + case sizeof(u32): + if (write) + sdio_writel(func, *(u32 *)data, addr, &ret); + else + *(u32 *)data = sdio_readl(func, addr, &ret); + break; + default: + brcmf_err("invalid size: %d\n", regsz); + break; + } + + if (ret) + brcmf_dbg(SDIO, "failed to %s data F%d@0x%05x, err: %d\n", + write ? "write" : "read", fn, addr, ret); + + return ret; +} + +static int brcmf_sdiod_regrw_helper(struct brcmf_sdio_dev *sdiodev, u32 addr, + u8 regsz, void *data, bool write) +{ + u8 func; + s32 retry = 0; + int ret; + + if (sdiodev->state == BRCMF_SDIOD_NOMEDIUM) + return -ENOMEDIUM; + + /* + * figure out how to read the register based on address range + * 0x00 ~ 0x7FF: function 0 CCCR and FBR + * 0x10000 ~ 0x1FFFF: function 1 miscellaneous registers + * The rest: function 1 silicon backplane core registers + */ + if ((addr & ~REG_F0_REG_MASK) == 0) + func = SDIO_FUNC_0; + else + func = SDIO_FUNC_1; + + do { + if (!write) + memset(data, 0, regsz); + /* for retry wait for 1 ms till bus get settled down */ + if (retry) + usleep_range(1000, 2000); + ret = brcmf_sdiod_request_data(sdiodev, func, addr, regsz, + data, write); + } while (ret != 0 && ret != -ENOMEDIUM && + retry++ < SDIOH_API_ACCESS_RETRY_LIMIT); + + if (ret == -ENOMEDIUM) + brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM); + else if (ret != 0) { + /* + * SleepCSR register access can fail when + * waking up the device so reduce this noise + * in the logs. + */ + if (addr != SBSDIO_FUNC1_SLEEPCSR) + brcmf_err("failed to %s data F%d@0x%05x, err: %d\n", + write ? "write" : "read", func, addr, ret); + else + brcmf_dbg(SDIO, "failed to %s data F%d@0x%05x, err: %d\n", + write ? "write" : "read", func, addr, ret); + } + return ret; +} + +static int +brcmf_sdiod_set_sbaddr_window(struct brcmf_sdio_dev *sdiodev, u32 address) +{ + int err = 0, i; + u8 addr[3]; + + if (sdiodev->state == BRCMF_SDIOD_NOMEDIUM) + return -ENOMEDIUM; + + addr[0] = (address >> 8) & SBSDIO_SBADDRLOW_MASK; + addr[1] = (address >> 16) & SBSDIO_SBADDRMID_MASK; + addr[2] = (address >> 24) & SBSDIO_SBADDRHIGH_MASK; + + for (i = 0; i < 3; i++) { + err = brcmf_sdiod_regrw_helper(sdiodev, + SBSDIO_FUNC1_SBADDRLOW + i, + sizeof(u8), &addr[i], true); + if (err) { + brcmf_err("failed at addr: 0x%0x\n", + SBSDIO_FUNC1_SBADDRLOW + i); + break; + } + } return err; } -u32 brcmf_sdiod_readl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret) +static int +brcmf_sdiod_addrprep(struct brcmf_sdio_dev *sdiodev, uint width, u32 *addr) { - u32 data = 0; - int retval; + uint bar0 = *addr & ~SBSDIO_SB_OFT_ADDR_MASK; + int err = 0; - retval = brcmf_sdiod_set_backplane_window(sdiodev, addr); - if (retval) - goto out; + if (bar0 != sdiodev->sbwad) { + err = brcmf_sdiod_set_sbaddr_window(sdiodev, bar0); + if (err) + return err; + + sdiodev->sbwad = bar0; + } - addr &= SBSDIO_SB_OFT_ADDR_MASK; - addr |= SBSDIO_SB_ACCESS_2_4B_FLAG; + *addr &= SBSDIO_SB_OFT_ADDR_MASK; - data = sdio_readl(sdiodev->func1, addr, &retval); + if (width == 4) + *addr |= SBSDIO_SB_ACCESS_2_4B_FLAG; + + return 0; +} + +u8 brcmf_sdiod_regrb(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret) +{ + u8 data; + int retval; + + brcmf_dbg(SDIO, "addr:0x%08x\n", addr); + retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data, + false); + brcmf_dbg(SDIO, "data:0x%02x\n", data); -out: if (ret) *ret = retval; return data; } -void brcmf_sdiod_writel(struct brcmf_sdio_dev *sdiodev, u32 addr, - u32 data, int *ret) +u32 brcmf_sdiod_regrl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret) { + u32 data = 0; int retval; - retval = brcmf_sdiod_set_backplane_window(sdiodev, addr); + brcmf_dbg(SDIO, "addr:0x%08x\n", addr); + retval = brcmf_sdiod_addrprep(sdiodev, sizeof(data), &addr); if (retval) - goto out; - - addr &= SBSDIO_SB_OFT_ADDR_MASK; - addr |= SBSDIO_SB_ACCESS_2_4B_FLAG; - - sdio_writel(sdiodev->func1, data, addr, &retval); + goto done; + retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data, + false); + brcmf_dbg(SDIO, "data:0x%08x\n", data); -out: +done: if (ret) *ret = retval; + + return data; } -static int brcmf_sdiod_skbuff_read(struct brcmf_sdio_dev *sdiodev, - struct sdio_func *func, u32 addr, - struct sk_buff *skb) +void brcmf_sdiod_regwb(struct brcmf_sdio_dev *sdiodev, u32 addr, + u8 data, int *ret) { - unsigned int req_sz; - int err; + int retval; - /* Single skb use the standard mmc interface */ - req_sz = skb->len + 3; - req_sz &= (uint)~3; + brcmf_dbg(SDIO, "addr:0x%08x, data:0x%02x\n", addr, data); + retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data, + true); + if (ret) + *ret = retval; +} - switch (func->num) { - case 1: - err = sdio_memcpy_fromio(func, ((u8 *)(skb->data)), addr, - req_sz); - break; - case 2: - err = sdio_readsb(func, ((u8 *)(skb->data)), addr, req_sz); - break; - default: - /* bail out as things are really fishy here */ - WARN(1, "invalid sdio function number: %d\n", func->num); - err = -ENOMEDIUM; - }; +void brcmf_sdiod_regwl(struct brcmf_sdio_dev *sdiodev, u32 addr, + u32 data, int *ret) +{ + int retval; - if (err == -ENOMEDIUM) - brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM); + brcmf_dbg(SDIO, "addr:0x%08x, data:0x%08x\n", addr, data); + retval = brcmf_sdiod_addrprep(sdiodev, sizeof(data), &addr); + if (retval) + goto done; + retval = brcmf_sdiod_regrw_helper(sdiodev, addr, sizeof(data), &data, + true); - return err; +done: + if (ret) + *ret = retval; } -static int brcmf_sdiod_skbuff_write(struct brcmf_sdio_dev *sdiodev, - struct sdio_func *func, u32 addr, - struct sk_buff *skb) +static int brcmf_sdiod_buffrw(struct brcmf_sdio_dev *sdiodev, uint fn, + bool write, u32 addr, struct sk_buff *pkt) { unsigned int req_sz; int err; /* Single skb use the standard mmc interface */ - req_sz = skb->len + 3; + req_sz = pkt->len + 3; req_sz &= (uint)~3; - err = sdio_memcpy_toio(func, addr, ((u8 *)(skb->data)), req_sz); - + if (write) + err = sdio_memcpy_toio(sdiodev->func[fn], addr, + ((u8 *)(pkt->data)), req_sz); + else if (fn == 1) + err = sdio_memcpy_fromio(sdiodev->func[fn], ((u8 *)(pkt->data)), + addr, req_sz); + else + /* function 2 read is FIFO operation */ + err = sdio_readsb(sdiodev->func[fn], ((u8 *)(pkt->data)), addr, + req_sz); if (err == -ENOMEDIUM) brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_NOMEDIUM); - return err; } /** * brcmf_sdiod_sglist_rw - SDIO interface function for block data access * @sdiodev: brcmfmac sdio device - * @func: SDIO function + * @fn: SDIO function number * @write: direction flag * @addr: dongle memory address as source/destination * @pkt: skb pointer @@ -354,8 +503,7 @@ static int brcmf_sdiod_skbuff_write(struct brcmf_sdio_dev *sdiodev, * stack for block data access. It assumes that the skb passed down by the * caller has already been padded and aligned. */ -static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, - struct sdio_func *func, +static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, uint fn, bool write, u32 addr, struct sk_buff_head *pktlist) { @@ -381,7 +529,7 @@ static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, req_sz = 0; skb_queue_walk(pktlist, pkt_next) req_sz += pkt_next->len; - req_sz = ALIGN(req_sz, func->cur_blksize); + req_sz = ALIGN(req_sz, sdiodev->func[fn]->cur_blksize); while (req_sz > PAGE_SIZE) { pkt_next = brcmu_pkt_buf_get_skb(PAGE_SIZE); if (pkt_next == NULL) { @@ -400,7 +548,7 @@ static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, target_list = &local_list; } - func_blk_sz = func->cur_blksize; + func_blk_sz = sdiodev->func[fn]->cur_blksize; max_req_sz = sdiodev->max_request_size; max_seg_cnt = min_t(unsigned short, sdiodev->max_segment_count, target_list->qlen); @@ -417,10 +565,10 @@ static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, mmc_dat.flags = write ? MMC_DATA_WRITE : MMC_DATA_READ; mmc_cmd.opcode = SD_IO_RW_EXTENDED; mmc_cmd.arg = write ? 1<<31 : 0; /* write flag */ - mmc_cmd.arg |= (func->num & 0x7) << 28; /* SDIO func num */ - mmc_cmd.arg |= 1 << 27; /* block mode */ + mmc_cmd.arg |= (fn & 0x7) << 28; /* SDIO func num */ + mmc_cmd.arg |= 1<<27; /* block mode */ /* for function 1 the addr will be incremented */ - mmc_cmd.arg |= (func->num == 1) ? 1 << 26 : 0; + mmc_cmd.arg |= (fn == 1) ? 1<<26 : 0; mmc_cmd.flags = MMC_RSP_SPI_R5 | MMC_RSP_R5 | MMC_CMD_ADTC; mmc_req.cmd = &mmc_cmd; mmc_req.data = &mmc_dat; @@ -466,11 +614,11 @@ static int brcmf_sdiod_sglist_rw(struct brcmf_sdio_dev *sdiodev, mmc_cmd.arg |= (addr & 0x1FFFF) << 9; /* address */ mmc_cmd.arg |= mmc_dat.blocks & 0x1FF; /* block count */ /* incrementing addr for function 1 */ - if (func->num == 1) + if (fn == 1) addr += req_sz; - mmc_set_data_timeout(&mmc_dat, func->card); - mmc_wait_for_req(func->card->host, &mmc_req); + mmc_set_data_timeout(&mmc_dat, sdiodev->func[fn]->card); + mmc_wait_for_req(sdiodev->func[fn]->card->host, &mmc_req); ret = mmc_cmd.error ? mmc_cmd.error : mmc_dat.error; if (ret == -ENOMEDIUM) { @@ -538,19 +686,16 @@ int brcmf_sdiod_recv_buf(struct brcmf_sdio_dev *sdiodev, u8 *buf, uint nbytes) int brcmf_sdiod_recv_pkt(struct brcmf_sdio_dev *sdiodev, struct sk_buff *pkt) { - u32 addr = sdiodev->cc_core->base; + u32 addr = sdiodev->sbwad; int err = 0; brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n", addr, pkt->len); - err = brcmf_sdiod_set_backplane_window(sdiodev, addr); + err = brcmf_sdiod_addrprep(sdiodev, 4, &addr); if (err) goto done; - addr &= SBSDIO_SB_OFT_ADDR_MASK; - addr |= SBSDIO_SB_ACCESS_2_4B_FLAG; - - err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr, pkt); + err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr, pkt); done: return err; @@ -561,28 +706,25 @@ int brcmf_sdiod_recv_chain(struct brcmf_sdio_dev *sdiodev, { struct sk_buff *glom_skb = NULL; struct sk_buff *skb; - u32 addr = sdiodev->cc_core->base; + u32 addr = sdiodev->sbwad; int err = 0; brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n", addr, pktq->qlen); - err = brcmf_sdiod_set_backplane_window(sdiodev, addr); + err = brcmf_sdiod_addrprep(sdiodev, 4, &addr); if (err) goto done; - addr &= SBSDIO_SB_OFT_ADDR_MASK; - addr |= SBSDIO_SB_ACCESS_2_4B_FLAG; - if (pktq->qlen == 1) - err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr, - pktq->next); + err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr, + pktq->next); else if (!sdiodev->sg_support) { glom_skb = brcmu_pkt_buf_get_skb(totlen); if (!glom_skb) return -ENOMEM; - err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func2, addr, - glom_skb); + err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, false, addr, + glom_skb); if (err) goto done; @@ -591,8 +733,8 @@ int brcmf_sdiod_recv_chain(struct brcmf_sdio_dev *sdiodev, skb_pull(glom_skb, skb->len); } } else - err = brcmf_sdiod_sglist_rw(sdiodev, sdiodev->func2, false, - addr, pktq); + err = brcmf_sdiod_sglist_rw(sdiodev, SDIO_FUNC_2, false, addr, + pktq); done: brcmu_pkt_buf_free_skb(glom_skb); @@ -602,11 +744,10 @@ int brcmf_sdiod_recv_chain(struct brcmf_sdio_dev *sdiodev, int brcmf_sdiod_send_buf(struct brcmf_sdio_dev *sdiodev, u8 *buf, uint nbytes) { struct sk_buff *mypkt; - u32 addr = sdiodev->cc_core->base; + u32 addr = sdiodev->sbwad; int err; mypkt = brcmu_pkt_buf_get_skb(nbytes); - if (!mypkt) { brcmf_err("brcmu_pkt_buf_get_skb failed: len %d\n", nbytes); @@ -615,49 +756,40 @@ int brcmf_sdiod_send_buf(struct brcmf_sdio_dev *sdiodev, u8 *buf, uint nbytes) memcpy(mypkt->data, buf, nbytes); - err = brcmf_sdiod_set_backplane_window(sdiodev, addr); - if (err) - return err; - - addr &= SBSDIO_SB_OFT_ADDR_MASK; - addr |= SBSDIO_SB_ACCESS_2_4B_FLAG; + err = brcmf_sdiod_addrprep(sdiodev, 4, &addr); if (!err) - err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func2, addr, - mypkt); + err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, true, addr, + mypkt); brcmu_pkt_buf_free_skb(mypkt); - return err; + } int brcmf_sdiod_send_pkt(struct brcmf_sdio_dev *sdiodev, struct sk_buff_head *pktq) { struct sk_buff *skb; - u32 addr = sdiodev->cc_core->base; + u32 addr = sdiodev->sbwad; int err; brcmf_dbg(SDIO, "addr = 0x%x, size = %d\n", addr, pktq->qlen); - err = brcmf_sdiod_set_backplane_window(sdiodev, addr); + err = brcmf_sdiod_addrprep(sdiodev, 4, &addr); if (err) return err; - addr &= SBSDIO_SB_OFT_ADDR_MASK; - addr |= SBSDIO_SB_ACCESS_2_4B_FLAG; - - if (pktq->qlen == 1 || !sdiodev->sg_support) { + if (pktq->qlen == 1 || !sdiodev->sg_support) skb_queue_walk(pktq, skb) { - err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func2, - addr, skb); + err = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_2, true, + addr, skb); if (err) break; } - } else { - err = brcmf_sdiod_sglist_rw(sdiodev, sdiodev->func2, true, - addr, pktq); - } + else + err = brcmf_sdiod_sglist_rw(sdiodev, SDIO_FUNC_2, true, addr, + pktq); return err; } @@ -666,7 +798,7 @@ int brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address, u8 *data, uint size) { - int err = 0; + int bcmerror = 0; struct sk_buff *pkt; u32 sdaddr; uint dsize; @@ -686,13 +818,13 @@ brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address, else dsize = size; - sdio_claim_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); /* Do the transfer(s) */ while (size) { /* Set the backplane window to include the start address */ - err = brcmf_sdiod_set_backplane_window(sdiodev, address); - if (err) + bcmerror = brcmf_sdiod_set_sbaddr_window(sdiodev, address); + if (bcmerror) break; brcmf_dbg(SDIO, "%s %d bytes at offset 0x%08x in window 0x%08x\n", @@ -703,17 +835,11 @@ brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address, sdaddr |= SBSDIO_SB_ACCESS_2_4B_FLAG; skb_put(pkt, dsize); - - if (write) { + if (write) memcpy(pkt->data, data, dsize); - err = brcmf_sdiod_skbuff_write(sdiodev, sdiodev->func1, - sdaddr, pkt); - } else { - err = brcmf_sdiod_skbuff_read(sdiodev, sdiodev->func1, - sdaddr, pkt); - } - - if (err) { + bcmerror = brcmf_sdiod_buffrw(sdiodev, SDIO_FUNC_1, write, + sdaddr, pkt); + if (bcmerror) { brcmf_err("membytes transfer failed\n"); break; } @@ -733,17 +859,24 @@ brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address, dev_kfree_skb(pkt); - sdio_release_host(sdiodev->func1); + /* Return the window to backplane enumeration space for core access */ + if (brcmf_sdiod_set_sbaddr_window(sdiodev, sdiodev->sbwad)) + brcmf_err("FAILED to set window back to 0x%x\n", + sdiodev->sbwad); - return err; + sdio_release_host(sdiodev->func[1]); + + return bcmerror; } -int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, struct sdio_func *func) +int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, uint fn) { + char t_func = (char)fn; brcmf_dbg(SDIO, "Enter\n"); - /* Issue abort cmd52 command through F0 */ - brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_ABORT, func->num, NULL); + /* issue abort cmd52 command through F0 */ + brcmf_sdiod_request_data(sdiodev, SDIO_FUNC_0, SDIO_CCCR_ABORT, + sizeof(t_func), &t_func, true); brcmf_dbg(SDIO, "Exit\n"); return 0; @@ -757,7 +890,7 @@ void brcmf_sdiod_sgtable_alloc(struct brcmf_sdio_dev *sdiodev) uint nents; int err; - func = sdiodev->func2; + func = sdiodev->func[2]; host = func->card->host; sdiodev->sg_support = host->max_segs > 1; max_blocks = min_t(uint, host->max_blk_count, 511u); @@ -818,17 +951,17 @@ static int brcmf_sdiod_freezer_on(struct brcmf_sdio_dev *sdiodev) brcmf_sdio_trigger_dpc(sdiodev->bus); wait_event(sdiodev->freezer->thread_freeze, atomic_read(expect) == sdiodev->freezer->frozen_count); - sdio_claim_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); res = brcmf_sdio_sleep(sdiodev->bus, true); - sdio_release_host(sdiodev->func1); + sdio_release_host(sdiodev->func[1]); return res; } static void brcmf_sdiod_freezer_off(struct brcmf_sdio_dev *sdiodev) { - sdio_claim_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); brcmf_sdio_sleep(sdiodev->bus, false); - sdio_release_host(sdiodev->func1); + sdio_release_host(sdiodev->func[1]); atomic_set(&sdiodev->freezer->freezing, 0); complete_all(&sdiodev->freezer->resumed); } @@ -878,19 +1011,19 @@ static int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev) brcmf_sdiod_freezer_detach(sdiodev); /* Disable Function 2 */ - sdio_claim_host(sdiodev->func2); - sdio_disable_func(sdiodev->func2); - sdio_release_host(sdiodev->func2); + sdio_claim_host(sdiodev->func[2]); + sdio_disable_func(sdiodev->func[2]); + sdio_release_host(sdiodev->func[2]); /* Disable Function 1 */ - sdio_claim_host(sdiodev->func1); - sdio_disable_func(sdiodev->func1); - sdio_release_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); + sdio_disable_func(sdiodev->func[1]); + sdio_release_host(sdiodev->func[1]); sg_free_table(&sdiodev->sgtable); sdiodev->sbwad = 0; - pm_runtime_allow(sdiodev->func1->card->host->parent); + pm_runtime_allow(sdiodev->func[1]->card->host->parent); return 0; } @@ -906,27 +1039,29 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) { int ret = 0; - sdio_claim_host(sdiodev->func1); + sdiodev->num_funcs = 2; - ret = sdio_set_block_size(sdiodev->func1, SDIO_FUNC1_BLOCKSIZE); + sdio_claim_host(sdiodev->func[1]); + + ret = sdio_set_block_size(sdiodev->func[1], SDIO_FUNC1_BLOCKSIZE); if (ret) { brcmf_err("Failed to set F1 blocksize\n"); - sdio_release_host(sdiodev->func1); + sdio_release_host(sdiodev->func[1]); goto out; } - ret = sdio_set_block_size(sdiodev->func2, SDIO_FUNC2_BLOCKSIZE); + ret = sdio_set_block_size(sdiodev->func[2], SDIO_FUNC2_BLOCKSIZE); if (ret) { brcmf_err("Failed to set F2 blocksize\n"); - sdio_release_host(sdiodev->func1); + sdio_release_host(sdiodev->func[1]); goto out; } /* increase F2 timeout */ - sdiodev->func2->enable_timeout = SDIO_WAIT_F2RDY; + sdiodev->func[2]->enable_timeout = SDIO_WAIT_F2RDY; /* Enable Function 1 */ - ret = sdio_enable_func(sdiodev->func1); - sdio_release_host(sdiodev->func1); + ret = sdio_enable_func(sdiodev->func[1]); + sdio_release_host(sdiodev->func[1]); if (ret) { brcmf_err("Failed to enable F1: err=%d\n", ret); goto out; @@ -942,7 +1077,7 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) ret = -ENODEV; goto out; } - brcmf_sdiod_host_fixup(sdiodev->func2->card->host); + brcmf_sdiod_host_fixup(sdiodev->func[2]->card->host); out: if (ret) brcmf_sdiod_remove(sdiodev); @@ -1004,10 +1139,6 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, brcmf_dbg(SDIO, "Function#: %d\n", func->num); dev = &func->dev; - - /* Set MMC_QUIRK_LENIENT_FN0 for this card */ - func->card->quirks |= MMC_QUIRK_LENIENT_FN0; - /* prohibit ACPI power management for this device */ brcmf_sdiod_acpi_set_power_manageable(dev, 0); @@ -1031,15 +1162,17 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, /* store refs to functions used. mmc_card does * not hold the F0 function pointer. */ - sdiodev->func1 = func->card->sdio_func[0]; - sdiodev->func2 = func; + sdiodev->func[0] = kmemdup(func, sizeof(*func), GFP_KERNEL); + sdiodev->func[0]->num = 0; + sdiodev->func[1] = func->card->sdio_func[0]; + sdiodev->func[2] = func; sdiodev->bus_if = bus_if; bus_if->bus_priv.sdio = sdiodev; bus_if->proto_type = BRCMF_PROTO_BCDC; dev_set_drvdata(&func->dev, bus_if); - dev_set_drvdata(&sdiodev->func1->dev, bus_if); - sdiodev->dev = &sdiodev->func1->dev; + dev_set_drvdata(&sdiodev->func[1]->dev, bus_if); + sdiodev->dev = &sdiodev->func[1]->dev; brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_DOWN); @@ -1055,7 +1188,8 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, fail: dev_set_drvdata(&func->dev, NULL); - dev_set_drvdata(&sdiodev->func1->dev, NULL); + dev_set_drvdata(&sdiodev->func[1]->dev, NULL); + kfree(sdiodev->func[0]); kfree(sdiodev); kfree(bus_if); return err; @@ -1084,10 +1218,11 @@ static void brcmf_ops_sdio_remove(struct sdio_func *func) /* only proceed with rest of cleanup if func 1 */ brcmf_sdiod_remove(sdiodev); - dev_set_drvdata(&sdiodev->func1->dev, NULL); - dev_set_drvdata(&sdiodev->func2->dev, NULL); + dev_set_drvdata(&sdiodev->func[1]->dev, NULL); + dev_set_drvdata(&sdiodev->func[2]->dev, NULL); kfree(bus_if); + kfree(sdiodev->func[0]); kfree(sdiodev); } @@ -1113,7 +1248,7 @@ static int brcmf_ops_sdio_suspend(struct device *dev) func = container_of(dev, struct sdio_func, dev); brcmf_dbg(SDIO, "Enter: F%d\n", func->num); - if (func->num != 1) + if (func->num != SDIO_FUNC_1) return 0; @@ -1130,7 +1265,7 @@ static int brcmf_ops_sdio_suspend(struct device *dev) else sdio_flags |= MMC_PM_WAKE_SDIO_IRQ; } - if (sdio_set_host_pm_flags(sdiodev->func1, sdio_flags)) + if (sdio_set_host_pm_flags(sdiodev->func[1], sdio_flags)) brcmf_err("Failed to set pm_flags %x\n", sdio_flags); return 0; } @@ -1142,7 +1277,7 @@ static int brcmf_ops_sdio_resume(struct device *dev) struct sdio_func *func = container_of(dev, struct sdio_func, dev); brcmf_dbg(SDIO, "Enter: F%d\n", func->num); - if (func->num != 2) + if (func->num != SDIO_FUNC_2) return 0; brcmf_sdiod_freezer_off(sdiodev); @@ -1165,7 +1300,6 @@ static struct sdio_driver brcmf_sdmmc_driver = { #ifdef CONFIG_PM_SLEEP .pm = &brcmf_sdio_pm_ops, #endif /* CONFIG_PM_SLEEP */ - .coredump = brcmf_dev_coredump, }, }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c index 372363a6e752e..3559fb5b8fb03 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c @@ -280,9 +280,9 @@ static void brcmf_btcoex_restore_part1(struct brcmf_btcoex_info *btci) /** * brcmf_btcoex_timerfunc() - BT coex timer callback */ -static void brcmf_btcoex_timerfunc(struct timer_list *t) +static void brcmf_btcoex_timerfunc(ulong data) { - struct brcmf_btcoex_info *bt_local = from_timer(bt_local, t, timer); + struct brcmf_btcoex_info *bt_local = (struct brcmf_btcoex_info *)data; brcmf_dbg(TRACE, "enter\n"); bt_local->timer_on = false; @@ -380,7 +380,7 @@ int brcmf_btcoex_attach(struct brcmf_cfg80211_info *cfg) /* Set up timer for BT */ btci->timer_on = false; btci->timeout = BRCMF_BTCOEX_OPPR_WIN_TIME; - timer_setup(&btci->timer, brcmf_btcoex_timerfunc, 0); + setup_timer(&btci->timer, brcmf_btcoex_timerfunc, (ulong)btci); btci->cfg = cfg; btci->saved_regs_part1 = false; btci->saved_regs_part2 = false; @@ -462,7 +462,7 @@ static void brcmf_btcoex_dhcp_end(struct brcmf_btcoex_info *btci) int brcmf_btcoex_set_mode(struct brcmf_cfg80211_vif *vif, enum brcmf_btcoex_mode mode, u16 duration) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(vif->wdev.wiphy); + struct brcmf_cfg80211_info *cfg = wiphy_priv(vif->wdev.wiphy); struct brcmf_btcoex_info *btci = cfg->btcoex; struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index c4965184cdf37..163ddc49f9511 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -71,7 +71,6 @@ struct brcmf_bus_dcmd { * @wowl_config: specify if dongle is configured for wowl when going to suspend * @get_ramsize: obtain size of device memory. * @get_memdump: obtain device memory dump in provided buffer. - * @get_fwname: obtain firmware name. * * This structure provides an abstract interface towards the * bus specific driver. For control messages to common driver @@ -88,8 +87,6 @@ struct brcmf_bus_ops { void (*wowl_config)(struct device *dev, bool enabled); size_t (*get_ramsize)(struct device *dev); int (*get_memdump)(struct device *dev, void *data, size_t len); - int (*get_fwname)(struct device *dev, const char *ext, - unsigned char *fw_name); }; @@ -140,7 +137,6 @@ struct brcmf_bus_stats { * @always_use_fws_queue: bus wants use queue also when fwsignal is inactive. * @wowl_supported: is wowl supported by bus driver. * @chiprev: revision of the dongle chip. - * @msgbuf: msgbuf protocol parameters provided by bus layer. */ struct brcmf_bus { union { @@ -228,13 +224,6 @@ int brcmf_bus_get_memdump(struct brcmf_bus *bus, void *data, size_t len) return bus->ops->get_memdump(bus->dev, data, len); } -static inline -int brcmf_bus_get_fwname(struct brcmf_bus *bus, const char *ext, - unsigned char *fw_name) -{ - return bus->ops->get_fwname(bus->dev, ext, fw_name); -} - /* * interface functions from common layer */ @@ -250,12 +239,11 @@ int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings); void brcmf_detach(struct device *dev); /* Indication from bus module that dongle should be reset */ void brcmf_dev_reset(struct device *dev); -/* Request from bus module to initiate a coredump */ -void brcmf_dev_coredump(struct device *dev); /* Configure the "global" bus state used by upper layers */ void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state); +int brcmf_bus_started(struct device *dev); s32 brcmf_iovar_data_set(struct device *dev, char *name, void *data, u32 len); void brcmf_bus_add_txhdrlen(struct device *dev, uint len); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index bbdc6000afb9b..083e5ce7eac73 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -472,6 +472,47 @@ send_key_to_dongle(struct brcmf_if *ifp, struct brcmf_wsec_key *key) return err; } +static s32 +brcmf_configure_arp_nd_offload(struct brcmf_if *ifp, bool enable) +{ + s32 err; + u32 mode; + + if (enable) + mode = BRCMF_ARP_OL_AGENT | BRCMF_ARP_OL_PEER_AUTO_REPLY; + else + mode = 0; + + /* Try to set and enable ARP offload feature, this may fail, then it */ + /* is simply not supported and err 0 will be returned */ + err = brcmf_fil_iovar_int_set(ifp, "arp_ol", mode); + if (err) { + brcmf_dbg(TRACE, "failed to set ARP offload mode to 0x%x, err = %d\n", + mode, err); + err = 0; + } else { + err = brcmf_fil_iovar_int_set(ifp, "arpoe", enable); + if (err) { + brcmf_dbg(TRACE, "failed to configure (%d) ARP offload err = %d\n", + enable, err); + err = 0; + } else + brcmf_dbg(TRACE, "successfully configured (%d) ARP offload to 0x%x\n", + enable, mode); + } + + err = brcmf_fil_iovar_int_set(ifp, "ndoe", enable); + if (err) { + brcmf_dbg(TRACE, "failed to configure (%d) ND offload err = %d\n", + enable, err); + err = 0; + } else + brcmf_dbg(TRACE, "successfully configured (%d) ND offload to 0x%x\n", + enable, mode); + + return err; +} + static void brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) { @@ -753,7 +794,7 @@ s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, static int brcmf_cfg80211_del_ap_iface(struct wiphy *wiphy, struct wireless_dev *wdev) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); struct net_device *ndev = wdev->netdev; struct brcmf_if *ifp = netdev_priv(ndev); int ret; @@ -786,7 +827,7 @@ static int brcmf_cfg80211_del_ap_iface(struct wiphy *wiphy, static int brcmf_cfg80211_del_iface(struct wiphy *wiphy, struct wireless_dev *wdev) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); struct net_device *ndev = wdev->netdev; if (ndev && ndev == cfg_to_ndev(cfg)) @@ -831,7 +872,7 @@ brcmf_cfg80211_change_iface(struct wiphy *wiphy, struct net_device *ndev, enum nl80211_iftype type, struct vif_params *params) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_vif *vif = ifp->vif; s32 infra = 0; @@ -1043,6 +1084,7 @@ brcmf_do_escan(struct brcmf_if *ifp, struct cfg80211_scan_request *request) { struct brcmf_cfg80211_info *cfg = ifp->drvr->config; s32 err; + u32 passive_scan; struct brcmf_scan_results *results; struct escan_info *escan = &cfg->escan_info; @@ -1050,7 +1092,13 @@ brcmf_do_escan(struct brcmf_if *ifp, struct cfg80211_scan_request *request) escan->ifp = ifp; escan->wiphy = cfg->wiphy; escan->escan_state = WL_ESCAN_STATE_SCANNING; - + passive_scan = cfg->active_scan ? 0 : 1; + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_PASSIVE_SCAN, + passive_scan); + if (err) { + brcmf_err("error (%d)\n", err); + return err; + } brcmf_scan_config_mpc(ifp, 0); results = (struct brcmf_scan_results *)cfg->escan_info.escan_buf; results->version = 0; @@ -1064,16 +1112,21 @@ brcmf_do_escan(struct brcmf_if *ifp, struct cfg80211_scan_request *request) } static s32 -brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +brcmf_cfg80211_escan(struct wiphy *wiphy, struct brcmf_cfg80211_vif *vif, + struct cfg80211_scan_request *request, + struct cfg80211_ssid *this_ssid) { + struct brcmf_if *ifp = vif->ifp; struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); - struct brcmf_cfg80211_vif *vif; - s32 err = 0; + struct cfg80211_ssid *ssids; + u32 passive_scan; + bool escan_req; + bool spec_scan; + s32 err; + struct brcmf_ssid_le ssid_le; + u32 SSID_len; - brcmf_dbg(TRACE, "Enter\n"); - vif = container_of(request->wdev, struct brcmf_cfg80211_vif, wdev); - if (!check_vif_up(vif)) - return -EIO; + brcmf_dbg(SCAN, "START ESCAN\n"); if (test_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status)) { brcmf_err("Scanning already: status (%lu)\n", cfg->scan_status); @@ -1089,8 +1142,8 @@ brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) cfg->scan_status); return -EAGAIN; } - if (test_bit(BRCMF_VIF_STATUS_CONNECTING, &vif->sme_state)) { - brcmf_err("Connecting: status (%lu)\n", vif->sme_state); + if (test_bit(BRCMF_VIF_STATUS_CONNECTING, &ifp->vif->sme_state)) { + brcmf_err("Connecting: status (%lu)\n", ifp->vif->sme_state); return -EAGAIN; } @@ -1098,38 +1151,96 @@ brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) if (vif == cfg->p2p.bss_idx[P2PAPI_BSSCFG_DEVICE].vif) vif = cfg->p2p.bss_idx[P2PAPI_BSSCFG_PRIMARY].vif; - brcmf_dbg(SCAN, "START ESCAN\n"); + escan_req = false; + if (request) { + /* scan bss */ + ssids = request->ssids; + escan_req = true; + } else { + /* scan in ibss */ + /* we don't do escan in ibss */ + ssids = this_ssid; + } cfg->scan_request = request; set_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status); + if (escan_req) { + cfg->escan_info.run = brcmf_run_escan; + err = brcmf_p2p_scan_prep(wiphy, request, vif); + if (err) + goto scan_out; - cfg->escan_info.run = brcmf_run_escan; - err = brcmf_p2p_scan_prep(wiphy, request, vif); - if (err) - goto scan_out; + err = brcmf_do_escan(vif->ifp, request); + if (err) + goto scan_out; + } else { + brcmf_dbg(SCAN, "ssid \"%s\", ssid_len (%d)\n", + ssids->ssid, ssids->ssid_len); + memset(&ssid_le, 0, sizeof(ssid_le)); + SSID_len = min_t(u8, sizeof(ssid_le.SSID), ssids->ssid_len); + ssid_le.SSID_len = cpu_to_le32(0); + spec_scan = false; + if (SSID_len) { + memcpy(ssid_le.SSID, ssids->ssid, SSID_len); + ssid_le.SSID_len = cpu_to_le32(SSID_len); + spec_scan = true; + } else + brcmf_dbg(SCAN, "Broadcast scan\n"); - err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_PRBREQ_FLAG, - request->ie, request->ie_len); - if (err) - goto scan_out; + passive_scan = cfg->active_scan ? 0 : 1; + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_PASSIVE_SCAN, + passive_scan); + if (err) { + brcmf_err("WLC_SET_PASSIVE_SCAN error (%d)\n", err); + goto scan_out; + } + brcmf_scan_config_mpc(ifp, 0); + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, &ssid_le, + sizeof(ssid_le)); + if (err) { + if (err == -EBUSY) + brcmf_dbg(INFO, "BUSY: scan for \"%s\" canceled\n", + ssid_le.SSID); + else + brcmf_err("WLC_SCAN error (%d)\n", err); - err = brcmf_do_escan(vif->ifp, request); - if (err) - goto scan_out; + brcmf_scan_config_mpc(ifp, 1); + goto scan_out; + } + } /* Arm scan timeout timer */ - mod_timer(&cfg->escan_timeout, - jiffies + msecs_to_jiffies(BRCMF_ESCAN_TIMER_INTERVAL_MS)); + mod_timer(&cfg->escan_timeout, jiffies + + BRCMF_ESCAN_TIMER_INTERVAL_MS * HZ / 1000); return 0; scan_out: - brcmf_err("scan error (%d)\n", err); clear_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status); cfg->scan_request = NULL; return err; } +static s32 +brcmf_cfg80211_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +{ + struct brcmf_cfg80211_vif *vif; + s32 err = 0; + + brcmf_dbg(TRACE, "Enter\n"); + vif = container_of(request->wdev, struct brcmf_cfg80211_vif, wdev); + if (!check_vif_up(vif)) + return -EIO; + + err = brcmf_cfg80211_escan(wiphy, vif, request, NULL); + + if (err) + brcmf_err("scan error (%d)\n", err); + + brcmf_dbg(TRACE, "Exit\n"); + return err; +} + static s32 brcmf_set_rts(struct net_device *ndev, u32 rts_threshold) { s32 err = 0; @@ -1264,7 +1375,7 @@ static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason) brcmf_dbg(TRACE, "Enter\n"); if (test_and_clear_bit(BRCMF_VIF_STATUS_CONNECTED, &vif->sme_state)) { - brcmf_dbg(INFO, "Call WLC_DISASSOC to stop excess roaming\n"); + brcmf_dbg(INFO, "Call WLC_DISASSOC to stop excess roaming\n "); err = brcmf_fil_cmd_data_set(vif->ifp, BRCMF_C_DISASSOC, NULL, 0); if (err) { @@ -2127,15 +2238,17 @@ static s32 brcmf_cfg80211_get_tx_power(struct wiphy *wiphy, struct wireless_dev *wdev, s32 *dbm) { - struct brcmf_cfg80211_vif *vif = wdev_to_vif(wdev); + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct net_device *ndev = cfg_to_ndev(cfg); + struct brcmf_if *ifp = netdev_priv(ndev); s32 qdbm = 0; s32 err; brcmf_dbg(TRACE, "Enter\n"); - if (!check_vif_up(vif)) + if (!check_vif_up(ifp->vif)) return -EIO; - err = brcmf_fil_iovar_int_get(vif->ifp, "qtxpower", &qdbm); + err = brcmf_fil_iovar_int_get(ifp, "qtxpower", &qdbm); if (err) { brcmf_err("error (%d)\n", err); goto done; @@ -2434,7 +2547,7 @@ static void brcmf_convert_sta_flags(u32 fw_sta_flags, struct station_info *si) struct nl80211_sta_flag_update *sfu; brcmf_dbg(TRACE, "flags %08x\n", fw_sta_flags); - si->filled |= BIT_ULL(NL80211_STA_INFO_STA_FLAGS); + si->filled |= BIT(NL80211_STA_INFO_STA_FLAGS); sfu = &si->sta_flags; sfu->mask = BIT(NL80211_STA_FLAG_WME) | BIT(NL80211_STA_FLAG_AUTHENTICATED) | @@ -2470,7 +2583,7 @@ static void brcmf_fill_bss_param(struct brcmf_if *ifp, struct station_info *si) brcmf_err("Failed to get bss info (%d)\n", err); goto out_kfree; } - si->filled |= BIT_ULL(NL80211_STA_INFO_BSS_PARAM); + si->filled |= BIT(NL80211_STA_INFO_BSS_PARAM); si->bss_param.beacon_interval = le16_to_cpu(buf->bss_le.beacon_period); si->bss_param.dtim_period = buf->bss_le.dtim_period; capability = le16_to_cpu(buf->bss_le.capability); @@ -2501,7 +2614,7 @@ brcmf_cfg80211_get_station_ibss(struct brcmf_if *ifp, brcmf_err("BRCMF_C_GET_RATE error (%d)\n", err); return err; } - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); + sinfo->filled |= BIT(NL80211_STA_INFO_TX_BITRATE); sinfo->txrate.legacy = rate * 5; memset(&scbval, 0, sizeof(scbval)); @@ -2512,7 +2625,7 @@ brcmf_cfg80211_get_station_ibss(struct brcmf_if *ifp, return err; } rssi = le32_to_cpu(scbval.val); - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL); sinfo->signal = rssi; err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_GET_PKTCNTS, &pktcnt, @@ -2521,10 +2634,10 @@ brcmf_cfg80211_get_station_ibss(struct brcmf_if *ifp, brcmf_err("BRCMF_C_GET_GET_PKTCNTS error (%d)\n", err); return err; } - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS) | - BIT_ULL(NL80211_STA_INFO_RX_DROP_MISC) | - BIT_ULL(NL80211_STA_INFO_TX_PACKETS) | - BIT_ULL(NL80211_STA_INFO_TX_FAILED); + sinfo->filled |= BIT(NL80211_STA_INFO_RX_PACKETS) | + BIT(NL80211_STA_INFO_RX_DROP_MISC) | + BIT(NL80211_STA_INFO_TX_PACKETS) | + BIT(NL80211_STA_INFO_TX_FAILED); sinfo->rx_packets = le32_to_cpu(pktcnt.rx_good_pkt); sinfo->rx_dropped_misc = le32_to_cpu(pktcnt.rx_bad_pkt); sinfo->tx_packets = le32_to_cpu(pktcnt.tx_good_pkt); @@ -2571,7 +2684,7 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, } } brcmf_dbg(TRACE, "version %d\n", le16_to_cpu(sta_info_le.ver)); - sinfo->filled = BIT_ULL(NL80211_STA_INFO_INACTIVE_TIME); + sinfo->filled = BIT(NL80211_STA_INFO_INACTIVE_TIME); sinfo->inactive_time = le32_to_cpu(sta_info_le.idle) * 1000; sta_flags = le32_to_cpu(sta_info_le.flags); brcmf_convert_sta_flags(sta_flags, sinfo); @@ -2581,33 +2694,33 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, else sinfo->sta_flags.set &= ~BIT(NL80211_STA_FLAG_TDLS_PEER); if (sta_flags & BRCMF_STA_ASSOC) { - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME); + sinfo->filled |= BIT(NL80211_STA_INFO_CONNECTED_TIME); sinfo->connected_time = le32_to_cpu(sta_info_le.in); brcmf_fill_bss_param(ifp, sinfo); } if (sta_flags & BRCMF_STA_SCBSTATS) { - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); + sinfo->filled |= BIT(NL80211_STA_INFO_TX_FAILED); sinfo->tx_failed = le32_to_cpu(sta_info_le.tx_failures); - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS); + sinfo->filled |= BIT(NL80211_STA_INFO_TX_PACKETS); sinfo->tx_packets = le32_to_cpu(sta_info_le.tx_pkts); sinfo->tx_packets += le32_to_cpu(sta_info_le.tx_mcast_pkts); - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS); + sinfo->filled |= BIT(NL80211_STA_INFO_RX_PACKETS); sinfo->rx_packets = le32_to_cpu(sta_info_le.rx_ucast_pkts); sinfo->rx_packets += le32_to_cpu(sta_info_le.rx_mcast_pkts); if (sinfo->tx_packets) { - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); + sinfo->filled |= BIT(NL80211_STA_INFO_TX_BITRATE); sinfo->txrate.legacy = le32_to_cpu(sta_info_le.tx_rate) / 100; } if (sinfo->rx_packets) { - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BITRATE); + sinfo->filled |= BIT(NL80211_STA_INFO_RX_BITRATE); sinfo->rxrate.legacy = le32_to_cpu(sta_info_le.rx_rate) / 100; } if (le16_to_cpu(sta_info_le.ver) >= 4) { - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES); + sinfo->filled |= BIT(NL80211_STA_INFO_TX_BYTES); sinfo->tx_bytes = le64_to_cpu(sta_info_le.tx_tot_bytes); - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES); + sinfo->filled |= BIT(NL80211_STA_INFO_RX_BYTES); sinfo->rx_bytes = le64_to_cpu(sta_info_le.rx_tot_bytes); } total_rssi = 0; @@ -2623,10 +2736,10 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, } } if (count_rssi) { - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CHAIN_SIGNAL); + sinfo->filled |= BIT(NL80211_STA_INFO_CHAIN_SIGNAL); sinfo->chains = count_rssi; - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL); total_rssi /= count_rssi; sinfo->signal = total_rssi; } else if (test_bit(BRCMF_VIF_STATUS_CONNECTED, @@ -2639,7 +2752,7 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, goto done; } else { rssi = le32_to_cpu(scb_val.val); - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->filled |= BIT(NL80211_STA_INFO_SIGNAL); sinfo->signal = rssi; brcmf_dbg(CONN, "RSSI %d dBm\n", rssi); } @@ -2728,8 +2841,9 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, struct brcmf_bss_info_le *bi) { struct wiphy *wiphy = cfg_to_wiphy(cfg); + struct ieee80211_channel *notify_channel; struct cfg80211_bss *bss; - enum nl80211_band band; + struct ieee80211_supported_band *band; struct brcmu_chan ch; u16 channel; u32 freq; @@ -2737,7 +2851,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, u16 notify_interval; u8 *notify_ie; size_t notify_ielen; - struct cfg80211_inform_bss bss_data = {}; + s32 notify_signal; if (le32_to_cpu(bi->length) > WL_BSS_INFO_MAX) { brcmf_err("Bss info is larger than buffer. Discarding\n"); @@ -2752,33 +2866,32 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, channel = bi->ctl_ch; if (channel <= CH_MAX_2G_CHANNEL) - band = NL80211_BAND_2GHZ; + band = wiphy->bands[NL80211_BAND_2GHZ]; else - band = NL80211_BAND_5GHZ; + band = wiphy->bands[NL80211_BAND_5GHZ]; - freq = ieee80211_channel_to_frequency(channel, band); - bss_data.chan = ieee80211_get_channel(wiphy, freq); - bss_data.scan_width = NL80211_BSS_CHAN_WIDTH_20; - bss_data.boottime_ns = ktime_to_ns(ktime_get_boottime()); + freq = ieee80211_channel_to_frequency(channel, band->band); + notify_channel = ieee80211_get_channel(wiphy, freq); notify_capability = le16_to_cpu(bi->capability); notify_interval = le16_to_cpu(bi->beacon_period); notify_ie = (u8 *)bi + le16_to_cpu(bi->ie_offset); notify_ielen = le32_to_cpu(bi->ie_length); - bss_data.signal = (s16)le16_to_cpu(bi->RSSI) * 100; + notify_signal = (s16)le16_to_cpu(bi->RSSI) * 100; brcmf_dbg(CONN, "bssid: %pM\n", bi->BSSID); brcmf_dbg(CONN, "Channel: %d(%d)\n", channel, freq); brcmf_dbg(CONN, "Capability: %X\n", notify_capability); brcmf_dbg(CONN, "Beacon interval: %d\n", notify_interval); - brcmf_dbg(CONN, "Signal: %d\n", bss_data.signal); + brcmf_dbg(CONN, "Signal: %d\n", notify_signal); - bss = cfg80211_inform_bss_data(wiphy, &bss_data, - CFG80211_BSS_FTYPE_UNKNOWN, - (const u8 *)bi->BSSID, - 0, notify_capability, - notify_interval, notify_ie, - notify_ielen, GFP_KERNEL); + bss = cfg80211_inform_bss(wiphy, notify_channel, + CFG80211_BSS_FTYPE_UNKNOWN, + (const u8 *)bi->BSSID, + 0, notify_capability, + notify_interval, notify_ie, + notify_ielen, notify_signal, + GFP_KERNEL); if (!bss) return -ENOMEM; @@ -2981,10 +3094,10 @@ static void brcmf_cfg80211_escan_timeout_worker(struct work_struct *work) brcmf_notify_escan_complete(cfg, cfg->escan_info.ifp, true, true); } -static void brcmf_escan_timeout(struct timer_list *t) +static void brcmf_escan_timeout(unsigned long data) { struct brcmf_cfg80211_info *cfg = - from_timer(cfg, t, escan_timeout); + (struct brcmf_cfg80211_info *)data; if (cfg->int_escan_map || cfg->scan_request) { brcmf_err("timer expired\n"); @@ -3148,7 +3261,9 @@ static void brcmf_init_escan(struct brcmf_cfg80211_info *cfg) brcmf_cfg80211_escan_handler); cfg->escan_info.escan_state = WL_ESCAN_STATE_IDLE; /* Init scan_timeout timer */ - timer_setup(&cfg->escan_timeout, brcmf_escan_timeout, 0); + init_timer(&cfg->escan_timeout); + cfg->escan_timeout.data = (unsigned long) cfg; + cfg->escan_timeout.function = brcmf_escan_timeout; INIT_WORK(&cfg->escan_timeout_work, brcmf_cfg80211_escan_timeout_worker); } @@ -3356,7 +3471,7 @@ brcmf_cfg80211_sched_scan_start(struct wiphy *wiphy, struct cfg80211_sched_scan_request *req) { struct brcmf_if *ifp = netdev_priv(ndev); - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); brcmf_dbg(SCAN, "Enter: n_match_sets=%d n_ssids=%d\n", req->n_match_sets, req->n_ssids); @@ -3466,8 +3581,6 @@ brcmf_wowl_nd_results(struct brcmf_if *ifp, const struct brcmf_event_msg *e, } netinfo = brcmf_get_netinfo_array(pfn_result); - if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) - netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; memcpy(cfg->wowl.nd->ssid.ssid, netinfo->SSID, netinfo->SSID_len); cfg->wowl.nd->ssid.ssid_len = netinfo->SSID_len; cfg->wowl.nd->n_channels = 1; @@ -5124,9 +5237,6 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X)) return -EINVAL; - if (conf->pmk_len > BRCMF_WSEC_MAX_PSK_LEN) - return -ERANGE; - return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len); } @@ -5190,19 +5300,6 @@ static struct cfg80211_ops brcmf_cfg80211_ops = { .del_pmk = brcmf_cfg80211_del_pmk, }; -struct cfg80211_ops *brcmf_cfg80211_get_ops(struct brcmf_mp_device *settings) -{ - struct cfg80211_ops *ops; - - ops = kmemdup(&brcmf_cfg80211_ops, sizeof(brcmf_cfg80211_ops), - GFP_KERNEL); - - if (ops && settings->roamoff) - ops->update_connect_params = NULL; - - return ops; -} - struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, enum nl80211_iftype type) { @@ -5368,8 +5465,6 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, conn_info->req_ie = kmemdup(cfg->extra_buf, conn_info->req_ie_len, GFP_KERNEL); - if (!conn_info->req_ie) - conn_info->req_ie_len = 0; } else { conn_info->req_ie_len = 0; conn_info->req_ie = NULL; @@ -5386,8 +5481,6 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, conn_info->resp_ie = kmemdup(cfg->extra_buf, conn_info->resp_ie_len, GFP_KERNEL); - if (!conn_info->resp_ie) - conn_info->resp_ie_len = 0; } else { conn_info->resp_ie_len = 0; conn_info->resp_ie = NULL; @@ -5511,7 +5604,7 @@ brcmf_notify_connect_status_ap(struct brcmf_cfg80211_info *cfg, static int generation; u32 event = e->event_code; u32 reason = e->reason; - struct station_info *sinfo; + struct station_info sinfo; brcmf_dbg(CONN, "event %s (%u), reason %d\n", brcmf_fweh_event_name(event), event, reason); @@ -5524,22 +5617,16 @@ brcmf_notify_connect_status_ap(struct brcmf_cfg80211_info *cfg, if (((event == BRCMF_E_ASSOC_IND) || (event == BRCMF_E_REASSOC_IND)) && (reason == BRCMF_E_STATUS_SUCCESS)) { + memset(&sinfo, 0, sizeof(sinfo)); if (!data) { brcmf_err("No IEs present in ASSOC/REASSOC_IND"); return -EINVAL; } - - sinfo = kzalloc(sizeof(*sinfo), GFP_KERNEL); - if (!sinfo) - return -ENOMEM; - - sinfo->assoc_req_ies = data; - sinfo->assoc_req_ies_len = e->datalen; + sinfo.assoc_req_ies = data; + sinfo.assoc_req_ies_len = e->datalen; generation++; - sinfo->generation = generation; - cfg80211_new_sta(ndev, e->addr, sinfo, GFP_KERNEL); - - kfree(sinfo); + sinfo.generation = generation; + cfg80211_new_sta(ndev, e->addr, &sinfo, GFP_KERNEL); } else if ((event == BRCMF_E_DISASSOC_IND) || (event == BRCMF_E_DEAUTH_IND) || (event == BRCMF_E_DEAUTH)) { @@ -5790,6 +5877,7 @@ static s32 wl_init_priv(struct brcmf_cfg80211_info *cfg) cfg->scan_request = NULL; cfg->pwr_save = true; + cfg->active_scan = true; /* we do active scan per default */ cfg->dongle_up = false; /* dongle is not up yet */ err = brcmf_init_priv_mem(cfg); if (err) @@ -5920,7 +6008,7 @@ static void brcmf_update_bw40_channel_flag(struct ieee80211_channel *channel, static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, u32 bw_cap[]) { - struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0); + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); struct ieee80211_supported_band *band; struct ieee80211_channel *channel; struct wiphy *wiphy; @@ -6010,8 +6098,7 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, * for subsequent chanspecs. */ channel->flags = IEEE80211_CHAN_NO_HT40 | - IEEE80211_CHAN_NO_80MHZ | - IEEE80211_CHAN_NO_160MHZ; + IEEE80211_CHAN_NO_80MHZ; ch.bw = BRCMU_CHAN_BW_20; cfg->d11inf.encchspec(&ch); chaninfo = ch.chspec; @@ -6036,7 +6123,7 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg) { - struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0); + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); struct ieee80211_supported_band *band; struct brcmf_fil_bwcap_le band_bwcap; struct brcmf_chanspec_list *list; @@ -6221,10 +6308,10 @@ static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, } } -static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) +static int brcmf_setup_wiphybands(struct wiphy *wiphy) { - struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0); - struct wiphy *wiphy; + struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); u32 nmode = 0; u32 vhtmode = 0; u32 bw_cap[2] = { WLC_BW_20MHZ_BIT, WLC_BW_20MHZ_BIT }; @@ -6314,16 +6401,6 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { .tx = 0xffff, .rx = BIT(IEEE80211_STYPE_ACTION >> 4) | BIT(IEEE80211_STYPE_PROBE_REQ >> 4) - }, - [NL80211_IFTYPE_AP] = { - .tx = 0xffff, - .rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) | - BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) | - BIT(IEEE80211_STYPE_PROBE_REQ >> 4) | - BIT(IEEE80211_STYPE_DISASSOC >> 4) | - BIT(IEEE80211_STYPE_AUTH >> 4) | - BIT(IEEE80211_STYPE_DEAUTH >> 4) | - BIT(IEEE80211_STYPE_ACTION >> 4) } }; @@ -6542,7 +6619,6 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy->flags |= WIPHY_FLAG_NETNS_OK | WIPHY_FLAG_PS_ON_BY_DEFAULT | - WIPHY_FLAG_HAVE_AP_SME | WIPHY_FLAG_OFFCHAN_TX | WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_TDLS)) @@ -6829,8 +6905,8 @@ static s32 brcmf_translate_country_code(struct brcmf_pub *drvr, char alpha2[2], static void brcmf_cfg80211_reg_notifier(struct wiphy *wiphy, struct regulatory_request *req) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); - struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0); + struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); struct brcmf_fil_country_le ccreq; s32 err; int i; @@ -6865,7 +6941,7 @@ static void brcmf_cfg80211_reg_notifier(struct wiphy *wiphy, brcmf_err("Firmware rejected country setting\n"); return; } - brcmf_setup_wiphybands(cfg); + brcmf_setup_wiphybands(wiphy); } static void brcmf_free_wiphy(struct wiphy *wiphy) @@ -6892,15 +6968,17 @@ static void brcmf_free_wiphy(struct wiphy *wiphy) if (wiphy->wowlan != &brcmf_wowlan_support) kfree(wiphy->wowlan); #endif + wiphy_free(wiphy); } struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, - struct cfg80211_ops *ops, + struct device *busdev, bool p2pdev_forced) { - struct wiphy *wiphy = drvr->wiphy; struct net_device *ndev = brcmf_get_ifp(drvr, 0)->ndev; struct brcmf_cfg80211_info *cfg; + struct wiphy *wiphy; + struct cfg80211_ops *ops; struct brcmf_cfg80211_vif *vif; struct brcmf_if *ifp; s32 err = 0; @@ -6912,13 +6990,26 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, return NULL; } - cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); - if (!cfg) { - brcmf_err("Could not allocate wiphy device\n"); + ops = kmemdup(&brcmf_cfg80211_ops, sizeof(*ops), GFP_KERNEL); + if (!ops) return NULL; + + ifp = netdev_priv(ndev); +#ifdef CONFIG_PM + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK)) + ops->set_rekey_data = brcmf_cfg80211_set_rekey_data; +#endif + wiphy = wiphy_new(ops, sizeof(struct brcmf_cfg80211_info)); + if (!wiphy) { + brcmf_err("Could not allocate wiphy device\n"); + goto ops_out; } + memcpy(wiphy->perm_addr, drvr->mac, ETH_ALEN); + set_wiphy_dev(wiphy, busdev); + cfg = wiphy_priv(wiphy); cfg->wiphy = wiphy; + cfg->ops = ops; cfg->pub = drvr; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); @@ -6927,7 +7018,6 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, if (IS_ERR(vif)) goto wiphy_out; - ifp = netdev_priv(ndev); vif->ifp = ifp; vif->wdev.netdev = ndev; ndev->ieee80211_ptr = &vif->wdev; @@ -6950,11 +7040,6 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->d11inf.io_type = (u8)io_type; brcmu_d11_attach(&cfg->d11inf); - /* regulatory notifer below needs access to cfg so - * assign it now. - */ - drvr->config = cfg; - err = brcmf_setup_wiphy(wiphy, ifp); if (err < 0) goto priv_out; @@ -6972,17 +7057,13 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cap = &wiphy->bands[NL80211_BAND_2GHZ]->ht_cap.cap; *cap |= IEEE80211_HT_CAP_SUP_WIDTH_20_40; } -#ifdef CONFIG_PM - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK)) - ops->set_rekey_data = brcmf_cfg80211_set_rekey_data; -#endif err = wiphy_register(wiphy); if (err < 0) { brcmf_err("Could not register wiphy device (%d)\n", err); goto priv_out; } - err = brcmf_setup_wiphybands(cfg); + err = brcmf_setup_wiphybands(wiphy); if (err) { brcmf_err("Setting wiphy bands failed (%d)\n", err); goto wiphy_unreg_out; @@ -6999,7 +7080,12 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, else *cap &= ~IEEE80211_HT_CAP_SUP_WIDTH_20_40; } - + /* p2p might require that "if-events" get processed by fweh. So + * activate the already registered event handlers now and activate + * the rest when initialization has completed. drvr->config needs to + * be assigned before activating events. + */ + drvr->config = cfg; err = brcmf_fweh_activate_events(ifp); if (err) { brcmf_err("FWEH activation failed (%d)\n", err); @@ -7067,7 +7153,8 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, ifp->vif = NULL; wiphy_out: brcmf_free_wiphy(wiphy); - kfree(cfg); +ops_out: + kfree(ops); return NULL; } @@ -7082,5 +7169,4 @@ void brcmf_cfg80211_detach(struct brcmf_cfg80211_info *cfg) kfree(cfg->ops); wl_deinit_priv(cfg); brcmf_free_wiphy(cfg->wiphy); - kfree(cfg); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 9a6287f084a92..7b2835e5e4344 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -283,6 +283,7 @@ struct brcmf_cfg80211_wowl { * @scan_status: scan activity on the dongle. * @pub: common driver information. * @channel: current channel. + * @active_scan: current scan mode. * @int_escan_map: bucket map for which internal e-scan is done. * @ibss_starter: indicates this sta is ibss starter. * @pwr_save: indicate whether dongle to support power save mode. @@ -315,6 +316,7 @@ struct brcmf_cfg80211_info { unsigned long scan_status; struct brcmf_pub *pub; u32 channel; + bool active_scan; u32 int_escan_map; bool ibss_starter; bool pwr_save; @@ -355,24 +357,20 @@ static inline struct wiphy *cfg_to_wiphy(struct brcmf_cfg80211_info *cfg) static inline struct brcmf_cfg80211_info *wiphy_to_cfg(struct wiphy *w) { - struct brcmf_pub *drvr = wiphy_priv(w); - return drvr->config; + return (struct brcmf_cfg80211_info *)(wiphy_priv(w)); } static inline struct brcmf_cfg80211_info *wdev_to_cfg(struct wireless_dev *wd) { - return wiphy_to_cfg(wd->wiphy); -} - -static inline struct brcmf_cfg80211_vif *wdev_to_vif(struct wireless_dev *wdev) -{ - return container_of(wdev, struct brcmf_cfg80211_vif, wdev); + return (struct brcmf_cfg80211_info *)(wdev_priv(wd)); } static inline struct net_device *cfg_to_ndev(struct brcmf_cfg80211_info *cfg) { - return brcmf_get_ifp(cfg->pub, 0)->ndev; + struct brcmf_cfg80211_vif *vif; + vif = list_first_entry(&cfg->vif_list, struct brcmf_cfg80211_vif, list); + return vif->wdev.netdev; } static inline struct brcmf_cfg80211_info *ndev_to_cfg(struct net_device *ndev) @@ -399,12 +397,11 @@ brcmf_cfg80211_connect_info *cfg_to_conn(struct brcmf_cfg80211_info *cfg) } struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, - struct cfg80211_ops *ops, + struct device *busdev, bool p2pdev_forced); void brcmf_cfg80211_detach(struct brcmf_cfg80211_info *cfg); s32 brcmf_cfg80211_up(struct net_device *ndev); s32 brcmf_cfg80211_down(struct net_device *ndev); -struct cfg80211_ops *brcmf_cfg80211_get_ops(struct brcmf_mp_device *settings); enum nl80211_iftype brcmf_cfg80211_get_iftype(struct brcmf_if *ifp); struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 927d62b3d41b8..c5d1a1cbf6019 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -464,12 +464,12 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, ci->ops->read32(ci->ctx, core->wrapbase + BCMA_IOCTL); } -char *brcmf_chip_name(u32 id, u32 rev, char *buf, uint len) +static char *brcmf_chip_name(uint chipid, char *buf, uint len) { const char *fmt; - fmt = ((id > 0xa000) || (id < 0x4000)) ? "BCM%d/%u" : "BCM%x/%u"; - snprintf(buf, len, fmt, id, rev); + fmt = ((chipid > 0xa000) || (chipid < 0x4000)) ? "%d" : "%x"; + snprintf(buf, len, fmt, chipid); return buf; } @@ -689,7 +689,6 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) case BRCM_CC_43525_CHIP_ID: case BRCM_CC_4365_CHIP_ID: case BRCM_CC_4366_CHIP_ID: - case BRCM_CC_43664_CHIP_ID: return 0x200000; case CY_CC_4373_CHIP_ID: return 0x160000; @@ -925,10 +924,10 @@ static int brcmf_chip_recognition(struct brcmf_chip_priv *ci) ci->pub.chiprev = (regdata & CID_REV_MASK) >> CID_REV_SHIFT; socitype = (regdata & CID_TYPE_MASK) >> CID_TYPE_SHIFT; - brcmf_chip_name(ci->pub.chip, ci->pub.chiprev, - ci->pub.name, sizeof(ci->pub.name)); - brcmf_dbg(INFO, "found %s chip: %s\n", - socitype == SOCI_SB ? "SB" : "AXI", ci->pub.name); + brcmf_chip_name(ci->pub.chip, ci->pub.name, sizeof(ci->pub.name)); + brcmf_dbg(INFO, "found %s chip: BCM%s, rev=%d\n", + socitype == SOCI_SB ? "SB" : "AXI", ci->pub.name, + ci->pub.chiprev); if (socitype == SOCI_SB) { if (ci->pub.chip != BRCM_CC_4329_CHIP_ID) { @@ -1339,7 +1338,6 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) switch (pub->chip) { case BRCM_CC_4354_CHIP_ID: case BRCM_CC_4356_CHIP_ID: - case BRCM_CC_4345_CHIP_ID: /* explicitly check SR engine enable bit */ pmu_cc3_mask = BIT(2); /* fall-through */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h index 0ae3b33bab627..dd0ec3eba6a98 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h @@ -45,7 +45,7 @@ struct brcmf_chip { u32 rambase; u32 ramsize; u32 srsize; - char name[12]; + char name[8]; }; /** @@ -93,6 +93,5 @@ void brcmf_chip_resetcore(struct brcmf_core *core, u32 prereset, u32 reset, void brcmf_chip_set_passive(struct brcmf_chip *ci); bool brcmf_chip_set_active(struct brcmf_chip *ci, u32 rstvec); bool brcmf_chip_sr_capable(struct brcmf_chip *pub); -char *brcmf_chip_name(u32 chipid, u32 chiprev, char *buf, uint len); #endif /* BRCMF_AXIDMP_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 8510d207ee87d..7a2b49587b4d3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include "core.h" @@ -29,13 +28,13 @@ #include "tracepoint.h" #include "common.h" #include "of.h" -#include "firmware.h" -#include "chip.h" MODULE_AUTHOR("Broadcom Corporation"); MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver."); MODULE_LICENSE("Dual BSD/GPL"); +const u8 ALLFFMAC[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; + #define BRCMF_DEFAULT_SCAN_CHANNEL_TIME 40 #define BRCMF_DEFAULT_SCAN_UNASSOC_TIME 40 @@ -50,7 +49,7 @@ MODULE_PARM_DESC(txglomsz, "Maximum tx packet chain size [SDIO]"); /* Debug level configuration. See debug.h for bits, sysfs modifiable */ int brcmf_msg_level; -module_param_named(debug, brcmf_msg_level, int, 0600); +module_param_named(debug, brcmf_msg_level, int, S_IRUSR | S_IWUSR); MODULE_PARM_DESC(debug, "Level of debug output"); static int brcmf_p2p_enable; @@ -63,7 +62,7 @@ MODULE_PARM_DESC(feature_disable, "Disable features"); static char brcmf_firmware_path[BRCMF_FW_ALTPATH_LEN]; module_param_string(alternative_fw_path, brcmf_firmware_path, - BRCMF_FW_ALTPATH_LEN, 0400); + BRCMF_FW_ALTPATH_LEN, S_IRUSR); MODULE_PARM_DESC(alternative_fw_path, "Alternative firmware path"); static int brcmf_fcmode; @@ -71,13 +70,9 @@ module_param_named(fcmode, brcmf_fcmode, int, 0); MODULE_PARM_DESC(fcmode, "Mode of firmware signalled flow control"); static int brcmf_roamoff; -module_param_named(roamoff, brcmf_roamoff, int, 0400); +module_param_named(roamoff, brcmf_roamoff, int, S_IRUSR); MODULE_PARM_DESC(roamoff, "Do not use internal roaming engine"); -static int brcmf_iapp_enable; -module_param_named(iapp, brcmf_iapp_enable, int, 0); -MODULE_PARM_DESC(iapp, "Enable partial support for the obsoleted Inter-Access Point Protocol"); - #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -109,104 +104,12 @@ void brcmf_c_set_joinpref_default(struct brcmf_if *ifp) brcmf_err("Set join_pref error (%d)\n", err); } -static int brcmf_c_download(struct brcmf_if *ifp, u16 flag, - struct brcmf_dload_data_le *dload_buf, - u32 len) -{ - s32 err; - - flag |= (DLOAD_HANDLER_VER << DLOAD_FLAG_VER_SHIFT); - dload_buf->flag = cpu_to_le16(flag); - dload_buf->dload_type = cpu_to_le16(DL_TYPE_CLM); - dload_buf->len = cpu_to_le32(len); - dload_buf->crc = cpu_to_le32(0); - len = sizeof(*dload_buf) + len - 1; - - err = brcmf_fil_iovar_data_set(ifp, "clmload", dload_buf, len); - - return err; -} - -static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) -{ - struct brcmf_bus *bus = ifp->drvr->bus_if; - struct brcmf_dload_data_le *chunk_buf; - const struct firmware *clm = NULL; - u8 clm_name[BRCMF_FW_NAME_LEN]; - u32 chunk_len; - u32 datalen; - u32 cumulative_len; - u16 dl_flag = DL_BEGIN; - u32 status; - s32 err; - - brcmf_dbg(TRACE, "Enter\n"); - - memset(clm_name, 0, sizeof(clm_name)); - err = brcmf_bus_get_fwname(bus, ".clm_blob", clm_name); - if (err) { - brcmf_err("get CLM blob file name failed (%d)\n", err); - return err; - } - - err = firmware_request_nowarn(&clm, clm_name, bus->dev); - if (err) { - brcmf_info("no clm_blob available (err=%d), device may have limited channels available\n", - err); - return 0; - } - - chunk_buf = kzalloc(sizeof(*chunk_buf) + MAX_CHUNK_LEN - 1, GFP_KERNEL); - if (!chunk_buf) { - err = -ENOMEM; - goto done; - } - - datalen = clm->size; - cumulative_len = 0; - do { - if (datalen > MAX_CHUNK_LEN) { - chunk_len = MAX_CHUNK_LEN; - } else { - chunk_len = datalen; - dl_flag |= DL_END; - } - memcpy(chunk_buf->data, clm->data + cumulative_len, chunk_len); - - err = brcmf_c_download(ifp, dl_flag, chunk_buf, chunk_len); - - dl_flag &= ~DL_BEGIN; - - cumulative_len += chunk_len; - datalen -= chunk_len; - } while ((datalen > 0) && (err == 0)); - - if (err) { - brcmf_err("clmload (%zu byte file) failed (%d); ", - clm->size, err); - /* Retrieve clmload_status and print */ - err = brcmf_fil_iovar_int_get(ifp, "clmload_status", &status); - if (err) - brcmf_err("get clmload_status failed (%d)\n", err); - else - brcmf_dbg(INFO, "clmload_status=%d\n", status); - err = -EIO; - } - - kfree(chunk_buf); -done: - release_firmware(clm); - return err; -} - int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) { s8 eventmask[BRCMF_EVENTING_MASK_LEN]; u8 buf[BRCMF_DCMD_SMLEN]; - struct brcmf_bus *bus; struct brcmf_rev_info_le revinfo; struct brcmf_rev_info *ri; - char *clmver; char *ptr; s32 err; @@ -217,21 +120,18 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) brcmf_err("Retreiving cur_etheraddr failed, %d\n", err); goto done; } - memcpy(ifp->drvr->wiphy->perm_addr, ifp->drvr->mac, ETH_ALEN); memcpy(ifp->drvr->mac, ifp->mac_addr, sizeof(ifp->drvr->mac)); - bus = ifp->drvr->bus_if; - ri = &ifp->drvr->revinfo; - err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_REVINFO, &revinfo, sizeof(revinfo)); + ri = &ifp->drvr->revinfo; if (err < 0) { brcmf_err("retrieving revision info failed, %d\n", err); - strlcpy(ri->chipname, "UNKNOWN", sizeof(ri->chipname)); } else { ri->vendorid = le32_to_cpu(revinfo.vendorid); ri->deviceid = le32_to_cpu(revinfo.deviceid); ri->radiorev = le32_to_cpu(revinfo.radiorev); + ri->chiprev = le32_to_cpu(revinfo.chiprev); ri->corerev = le32_to_cpu(revinfo.corerev); ri->boardid = le32_to_cpu(revinfo.boardid); ri->boardvendor = le32_to_cpu(revinfo.boardvendor); @@ -239,31 +139,15 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) ri->driverrev = le32_to_cpu(revinfo.driverrev); ri->ucoderev = le32_to_cpu(revinfo.ucoderev); ri->bus = le32_to_cpu(revinfo.bus); + ri->chipnum = le32_to_cpu(revinfo.chipnum); ri->phytype = le32_to_cpu(revinfo.phytype); ri->phyrev = le32_to_cpu(revinfo.phyrev); ri->anarev = le32_to_cpu(revinfo.anarev); ri->chippkg = le32_to_cpu(revinfo.chippkg); ri->nvramrev = le32_to_cpu(revinfo.nvramrev); - - /* use revinfo if not known yet */ - if (!bus->chip) { - bus->chip = le32_to_cpu(revinfo.chipnum); - bus->chiprev = le32_to_cpu(revinfo.chiprev); - } } ri->result = err; - if (bus->chip) - brcmf_chip_name(bus->chip, bus->chiprev, - ri->chipname, sizeof(ri->chipname)); - - /* Do any CLM downloading */ - err = brcmf_c_process_clm_blob(ifp); - if (err < 0) { - brcmf_err("download CLM blob file failed, %d\n", err); - goto done; - } - /* query for 'ver' to get version info from firmware */ memset(buf, 0, sizeof(buf)); strcpy(buf, "ver"); @@ -277,30 +161,12 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) strsep(&ptr, "\n"); /* Print fw version info */ - brcmf_info("Firmware: %s %s\n", ri->chipname, buf); + brcmf_info("Firmware version = %s\n", buf); /* locate firmware version number for ethtool */ ptr = strrchr(buf, ' ') + 1; strlcpy(ifp->drvr->fwver, ptr, sizeof(ifp->drvr->fwver)); - /* Query for 'clmver' to get CLM version info from firmware */ - memset(buf, 0, sizeof(buf)); - err = brcmf_fil_iovar_data_get(ifp, "clmver", buf, sizeof(buf)); - if (err) { - brcmf_dbg(TRACE, "retrieving clmver failed, %d\n", err); - } else { - clmver = (char *)buf; - /* store CLM version for adding it to revinfo debugfs file */ - memcpy(ifp->drvr->clmver, clmver, sizeof(ifp->drvr->clmver)); - - /* Replace all newline/linefeed characters with space - * character - */ - strreplace(clmver, '\n', ' '); - - brcmf_dbg(INFO, "CLM version = %s\n", clmver); - } - /* set mpc */ err = brcmf_fil_iovar_int_set(ifp, "mpc", 1); if (err) { @@ -345,6 +211,9 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) /* Enable tx beamforming, errors can be ignored (not supported) */ (void)brcmf_fil_iovar_int_set(ifp, "txbf", 1); + + /* do bus specific preinit here */ + err = brcmf_bus_preinit(ifp->drvr->bus_if); done: return err; } @@ -418,7 +287,6 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->feature_disable = brcmf_feature_disable; settings->fcmode = brcmf_fcmode; settings->roamoff = !!brcmf_roamoff; - settings->iapp = !!brcmf_iapp_enable; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif @@ -492,6 +360,9 @@ static int __init brcmfmac_module_init(void) { int err; + /* Initialize debug system first */ + brcmf_debugfs_init(); + /* Get the platform data (if available) for our devices */ err = platform_driver_probe(&brcmf_pd, brcmf_common_pd_probe); if (err == -ENODEV) @@ -503,6 +374,7 @@ static int __init brcmfmac_module_init(void) /* Continue the initialization by registering the different busses */ err = brcmf_core_init(); if (err) { + brcmf_debugfs_exit(); if (brcmfmac_pdata) platform_driver_unregister(&brcmf_pd); } @@ -515,6 +387,7 @@ static void __exit brcmfmac_module_exit(void) brcmf_core_exit(); if (brcmfmac_pdata) platform_driver_unregister(&brcmf_pd); + brcmf_debugfs_exit(); } module_init(brcmfmac_module_init); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index a34642cb4d2fb..a62f8e70b3207 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -19,6 +19,8 @@ #include #include "fwil_types.h" +extern const u8 ALLFFMAC[ETH_ALEN]; + #define BRCMF_FW_ALTPATH_LEN 256 /* Definitions for the module global and device specific settings are defined @@ -56,7 +58,6 @@ struct brcmf_mp_device { unsigned int feature_disable; int fcmode; bool roamoff; - bool iapp; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; union { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 584e05fdca6ad..5cc3a07dda9e6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -72,43 +71,6 @@ struct brcmf_if *brcmf_get_ifp(struct brcmf_pub *drvr, int ifidx) return ifp; } -void brcmf_configure_arp_nd_offload(struct brcmf_if *ifp, bool enable) -{ - s32 err; - u32 mode; - - if (enable) - mode = BRCMF_ARP_OL_AGENT | BRCMF_ARP_OL_PEER_AUTO_REPLY; - else - mode = 0; - - /* Try to set and enable ARP offload feature, this may fail, then it */ - /* is simply not supported and err 0 will be returned */ - err = brcmf_fil_iovar_int_set(ifp, "arp_ol", mode); - if (err) { - brcmf_dbg(TRACE, "failed to set ARP offload mode to 0x%x, err = %d\n", - mode, err); - } else { - err = brcmf_fil_iovar_int_set(ifp, "arpoe", enable); - if (err) { - brcmf_dbg(TRACE, "failed to configure (%d) ARP offload err = %d\n", - enable, err); - } else { - brcmf_dbg(TRACE, "successfully configured (%d) ARP offload to 0x%x\n", - enable, mode); - } - } - - err = brcmf_fil_iovar_int_set(ifp, "ndoe", enable); - if (err) { - brcmf_dbg(TRACE, "failed to configure (%d) ND offload err = %d\n", - enable, err); - } else { - brcmf_dbg(TRACE, "successfully configured (%d) ND offload to 0x%x\n", - enable, mode); - } -} - static void _brcmf_set_multicast_list(struct work_struct *work) { struct brcmf_if *ifp; @@ -172,7 +134,6 @@ static void _brcmf_set_multicast_list(struct work_struct *work) if (err < 0) brcmf_err("Setting BRCMF_C_SET_PROMISC failed, %d\n", err); - brcmf_configure_arp_nd_offload(ifp, !cmd_value); } #if IS_ENABLED(CONFIG_IPV6) @@ -231,37 +192,6 @@ static void brcmf_netdev_set_multicast_list(struct net_device *ndev) schedule_work(&ifp->multicast_work); } -/** - * brcmf_skb_is_iapp - checks if skb is an IAPP packet - * - * @skb: skb to check - */ -static bool brcmf_skb_is_iapp(struct sk_buff *skb) -{ - static const u8 iapp_l2_update_packet[6] __aligned(2) = { - 0x00, 0x01, 0xaf, 0x81, 0x01, 0x00, - }; - unsigned char *eth_data; -#if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) - const u16 *a, *b; -#endif - - if (skb->len - skb->mac_len != 6 || - !is_multicast_ether_addr(eth_hdr(skb)->h_dest)) - return false; - - eth_data = skb_mac_header(skb) + ETH_HLEN; -#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) - return !(((*(const u32 *)eth_data) ^ (*(const u32 *)iapp_l2_update_packet)) | - ((*(const u16 *)(eth_data + 4)) ^ (*(const u16 *)(iapp_l2_update_packet + 4)))); -#else - a = (const u16 *)eth_data; - b = (const u16 *)iapp_l2_update_packet; - - return !((a[0] ^ b[0]) | (a[1] ^ b[1]) | (a[2] ^ b[2])); -#endif -} - static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, struct net_device *ndev) { @@ -282,23 +212,6 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, goto done; } - /* Some recent Broadcom's firmwares disassociate STA when they receive - * an 802.11f ADD frame. This behavior can lead to a local DoS security - * issue. Attacker may trigger disassociation of any STA by sending a - * proper Ethernet frame to the wireless interface. - * - * Moreover this feature may break AP interfaces in some specific - * setups. This applies e.g. to the bridge with hairpin mode enabled and - * IFLA_BRPORT_MCAST_TO_UCAST set. IAPP packet generated by a firmware - * will get passed back to the wireless interface and cause immediate - * disassociation of a just-connected STA. - */ - if (!drvr->settings->iapp && brcmf_skb_is_iapp(skb)) { - dev_kfree_skb(skb); - ret = -EINVAL; - goto done; - } - /* Make sure there's enough writeable headroom */ if (skb_headroom(skb) < drvr->hdrlen || skb_header_cloned(skb)) { head_delta = max_t(int, drvr->hdrlen - skb_headroom(skb), 0); @@ -374,15 +287,6 @@ void brcmf_txflowblock_if(struct brcmf_if *ifp, void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb) { - /* Most of Broadcom's firmwares send 802.11f ADD frame every time a new - * STA connects to the AP interface. This is an obsoleted standard most - * users don't use, so don't pass these frames up unless requested. - */ - if (!ifp->drvr->settings->iapp && brcmf_skb_is_iapp(skb)) { - brcmu_pkt_buf_free_skb(skb); - return; - } - if (skb->pkt_type == PACKET_MULTICAST) ifp->ndev->stats.multicast++; @@ -405,30 +309,6 @@ void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb) netif_rx_ni(skb); } -void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb) -{ - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MONITOR_FMT_RADIOTAP)) { - /* Do nothing */ - } else { - struct ieee80211_radiotap_header *radiotap; - - /* TODO: use RX status to fill some radiotap data */ - radiotap = skb_push(skb, sizeof(*radiotap)); - memset(radiotap, 0, sizeof(*radiotap)); - radiotap->it_len = cpu_to_le16(sizeof(*radiotap)); - - /* TODO: 4 bytes with receive status? */ - skb->len -= 4; - } - - skb->dev = ifp->ndev; - skb_reset_mac_header(skb); - skb->pkt_type = PACKET_OTHERHOST; - skb->protocol = htons(ETH_P_802_2); - - brcmf_netif_rx(ifp, skb); -} - static int brcmf_rx_hdrpull(struct brcmf_pub *drvr, struct sk_buff *skb, struct brcmf_if **ifp) { @@ -464,8 +344,7 @@ void brcmf_rx_frame(struct device *dev, struct sk_buff *skb, bool handle_event) } else { /* Process special event packets */ if (handle_event) - brcmf_fweh_process_skb(ifp->drvr, skb, - BCMILCP_SUBTYPE_VENDOR_LONG); + brcmf_fweh_process_skb(ifp->drvr, skb); brcmf_netif_rx(ifp, skb); } @@ -482,7 +361,7 @@ void brcmf_rx_event(struct device *dev, struct sk_buff *skb) if (brcmf_rx_hdrpull(drvr, skb, &ifp)) return; - brcmf_fweh_process_skb(ifp->drvr, skb, 0); + brcmf_fweh_process_skb(ifp->drvr, skb); brcmu_pkt_buf_free_skb(skb); } @@ -784,17 +663,17 @@ static void brcmf_del_if(struct brcmf_pub *drvr, s32 bsscfgidx, bool rtnl_locked) { struct brcmf_if *ifp; - int ifidx; ifp = drvr->iflist[bsscfgidx]; + drvr->iflist[bsscfgidx] = NULL; if (!ifp) { brcmf_err("Null interface, bsscfgidx=%d\n", bsscfgidx); return; } brcmf_dbg(TRACE, "Enter, bsscfgidx=%d, ifidx=%d\n", bsscfgidx, ifp->ifidx); - ifidx = ifp->ifidx; - + if (drvr->if2bss[ifp->ifidx] == bsscfgidx) + drvr->if2bss[ifp->ifidx] = BRCMF_BSSIDX_INVALID; if (ifp->ndev) { if (bsscfgidx == 0) { if (ifp->ndev->netdev_ops == &brcmf_netdev_ops_pri) { @@ -822,10 +701,6 @@ static void brcmf_del_if(struct brcmf_pub *drvr, s32 bsscfgidx, brcmf_p2p_ifp_removed(ifp, rtnl_locked); kfree(ifp); } - - drvr->iflist[bsscfgidx] = NULL; - if (drvr->if2bss[ifidx] == bsscfgidx) - drvr->if2bss[ifidx] = BRCMF_BSSIDX_INVALID; } void brcmf_remove_interface(struct brcmf_if *ifp, bool rtnl_locked) @@ -1001,6 +876,55 @@ static int brcmf_inet6addr_changed(struct notifier_block *nb, } #endif +int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings) +{ + struct brcmf_pub *drvr = NULL; + int ret = 0; + int i; + + brcmf_dbg(TRACE, "Enter\n"); + + /* Allocate primary brcmf_info */ + drvr = kzalloc(sizeof(struct brcmf_pub), GFP_ATOMIC); + if (!drvr) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(drvr->if2bss); i++) + drvr->if2bss[i] = BRCMF_BSSIDX_INVALID; + + mutex_init(&drvr->proto_block); + + /* Link to bus module */ + drvr->hdrlen = 0; + drvr->bus_if = dev_get_drvdata(dev); + drvr->bus_if->drvr = drvr; + drvr->settings = settings; + + /* attach debug facilities */ + brcmf_debug_attach(drvr); + + /* Attach and link in the protocol */ + ret = brcmf_proto_attach(drvr); + if (ret != 0) { + brcmf_err("brcmf_prot_attach failed\n"); + goto fail; + } + + /* Attach to events important for core code */ + brcmf_fweh_register(drvr, BRCMF_E_PSM_WATCHDOG, + brcmf_psm_watchdog_notify); + + /* attach firmware event handler */ + brcmf_fweh_attach(drvr); + + return ret; + +fail: + brcmf_detach(dev); + + return ret; +} + static int brcmf_revinfo_read(struct seq_file *s, void *data) { struct brcmf_bus *bus_if = dev_get_drvdata(s->private); @@ -1011,7 +935,8 @@ static int brcmf_revinfo_read(struct seq_file *s, void *data) seq_printf(s, "vendorid: 0x%04x\n", ri->vendorid); seq_printf(s, "deviceid: 0x%04x\n", ri->deviceid); seq_printf(s, "radiorev: %s\n", brcmu_dotrev_str(ri->radiorev, drev)); - seq_printf(s, "chip: %s\n", ri->chipname); + seq_printf(s, "chipnum: %u (%x)\n", ri->chipnum, ri->chipnum); + seq_printf(s, "chiprev: %u\n", ri->chiprev); seq_printf(s, "chippkg: %u\n", ri->chippkg); seq_printf(s, "corerev: %u\n", ri->corerev); seq_printf(s, "boardid: 0x%04x\n", ri->boardid); @@ -1025,15 +950,14 @@ static int brcmf_revinfo_read(struct seq_file *s, void *data) seq_printf(s, "anarev: %u\n", ri->anarev); seq_printf(s, "nvramrev: %08x\n", ri->nvramrev); - seq_printf(s, "clmver: %s\n", bus_if->drvr->clmver); - return 0; } -static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops) +int brcmf_bus_started(struct device *dev) { int ret = -1; - struct brcmf_bus *bus_if = drvr->bus_if; + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pub *drvr = bus_if->drvr; struct brcmf_if *ifp; struct brcmf_if *p2p_ifp; @@ -1049,16 +973,20 @@ static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops) /* signal bus ready */ brcmf_bus_change_state(bus_if, BRCMF_BUS_UP); - /* do bus specific preinit here */ - ret = brcmf_bus_preinit(bus_if); - if (ret < 0) - goto fail; - /* Bus is ready, do any initialization */ ret = brcmf_c_preinit_dcmds(ifp); if (ret < 0) goto fail; + brcmf_debugfs_add_entry(drvr, "revinfo", brcmf_revinfo_read); + + /* assure we have chipid before feature attach */ + if (!bus_if->chip) { + bus_if->chip = drvr->revinfo.chipnum; + bus_if->chiprev = drvr->revinfo.chiprev; + brcmf_dbg(INFO, "firmware revinfo: chip %x (%d) rev %d\n", + bus_if->chip, bus_if->chip, bus_if->chiprev); + } brcmf_feat_attach(drvr); ret = brcmf_proto_init_done(drvr); @@ -1067,7 +995,7 @@ static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops) brcmf_proto_add_if(drvr, ifp); - drvr->config = brcmf_cfg80211_attach(drvr, ops, + drvr->config = brcmf_cfg80211_attach(drvr, bus_if->dev, drvr->settings->p2p_enable); if (drvr->config == NULL) { ret = -ENOMEM; @@ -1101,11 +1029,6 @@ static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops) #endif #endif /* CONFIG_INET */ - /* populate debugfs */ - brcmf_debugfs_add_entry(drvr, "revinfo", brcmf_revinfo_read); - brcmf_feat_debugfs_create(drvr); - brcmf_proto_debugfs_create(drvr); - return 0; fail: @@ -1125,69 +1048,6 @@ static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops) return ret; } -int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings) -{ - struct wiphy *wiphy; - struct cfg80211_ops *ops; - struct brcmf_pub *drvr = NULL; - int ret = 0; - int i; - - brcmf_dbg(TRACE, "Enter\n"); - - ops = brcmf_cfg80211_get_ops(settings); - if (!ops) - return -ENOMEM; - - wiphy = wiphy_new(ops, sizeof(*drvr)); - if (!wiphy) - return -ENOMEM; - - set_wiphy_dev(wiphy, dev); - drvr = wiphy_priv(wiphy); - drvr->wiphy = wiphy; - - for (i = 0; i < ARRAY_SIZE(drvr->if2bss); i++) - drvr->if2bss[i] = BRCMF_BSSIDX_INVALID; - - mutex_init(&drvr->proto_block); - - /* Link to bus module */ - drvr->hdrlen = 0; - drvr->bus_if = dev_get_drvdata(dev); - drvr->bus_if->drvr = drvr; - drvr->settings = settings; - - /* Attach and link in the protocol */ - ret = brcmf_proto_attach(drvr); - if (ret != 0) { - brcmf_err("brcmf_prot_attach failed\n"); - goto fail; - } - - /* Attach to events important for core code */ - brcmf_fweh_register(drvr, BRCMF_E_PSM_WATCHDOG, - brcmf_psm_watchdog_notify); - - /* attach firmware event handler */ - brcmf_fweh_attach(drvr); - - ret = brcmf_bus_started(drvr, ops); - if (ret != 0) { - brcmf_err("dongle is not responding: err=%d\n", ret); - goto fail; - } - - drvr->config->ops = ops; - return 0; - -fail: - brcmf_detach(dev); - kfree(ops); - - return ret; -} - void brcmf_bus_add_txhdrlen(struct device *dev, uint len) { struct brcmf_bus *bus_if = dev_get_drvdata(dev); @@ -1210,14 +1070,6 @@ void brcmf_dev_reset(struct device *dev) brcmf_fil_cmd_int_set(drvr->iflist[0], BRCMF_C_TERMINATED, 1); } -void brcmf_dev_coredump(struct device *dev) -{ - struct brcmf_bus *bus_if = dev_get_drvdata(dev); - - if (brcmf_debug_create_memdump(bus_if, NULL, 0) < 0) - brcmf_dbg(TRACE, "failed to create coredump\n"); -} - void brcmf_detach(struct device *dev) { s32 i; @@ -1244,26 +1096,19 @@ void brcmf_detach(struct device *dev) brcmf_bus_change_state(bus_if, BRCMF_BUS_DOWN); - brcmf_proto_detach_pre_delif(drvr); - - if (drvr->mon_if) { - brcmf_net_detach(drvr->mon_if->ndev, false); - drvr->mon_if = NULL; - } - /* make sure primary interface removed last */ for (i = BRCMF_MAX_IFS-1; i > -1; i--) brcmf_remove_interface(drvr->iflist[i], false); brcmf_cfg80211_detach(drvr->config); - drvr->config = NULL; brcmf_bus_stop(drvr->bus_if); - brcmf_proto_detach_post_delif(drvr); + brcmf_proto_detach(drvr); + brcmf_debug_detach(drvr); bus_if->drvr = NULL; - wiphy_free(drvr->wiphy); + kfree(drvr); } s32 brcmf_iovar_data_set(struct device *dev, char *name, void *data, u32 len) @@ -1300,12 +1145,6 @@ void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state) int ifidx; brcmf_dbg(TRACE, "%d -> %d\n", bus->state, state); - - if (!drvr) { - brcmf_dbg(INFO, "ignoring transition, bus not attached yet\n"); - return; - } - bus->state = state; if (state == BRCMF_BUS_UP) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index dcf6e27cc16f2..a4dd313140f37 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -87,6 +87,7 @@ struct brcmf_rev_info { u32 vendorid; u32 deviceid; u32 radiorev; + u32 chiprev; u32 corerev; u32 boardid; u32 boardvendor; @@ -94,7 +95,7 @@ struct brcmf_rev_info { u32 driverrev; u32 ucoderev; u32 bus; - char chipname[12]; + u32 chipnum; u32 phytype; u32 phyrev; u32 anarev; @@ -107,7 +108,6 @@ struct brcmf_pub { /* Linkage ponters */ struct brcmf_bus *bus_if; struct brcmf_proto *proto; - struct wiphy *wiphy; struct brcmf_cfg80211_info *config; /* Internal brcmf items */ @@ -121,7 +121,6 @@ struct brcmf_pub { struct brcmf_if *iflist[BRCMF_MAX_IFS]; s32 if2bss[BRCMF_MAX_IFS]; - struct brcmf_if *mon_if; struct mutex proto_block; unsigned char proto_buf[BRCMF_DCMD_MAXLEN]; @@ -142,8 +141,6 @@ struct brcmf_pub { struct notifier_block inetaddr_notifier; struct notifier_block inet6addr_notifier; struct brcmf_mp_device *settings; - - u8 clmver[BRCMF_DCMD_SMLEN]; }; /* forward declarations */ @@ -182,7 +179,6 @@ enum brcmf_netif_stop_reason { * @netif_stop_lock: spinlock for update netif_stop from multiple sources. * @pend_8021x_cnt: tracks outstanding number of 802.1x frames. * @pend_8021x_wait: used for signalling change in count. - * @fwil_fwerr: flag indicating fwil layer should return firmware error codes. */ struct brcmf_if { struct brcmf_pub *drvr; @@ -200,7 +196,6 @@ struct brcmf_if { wait_queue_head_t pend_8021x_wait; struct in6_addr ipv6_addr_tbl[NDOL_MAX_ENTRIES]; u8 ipv6addr_idx; - bool fwil_fwerr; }; int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); @@ -208,7 +203,6 @@ int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); /* Return pointer to interface name */ char *brcmf_ifname(struct brcmf_if *ifp); struct brcmf_if *brcmf_get_ifp(struct brcmf_pub *drvr, int ifidx); -void brcmf_configure_arp_nd_offload(struct brcmf_if *ifp, bool enable); int brcmf_net_attach(struct brcmf_if *ifp, bool rtnl_locked); struct brcmf_if *brcmf_add_if(struct brcmf_pub *drvr, s32 bsscfgidx, s32 ifidx, bool is_p2pdev, const char *name, u8 *mac_addr); @@ -217,7 +211,6 @@ void brcmf_txflowblock_if(struct brcmf_if *ifp, enum brcmf_netif_stop_reason reason, bool state); void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success); void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb); -void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_net_setcarrier(struct brcmf_if *ifp, bool on); int __init brcmf_core_init(void); void __exit brcmf_core_exit(void); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c index 489b5dfdf5b9d..2d3e5e263a326 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c @@ -25,6 +25,8 @@ #include "fweh.h" #include "debug.h" +static struct dentry *root_folder; + int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, size_t len) { @@ -40,8 +42,7 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, if (!dump) return -ENOMEM; - if (data && len > 0) - memcpy(dump, data, len); + memcpy(dump, data, len); err = brcmf_bus_get_memdump(bus, dump + len, ramsize); if (err) { vfree(dump); @@ -53,9 +54,44 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, return 0; } +void brcmf_debugfs_init(void) +{ + root_folder = debugfs_create_dir(KBUILD_MODNAME, NULL); + if (IS_ERR(root_folder)) + root_folder = NULL; +} + +void brcmf_debugfs_exit(void) +{ + if (!root_folder) + return; + + debugfs_remove_recursive(root_folder); + root_folder = NULL; +} + +int brcmf_debug_attach(struct brcmf_pub *drvr) +{ + struct device *dev = drvr->bus_if->dev; + + if (!root_folder) + return -ENODEV; + + drvr->dbgfs_dir = debugfs_create_dir(dev_name(dev), root_folder); + return PTR_ERR_OR_ZERO(drvr->dbgfs_dir); +} + +void brcmf_debug_detach(struct brcmf_pub *drvr) +{ + brcmf_fweh_unregister(drvr, BRCMF_E_PSM_WATCHDOG); + + if (!IS_ERR_OR_NULL(drvr->dbgfs_dir)) + debugfs_remove_recursive(drvr->dbgfs_dir); +} + struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr) { - return drvr->wiphy->debugfsdir; + return drvr->dbgfs_dir; } int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, @@ -63,8 +99,7 @@ int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, { struct dentry *e; - WARN(!drvr->wiphy->debugfsdir, "wiphy not (yet) registered\n"); e = debugfs_create_devm_seqfile(drvr->bus_if->dev, fn, - drvr->wiphy->debugfsdir, read_fn); + drvr->dbgfs_dir, read_fn); return PTR_ERR_OR_ZERO(e); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index cfed0626bf5a6..35919d9e8e139 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -113,12 +113,29 @@ extern int brcmf_msg_level; struct brcmf_bus; struct brcmf_pub; #ifdef DEBUG +void brcmf_debugfs_init(void); +void brcmf_debugfs_exit(void); +int brcmf_debug_attach(struct brcmf_pub *drvr); +void brcmf_debug_detach(struct brcmf_pub *drvr); struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr); int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, int (*read_fn)(struct seq_file *seq, void *data)); int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, size_t len); #else +static inline void brcmf_debugfs_init(void) +{ +} +static inline void brcmf_debugfs_exit(void) +{ +} +static inline int brcmf_debug_attach(struct brcmf_pub *drvr) +{ + return 0; +} +static inline void brcmf_debug_detach(struct brcmf_pub *drvr) +{ +} static inline int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, int (*read_fn)(struct seq_file *seq, void *data)) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 4c5a3995dc352..53ae302599899 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -48,8 +48,6 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_MBSS, "mbss" }, { BRCMF_FEAT_MCHAN, "mchan" }, { BRCMF_FEAT_P2P, "p2p" }, - { BRCMF_FEAT_MONITOR, "monitor" }, - { BRCMF_FEAT_MONITOR_FMT_RADIOTAP, "rtap" }, }; #ifdef DEBUG @@ -93,42 +91,6 @@ static int brcmf_feat_debugfs_read(struct seq_file *seq, void *data) } #endif /* DEBUG */ -struct brcmf_feat_fwfeat { - const char * const fwid; - u32 feat_flags; -}; - -static const struct brcmf_feat_fwfeat brcmf_feat_fwfeat_map[] = { - /* brcmfmac43602-pcie.ap.bin from linux-firmware.git commit ea1178515b88 */ - { "01-6cb8e269", BIT(BRCMF_FEAT_MONITOR) }, - /* brcmfmac4366b-pcie.bin from linux-firmware.git commit 52442afee990 */ - { "01-c47a91a4", BIT(BRCMF_FEAT_MONITOR) }, -}; - -static void brcmf_feat_firmware_overrides(struct brcmf_pub *drv) -{ - const struct brcmf_feat_fwfeat *e; - u32 feat_flags = 0; - int i; - - for (i = 0; i < ARRAY_SIZE(brcmf_feat_fwfeat_map); i++) { - e = &brcmf_feat_fwfeat_map[i]; - if (!strcmp(e->fwid, drv->fwver)) { - feat_flags = e->feat_flags; - break; - } - } - - if (!feat_flags) - return; - - for (i = 0; i < BRCMF_FEAT_LAST; i++) - if (feat_flags & BIT(i)) - brcmf_dbg(INFO, "enabling firmware feature: %s\n", - brcmf_feat_names[i]); - drv->feat_flags |= feat_flags; -} - /** * brcmf_feat_iovar_int_get() - determine feature through iovar query. * @@ -142,9 +104,6 @@ static void brcmf_feat_iovar_int_get(struct brcmf_if *ifp, u32 data; int err; - /* we need to know firmware error */ - ifp->fwil_fwerr = true; - err = brcmf_fil_iovar_int_get(ifp, name, &data); if (err == 0) { brcmf_dbg(INFO, "enabling feature: %s\n", brcmf_feat_names[id]); @@ -153,8 +112,6 @@ static void brcmf_feat_iovar_int_get(struct brcmf_if *ifp, brcmf_dbg(TRACE, "%s feature check failed: %d\n", brcmf_feat_names[id], err); } - - ifp->fwil_fwerr = false; } static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp, @@ -163,9 +120,6 @@ static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp, { int err; - /* we need to know firmware error */ - ifp->fwil_fwerr = true; - err = brcmf_fil_iovar_data_set(ifp, name, data, len); if (err != -BRCMF_FW_UNSUPPORTED) { brcmf_dbg(INFO, "enabling feature: %s\n", brcmf_feat_names[id]); @@ -174,23 +128,15 @@ static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp, brcmf_dbg(TRACE, "%s feature check failed: %d\n", brcmf_feat_names[id], err); } - - ifp->fwil_fwerr = false; } -#define MAX_CAPS_BUFFER_SIZE 768 static void brcmf_feat_firmware_capabilities(struct brcmf_if *ifp) { - char caps[MAX_CAPS_BUFFER_SIZE]; + char caps[256]; enum brcmf_feat_id id; - int i, err; - - err = brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps)); - if (err) { - brcmf_err("could not get firmware cap (%d)\n", err); - return; - } + int i; + brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps)); brcmf_dbg(INFO, "[ %s]\n", caps); for (i = 0; i < ARRAY_SIZE(brcmf_fwcap_map); i++) { @@ -203,41 +149,6 @@ static void brcmf_feat_firmware_capabilities(struct brcmf_if *ifp) } } -/** - * brcmf_feat_fwcap_debugfs_read() - expose firmware capabilities to debugfs. - * - * @seq: sequence for debugfs entry. - * @data: raw data pointer. - */ -static int brcmf_feat_fwcap_debugfs_read(struct seq_file *seq, void *data) -{ - struct brcmf_bus *bus_if = dev_get_drvdata(seq->private); - struct brcmf_if *ifp = brcmf_get_ifp(bus_if->drvr, 0); - char caps[MAX_CAPS_BUFFER_SIZE + 1] = { }; - char *tmp; - int err; - - err = brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps)); - if (err) { - brcmf_err("could not get firmware cap (%d)\n", err); - return err; - } - - /* Put every capability in a new line */ - for (tmp = caps; *tmp; tmp++) { - if (*tmp == ' ') - *tmp = '\n'; - } - - /* Usually there is a space at the end of capabilities string */ - seq_printf(seq, "%s", caps); - /* So make sure we don't print two line breaks */ - if (tmp > caps && *(tmp - 1) != '\n') - seq_printf(seq, "\n"); - - return 0; -} - void brcmf_feat_attach(struct brcmf_pub *drvr) { struct brcmf_if *ifp = brcmf_get_ifp(drvr, 0); @@ -289,8 +200,6 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) } brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa"); - brcmf_feat_firmware_overrides(drvr); - /* set chip related quirks */ switch (drvr->bus_if->chip) { case BRCM_CC_43236_CHIP_ID: @@ -303,12 +212,8 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) /* no quirks */ break; } -} -void brcmf_feat_debugfs_create(struct brcmf_pub *drvr) -{ brcmf_debugfs_add_entry(drvr, "features", brcmf_feat_debugfs_read); - brcmf_debugfs_add_entry(drvr, "fwcap", brcmf_feat_fwcap_debugfs_read); } bool brcmf_feat_is_enabled(struct brcmf_if *ifp, enum brcmf_feat_id id) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 0b4974df353a5..1ab4f16171120 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -33,8 +33,6 @@ * MFP: 802.11w Management Frame Protection. * GSCAN: enhanced scan offload feature. * FWSUP: Firmware supplicant. - * MONITOR: firmware can pass monitor packets to host. - * MONITOR_FMT_RADIOTAP: firmware provides monitor packets with radiotap header */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -50,9 +48,7 @@ BRCMF_FEAT_DEF(WOWL_ARP_ND) \ BRCMF_FEAT_DEF(MFP) \ BRCMF_FEAT_DEF(GSCAN) \ - BRCMF_FEAT_DEF(FWSUP) \ - BRCMF_FEAT_DEF(MONITOR) \ - BRCMF_FEAT_DEF(MONITOR_FMT_RADIOTAP) + BRCMF_FEAT_DEF(FWSUP) /* * Quirks: @@ -93,13 +89,6 @@ enum brcmf_feat_quirk { */ void brcmf_feat_attach(struct brcmf_pub *drvr); -/** - * brcmf_feat_debugfs_create() - create debugfs entries. - * - * @drvr: driver instance. - */ -void brcmf_feat_debugfs_create(struct brcmf_pub *drvr); - /** * brcmf_feat_is_enabled() - query feature. * diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c index 9927079a9ace4..091b52979e030 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c @@ -25,7 +25,6 @@ #include "firmware.h" #include "core.h" #include "common.h" -#include "chip.h" #define BRCMF_FW_MAX_NVRAM_SIZE 64000 #define BRCMF_FW_NVRAM_DEVPATH_LEN 19 /* devpath0=pcie/1/4/ */ @@ -438,31 +437,18 @@ void brcmf_fw_nvram_free(void *nvram) struct brcmf_fw { struct device *dev; - struct brcmf_fw_request *req; - u32 curpos; - void (*done)(struct device *dev, int err, struct brcmf_fw_request *req); + u16 flags; + const struct firmware *code; + const char *nvram_name; + u16 domain_nr; + u16 bus_nr; + void (*done)(struct device *dev, int err, const struct firmware *fw, + void *nvram_image, u32 nvram_len); }; -static void brcmf_fw_request_done(const struct firmware *fw, void *ctx); - -static void brcmf_fw_free_request(struct brcmf_fw_request *req) -{ - struct brcmf_fw_item *item; - int i; - - for (i = 0, item = &req->items[0]; i < req->n_items; i++, item++) { - if (item->type == BRCMF_FW_TYPE_BINARY) - release_firmware(item->binary); - else if (item->type == BRCMF_FW_TYPE_NVRAM) - brcmf_fw_nvram_free(item->nv_data.data); - } - kfree(req); -} - -static int brcmf_fw_request_nvram_done(const struct firmware *fw, void *ctx) +static void brcmf_fw_request_nvram_done(const struct firmware *fw, void *ctx) { struct brcmf_fw *fwctx = ctx; - struct brcmf_fw_item *cur; u32 nvram_length = 0; void *nvram = NULL; u8 *data = NULL; @@ -470,154 +456,83 @@ static int brcmf_fw_request_nvram_done(const struct firmware *fw, void *ctx) bool raw_nvram; brcmf_dbg(TRACE, "enter: dev=%s\n", dev_name(fwctx->dev)); - - cur = &fwctx->req->items[fwctx->curpos]; - if (fw && fw->data) { data = (u8 *)fw->data; data_len = fw->size; raw_nvram = false; } else { data = bcm47xx_nvram_get_contents(&data_len); - if (!data && !(cur->flags & BRCMF_FW_REQF_OPTIONAL)) + if (!data && !(fwctx->flags & BRCMF_FW_REQ_NV_OPTIONAL)) goto fail; raw_nvram = true; } if (data) nvram = brcmf_fw_nvram_strip(data, data_len, &nvram_length, - fwctx->req->domain_nr, - fwctx->req->bus_nr); + fwctx->domain_nr, fwctx->bus_nr); if (raw_nvram) bcm47xx_nvram_release_contents(data); release_firmware(fw); - if (!nvram && !(cur->flags & BRCMF_FW_REQF_OPTIONAL)) + if (!nvram && !(fwctx->flags & BRCMF_FW_REQ_NV_OPTIONAL)) goto fail; - brcmf_dbg(TRACE, "nvram %p len %d\n", nvram, nvram_length); - cur->nv_data.data = nvram; - cur->nv_data.len = nvram_length; - return 0; + fwctx->done(fwctx->dev, 0, fwctx->code, nvram, nvram_length); + kfree(fwctx); + return; fail: - return -ENOENT; -} - -static int brcmf_fw_request_next_item(struct brcmf_fw *fwctx, bool async) -{ - struct brcmf_fw_item *cur; - const struct firmware *fw = NULL; - int ret; - - cur = &fwctx->req->items[fwctx->curpos]; - - brcmf_dbg(TRACE, "%srequest for %s\n", async ? "async " : "", - cur->path); - - if (async) - ret = request_firmware_nowait(THIS_MODULE, true, cur->path, - fwctx->dev, GFP_KERNEL, fwctx, - brcmf_fw_request_done); - else - ret = request_firmware(&fw, cur->path, fwctx->dev); - - if (ret < 0) { - brcmf_fw_request_done(NULL, fwctx); - } else if (!async && fw) { - brcmf_dbg(TRACE, "firmware %s %sfound\n", cur->path, - fw ? "" : "not "); - if (cur->type == BRCMF_FW_TYPE_BINARY) - cur->binary = fw; - else if (cur->type == BRCMF_FW_TYPE_NVRAM) - brcmf_fw_request_nvram_done(fw, fwctx); - else - release_firmware(fw); - - return -EAGAIN; - } - return 0; + brcmf_dbg(TRACE, "failed: dev=%s\n", dev_name(fwctx->dev)); + release_firmware(fwctx->code); + fwctx->done(fwctx->dev, -ENOENT, NULL, NULL, 0); + kfree(fwctx); } -static void brcmf_fw_request_done(const struct firmware *fw, void *ctx) +static void brcmf_fw_request_code_done(const struct firmware *fw, void *ctx) { struct brcmf_fw *fwctx = ctx; - struct brcmf_fw_item *cur; int ret = 0; - cur = &fwctx->req->items[fwctx->curpos]; - - brcmf_dbg(TRACE, "enter: firmware %s %sfound\n", cur->path, - fw ? "" : "not "); - - if (!fw) + brcmf_dbg(TRACE, "enter: dev=%s\n", dev_name(fwctx->dev)); + if (!fw) { ret = -ENOENT; - - switch (cur->type) { - case BRCMF_FW_TYPE_NVRAM: - ret = brcmf_fw_request_nvram_done(fw, fwctx); - break; - case BRCMF_FW_TYPE_BINARY: - cur->binary = fw; - break; - default: - /* something fishy here so bail out early */ - brcmf_err("unknown fw type: %d\n", cur->type); - release_firmware(fw); - ret = -EINVAL; goto fail; } - - if (ret < 0 && !(cur->flags & BRCMF_FW_REQF_OPTIONAL)) - goto fail; - - do { - if (++fwctx->curpos == fwctx->req->n_items) { - ret = 0; - goto done; - } - - ret = brcmf_fw_request_next_item(fwctx, false); - } while (ret == -EAGAIN); - + /* only requested code so done here */ + if (!(fwctx->flags & BRCMF_FW_REQUEST_NVRAM)) + goto done; + + fwctx->code = fw; + ret = request_firmware_nowait(THIS_MODULE, true, fwctx->nvram_name, + fwctx->dev, GFP_KERNEL, fwctx, + brcmf_fw_request_nvram_done); + + /* pass NULL to nvram callback for bcm47xx fallback */ + if (ret) + brcmf_fw_request_nvram_done(NULL, fwctx); return; fail: - brcmf_dbg(TRACE, "failed err=%d: dev=%s, fw=%s\n", ret, - dev_name(fwctx->dev), cur->path); - brcmf_fw_free_request(fwctx->req); - fwctx->req = NULL; + brcmf_dbg(TRACE, "failed: dev=%s\n", dev_name(fwctx->dev)); done: - fwctx->done(fwctx->dev, ret, fwctx->req); + fwctx->done(fwctx->dev, ret, fw, NULL, 0); kfree(fwctx); } -static bool brcmf_fw_request_is_valid(struct brcmf_fw_request *req) -{ - struct brcmf_fw_item *item; - int i; - - if (!req->n_items) - return false; - - for (i = 0, item = &req->items[0]; i < req->n_items; i++, item++) { - if (!item->path) - return false; - } - return true; -} - -int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req, - void (*fw_cb)(struct device *dev, int err, - struct brcmf_fw_request *req)) +int brcmf_fw_get_firmwares_pcie(struct device *dev, u16 flags, + const char *code, const char *nvram, + void (*fw_cb)(struct device *dev, int err, + const struct firmware *fw, + void *nvram_image, u32 nvram_len), + u16 domain_nr, u16 bus_nr) { struct brcmf_fw *fwctx; brcmf_dbg(TRACE, "enter: dev=%s\n", dev_name(dev)); - if (!fw_cb) + if (!fw_cb || !code) return -EINVAL; - if (!brcmf_fw_request_is_valid(req)) + if ((flags & BRCMF_FW_REQUEST_NVRAM) && !nvram) return -EINVAL; fwctx = kzalloc(sizeof(*fwctx), GFP_KERNEL); @@ -625,26 +540,35 @@ int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req, return -ENOMEM; fwctx->dev = dev; - fwctx->req = req; + fwctx->flags = flags; fwctx->done = fw_cb; + if (flags & BRCMF_FW_REQUEST_NVRAM) + fwctx->nvram_name = nvram; + fwctx->domain_nr = domain_nr; + fwctx->bus_nr = bus_nr; + + return request_firmware_nowait(THIS_MODULE, true, code, dev, + GFP_KERNEL, fwctx, + brcmf_fw_request_code_done); +} - brcmf_fw_request_next_item(fwctx, true); - return 0; +int brcmf_fw_get_firmwares(struct device *dev, u16 flags, + const char *code, const char *nvram, + void (*fw_cb)(struct device *dev, int err, + const struct firmware *fw, + void *nvram_image, u32 nvram_len)) +{ + return brcmf_fw_get_firmwares_pcie(dev, flags, code, nvram, fw_cb, 0, + 0); } -struct brcmf_fw_request * -brcmf_fw_alloc_request(u32 chip, u32 chiprev, - const struct brcmf_firmware_mapping mapping_table[], - u32 table_size, struct brcmf_fw_name *fwnames, - u32 n_fwnames) +int brcmf_fw_map_chip_to_name(u32 chip, u32 chiprev, + struct brcmf_firmware_mapping mapping_table[], + u32 table_size, char fw_name[BRCMF_FW_NAME_LEN], + char nvram_name[BRCMF_FW_NAME_LEN]) { - struct brcmf_fw_request *fwreq; - char chipname[12]; - const char *mp_path; - size_t mp_path_len; - u32 i, j; - char end = '\0'; - size_t reqsz; + u32 i; + char end; for (i = 0; i < table_size; i++) { if (mapping_table[i].chipid == chip && @@ -654,44 +578,32 @@ brcmf_fw_alloc_request(u32 chip, u32 chiprev, if (i == table_size) { brcmf_err("Unknown chipid %d [%d]\n", chip, chiprev); - return NULL; + return -ENODEV; } - reqsz = sizeof(*fwreq) + n_fwnames * sizeof(struct brcmf_fw_item); - fwreq = kzalloc(reqsz, GFP_KERNEL); - if (!fwreq) - return NULL; - - brcmf_chip_name(chip, chiprev, chipname, sizeof(chipname)); - - brcmf_info("using %s for chip %s\n", - mapping_table[i].fw_base, chipname); - - mp_path = brcmf_mp_global.firmware_path; - mp_path_len = strnlen(mp_path, BRCMF_FW_ALTPATH_LEN); - if (mp_path_len) - end = mp_path[mp_path_len - 1]; - - fwreq->n_items = n_fwnames; - - for (j = 0; j < n_fwnames; j++) { - fwreq->items[j].path = fwnames[j].path; - /* check if firmware path is provided by module parameter */ - if (brcmf_mp_global.firmware_path[0] != '\0') { - strlcpy(fwnames[j].path, mp_path, + /* check if firmware path is provided by module parameter */ + if (brcmf_mp_global.firmware_path[0] != '\0') { + strlcpy(fw_name, brcmf_mp_global.firmware_path, + BRCMF_FW_NAME_LEN); + if ((nvram_name) && (mapping_table[i].nvram)) + strlcpy(nvram_name, brcmf_mp_global.firmware_path, BRCMF_FW_NAME_LEN); - if (end != '/') { - strlcat(fwnames[j].path, "/", - BRCMF_FW_NAME_LEN); - } + end = brcmf_mp_global.firmware_path[ + strlen(brcmf_mp_global.firmware_path) - 1]; + if (end != '/') { + strlcat(fw_name, "/", BRCMF_FW_NAME_LEN); + if ((nvram_name) && (mapping_table[i].nvram)) + strlcat(nvram_name, "/", BRCMF_FW_NAME_LEN); } - strlcat(fwnames[j].path, mapping_table[i].fw_base, - BRCMF_FW_NAME_LEN); - strlcat(fwnames[j].path, fwnames[j].extension, - BRCMF_FW_NAME_LEN); - fwreq->items[j].path = fwnames[j].path; } + strlcat(fw_name, mapping_table[i].fw, BRCMF_FW_NAME_LEN); + if ((nvram_name) && (mapping_table[i].nvram)) + strlcat(nvram_name, mapping_table[i].nvram, BRCMF_FW_NAME_LEN); - return fwreq; + brcmf_info("using %s for chip %#08x(%d) rev %#08x\n", + fw_name, chip, chip, chiprev); + + return 0; } + diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h index 2893e56910f02..8fa4b7e1ab3db 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h @@ -16,7 +16,10 @@ #ifndef BRCMFMAC_FIRMWARE_H #define BRCMFMAC_FIRMWARE_H -#define BRCMF_FW_REQF_OPTIONAL 0x0001 +#define BRCMF_FW_REQUEST 0x000F +#define BRCMF_FW_REQUEST_NVRAM 0x0001 +#define BRCMF_FW_REQ_FLAGS 0x00F0 +#define BRCMF_FW_REQ_NV_OPTIONAL 0x0010 #define BRCMF_FW_NAME_LEN 320 @@ -35,62 +38,49 @@ struct brcmf_firmware_mapping { u32 chipid; u32 revmask; - const char *fw_base; + const char *fw; + const char *nvram; }; -#define BRCMF_FW_DEF(fw_name, fw_base) \ -static const char BRCM_ ## fw_name ## _FIRMWARE_BASENAME[] = \ - BRCMF_FW_DEFAULT_PATH fw_base; \ -MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH fw_base ".bin") +#define BRCMF_FW_NVRAM_DEF(fw_nvram_name, fw, nvram) \ +static const char BRCM_ ## fw_nvram_name ## _FIRMWARE_NAME[] = \ + BRCMF_FW_DEFAULT_PATH fw; \ +static const char BRCM_ ## fw_nvram_name ## _NVRAM_NAME[] = \ + BRCMF_FW_DEFAULT_PATH nvram; \ +MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH fw); -#define BRCMF_FW_ENTRY(chipid, mask, name) \ - { chipid, mask, BRCM_ ## name ## _FIRMWARE_BASENAME } - -void brcmf_fw_nvram_free(void *nvram); - -enum brcmf_fw_type { - BRCMF_FW_TYPE_BINARY, - BRCMF_FW_TYPE_NVRAM -}; - -struct brcmf_fw_item { - const char *path; - enum brcmf_fw_type type; - u16 flags; - union { - const struct firmware *binary; - struct { - void *data; - u32 len; - } nv_data; - }; -}; - -struct brcmf_fw_request { - u16 domain_nr; - u16 bus_nr; - u32 n_items; - struct brcmf_fw_item items[0]; -}; +#define BRCMF_FW_DEF(fw_name, fw) \ +static const char BRCM_ ## fw_name ## _FIRMWARE_NAME[] = \ + BRCMF_FW_DEFAULT_PATH fw; \ +MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH fw) \ -struct brcmf_fw_name { - const char *extension; - char *path; -}; +#define BRCMF_FW_NVRAM_ENTRY(chipid, mask, name) \ + { chipid, mask, \ + BRCM_ ## name ## _FIRMWARE_NAME, BRCM_ ## name ## _NVRAM_NAME } -struct brcmf_fw_request * -brcmf_fw_alloc_request(u32 chip, u32 chiprev, - const struct brcmf_firmware_mapping mapping_table[], - u32 table_size, struct brcmf_fw_name *fwnames, - u32 n_fwnames); +#define BRCMF_FW_ENTRY(chipid, mask, name) \ + { chipid, mask, BRCM_ ## name ## _FIRMWARE_NAME, NULL } +int brcmf_fw_map_chip_to_name(u32 chip, u32 chiprev, + struct brcmf_firmware_mapping mapping_table[], + u32 table_size, char fw_name[BRCMF_FW_NAME_LEN], + char nvram_name[BRCMF_FW_NAME_LEN]); +void brcmf_fw_nvram_free(void *nvram); /* * Request firmware(s) asynchronously. When the asynchronous request * fails it will not use the callback, but call device_release_driver() * instead which will call the driver .remove() callback. */ -int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req, +int brcmf_fw_get_firmwares_pcie(struct device *dev, u16 flags, + const char *code, const char *nvram, + void (*fw_cb)(struct device *dev, int err, + const struct firmware *fw, + void *nvram_image, u32 nvram_len), + u16 domain_nr, u16 bus_nr); +int brcmf_fw_get_firmwares(struct device *dev, u16 flags, + const char *code, const char *nvram, void (*fw_cb)(struct device *dev, int err, - struct brcmf_fw_request *req)); + const struct firmware *fw, + void *nvram_image, u32 nvram_len)); #endif /* BRCMFMAC_FIRMWARE_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c index d0d8b32af7d05..d0b738da24582 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c @@ -46,8 +46,6 @@ static const u8 brcmf_flowring_prio2fifo[] = { 3 }; -static const u8 ALLFFMAC[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - static bool brcmf_flowring_is_tdls_mac(struct brcmf_flowring *flow, u8 mac[ETH_ALEN]) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index e7eaa57d11d98..ef72baf6dd969 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -257,6 +257,11 @@ static void brcmf_fweh_event_worker(struct work_struct *work) brcmf_dbg_hex_dump(BRCMF_EVENT_ON(), event->data, min_t(u32, emsg.datalen, 64), "event payload, len=%d\n", emsg.datalen); + if (emsg.datalen > event->datalen) { + brcmf_err("event invalid length header=%d, msg=%d\n", + event->datalen, emsg.datalen); + goto event_free; + } /* special handling of interface event */ if (event->code == BRCMF_E_IF) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index ebd66fe0d949c..816f80ea925b1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -211,7 +211,7 @@ enum brcmf_fweh_event_code { */ #define BRCM_OUI "\x00\x10\x18" #define BCMILCP_BCM_SUBTYPE_EVENT 1 -#define BCMILCP_SUBTYPE_VENDOR_LONG 32769 + /** * struct brcm_ethhdr - broadcom specific ether header. @@ -334,10 +334,10 @@ void brcmf_fweh_process_event(struct brcmf_pub *drvr, void brcmf_fweh_p2pdev_setup(struct brcmf_if *ifp, bool ongoing); static inline void brcmf_fweh_process_skb(struct brcmf_pub *drvr, - struct sk_buff *skb, u16 stype) + struct sk_buff *skb) { struct brcmf_event *event_packet; - u16 subtype, usr_stype; + u16 usr_stype; /* only process events when protocol matches */ if (skb->protocol != cpu_to_be16(ETH_P_LINK_CTL)) @@ -346,16 +346,8 @@ static inline void brcmf_fweh_process_skb(struct brcmf_pub *drvr, if ((skb->len + ETH_HLEN) < sizeof(*event_packet)) return; - event_packet = (struct brcmf_event *)skb_mac_header(skb); - - /* check subtype if needed */ - if (unlikely(stype)) { - subtype = get_unaligned_be16(&event_packet->hdr.subtype); - if (subtype != stype) - return; - } - /* check for BRCM oui match */ + event_packet = (struct brcmf_event *)skb_mac_header(skb); if (memcmp(BRCM_OUI, &event_packet->hdr.oui[0], sizeof(event_packet->hdr.oui))) return; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c index 802d7cb73b80f..f6a2df94dba72 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.c @@ -107,7 +107,7 @@ static s32 brcmf_fil_cmd_data(struct brcmf_if *ifp, u32 cmd, void *data, u32 len, bool set) { struct brcmf_pub *drvr = ifp->drvr; - s32 err, fwerr; + s32 err; if (drvr->bus_if->state != BRCMF_BUS_UP) { brcmf_err("bus is down. we have nothing to do.\n"); @@ -117,21 +117,15 @@ brcmf_fil_cmd_data(struct brcmf_if *ifp, u32 cmd, void *data, u32 len, bool set) if (data != NULL) len = min_t(uint, len, BRCMF_DCMD_MAXLEN); if (set) - err = brcmf_proto_set_dcmd(drvr, ifp->ifidx, cmd, - data, len, &fwerr); + err = brcmf_proto_set_dcmd(drvr, ifp->ifidx, cmd, data, len); else - err = brcmf_proto_query_dcmd(drvr, ifp->ifidx, cmd, - data, len, &fwerr); - - if (err) { - brcmf_dbg(FIL, "Failed: error=%d\n", err); - } else if (fwerr < 0) { - brcmf_dbg(FIL, "Firmware error: %s (%d)\n", - brcmf_fil_get_errstr((u32)(-fwerr)), fwerr); - err = -EBADE; - } - if (ifp->fwil_fwerr) - return fwerr; + err = brcmf_proto_query_dcmd(drvr, ifp->ifidx, cmd, data, len); + + if (err >= 0) + return 0; + + brcmf_dbg(FIL, "Failed: %s (%d)\n", + brcmf_fil_get_errstr((u32)(-err)), err); return err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index d5bb81e887624..e0d22fedb2b45 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -32,30 +32,11 @@ #define BRCMF_BSS_INFO_VERSION 109 /* curr ver of brcmf_bss_info_le struct */ #define BRCMF_BSS_RSSI_ON_CHANNEL 0x0002 -#define BRCMF_STA_BRCM 0x00000001 /* Running a Broadcom driver */ -#define BRCMF_STA_WME 0x00000002 /* WMM association */ -#define BRCMF_STA_NONERP 0x00000004 /* No ERP */ -#define BRCMF_STA_AUTHE 0x00000008 /* Authenticated */ -#define BRCMF_STA_ASSOC 0x00000010 /* Associated */ -#define BRCMF_STA_AUTHO 0x00000020 /* Authorized */ -#define BRCMF_STA_WDS 0x00000040 /* Wireless Distribution System */ -#define BRCMF_STA_WDS_LINKUP 0x00000080 /* WDS traffic/probes flowing properly */ -#define BRCMF_STA_PS 0x00000100 /* STA is in power save mode from AP's viewpoint */ -#define BRCMF_STA_APSD_BE 0x00000200 /* APSD delv/trigger for AC_BE is default enabled */ -#define BRCMF_STA_APSD_BK 0x00000400 /* APSD delv/trigger for AC_BK is default enabled */ -#define BRCMF_STA_APSD_VI 0x00000800 /* APSD delv/trigger for AC_VI is default enabled */ -#define BRCMF_STA_APSD_VO 0x00001000 /* APSD delv/trigger for AC_VO is default enabled */ -#define BRCMF_STA_N_CAP 0x00002000 /* STA 802.11n capable */ -#define BRCMF_STA_SCBSTATS 0x00004000 /* Per STA debug stats */ -#define BRCMF_STA_AMPDU_CAP 0x00008000 /* STA AMPDU capable */ -#define BRCMF_STA_AMSDU_CAP 0x00010000 /* STA AMSDU capable */ -#define BRCMF_STA_MIMO_PS 0x00020000 /* mimo ps mode is enabled */ -#define BRCMF_STA_MIMO_RTS 0x00040000 /* send rts in mimo ps mode */ -#define BRCMF_STA_RIFS_CAP 0x00080000 /* rifs enabled */ -#define BRCMF_STA_VHT_CAP 0x00100000 /* STA VHT(11ac) capable */ -#define BRCMF_STA_WPS 0x00200000 /* WPS state */ -#define BRCMF_STA_DWDS_CAP 0x01000000 /* DWDS CAP */ -#define BRCMF_STA_DWDS 0x02000000 /* DWDS active */ +#define BRCMF_STA_WME 0x00000002 /* WMM association */ +#define BRCMF_STA_AUTHE 0x00000008 /* Authenticated */ +#define BRCMF_STA_ASSOC 0x00000010 /* Associated */ +#define BRCMF_STA_AUTHO 0x00000020 /* Authorized */ +#define BRCMF_STA_SCBSTATS 0x00004000 /* Per STA debug stats */ /* size of brcmf_scan_params not including variable length array */ #define BRCMF_SCAN_PARAMS_FIXED_SIZE 64 @@ -174,23 +155,6 @@ #define BRCMF_MFP_CAPABLE 1 #define BRCMF_MFP_REQUIRED 2 -#define BRCMF_VHT_CAP_MCS_MAP_NSS_MAX 8 - -/* MAX_CHUNK_LEN is the maximum length for data passing to firmware in each - * ioctl. It is relatively small because firmware has small maximum size input - * playload restriction for ioctls. - */ -#define MAX_CHUNK_LEN 1400 - -#define DLOAD_HANDLER_VER 1 /* Downloader version */ -#define DLOAD_FLAG_VER_MASK 0xf000 /* Downloader version mask */ -#define DLOAD_FLAG_VER_SHIFT 12 /* Downloader version shift */ - -#define DL_BEGIN 0x0002 -#define DL_END 0x0004 - -#define DL_TYPE_CLM 2 - /* join preference types for join_pref iovar */ enum brcmf_join_pref_types { BRCMF_JOIN_PREF_RSSI = 1, @@ -552,8 +516,6 @@ struct brcmf_sta_info_le { /* w/hi bit set if basic */ __le32 in; /* seconds elapsed since associated */ __le32 listen_interval_inms; /* Min Listen interval in ms for STA */ - - /* Fields valid for ver >= 3 */ __le32 tx_pkts; /* # of packets transmitted */ __le32 tx_failures; /* # of packets failed */ __le32 rx_ucast_pkts; /* # of unicast packets received */ @@ -562,8 +524,6 @@ struct brcmf_sta_info_le { __le32 rx_rate; /* Rate of last successful rx frame */ __le32 rx_decrypt_succeeds; /* # of packet decrypted successfully */ __le32 rx_decrypt_failures; /* # of packet decrypted failed */ - - /* Fields valid for ver >= 4 */ __le32 tx_tot_pkts; /* # of tx pkts (ucast + mcast) */ __le32 rx_tot_pkts; /* # of data packets recvd (uni + mcast) */ __le32 tx_mcast_pkts; /* # of mcast pkts txed */ @@ -600,14 +560,6 @@ struct brcmf_sta_info_le { */ __le32 rx_pkts_retried; /* # rx with retry bit set */ __le32 tx_rate_fallback; /* lowest fallback TX rate */ - - /* Fields valid for ver >= 5 */ - struct { - __le32 count; /* # rates in this set */ - u8 rates[BRCMF_MAXRATES_IN_SET]; /* rates in 500kbps units w/hi bit set if basic */ - u8 mcs[BRCMF_MCSSET_LEN]; /* supported mcs index bit map */ - __le16 vht_mcs[BRCMF_VHT_CAP_MCS_MAP_NSS_MAX]; /* supported mcs index bit map per nss */ - } rateset_adv; }; struct brcmf_chanspec_list { @@ -874,22 +826,6 @@ struct brcmf_pno_macaddr_le { u8 mac[ETH_ALEN]; }; -/** - * struct brcmf_dload_data_le - data passing to firmware for downloading - * @flag: flags related to download data. - * @dload_type: type of download data. - * @len: length in bytes of download data. - * @crc: crc of download data. - * @data: download data. - */ -struct brcmf_dload_data_le { - __le16 flag; - __le16 dload_type; - __le32 len; - __le32 crc; - u8 data[1]; -}; - /** * struct brcmf_pno_bssid_le - bssid configuration for PNO scan. * diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 1de8497d92b8a..f59642b2c935a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -579,6 +579,24 @@ static bool brcmf_fws_ifidx_match(struct sk_buff *skb, void *arg) return ifidx == *(int *)arg; } +static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q, + int ifidx) +{ + bool (*matchfn)(struct sk_buff *, void *) = NULL; + struct sk_buff *skb; + int prec; + + if (ifidx != -1) + matchfn = brcmf_fws_ifidx_match; + for (prec = 0; prec < q->num_prec; prec++) { + skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); + while (skb) { + brcmu_pkt_buf_free_skb(skb); + skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); + } + } +} + static void brcmf_fws_hanger_init(struct brcmf_fws_hanger *hanger) { int i; @@ -650,28 +668,6 @@ static inline int brcmf_fws_hanger_poppkt(struct brcmf_fws_hanger *h, return 0; } -static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q, - int ifidx) -{ - bool (*matchfn)(struct sk_buff *, void *) = NULL; - struct sk_buff *skb; - int prec; - u32 hslot; - - if (ifidx != -1) - matchfn = brcmf_fws_ifidx_match; - for (prec = 0; prec < q->num_prec; prec++) { - skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); - while (skb) { - hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); - brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb, - true); - brcmu_pkt_buf_free_skb(skb); - skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); - } - } -} - static int brcmf_fws_hanger_mark_suppressed(struct brcmf_fws_hanger *h, u32 slot_id) { @@ -2172,8 +2168,6 @@ void brcmf_fws_del_interface(struct brcmf_if *ifp) brcmf_fws_lock(fws); ifp->fws_desc = NULL; brcmf_dbg(TRACE, "deleting %s\n", entry->name); - brcmf_fws_macdesc_cleanup(fws, &fws->desc.iface[ifp->ifidx], - ifp->ifidx); brcmf_fws_macdesc_deinit(entry); brcmf_fws_cleanup(fws, ifp->ifidx); brcmf_fws_unlock(fws); @@ -2405,30 +2399,26 @@ struct brcmf_fws_info *brcmf_fws_attach(struct brcmf_pub *drvr) brcmu_pktq_init(&fws->desc.other.psq, BRCMF_FWS_PSQ_PREC_COUNT, BRCMF_FWS_PSQ_LEN); + /* create debugfs file for statistics */ + brcmf_debugfs_add_entry(drvr, "fws_stats", + brcmf_debugfs_fws_stats_read); + brcmf_dbg(INFO, "%s bdcv2 tlv signaling [%x]\n", fws->fw_signals ? "enabled" : "disabled", tlv); return fws; fail: - brcmf_fws_detach_pre_delif(fws); - brcmf_fws_detach_post_delif(fws); + brcmf_fws_detach(fws); return ERR_PTR(rc); } -void brcmf_fws_detach_pre_delif(struct brcmf_fws_info *fws) +void brcmf_fws_detach(struct brcmf_fws_info *fws) { if (!fws) return; - if (fws->fws_wq) { - destroy_workqueue(fws->fws_wq); - fws->fws_wq = NULL; - } -} -void brcmf_fws_detach_post_delif(struct brcmf_fws_info *fws) -{ - if (!fws) - return; + if (fws->fws_wq) + destroy_workqueue(fws->fws_wq); /* cleanup */ brcmf_fws_lock(fws); @@ -2439,13 +2429,6 @@ void brcmf_fws_detach_post_delif(struct brcmf_fws_info *fws) kfree(fws); } -void brcmf_fws_debugfs_create(struct brcmf_pub *drvr) -{ - /* create debugfs file for statistics */ - brcmf_debugfs_add_entry(drvr, "fws_stats", - brcmf_debugfs_fws_stats_read); -} - bool brcmf_fws_queue_skbs(struct brcmf_fws_info *fws) { return !fws->avoid_queueing; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h index 749c06dcdc17d..ba07bd9720020 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h @@ -19,9 +19,7 @@ #define FWSIGNAL_H_ struct brcmf_fws_info *brcmf_fws_attach(struct brcmf_pub *drvr); -void brcmf_fws_detach_pre_delif(struct brcmf_fws_info *fws); -void brcmf_fws_detach_post_delif(struct brcmf_fws_info *fws); -void brcmf_fws_debugfs_create(struct brcmf_pub *drvr); +void brcmf_fws_detach(struct brcmf_fws_info *fws); bool brcmf_fws_queue_skbs(struct brcmf_fws_info *fws); bool brcmf_fws_fc_active(struct brcmf_fws_info *fws); void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index ee922b0525610..d2c834c3b2fc9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -69,8 +69,6 @@ #define BRCMF_MSGBUF_MAX_EVENTBUF_POST 8 #define BRCMF_MSGBUF_PKT_FLAGS_FRAME_802_3 0x01 -#define BRCMF_MSGBUF_PKT_FLAGS_FRAME_802_11 0x02 -#define BRCMF_MSGBUF_PKT_FLAGS_FRAME_MASK 0x07 #define BRCMF_MSGBUF_PKT_FLAGS_PRIO_SHIFT 5 #define BRCMF_MSGBUF_TX_FLUSH_CNT1 32 @@ -479,7 +477,7 @@ static void brcmf_msgbuf_ioctl_resp_wake(struct brcmf_msgbuf *msgbuf) static int brcmf_msgbuf_query_dcmd(struct brcmf_pub *drvr, int ifidx, - uint cmd, void *buf, uint len, int *fwerr) + uint cmd, void *buf, uint len) { struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; struct sk_buff *skb = NULL; @@ -487,7 +485,6 @@ static int brcmf_msgbuf_query_dcmd(struct brcmf_pub *drvr, int ifidx, int err; brcmf_dbg(MSGBUF, "ifidx=%d, cmd=%d, len=%d\n", ifidx, cmd, len); - *fwerr = 0; msgbuf->ctl_completed = false; err = brcmf_msgbuf_tx_ioctl(drvr, ifidx, cmd, buf, len); if (err) @@ -511,15 +508,14 @@ static int brcmf_msgbuf_query_dcmd(struct brcmf_pub *drvr, int ifidx, } brcmu_pkt_buf_free_skb(skb); - *fwerr = msgbuf->ioctl_resp_status; - return 0; + return msgbuf->ioctl_resp_status; } static int brcmf_msgbuf_set_dcmd(struct brcmf_pub *drvr, int ifidx, - uint cmd, void *buf, uint len, int *fwerr) + uint cmd, void *buf, uint len) { - return brcmf_msgbuf_query_dcmd(drvr, ifidx, cmd, buf, len, fwerr); + return brcmf_msgbuf_query_dcmd(drvr, ifidx, cmd, buf, len); } @@ -1116,7 +1112,7 @@ static void brcmf_msgbuf_process_event(struct brcmf_msgbuf *msgbuf, void *buf) skb->protocol = eth_type_trans(skb, ifp->ndev); - brcmf_fweh_process_skb(ifp->drvr, skb, 0); + brcmf_fweh_process_skb(ifp->drvr, skb); exit: brcmu_pkt_buf_free_skb(skb); @@ -1130,7 +1126,6 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) struct sk_buff *skb; u16 data_offset; u16 buflen; - u16 flags; u32 idx; struct brcmf_if *ifp; @@ -1140,7 +1135,6 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) data_offset = le16_to_cpu(rx_complete->data_offset); buflen = le16_to_cpu(rx_complete->data_len); idx = le32_to_cpu(rx_complete->msg.request_id); - flags = le16_to_cpu(rx_complete->flags); skb = brcmf_msgbuf_get_pktid(msgbuf->drvr->bus_if->dev, msgbuf->rx_pktids, idx); @@ -1154,20 +1148,6 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) skb_trim(skb, buflen); - if ((flags & BRCMF_MSGBUF_PKT_FLAGS_FRAME_MASK) == - BRCMF_MSGBUF_PKT_FLAGS_FRAME_802_11) { - ifp = msgbuf->drvr->mon_if; - - if (!ifp) { - brcmf_err("Received unexpected monitor pkt\n"); - brcmu_pkt_buf_free_skb(skb); - return; - } - - brcmf_netif_mon_rx(ifp, skb); - return; - } - ifp = brcmf_get_ifp(msgbuf->drvr, rx_complete->msg.ifidx); if (!ifp || !ifp->ndev) { brcmf_err("Received pkt for invalid ifidx %d\n", @@ -1436,11 +1416,6 @@ static int brcmf_msgbuf_stats_read(struct seq_file *seq, void *data) } #endif -static void brcmf_msgbuf_debugfs_create(struct brcmf_pub *drvr) -{ - brcmf_debugfs_add_entry(drvr, "msgbuf_stats", brcmf_msgbuf_stats_read); -} - int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr) { struct brcmf_bus_msgbuf *if_msgbuf; @@ -1495,7 +1470,6 @@ int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr) drvr->proto->delete_peer = brcmf_msgbuf_delete_peer; drvr->proto->add_tdls_peer = brcmf_msgbuf_add_tdls_peer; drvr->proto->rxreorder = brcmf_msgbuf_rxreorder; - drvr->proto->debugfs_create = brcmf_msgbuf_debugfs_create; drvr->proto->pd = msgbuf; init_waitqueue_head(&msgbuf->ioctl_resp_wait); @@ -1504,9 +1478,8 @@ int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr) (struct brcmf_commonring **)if_msgbuf->commonrings; msgbuf->flowrings = (struct brcmf_commonring **)if_msgbuf->flowrings; msgbuf->max_flowrings = if_msgbuf->max_flowrings; - msgbuf->flowring_dma_handle = - kcalloc(msgbuf->max_flowrings, - sizeof(*msgbuf->flowring_dma_handle), GFP_KERNEL); + msgbuf->flowring_dma_handle = kzalloc(msgbuf->max_flowrings * + sizeof(*msgbuf->flowring_dma_handle), GFP_KERNEL); if (!msgbuf->flowring_dma_handle) goto fail; @@ -1550,6 +1523,8 @@ int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr) spin_lock_init(&msgbuf->flowring_work_lock); INIT_LIST_HEAD(&msgbuf->work_queue); + brcmf_debugfs_add_entry(drvr, "msgbuf_stats", brcmf_msgbuf_stats_read); + return 0; fail: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h index 692235d252779..f93ba6be1ef87 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h @@ -27,10 +27,8 @@ #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE 40 #define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE 32 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_ITEMSIZE 24 -#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE_PRE_V7 16 -#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE 24 -#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE_PRE_V7 32 -#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE 40 +#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE 16 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE 32 #define BRCMF_H2D_TXFLOWRING_ITEMSIZE 48 struct msgbuf_buf_addr { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 456a1bf008b3d..450f2216fac2d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -74,7 +74,7 @@ #define P2P_AF_MAX_WAIT_TIME msecs_to_jiffies(2000) #define P2P_INVALID_CHANNEL -1 #define P2P_CHANNEL_SYNC_RETRY 5 -#define P2P_AF_FRM_SCAN_MAX_WAIT msecs_to_jiffies(450) +#define P2P_AF_FRM_SCAN_MAX_WAIT msecs_to_jiffies(1500) #define P2P_DEFAULT_SLEEP_TIME_VSDB 200 /* WiFi P2P Public Action Frame OUI Subtypes */ @@ -690,7 +690,10 @@ static s32 brcmf_p2p_escan(struct brcmf_p2p_info *p2p, u32 num_chans, /* determine the scan engine parameters */ sparams->bss_type = DOT11_BSSTYPE_ANY; - sparams->scan_type = BRCMF_SCANTYPE_ACTIVE; + if (p2p->cfg->active_scan) + sparams->scan_type = 0; + else + sparams->scan_type = 1; eth_broadcast_addr(sparams->bssid); sparams->home_time = cpu_to_le32(P2PAPI_SCAN_HOME_TIME_MS); @@ -879,7 +882,7 @@ int brcmf_p2p_scan_prep(struct wiphy *wiphy, { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_p2p_info *p2p = &cfg->p2p; - int err; + int err = 0; if (brcmf_p2p_scan_is_p2p_request(request)) { /* find my listen channel */ @@ -902,7 +905,9 @@ int brcmf_p2p_scan_prep(struct wiphy *wiphy, /* override .run_escan() callback. */ cfg->escan_info.run = brcmf_p2p_run_escan; } - return 0; + err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_PRBREQ_FLAG, + request->ie, request->ie_len); + return err; } @@ -1058,7 +1063,7 @@ static s32 brcmf_p2p_act_frm_search(struct brcmf_p2p_info *p2p, u16 channel) channel_cnt = AF_PEER_SEARCH_CNT; else channel_cnt = SOCIAL_CHAN_CNT; - default_chan_list = kcalloc(channel_cnt, sizeof(*default_chan_list), + default_chan_list = kzalloc(channel_cnt * sizeof(*default_chan_list), GFP_KERNEL); if (default_chan_list == NULL) { brcmf_err("channel list allocation failed\n"); @@ -1134,6 +1139,7 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) { struct afx_hdl *afx_hdl = &p2p->afx_hdl; struct brcmf_cfg80211_vif *pri_vif; + unsigned long duration; s32 retry; brcmf_dbg(TRACE, "Enter\n"); @@ -1149,6 +1155,7 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) * pending action frame tx is cancelled. */ retry = 0; + duration = msecs_to_jiffies(P2P_AF_FRM_SCAN_MAX_WAIT); while ((retry < P2P_CHANNEL_SYNC_RETRY) && (afx_hdl->peer_chan == P2P_INVALID_CHANNEL)) { afx_hdl->is_listen = false; @@ -1156,8 +1163,7 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) retry); /* search peer on peer's listen channel */ schedule_work(&afx_hdl->afx_work); - wait_for_completion_timeout(&afx_hdl->act_frm_scan, - P2P_AF_FRM_SCAN_MAX_WAIT); + wait_for_completion_timeout(&afx_hdl->act_frm_scan, duration); if ((afx_hdl->peer_chan != P2P_INVALID_CHANNEL) || (!test_bit(BRCMF_P2P_STATUS_FINDING_COMMON_CHANNEL, &p2p->status))) @@ -1170,7 +1176,7 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) afx_hdl->is_listen = true; schedule_work(&afx_hdl->afx_work); wait_for_completion_timeout(&afx_hdl->act_frm_scan, - P2P_AF_FRM_SCAN_MAX_WAIT); + duration); } if ((afx_hdl->peer_chan != P2P_INVALID_CHANNEL) || (!test_bit(BRCMF_P2P_STATUS_FINDING_COMMON_CHANNEL, @@ -1457,12 +1463,10 @@ int brcmf_p2p_notify_action_tx_complete(struct brcmf_if *ifp, return 0; if (e->event_code == BRCMF_E_ACTION_FRAME_COMPLETE) { - if (e->status == BRCMF_E_STATUS_SUCCESS) { + if (e->status == BRCMF_E_STATUS_SUCCESS) set_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status); - if (!p2p->wait_for_offchan_complete) - complete(&p2p->send_af_done); - } else { + else { set_bit(BRCMF_P2P_STATUS_ACTION_TX_NOACK, &p2p->status); /* If there is no ack, we don't need to wait for * WLC_E_ACTION_FRAME_OFFCHAN_COMPLETE event @@ -1513,17 +1517,6 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, p2p->af_sent_channel = le32_to_cpu(af_params->channel); p2p->af_tx_sent_jiffies = jiffies; - if (test_bit(BRCMF_P2P_STATUS_DISCOVER_LISTEN, &p2p->status) && - p2p->af_sent_channel == - ieee80211_frequency_to_channel(p2p->remain_on_channel.center_freq)) - p2p->wait_for_offchan_complete = false; - else - p2p->wait_for_offchan_complete = true; - - brcmf_dbg(TRACE, "Waiting for %s tx completion event\n", - (p2p->wait_for_offchan_complete) ? - "off-channel" : "on-channel"); - timeout = wait_for_completion_timeout(&p2p->send_af_done, P2P_AF_MAX_WAIT_TIME); @@ -1858,6 +1851,7 @@ s32 brcmf_p2p_notify_rx_mgmt_p2p_probereq(struct brcmf_if *ifp, struct afx_hdl *afx_hdl = &p2p->afx_hdl; struct brcmf_cfg80211_vif *vif = ifp->vif; struct brcmf_rx_mgmt_data *rxframe = (struct brcmf_rx_mgmt_data *)data; + u16 chanspec = be16_to_cpu(rxframe->chanspec); struct brcmu_chan ch; u8 *mgmt_frame; u32 mgmt_frame_len; @@ -1910,7 +1904,7 @@ s32 brcmf_p2p_notify_rx_mgmt_p2p_probereq(struct brcmf_if *ifp, cfg80211_rx_mgmt(&vif->wdev, freq, 0, mgmt_frame, mgmt_frame_len, 0); brcmf_dbg(INFO, "mgmt_frame_len (%d) , e->datalen (%d), chanspec (%04x), freq (%d)\n", - mgmt_frame_len, e->datalen, ch.chspec, freq); + mgmt_frame_len, e->datalen, chanspec, freq); return 0; } @@ -2085,13 +2079,6 @@ static struct wireless_dev *brcmf_p2p_create_p2pdev(struct brcmf_p2p_info *p2p, } pri_ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp; - - /* firmware requires unique mac address for p2pdev interface */ - if (addr && ether_addr_equal(addr, pri_ifp->mac_addr)) { - brcmf_err("discovery vif must be different from primary interface\n"); - return ERR_PTR(-EINVAL); - } - brcmf_p2p_generate_bss_mac(p2p, addr); brcmf_p2p_set_firmware(pri_ifp, p2p->dev_addr); @@ -2246,7 +2233,7 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, */ int brcmf_p2p_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); struct brcmf_p2p_info *p2p = &cfg->p2p; struct brcmf_cfg80211_vif *vif; enum nl80211_iftype iftype; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h index 39f0d02180882..0e8b34d2d85cb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h @@ -124,7 +124,6 @@ struct afx_hdl { * @gon_req_action: about to send go negotiation requets frame. * @block_gon_req_tx: drop tx go negotiation requets frame. * @p2pdev_dynamically: is p2p device if created by module param or supplicant. - * @wait_for_offchan_complete: wait for off-channel tx completion event. */ struct brcmf_p2p_info { struct brcmf_cfg80211_info *cfg; @@ -145,7 +144,6 @@ struct brcmf_p2p_info { bool gon_req_action; bool block_gon_req_tx; bool p2pdev_dynamically; - bool wait_for_offchan_complete; }; s32 brcmf_p2p_attach(struct brcmf_cfg80211_info *cfg, bool p2pdev_forced); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 4fffa6988087b..e6e9b00b79d71 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -46,37 +46,36 @@ enum brcmf_pcie_state { BRCMFMAC_PCIE_STATE_UP }; -BRCMF_FW_DEF(43602, "brcmfmac43602-pcie"); -BRCMF_FW_DEF(4350, "brcmfmac4350-pcie"); -BRCMF_FW_DEF(4350C, "brcmfmac4350c2-pcie"); -BRCMF_FW_DEF(4356, "brcmfmac4356-pcie"); -BRCMF_FW_DEF(43570, "brcmfmac43570-pcie"); -BRCMF_FW_DEF(4358, "brcmfmac4358-pcie"); -BRCMF_FW_DEF(4359, "brcmfmac4359-pcie"); -BRCMF_FW_DEF(4365B, "brcmfmac4365b-pcie"); -BRCMF_FW_DEF(4365C, "brcmfmac4365c-pcie"); -BRCMF_FW_DEF(4366B, "brcmfmac4366b-pcie"); -BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie"); -BRCMF_FW_DEF(4371, "brcmfmac4371-pcie"); - -static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { - BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602), - BRCMF_FW_ENTRY(BRCM_CC_43465_CHIP_ID, 0xFFFFFFF0, 4366C), - BRCMF_FW_ENTRY(BRCM_CC_4350_CHIP_ID, 0x000000FF, 4350C), - BRCMF_FW_ENTRY(BRCM_CC_4350_CHIP_ID, 0xFFFFFF00, 4350), - BRCMF_FW_ENTRY(BRCM_CC_43525_CHIP_ID, 0xFFFFFFF0, 4365C), - BRCMF_FW_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356), - BRCMF_FW_ENTRY(BRCM_CC_43567_CHIP_ID, 0xFFFFFFFF, 43570), - BRCMF_FW_ENTRY(BRCM_CC_43569_CHIP_ID, 0xFFFFFFFF, 43570), - BRCMF_FW_ENTRY(BRCM_CC_43570_CHIP_ID, 0xFFFFFFFF, 43570), - BRCMF_FW_ENTRY(BRCM_CC_4358_CHIP_ID, 0xFFFFFFFF, 4358), - BRCMF_FW_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359), - BRCMF_FW_ENTRY(BRCM_CC_4365_CHIP_ID, 0x0000000F, 4365B), - BRCMF_FW_ENTRY(BRCM_CC_4365_CHIP_ID, 0xFFFFFFF0, 4365C), - BRCMF_FW_ENTRY(BRCM_CC_4366_CHIP_ID, 0x0000000F, 4366B), - BRCMF_FW_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C), - BRCMF_FW_ENTRY(BRCM_CC_43664_CHIP_ID, 0xFFFFFFF0, 4366C), - BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), +BRCMF_FW_NVRAM_DEF(43602, "brcmfmac43602-pcie.bin", "brcmfmac43602-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4350, "brcmfmac4350-pcie.bin", "brcmfmac4350-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4350C, "brcmfmac4350c2-pcie.bin", "brcmfmac4350c2-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4356, "brcmfmac4356-pcie.bin", "brcmfmac4356-pcie.txt"); +BRCMF_FW_NVRAM_DEF(43570, "brcmfmac43570-pcie.bin", "brcmfmac43570-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4358, "brcmfmac4358-pcie.bin", "brcmfmac4358-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4359, "brcmfmac4359-pcie.bin", "brcmfmac4359-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4365B, "brcmfmac4365b-pcie.bin", "brcmfmac4365b-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4365C, "brcmfmac4365c-pcie.bin", "brcmfmac4365c-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4366B, "brcmfmac4366b-pcie.bin", "brcmfmac4366b-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4366C, "brcmfmac4366c-pcie.bin", "brcmfmac4366c-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4371, "brcmfmac4371-pcie.bin", "brcmfmac4371-pcie.txt"); + +static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43465_CHIP_ID, 0xFFFFFFF0, 4366C), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4350_CHIP_ID, 0x000000FF, 4350C), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4350_CHIP_ID, 0xFFFFFF00, 4350), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43525_CHIP_ID, 0xFFFFFFF0, 4365C), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43567_CHIP_ID, 0xFFFFFFFF, 43570), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43569_CHIP_ID, 0xFFFFFFFF, 43570), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43570_CHIP_ID, 0xFFFFFFFF, 43570), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4358_CHIP_ID, 0xFFFFFFFF, 4358), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4365_CHIP_ID, 0x0000000F, 4365B), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4365_CHIP_ID, 0xFFFFFFF0, 4365C), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0x0000000F, 4366B), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), }; #define BRCMF_PCIE_FW_UP_TIMEOUT 2000 /* msec */ @@ -105,8 +104,7 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_PCIE2REG_MAILBOXMASK 0x4C #define BRCMF_PCIE_PCIE2REG_CONFIGADDR 0x120 #define BRCMF_PCIE_PCIE2REG_CONFIGDATA 0x124 -#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0 0x140 -#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1 0x144 +#define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX 0x140 #define BRCMF_PCIE2_INTA 0x01 #define BRCMF_PCIE2_INTB 0x02 @@ -136,13 +134,11 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_PCIE_MB_INT_D2H3_DB0 | \ BRCMF_PCIE_MB_INT_D2H3_DB1) -#define BRCMF_PCIE_SHARED_VERSION_7 7 #define BRCMF_PCIE_MIN_SHARED_VERSION 5 -#define BRCMF_PCIE_MAX_SHARED_VERSION BRCMF_PCIE_SHARED_VERSION_7 +#define BRCMF_PCIE_MAX_SHARED_VERSION 6 #define BRCMF_PCIE_SHARED_VERSION_MASK 0x00FF #define BRCMF_PCIE_SHARED_DMA_INDEX 0x10000 #define BRCMF_PCIE_SHARED_DMA_2B_IDX 0x100000 -#define BRCMF_PCIE_SHARED_HOSTRDY_DB1 0x10000000 #define BRCMF_PCIE_FLAGS_HTOD_SPLIT 0x4000 #define BRCMF_PCIE_FLAGS_DTOH_SPLIT 0x8000 @@ -182,7 +178,6 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_D2H_DEV_D3_ACK 0x00000001 #define BRCMF_D2H_DEV_DS_ENTER_REQ 0x00000002 #define BRCMF_D2H_DEV_DS_EXIT_NOTE 0x00000004 -#define BRCMF_D2H_DEV_FWHALT 0x10000000 #define BRCMF_H2D_HOST_D3_INFORM 0x00000001 #define BRCMF_H2D_HOST_DS_ACK 0x00000002 @@ -320,14 +315,6 @@ static const u32 brcmf_ring_max_item[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM }; -static const u32 brcmf_ring_itemsize_pre_v7[BRCMF_NROF_COMMON_MSGRINGS] = { - BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE, - BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE, - BRCMF_D2H_MSGRING_CONTROL_COMPLETE_ITEMSIZE, - BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE_PRE_V7, - BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE_PRE_V7 -}; - static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE, BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE, @@ -718,10 +705,6 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) devinfo->mbdata_completed = true; wake_up(&devinfo->mbdata_resp_wait); } - if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) { - brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); - brcmf_dev_coredump(&devinfo->pdev->dev); - } } @@ -798,12 +781,6 @@ static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo) BRCMF_PCIE_MB_INT_FN0_1); } -static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo) -{ - if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) - brcmf_pcie_write_reg32(devinfo, - BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1); -} static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg) { @@ -946,7 +923,7 @@ static int brcmf_pcie_ring_mb_ring_bell(void *ctx) brcmf_dbg(PCIE, "RING !\n"); /* Any arbitrary value will do, lets use 1 */ - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1); + brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX, 1); return 0; } @@ -1021,14 +998,8 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, struct brcmf_pcie_ringbuf *ring; u32 size; u32 addr; - const u32 *ring_itemsize_array; - - if (devinfo->shared.version < BRCMF_PCIE_SHARED_VERSION_7) - ring_itemsize_array = brcmf_ring_itemsize_pre_v7; - else - ring_itemsize_array = brcmf_ring_itemsize; - size = brcmf_ring_max_item[ring_id] * ring_itemsize_array[ring_id]; + size = brcmf_ring_max_item[ring_id] * brcmf_ring_itemsize[ring_id]; dma_buf = brcmf_pcie_init_dmabuffer_for_device(devinfo, size, tcm_ring_phys_addr + BRCMF_RING_MEM_BASE_ADDR_OFFSET, &dma_handle); @@ -1038,7 +1009,7 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, addr = tcm_ring_phys_addr + BRCMF_RING_MAX_ITEM_OFFSET; brcmf_pcie_write_tcm16(devinfo, addr, brcmf_ring_max_item[ring_id]); addr = tcm_ring_phys_addr + BRCMF_RING_LEN_ITEMS_OFFSET; - brcmf_pcie_write_tcm16(devinfo, addr, ring_itemsize_array[ring_id]); + brcmf_pcie_write_tcm16(devinfo, addr, brcmf_ring_itemsize[ring_id]); ring = kzalloc(sizeof(*ring), GFP_KERNEL); if (!ring) { @@ -1047,7 +1018,7 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, return NULL; } brcmf_commonring_config(&ring->commonring, brcmf_ring_max_item[ring_id], - ring_itemsize_array[ring_id], dma_buf); + brcmf_ring_itemsize[ring_id], dma_buf); ring->dma_handle = dma_handle; ring->devinfo = devinfo; brcmf_commonring_register_cb(&ring->commonring, @@ -1280,14 +1251,14 @@ static int brcmf_pcie_init_scratchbuffers(struct brcmf_pciedev_info *devinfo) u64 address; u32 addr; - devinfo->shared.scratch = - dma_zalloc_coherent(&devinfo->pdev->dev, - BRCMF_DMA_D2H_SCRATCH_BUF_LEN, - &devinfo->shared.scratch_dmahandle, - GFP_KERNEL); + devinfo->shared.scratch = dma_alloc_coherent(&devinfo->pdev->dev, + BRCMF_DMA_D2H_SCRATCH_BUF_LEN, + &devinfo->shared.scratch_dmahandle, GFP_KERNEL); if (!devinfo->shared.scratch) goto fail; + memset(devinfo->shared.scratch, 0, BRCMF_DMA_D2H_SCRATCH_BUF_LEN); + addr = devinfo->shared.tcm_base_address + BRCMF_SHARED_DMA_SCRATCH_ADDR_OFFSET; address = (u64)devinfo->shared.scratch_dmahandle; @@ -1297,14 +1268,14 @@ static int brcmf_pcie_init_scratchbuffers(struct brcmf_pciedev_info *devinfo) BRCMF_SHARED_DMA_SCRATCH_LEN_OFFSET; brcmf_pcie_write_tcm32(devinfo, addr, BRCMF_DMA_D2H_SCRATCH_BUF_LEN); - devinfo->shared.ringupd = - dma_zalloc_coherent(&devinfo->pdev->dev, - BRCMF_DMA_D2H_RINGUPD_BUF_LEN, - &devinfo->shared.ringupd_dmahandle, - GFP_KERNEL); + devinfo->shared.ringupd = dma_alloc_coherent(&devinfo->pdev->dev, + BRCMF_DMA_D2H_RINGUPD_BUF_LEN, + &devinfo->shared.ringupd_dmahandle, GFP_KERNEL); if (!devinfo->shared.ringupd) goto fail; + memset(devinfo->shared.ringupd, 0, BRCMF_DMA_D2H_RINGUPD_BUF_LEN); + addr = devinfo->shared.tcm_base_address + BRCMF_SHARED_DMA_RINGUPD_ADDR_OFFSET; address = (u64)devinfo->shared.ringupd_dmahandle; @@ -1379,25 +1350,6 @@ static int brcmf_pcie_get_memdump(struct device *dev, void *data, size_t len) return 0; } -static -int brcmf_pcie_get_fwname(struct device *dev, const char *ext, u8 *fw_name) -{ - struct brcmf_bus *bus_if = dev_get_drvdata(dev); - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ext, fw_name }, - }; - - fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev, - brcmf_pcie_fwnames, - ARRAY_SIZE(brcmf_pcie_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return -ENOMEM; - - kfree(fwreq); - return 0; -} static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .txdata = brcmf_pcie_tx, @@ -1407,7 +1359,6 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .wowl_config = brcmf_pcie_wowl_config, .get_ramsize = brcmf_pcie_get_ramsize, .get_memdump = brcmf_pcie_get_memdump, - .get_fwname = brcmf_pcie_get_fwname, }; @@ -1611,6 +1562,24 @@ static void brcmf_pcie_release_resource(struct brcmf_pciedev_info *devinfo) } +static int brcmf_pcie_attach_bus(struct brcmf_pciedev_info *devinfo) +{ + int ret; + + /* Attach to the common driver interface */ + ret = brcmf_attach(&devinfo->pdev->dev, devinfo->settings); + if (ret) { + brcmf_err("brcmf_attach failed\n"); + } else { + ret = brcmf_bus_started(&devinfo->pdev->dev); + if (ret) + brcmf_err("dongle is not responding\n"); + } + + return ret; +} + + static u32 brcmf_pcie_buscore_prep_addr(const struct pci_dev *pdev, u32 addr) { u32 ret_addr; @@ -1681,19 +1650,15 @@ static const struct brcmf_buscore_ops brcmf_pcie_buscore_ops = { .write32 = brcmf_pcie_buscore_write32, }; -#define BRCMF_PCIE_FW_CODE 0 -#define BRCMF_PCIE_FW_NVRAM 1 - static void brcmf_pcie_setup(struct device *dev, int ret, - struct brcmf_fw_request *fwreq) + const struct firmware *fw, + void *nvram, u32 nvram_len) { - const struct firmware *fw; - void *nvram; struct brcmf_bus *bus; struct brcmf_pciedev *pcie_bus_dev; struct brcmf_pciedev_info *devinfo; struct brcmf_commonring **flowrings; - u32 i, nvram_len; + u32 i; /* check firmware loading result */ if (ret) @@ -1704,11 +1669,6 @@ static void brcmf_pcie_setup(struct device *dev, int ret, devinfo = pcie_bus_dev->devinfo; brcmf_pcie_attach(devinfo); - fw = fwreq->items[BRCMF_PCIE_FW_CODE].binary; - nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data; - nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len; - kfree(fwreq); - /* Some of the firmwares have the size of the memory of the device * defined inside the firmware. This is because part of the memory in * the device is shared and the devision is determined by FW. Parse @@ -1756,8 +1716,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, init_waitqueue_head(&devinfo->mbdata_resp_wait); brcmf_pcie_intr_enable(devinfo); - brcmf_pcie_hostready(devinfo); - if (brcmf_attach(&devinfo->pdev->dev, devinfo->settings) == 0) + if (brcmf_pcie_attach_bus(devinfo) == 0) return; brcmf_pcie_bus_console_read(devinfo); @@ -1766,42 +1725,20 @@ static void brcmf_pcie_setup(struct device *dev, int ret, device_release_driver(dev); } -static struct brcmf_fw_request * -brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) -{ - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ".bin", devinfo->fw_name }, - { ".txt", devinfo->nvram_name }, - }; - - fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev, - brcmf_pcie_fwnames, - ARRAY_SIZE(brcmf_pcie_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return NULL; - - fwreq->items[BRCMF_PCIE_FW_CODE].type = BRCMF_FW_TYPE_BINARY; - fwreq->items[BRCMF_PCIE_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM; - fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL; - /* NVRAM reserves PCI domain 0 for Broadcom's SDK faked bus */ - fwreq->domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1; - fwreq->bus_nr = devinfo->pdev->bus->number; - - return fwreq; -} - static int brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id) { int ret; - struct brcmf_fw_request *fwreq; struct brcmf_pciedev_info *devinfo; struct brcmf_pciedev *pcie_bus_dev; struct brcmf_bus *bus; + u16 domain_nr; + u16 bus_nr; - brcmf_dbg(PCIE, "Enter %x:%x\n", pdev->vendor, pdev->device); + domain_nr = pci_domain_nr(pdev->bus) + 1; + bus_nr = pdev->bus->number; + brcmf_dbg(PCIE, "Enter %x:%x (%d/%d)\n", pdev->vendor, pdev->device, + domain_nr, bus_nr); ret = -ENOMEM; devinfo = kzalloc(sizeof(*devinfo), GFP_KERNEL); @@ -1855,19 +1792,19 @@ brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id) bus->wowl_supported = pci_pme_capable(pdev, PCI_D3hot); dev_set_drvdata(&pdev->dev, bus); - fwreq = brcmf_pcie_prepare_fw_request(devinfo); - if (!fwreq) { - ret = -ENOMEM; - goto fail_bus; - } - - ret = brcmf_fw_get_firmwares(bus->dev, fwreq, brcmf_pcie_setup); - if (ret < 0) { - kfree(fwreq); + ret = brcmf_fw_map_chip_to_name(devinfo->ci->chip, devinfo->ci->chiprev, + brcmf_pcie_fwnames, + ARRAY_SIZE(brcmf_pcie_fwnames), + devinfo->fw_name, devinfo->nvram_name); + if (ret) goto fail_bus; - } - return 0; + ret = brcmf_fw_get_firmwares_pcie(bus->dev, BRCMF_FW_REQUEST_NVRAM | + BRCMF_FW_REQ_NV_OPTIONAL, + devinfo->fw_name, devinfo->nvram_name, + brcmf_pcie_setup, domain_nr, bus_nr); + if (ret == 0) + return 0; fail_bus: kfree(bus->msgbuf); kfree(bus); @@ -1980,7 +1917,6 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); brcmf_bus_change_state(bus, BRCMF_BUS_UP); brcmf_pcie_intr_enable(devinfo); - brcmf_pcie_hostready(devinfo); return 0; } @@ -2050,7 +1986,6 @@ static struct pci_driver brcmf_pciedrvr = { #ifdef CONFIG_PM .driver.pm = &brcmf_pciedrvr_pm, #endif - .driver.coredump = brcmf_dev_coredump, }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c index 74e6fdbd3a2b8..d26ff219ef66f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.c @@ -54,8 +54,7 @@ int brcmf_proto_attach(struct brcmf_pub *drvr) if (!proto->tx_queue_data || (proto->hdrpull == NULL) || (proto->query_dcmd == NULL) || (proto->set_dcmd == NULL) || (proto->configure_addr_mode == NULL) || - (proto->delete_peer == NULL) || (proto->add_tdls_peer == NULL) || - (proto->debugfs_create == NULL)) { + (proto->delete_peer == NULL) || (proto->add_tdls_peer == NULL)) { brcmf_err("Not all proto handlers have been installed\n"); goto fail; } @@ -67,22 +66,16 @@ int brcmf_proto_attach(struct brcmf_pub *drvr) return -ENOMEM; } -void brcmf_proto_detach_post_delif(struct brcmf_pub *drvr) +void brcmf_proto_detach(struct brcmf_pub *drvr) { brcmf_dbg(TRACE, "Enter\n"); if (drvr->proto) { if (drvr->bus_if->proto_type == BRCMF_PROTO_BCDC) - brcmf_proto_bcdc_detach_post_delif(drvr); + brcmf_proto_bcdc_detach(drvr); else if (drvr->bus_if->proto_type == BRCMF_PROTO_MSGBUF) brcmf_proto_msgbuf_detach(drvr); kfree(drvr->proto); drvr->proto = NULL; } } - -void brcmf_proto_detach_pre_delif(struct brcmf_pub *drvr) -{ - if (drvr->proto && drvr->bus_if->proto_type == BRCMF_PROTO_BCDC) - brcmf_proto_bcdc_detach_pre_delif(drvr); -} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h index 72355aea90287..2404f8a7c31c9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/proto.h @@ -30,9 +30,9 @@ struct brcmf_proto { int (*hdrpull)(struct brcmf_pub *drvr, bool do_fws, struct sk_buff *skb, struct brcmf_if **ifp); int (*query_dcmd)(struct brcmf_pub *drvr, int ifidx, uint cmd, - void *buf, uint len, int *fwerr); + void *buf, uint len); int (*set_dcmd)(struct brcmf_pub *drvr, int ifidx, uint cmd, void *buf, - uint len, int *fwerr); + uint len); int (*tx_queue_data)(struct brcmf_pub *drvr, int ifidx, struct sk_buff *skb); int (*txdata)(struct brcmf_pub *drvr, int ifidx, u8 offset, @@ -48,14 +48,12 @@ struct brcmf_proto { void (*del_if)(struct brcmf_if *ifp); void (*reset_if)(struct brcmf_if *ifp); int (*init_done)(struct brcmf_pub *drvr); - void (*debugfs_create)(struct brcmf_pub *drvr); void *pd; }; int brcmf_proto_attach(struct brcmf_pub *drvr); -void brcmf_proto_detach_pre_delif(struct brcmf_pub *drvr); -void brcmf_proto_detach_post_delif(struct brcmf_pub *drvr); +void brcmf_proto_detach(struct brcmf_pub *drvr); static inline int brcmf_proto_hdrpull(struct brcmf_pub *drvr, bool do_fws, struct sk_buff *skb, @@ -73,16 +71,14 @@ static inline int brcmf_proto_hdrpull(struct brcmf_pub *drvr, bool do_fws, return drvr->proto->hdrpull(drvr, do_fws, skb, ifp); } static inline int brcmf_proto_query_dcmd(struct brcmf_pub *drvr, int ifidx, - uint cmd, void *buf, uint len, - int *fwerr) + uint cmd, void *buf, uint len) { - return drvr->proto->query_dcmd(drvr, ifidx, cmd, buf, len,fwerr); + return drvr->proto->query_dcmd(drvr, ifidx, cmd, buf, len); } static inline int brcmf_proto_set_dcmd(struct brcmf_pub *drvr, int ifidx, - uint cmd, void *buf, uint len, - int *fwerr) + uint cmd, void *buf, uint len) { - return drvr->proto->set_dcmd(drvr, ifidx, cmd, buf, len, fwerr); + return drvr->proto->set_dcmd(drvr, ifidx, cmd, buf, len); } static inline int brcmf_proto_tx_queue_data(struct brcmf_pub *drvr, int ifidx, @@ -158,10 +154,4 @@ brcmf_proto_init_done(struct brcmf_pub *drvr) return drvr->proto->init_done(drvr); } -static inline void -brcmf_proto_debugfs_create(struct brcmf_pub *drvr) -{ - drvr->proto->debugfs_create(drvr); -} - #endif /* BRCMFMAC_PROTO_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index abaed2fa2defd..4c28b04ea6053 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -49,10 +49,6 @@ #define DCMD_RESP_TIMEOUT msecs_to_jiffies(2500) #define CTL_DONE_TIMEOUT msecs_to_jiffies(2500) -/* watermark expressed in number of words */ -#define DEFAULT_F2_WATERMARK 0x8 -#define CY_4373_F2_WATERMARK 0x40 - #ifdef DEBUG #define BRCMF_TRAP_INFO_SIZE 80 @@ -142,8 +138,6 @@ struct rte_console { /* 1: isolate internal sdio signals, put external pads in tri-state; requires * sdio bus power cycle to clear (rev 9) */ #define SBSDIO_DEVCTL_PADS_ISO 0x08 -/* 1: enable F2 Watermark */ -#define SBSDIO_DEVCTL_F2WM_ENAB 0x10 /* Force SD->SB reset mapping (rev 11) */ #define SBSDIO_DEVCTL_SB_RST_CTL 0x30 /* Determined by CoreControl bit */ @@ -165,8 +159,8 @@ struct rte_console { /* manfid tuple length, include tuple, link bytes */ #define SBSDIO_CIS_MANFID_TUPLE_LEN 6 -#define SD_REG(field) \ - (offsetof(struct sdpcmd_regs, field)) +#define CORE_BUS_REG(base, field) \ + (base + offsetof(struct sdpcmd_regs, field)) /* SDIO function 1 register CHIPCLKCSR */ /* Force ALP request to backplane */ @@ -266,11 +260,10 @@ struct rte_console { #define I_HMB_HOST_INT I_HMB_SW3 /* Miscellaneous Interrupt */ /* tohostmailboxdata */ -#define HMB_DATA_NAKHANDLED 0x0001 /* retransmit NAK'd frame */ -#define HMB_DATA_DEVREADY 0x0002 /* talk to host after enable */ -#define HMB_DATA_FC 0x0004 /* per prio flowcontrol update flag */ -#define HMB_DATA_FWREADY 0x0008 /* fw ready for protocol activity */ -#define HMB_DATA_FWHALT 0x0010 /* firmware halted */ +#define HMB_DATA_NAKHANDLED 1 /* retransmit NAK'd frame */ +#define HMB_DATA_DEVREADY 2 /* talk to host after enable */ +#define HMB_DATA_FC 4 /* per prio flowcontrol update flag */ +#define HMB_DATA_FWREADY 8 /* fw ready for protocol activity */ #define HMB_DATA_FCDATA_MASK 0xff000000 #define HMB_DATA_FCDATA_SHIFT 24 @@ -442,7 +435,6 @@ struct brcmf_sdio_count { struct brcmf_sdio { struct brcmf_sdio_dev *sdiodev; /* sdio device handler */ struct brcmf_chip *ci; /* Chip info struct */ - struct brcmf_core *sdio_core; /* sdio core info struct */ u32 hostintmask; /* Copy of Host Interrupt Mask */ atomic_t intstatus; /* Intstatus bits (events) pending */ @@ -606,44 +598,47 @@ static const struct sdiod_drive_str sdiod_drvstr_tab2_3v3[] = { {4, 0x1} }; -BRCMF_FW_DEF(43143, "brcmfmac43143-sdio"); -BRCMF_FW_DEF(43241B0, "brcmfmac43241b0-sdio"); -BRCMF_FW_DEF(43241B4, "brcmfmac43241b4-sdio"); -BRCMF_FW_DEF(43241B5, "brcmfmac43241b5-sdio"); -BRCMF_FW_DEF(4329, "brcmfmac4329-sdio"); -BRCMF_FW_DEF(4330, "brcmfmac4330-sdio"); -BRCMF_FW_DEF(4334, "brcmfmac4334-sdio"); -BRCMF_FW_DEF(43340, "brcmfmac43340-sdio"); -BRCMF_FW_DEF(4335, "brcmfmac4335-sdio"); -BRCMF_FW_DEF(43362, "brcmfmac43362-sdio"); -BRCMF_FW_DEF(4339, "brcmfmac4339-sdio"); -BRCMF_FW_DEF(43430A0, "brcmfmac43430a0-sdio"); +BRCMF_FW_NVRAM_DEF(43143, "brcmfmac43143-sdio.bin", "brcmfmac43143-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43241B0, "brcmfmac43241b0-sdio.bin", + "brcmfmac43241b0-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43241B4, "brcmfmac43241b4-sdio.bin", + "brcmfmac43241b4-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43241B5, "brcmfmac43241b5-sdio.bin", + "brcmfmac43241b5-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4329, "brcmfmac4329-sdio.bin", "brcmfmac4329-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4330, "brcmfmac4330-sdio.bin", "brcmfmac4330-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4334, "brcmfmac4334-sdio.bin", "brcmfmac4334-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43340, "brcmfmac43340-sdio.bin", "brcmfmac43340-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4335, "brcmfmac4335-sdio.bin", "brcmfmac4335-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43362, "brcmfmac43362-sdio.bin", "brcmfmac43362-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4339, "brcmfmac4339-sdio.bin", "brcmfmac4339-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43430A0, "brcmfmac43430a0-sdio.bin", "brcmfmac43430a0-sdio.txt"); /* Note the names are not postfixed with a1 for backward compatibility */ -BRCMF_FW_DEF(43430A1, "brcmfmac43430-sdio"); -BRCMF_FW_DEF(43455, "brcmfmac43455-sdio"); -BRCMF_FW_DEF(4354, "brcmfmac4354-sdio"); -BRCMF_FW_DEF(4356, "brcmfmac4356-sdio"); -BRCMF_FW_DEF(4373, "brcmfmac4373-sdio"); - -static const struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { - BRCMF_FW_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143), - BRCMF_FW_ENTRY(BRCM_CC_43241_CHIP_ID, 0x0000001F, 43241B0), - BRCMF_FW_ENTRY(BRCM_CC_43241_CHIP_ID, 0x00000020, 43241B4), - BRCMF_FW_ENTRY(BRCM_CC_43241_CHIP_ID, 0xFFFFFFC0, 43241B5), - BRCMF_FW_ENTRY(BRCM_CC_4329_CHIP_ID, 0xFFFFFFFF, 4329), - BRCMF_FW_ENTRY(BRCM_CC_4330_CHIP_ID, 0xFFFFFFFF, 4330), - BRCMF_FW_ENTRY(BRCM_CC_4334_CHIP_ID, 0xFFFFFFFF, 4334), - BRCMF_FW_ENTRY(BRCM_CC_43340_CHIP_ID, 0xFFFFFFFF, 43340), - BRCMF_FW_ENTRY(BRCM_CC_43341_CHIP_ID, 0xFFFFFFFF, 43340), - BRCMF_FW_ENTRY(BRCM_CC_4335_CHIP_ID, 0xFFFFFFFF, 4335), - BRCMF_FW_ENTRY(BRCM_CC_43362_CHIP_ID, 0xFFFFFFFE, 43362), - BRCMF_FW_ENTRY(BRCM_CC_4339_CHIP_ID, 0xFFFFFFFF, 4339), - BRCMF_FW_ENTRY(BRCM_CC_43430_CHIP_ID, 0x00000001, 43430A0), - BRCMF_FW_ENTRY(BRCM_CC_43430_CHIP_ID, 0xFFFFFFFE, 43430A1), - BRCMF_FW_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFFC0, 43455), - BRCMF_FW_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354), - BRCMF_FW_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356), - BRCMF_FW_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373) +BRCMF_FW_NVRAM_DEF(43430A1, "brcmfmac43430-sdio.bin", "brcmfmac43430-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43455, "brcmfmac43455-sdio.bin", "brcmfmac43455-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4354, "brcmfmac4354-sdio.bin", "brcmfmac4354-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4356, "brcmfmac4356-sdio.bin", "brcmfmac4356-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4373, "brcmfmac4373-sdio.bin", "brcmfmac4373-sdio.txt"); + +static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43241_CHIP_ID, 0x0000001F, 43241B0), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43241_CHIP_ID, 0x00000020, 43241B4), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43241_CHIP_ID, 0xFFFFFFC0, 43241B5), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4329_CHIP_ID, 0xFFFFFFFF, 4329), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4330_CHIP_ID, 0xFFFFFFFF, 4330), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4334_CHIP_ID, 0xFFFFFFFF, 4334), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43340_CHIP_ID, 0xFFFFFFFF, 43340), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43341_CHIP_ID, 0xFFFFFFFF, 43340), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4335_CHIP_ID, 0xFFFFFFFF, 4335), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43362_CHIP_ID, 0xFFFFFFFE, 43362), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4339_CHIP_ID, 0xFFFFFFFF, 4339), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43430_CHIP_ID, 0x00000001, 43430A0), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43430_CHIP_ID, 0xFFFFFFFE, 43430A1), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFFC0, 43455), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356), + BRCMF_FW_NVRAM_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373) }; static void pkt_align(struct sk_buff *p, int len, int align) @@ -663,6 +658,32 @@ static bool data_ok(struct brcmf_sdio *bus) ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0; } +/* + * Reads a register in the SDIO hardware block. This block occupies a series of + * adresses on the 32 bit backplane bus. + */ +static int r_sdreg32(struct brcmf_sdio *bus, u32 *regvar, u32 offset) +{ + struct brcmf_core *core; + int ret; + + core = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV); + *regvar = brcmf_sdiod_regrl(bus->sdiodev, core->base + offset, &ret); + + return ret; +} + +static int w_sdreg32(struct brcmf_sdio *bus, u32 regval, u32 reg_offset) +{ + struct brcmf_core *core; + int ret; + + core = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV); + brcmf_sdiod_regwl(bus->sdiodev, core->base + reg_offset, regval, &ret); + + return ret; +} + static int brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on) { @@ -673,15 +694,10 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on) brcmf_dbg(TRACE, "Enter: on=%d\n", on); - sdio_retune_crc_disable(bus->sdiodev->func1); - - /* Cannot re-tune if device is asleep; defer till we're awake */ - if (on) - sdio_retune_hold_now(bus->sdiodev->func1); - wr_val = (on << SBSDIO_FUNC1_SLEEPCSR_KSO_SHIFT); /* 1st KSO write goes to AOS wake up core if device is asleep */ - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, wr_val, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, + wr_val, &err); if (on) { /* device WAKEUP through KSO: @@ -707,7 +723,7 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on) * just one write attempt may fail, * read it back until it matches written value */ - rd_val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, + rd_val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, &err); if (!err) { if ((rd_val & bmask) == cmp_val) @@ -717,11 +733,9 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on) /* bail out upon subsequent access errors */ if (err && (err_cnt++ > BRCMF_SDIO_MAX_ACCESS_ERRORS)) break; - udelay(KSO_WAIT_US); - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, wr_val, - &err); - + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, + wr_val, &err); } while (try_cnt++ < MAX_KSO_ATTEMPTS); if (try_cnt > 2) @@ -731,11 +745,6 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on) if (try_cnt > MAX_KSO_ATTEMPTS) brcmf_err("max tries: rd_val=0x%x err=%d\n", rd_val, err); - if (on) - sdio_retune_release(bus->sdiodev->func1); - - sdio_retune_crc_enable(bus->sdiodev->func1); - return err; } @@ -762,15 +771,15 @@ static int brcmf_sdio_htclk(struct brcmf_sdio *bus, bool on, bool pendok) clkreq = bus->alp_only ? SBSDIO_ALP_AVAIL_REQ : SBSDIO_HT_AVAIL_REQ; - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - clkreq, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + clkreq, &err); if (err) { brcmf_err("HT Avail request error: %d\n", err); return -EBADE; } /* Check current status */ - clkctl = brcmf_sdiod_readb(bus->sdiodev, + clkctl = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); if (err) { brcmf_err("HT Avail read error: %d\n", err); @@ -780,34 +789,35 @@ static int brcmf_sdio_htclk(struct brcmf_sdio *bus, bool on, bool pendok) /* Go to pending and await interrupt if appropriate */ if (!SBSDIO_CLKAV(clkctl, bus->alp_only) && pendok) { /* Allow only clock-available interrupt */ - devctl = brcmf_sdiod_readb(bus->sdiodev, + devctl = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_DEVICE_CTL, &err); if (err) { - brcmf_err("Devctl error setting CA: %d\n", err); + brcmf_err("Devctl error setting CA: %d\n", + err); return -EBADE; } devctl |= SBSDIO_DEVCTL_CA_INT_ONLY; - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL, - devctl, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL, + devctl, &err); brcmf_dbg(SDIO, "CLKCTL: set PENDING\n"); bus->clkstate = CLK_PENDING; return 0; } else if (bus->clkstate == CLK_PENDING) { /* Cancel CA-only interrupt filter */ - devctl = brcmf_sdiod_readb(bus->sdiodev, + devctl = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_DEVICE_CTL, &err); devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY; - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL, - devctl, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL, + devctl, &err); } /* Otherwise, wait here (polling) for HT Avail */ timeout = jiffies + msecs_to_jiffies(PMU_MAX_TRANSITION_DLY/1000); while (!SBSDIO_CLKAV(clkctl, bus->alp_only)) { - clkctl = brcmf_sdiod_readb(bus->sdiodev, + clkctl = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); if (time_after(jiffies, timeout)) @@ -841,16 +851,16 @@ static int brcmf_sdio_htclk(struct brcmf_sdio *bus, bool on, bool pendok) if (bus->clkstate == CLK_PENDING) { /* Cancel CA-only interrupt filter */ - devctl = brcmf_sdiod_readb(bus->sdiodev, + devctl = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_DEVICE_CTL, &err); devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY; - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_DEVICE_CTL, - devctl, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL, + devctl, &err); } bus->clkstate = CLK_SDONLY; - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - clkreq, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + clkreq, &err); brcmf_dbg(SDIO, "CLKCTL: turned OFF\n"); if (err) { brcmf_err("Failed access turning clock off: %d\n", @@ -940,14 +950,14 @@ brcmf_sdio_bus_sleep(struct brcmf_sdio *bus, bool sleep, bool pendok) /* Going to sleep */ if (sleep) { - clkcsr = brcmf_sdiod_readb(bus->sdiodev, + clkcsr = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); if ((clkcsr & SBSDIO_CSR_MASK) == 0) { brcmf_dbg(SDIO, "no clock, set ALP\n"); - brcmf_sdiod_writeb(bus->sdiodev, - SBSDIO_FUNC1_CHIPCLKCSR, - SBSDIO_ALP_AVAIL_REQ, &err); + brcmf_sdiod_regwb(bus->sdiodev, + SBSDIO_FUNC1_CHIPCLKCSR, + SBSDIO_ALP_AVAIL_REQ, &err); } err = brcmf_sdio_kso_control(bus, false); } else { @@ -993,7 +1003,7 @@ static int brcmf_sdio_readshared(struct brcmf_sdio *bus, struct sdpcm_shared_le sh_le; __le32 addr_le; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_bus_sleep(bus, false, false); /* @@ -1027,7 +1037,7 @@ static int brcmf_sdio_readshared(struct brcmf_sdio *bus, if (rv < 0) goto fail; - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); /* Endianness */ sh->flags = le32_to_cpu(sh_le.flags); @@ -1049,7 +1059,7 @@ static int brcmf_sdio_readshared(struct brcmf_sdio *bus, fail: brcmf_err("unable to obtain sdpcm_shared info: rv=%d (addr=0x%x)\n", rv, addr); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); return rv; } @@ -1068,8 +1078,6 @@ static void brcmf_sdio_get_console_addr(struct brcmf_sdio *bus) static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) { - struct brcmf_sdio_dev *sdiod = bus->sdiodev; - struct brcmf_core *core = bus->sdio_core; u32 intstatus = 0; u32 hmb_data; u8 fcbits; @@ -1078,22 +1086,14 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) brcmf_dbg(SDIO, "Enter\n"); /* Read mailbox data and ack that we did so */ - hmb_data = brcmf_sdiod_readl(sdiod, - core->base + SD_REG(tohostmailboxdata), - &ret); - - if (!ret) - brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailbox), - SMB_INT_ACK, &ret); + ret = r_sdreg32(bus, &hmb_data, + offsetof(struct sdpcmd_regs, tohostmailboxdata)); + if (ret == 0) + w_sdreg32(bus, SMB_INT_ACK, + offsetof(struct sdpcmd_regs, tosbmailbox)); bus->sdcnt.f1regdata += 2; - /* dongle indicates the firmware has halted/crashed */ - if (hmb_data & HMB_DATA_FWHALT) { - brcmf_err("mailbox indicates firmware halted\n"); - brcmf_dev_coredump(&sdiod->func1->dev); - } - /* Dongle recomposed rx frames, accept them again */ if (hmb_data & HMB_DATA_NAKHANDLED) { brcmf_dbg(SDIO, "Dongle reports NAK handled, expect rtx of %d\n", @@ -1151,7 +1151,6 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) HMB_DATA_NAKHANDLED | HMB_DATA_FC | HMB_DATA_FWREADY | - HMB_DATA_FWHALT | HMB_DATA_FCDATA_MASK | HMB_DATA_VERSION_MASK)) brcmf_err("Unknown mailbox data content: 0x%02x\n", hmb_data); @@ -1161,8 +1160,6 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) static void brcmf_sdio_rxfail(struct brcmf_sdio *bus, bool abort, bool rtx) { - struct brcmf_sdio_dev *sdiod = bus->sdiodev; - struct brcmf_core *core = bus->sdio_core; uint retries = 0; u16 lastrbc; u8 hi, lo; @@ -1173,18 +1170,18 @@ static void brcmf_sdio_rxfail(struct brcmf_sdio *bus, bool abort, bool rtx) rtx ? ", send NAK" : ""); if (abort) - brcmf_sdiod_abort(bus->sdiodev, bus->sdiodev->func2); + brcmf_sdiod_abort(bus->sdiodev, SDIO_FUNC_2); - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_RF_TERM, - &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_FRAMECTRL, + SFC_RF_TERM, &err); bus->sdcnt.f1regdata++; /* Wait until the packet has been flushed (device/FIFO stable) */ for (lastrbc = retries = 0xffff; retries > 0; retries--) { - hi = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_RFRAMEBCHI, - &err); - lo = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_RFRAMEBCLO, - &err); + hi = brcmf_sdiod_regrb(bus->sdiodev, + SBSDIO_FUNC1_RFRAMEBCHI, &err); + lo = brcmf_sdiod_regrb(bus->sdiodev, + SBSDIO_FUNC1_RFRAMEBCLO, &err); bus->sdcnt.f1regdata += 2; if ((hi == 0) && (lo == 0)) @@ -1204,8 +1201,8 @@ static void brcmf_sdio_rxfail(struct brcmf_sdio *bus, bool abort, bool rtx) if (rtx) { bus->sdcnt.rxrtx++; - brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailbox), - SMB_NAK, &err); + err = w_sdreg32(bus, SMB_NAK, + offsetof(struct sdpcmd_regs, tosbmailbox)); bus->sdcnt.f1regdata++; if (err == 0) @@ -1225,13 +1222,13 @@ static void brcmf_sdio_txfail(struct brcmf_sdio *bus) brcmf_err("sdio error, abort command and terminate frame\n"); bus->sdcnt.tx_sderrs++; - brcmf_sdiod_abort(sdiodev, sdiodev->func2); - brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_WF_TERM, NULL); + brcmf_sdiod_abort(sdiodev, SDIO_FUNC_2); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_FRAMECTRL, SFC_WF_TERM, NULL); bus->sdcnt.f1regdata++; for (i = 0; i < 3; i++) { - hi = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_WFRAMEBCHI, NULL); - lo = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_WFRAMEBCLO, NULL); + hi = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_WFRAMEBCHI, NULL); + lo = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_WFRAMEBCLO, NULL); bus->sdcnt.f1regdata += 2; if ((hi == 0) && (lo == 0)) break; @@ -1581,10 +1578,10 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq) * read directly into the chained packet, or allocate a large * packet and and copy into the chain. */ - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); errcode = brcmf_sdiod_recv_chain(bus->sdiodev, &bus->glom, dlen); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); bus->sdcnt.f2rxdata++; /* On failure, kill the superframe */ @@ -1592,11 +1589,11 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq) brcmf_err("glom read of %d bytes failed: %d\n", dlen, errcode); - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_rxfail(bus, true, false); bus->sdcnt.rxglomfail++; brcmf_sdio_free_glom(bus); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); return 0; } @@ -1606,10 +1603,10 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq) rd_new.seq_num = rxseq; rd_new.len = dlen; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); errcode = brcmf_sdio_hdparse(bus, pfirst->data, &rd_new, BRCMF_SDIO_FT_SUPER); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); bus->cur_read.len = rd_new.len_nxtfrm << 4; /* Remove superframe header, remember offset */ @@ -1625,10 +1622,10 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq) rd_new.len = pnext->len; rd_new.seq_num = rxseq++; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); errcode = brcmf_sdio_hdparse(bus, pnext->data, &rd_new, BRCMF_SDIO_FT_SUB); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); brcmf_dbg_hex_dump(BRCMF_GLOM_ON(), pnext->data, 32, "subframe:\n"); @@ -1637,11 +1634,11 @@ static u8 brcmf_sdio_rxglom(struct brcmf_sdio *bus, u8 rxseq) if (errcode) { /* Terminate frame on error */ - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_rxfail(bus, true, false); bus->sdcnt.rxglomfail++; brcmf_sdio_free_glom(bus); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); bus->cur_read.len = 0; return 0; } @@ -1722,7 +1719,8 @@ brcmf_sdio_read_control(struct brcmf_sdio *bus, u8 *hdr, uint len, uint doff) u8 *buf = NULL, *rbuf; int sdret; - brcmf_dbg(SDIO, "Enter\n"); + brcmf_dbg(TRACE, "Enter\n"); + if (bus->rxblen) buf = vzalloc(bus->rxblen); if (!buf) @@ -1825,7 +1823,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) struct brcmf_sdio_hdrinfo *rd = &bus->cur_read, rd_new; u8 head_read = 0; - brcmf_dbg(SDIO, "Enter\n"); + brcmf_dbg(TRACE, "Enter\n"); /* Not finished unless we encounter no more frames indication */ bus->rxpending = true; @@ -1848,7 +1846,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) rd->len_left = rd->len; /* read header first for unknow frame length */ - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); if (!rd->len) { ret = brcmf_sdiod_recv_buf(bus->sdiodev, bus->rxhdr, BRCMF_FIRSTREAD); @@ -1858,7 +1856,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) ret); bus->sdcnt.rx_hdrfail++; brcmf_sdio_rxfail(bus, true, true); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); continue; } @@ -1868,7 +1866,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) if (brcmf_sdio_hdparse(bus, bus->rxhdr, rd, BRCMF_SDIO_FT_NORMAL)) { - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); if (!bus->rxpending) break; else @@ -1884,7 +1882,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) rd->len_nxtfrm = 0; /* treat all packet as event if we don't know */ rd->channel = SDPCM_EVENT_CHANNEL; - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); continue; } rd->len_left = rd->len > BRCMF_FIRSTREAD ? @@ -1901,7 +1899,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) brcmf_err("brcmu_pkt_buf_get_skb failed\n"); brcmf_sdio_rxfail(bus, false, RETRYCHAN(rd->channel)); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); continue; } skb_pull(pkt, head_read); @@ -1909,16 +1907,16 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) ret = brcmf_sdiod_recv_pkt(bus->sdiodev, pkt); bus->sdcnt.f2rxdata++; - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); if (ret < 0) { brcmf_err("read %d bytes from channel %d failed: %d\n", rd->len, rd->channel, ret); brcmu_pkt_buf_free_skb(pkt); - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_rxfail(bus, true, RETRYCHAN(rd->channel)); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); continue; } @@ -1929,7 +1927,7 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) } else { memcpy(bus->rxhdr, pkt->data, SDPCM_HDRLEN); rd_new.seq_num = rd->seq_num; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); if (brcmf_sdio_hdparse(bus, bus->rxhdr, &rd_new, BRCMF_SDIO_FT_NORMAL)) { rd->len = 0; @@ -1942,11 +1940,11 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) roundup(rd_new.len, 16) >> 4); rd->len = 0; brcmf_sdio_rxfail(bus, true, true); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); brcmu_pkt_buf_free_skb(pkt); continue; } - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); rd->len_nxtfrm = rd_new.len_nxtfrm; rd->channel = rd_new.channel; rd->dat_offset = rd_new.dat_offset; @@ -1962,9 +1960,9 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) rd_new.seq_num); /* Force retry w/normal header read */ rd->len = 0; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_rxfail(bus, false, true); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); brcmu_pkt_buf_free_skb(pkt); continue; } @@ -1987,9 +1985,9 @@ static uint brcmf_sdio_readframes(struct brcmf_sdio *bus, uint maxframes) } else { brcmf_err("%s: glom superframe w/o " "descriptor!\n", __func__); - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_rxfail(bus, false, false); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); } /* prepare the descriptor for the next read */ rd->len = rd->len_nxtfrm << 4; @@ -2087,7 +2085,7 @@ static int brcmf_sdio_txpkt_prep_sg(struct brcmf_sdio *bus, int ntail, ret; sdiodev = bus->sdiodev; - blksize = sdiodev->func2->cur_blksize; + blksize = sdiodev->func[SDIO_FUNC_2]->cur_blksize; /* sg entry alignment should be a divisor of block size */ WARN_ON(blksize % bus->sgentry_align); @@ -2266,14 +2264,14 @@ static int brcmf_sdio_txpkt(struct brcmf_sdio *bus, struct sk_buff_head *pktq, if (ret) goto done; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); ret = brcmf_sdiod_send_pkt(bus->sdiodev, pktq); bus->sdcnt.f2txdata++; if (ret < 0) brcmf_sdio_txfail(bus); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); done: brcmf_sdio_txpkt_postp(bus, pktq); @@ -2291,7 +2289,6 @@ static uint brcmf_sdio_sendfromq(struct brcmf_sdio *bus, uint maxframes) { struct sk_buff *pkt; struct sk_buff_head pktq; - u32 intstat_addr = bus->sdio_core->base + SD_REG(intstatus); u32 intstatus = 0; int ret = 0, prec_out, i; uint cnt = 0; @@ -2329,11 +2326,11 @@ static uint brcmf_sdio_sendfromq(struct brcmf_sdio *bus, uint maxframes) /* In poll mode, need to check for other events */ if (!bus->intr) { /* Check device status, signal pending interrupt */ - sdio_claim_host(bus->sdiodev->func1); - intstatus = brcmf_sdiod_readl(bus->sdiodev, - intstat_addr, &ret); - sdio_release_host(bus->sdiodev->func1); - + sdio_claim_host(bus->sdiodev->func[1]); + ret = r_sdreg32(bus, &intstatus, + offsetof(struct sdpcmd_regs, + intstatus)); + sdio_release_host(bus->sdiodev->func[1]); bus->sdcnt.f2txdata++; if (ret != 0) break; @@ -2360,7 +2357,7 @@ static int brcmf_sdio_tx_ctrlframe(struct brcmf_sdio *bus, u8 *frame, u16 len) struct brcmf_sdio_hdrinfo hd_info = {0}; int ret; - brcmf_dbg(SDIO, "Enter\n"); + brcmf_dbg(TRACE, "Enter\n"); /* Back the pointer to make room for bus header */ frame -= bus->tx_hdrlen; @@ -2416,13 +2413,12 @@ static int brcmf_sdio_tx_ctrlframe(struct brcmf_sdio *bus, u8 *frame, u16 len) static void brcmf_sdio_bus_stop(struct device *dev) { - struct brcmf_bus *bus_if = dev_get_drvdata(dev); - struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; - struct brcmf_sdio *bus = sdiodev->bus; - struct brcmf_core *core = bus->sdio_core; u32 local_hostintmask; u8 saveclk; int err; + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + struct brcmf_sdio *bus = sdiodev->bus; brcmf_dbg(TRACE, "Enter\n"); @@ -2433,37 +2429,35 @@ static void brcmf_sdio_bus_stop(struct device *dev) } if (sdiodev->state != BRCMF_SDIOD_NOMEDIUM) { - sdio_claim_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); /* Enable clock for device interrupts */ brcmf_sdio_bus_sleep(bus, false, false); /* Disable and clear interrupts at the chip level also */ - brcmf_sdiod_writel(sdiodev, core->base + SD_REG(hostintmask), - 0, NULL); - + w_sdreg32(bus, 0, offsetof(struct sdpcmd_regs, hostintmask)); local_hostintmask = bus->hostintmask; bus->hostintmask = 0; /* Force backplane clocks to assure F2 interrupt propagates */ - saveclk = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); if (!err) - brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - (saveclk | SBSDIO_FORCE_HT), &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | SBSDIO_FORCE_HT), &err); if (err) brcmf_err("Failed to force clock for F2: err %d\n", err); /* Turn off the bus (F2), free any pending packets */ brcmf_dbg(INTR, "disable SDIO interrupts\n"); - sdio_disable_func(sdiodev->func2); + sdio_disable_func(sdiodev->func[SDIO_FUNC_2]); /* Clear any pending interrupts now that F2 is disabled */ - brcmf_sdiod_writel(sdiodev, core->base + SD_REG(intstatus), - local_hostintmask, NULL); + w_sdreg32(bus, local_hostintmask, + offsetof(struct sdpcmd_regs, intstatus)); - sdio_release_host(sdiodev->func1); + sdio_release_host(sdiodev->func[1]); } /* Clear the data packet queues */ brcmu_pktq_flush(&bus->txq, true, NULL, NULL); @@ -2501,14 +2495,15 @@ static inline void brcmf_sdio_clrintr(struct brcmf_sdio *bus) static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus) { - struct brcmf_core *core = bus->sdio_core; + struct brcmf_core *buscore; u32 addr; unsigned long val; int ret; - addr = core->base + SD_REG(intstatus); + buscore = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV); + addr = buscore->base + offsetof(struct sdpcmd_regs, intstatus); - val = brcmf_sdiod_readl(bus->sdiodev, addr, &ret); + val = brcmf_sdiod_regrl(bus->sdiodev, addr, &ret); bus->sdcnt.f1regdata++; if (ret != 0) return ret; @@ -2518,7 +2513,7 @@ static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus) /* Clear interrupts */ if (val) { - brcmf_sdiod_writel(bus->sdiodev, addr, val, &ret); + brcmf_sdiod_regwl(bus->sdiodev, addr, val, &ret); bus->sdcnt.f1regdata++; atomic_or(val, &bus->intstatus); } @@ -2528,17 +2523,15 @@ static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus) static void brcmf_sdio_dpc(struct brcmf_sdio *bus) { - struct brcmf_sdio_dev *sdiod = bus->sdiodev; u32 newstatus = 0; - u32 intstat_addr = bus->sdio_core->base + SD_REG(intstatus); unsigned long intstatus; uint txlimit = bus->txbound; /* Tx frames to send before resched */ uint framecnt; /* Temporary counter of tx/rx frames */ int err = 0; - brcmf_dbg(SDIO, "Enter\n"); + brcmf_dbg(TRACE, "Enter\n"); - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); /* If waiting for HTAVAIL, check status */ if (!bus->sr_enabled && bus->clkstate == CLK_PENDING) { @@ -2546,23 +2539,23 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) #ifdef DEBUG /* Check for inconsistent device control */ - devctl = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_DEVICE_CTL, - &err); + devctl = brcmf_sdiod_regrb(bus->sdiodev, + SBSDIO_DEVICE_CTL, &err); #endif /* DEBUG */ /* Read CSR, if clock on switch to AVAIL, else ignore */ - clkctl = brcmf_sdiod_readb(bus->sdiodev, + clkctl = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); brcmf_dbg(SDIO, "DPC: PENDING, devctl 0x%02x clkctl 0x%02x\n", devctl, clkctl); if (SBSDIO_HTAV(clkctl)) { - devctl = brcmf_sdiod_readb(bus->sdiodev, + devctl = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_DEVICE_CTL, &err); devctl &= ~SBSDIO_DEVCTL_CA_INT_ONLY; - brcmf_sdiod_writeb(bus->sdiodev, - SBSDIO_DEVICE_CTL, devctl, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_DEVICE_CTL, + devctl, &err); bus->clkstate = CLK_AVAIL; } } @@ -2585,10 +2578,11 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) */ if (intstatus & I_HMB_FC_CHANGE) { intstatus &= ~I_HMB_FC_CHANGE; - brcmf_sdiod_writel(sdiod, intstat_addr, I_HMB_FC_CHANGE, &err); - - newstatus = brcmf_sdiod_readl(sdiod, intstat_addr, &err); + err = w_sdreg32(bus, I_HMB_FC_CHANGE, + offsetof(struct sdpcmd_regs, intstatus)); + err = r_sdreg32(bus, &newstatus, + offsetof(struct sdpcmd_regs, intstatus)); bus->sdcnt.f1regdata += 2; atomic_set(&bus->fcstate, !!(newstatus & (I_HMB_FC_STATE | I_HMB_FC_CHANGE))); @@ -2601,7 +2595,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) intstatus |= brcmf_sdio_hostmail(bus); } - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); /* Generally don't ask for these, can get CRC errors... */ if (intstatus & I_WR_OOSYNC) { @@ -2621,7 +2615,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) /* Would be active due to wake-wlan in gSPI */ if (intstatus & I_CHIPACTIVE) { - brcmf_dbg(SDIO, "Dongle reports CHIPACTIVE\n"); + brcmf_dbg(INFO, "Dongle reports CHIPACTIVE\n"); intstatus &= ~I_CHIPACTIVE; } @@ -2644,7 +2638,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) if (bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL) && data_ok(bus)) { - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); if (bus->ctrl_frame_stat) { err = brcmf_sdio_tx_ctrlframe(bus, bus->ctrl_frame_buf, bus->ctrl_frame_len); @@ -2652,7 +2646,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) wmb(); bus->ctrl_frame_stat = false; } - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); brcmf_sdio_wait_event_wakeup(bus); } /* Send queued frames (limit 1 if rx may still be pending) */ @@ -2668,14 +2662,14 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) brcmf_err("failed backplane access over SDIO, halting operation\n"); atomic_set(&bus->intstatus, 0); if (bus->ctrl_frame_stat) { - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); if (bus->ctrl_frame_stat) { bus->ctrl_frame_err = -ENODEV; wmb(); bus->ctrl_frame_stat = false; brcmf_sdio_wait_event_wakeup(bus); } - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); } } else if (atomic_read(&bus->intstatus) || atomic_read(&bus->ipend) > 0 || @@ -2890,13 +2884,13 @@ brcmf_sdio_bus_txctl(struct device *dev, unsigned char *msg, uint msglen) CTL_DONE_TIMEOUT); ret = 0; if (bus->ctrl_frame_stat) { - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); if (bus->ctrl_frame_stat) { brcmf_dbg(SDIO, "ctrl_frame timeout\n"); bus->ctrl_frame_stat = false; ret = -ETIMEDOUT; } - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); } if (!ret) { brcmf_dbg(SDIO, "ctrl_frame complete, err=%d\n", @@ -3020,7 +3014,7 @@ static int brcmf_sdio_assert_info(struct seq_file *seq, struct brcmf_sdio *bus, return 0; } - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); if (sh->assert_file_addr != 0) { error = brcmf_sdiod_ramrw(bus->sdiodev, false, sh->assert_file_addr, (u8 *)file, 80); @@ -3033,7 +3027,7 @@ static int brcmf_sdio_assert_info(struct seq_file *seq, struct brcmf_sdio *bus, if (error < 0) return error; } - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); seq_printf(seq, "dongle assert: %s:%d: assert(%s)\n", file, sh->assert_line, expr); @@ -3307,7 +3301,7 @@ static int brcmf_sdio_download_firmware(struct brcmf_sdio *bus, int bcmerror; u32 rstvec; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_clkctl(bus, CLK_AVAIL, false); rstvec = get_unaligned_le32(fw->data); @@ -3336,7 +3330,7 @@ static int brcmf_sdio_download_firmware(struct brcmf_sdio *bus, err: brcmf_sdio_clkctl(bus, CLK_SDONLY, false); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); return bcmerror; } @@ -3347,31 +3341,31 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus) brcmf_dbg(TRACE, "Enter\n"); - val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err); + val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err); if (err) { brcmf_err("error reading SBSDIO_FUNC1_WAKEUPCTRL\n"); return; } val |= 1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT; - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, val, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, val, &err); if (err) { brcmf_err("error writing SBSDIO_FUNC1_WAKEUPCTRL\n"); return; } /* Add CMD14 Support */ - brcmf_sdiod_func0_wb(bus->sdiodev, SDIO_CCCR_BRCM_CARDCAP, - (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | - SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT), - &err); + brcmf_sdiod_regwb(bus->sdiodev, SDIO_CCCR_BRCM_CARDCAP, + (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | + SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT), + &err); if (err) { brcmf_err("error writing SDIO_CCCR_BRCM_CARDCAP\n"); return; } - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - SBSDIO_FORCE_HT, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + SBSDIO_FORCE_HT, &err); if (err) { brcmf_err("error writing SBSDIO_FUNC1_CHIPCLKCSR\n"); return; @@ -3385,17 +3379,16 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus) /* enable KSO bit */ static int brcmf_sdio_kso_init(struct brcmf_sdio *bus) { - struct brcmf_core *core = bus->sdio_core; u8 val; int err = 0; brcmf_dbg(TRACE, "Enter\n"); /* KSO bit added in SDIO core rev 12 */ - if (core->rev < 12) + if (brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV)->rev < 12) return 0; - val = brcmf_sdiod_readb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, &err); + val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, &err); if (err) { brcmf_err("error reading SBSDIO_FUNC1_SLEEPCSR\n"); return err; @@ -3404,8 +3397,8 @@ static int brcmf_sdio_kso_init(struct brcmf_sdio *bus) if (!(val & SBSDIO_FUNC1_SLEEPCSR_KSO_MASK)) { val |= (SBSDIO_FUNC1_SLEEPCSR_KSO_EN << SBSDIO_FUNC1_SLEEPCSR_KSO_SHIFT); - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, - val, &err); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, + val, &err); if (err) { brcmf_err("error writing SBSDIO_FUNC1_SLEEPCSR\n"); return err; @@ -3421,30 +3414,15 @@ static int brcmf_sdio_bus_preinit(struct device *dev) struct brcmf_bus *bus_if = dev_get_drvdata(dev); struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; struct brcmf_sdio *bus = sdiodev->bus; - struct brcmf_core *core = bus->sdio_core; uint pad_size; u32 value; int err; - /* maxctl provided by common layer */ - if (WARN_ON(!bus_if->maxctl)) - return -EINVAL; - - /* Allocate control receive buffer */ - bus_if->maxctl += bus->roundup; - value = roundup((bus_if->maxctl + SDPCM_HDRLEN), ALIGNMENT); - value += bus->head_align; - bus->rxbuf = kmalloc(value, GFP_ATOMIC); - if (bus->rxbuf) - bus->rxblen = value; - - brcmf_sdio_debugfs_create(bus); - /* the commands below use the terms tx and rx from * a device perspective, ie. bus:txglom affects the * bus transfers from device to host. */ - if (core->rev < 12) { + if (brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV)->rev < 12) { /* for sdio core rev < 12, disable txgloming */ value = 0; err = brcmf_iovar_data_set(dev, "bus:txglom", &value, @@ -3465,7 +3443,7 @@ static int brcmf_sdio_bus_preinit(struct device *dev) if (sdiodev->sg_support) { bus->txglom = false; value = 1; - pad_size = bus->sdiodev->func2->cur_blksize << 1; + pad_size = bus->sdiodev->func[2]->cur_blksize << 1; err = brcmf_iovar_data_set(bus->sdiodev->dev, "bus:rxglom", &value, sizeof(u32)); if (err < 0) { @@ -3507,7 +3485,7 @@ static int brcmf_sdio_bus_get_memdump(struct device *dev, void *data, address = bus->ci->rambase; offset = err = 0; - sdio_claim_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); while (offset < mem_size) { len = ((offset + MEMBLOCK) < mem_size) ? MEMBLOCK : mem_size - offset; @@ -3523,7 +3501,7 @@ static int brcmf_sdio_bus_get_memdump(struct device *dev, void *data, } done: - sdio_release_host(sdiodev->func1); + sdio_release_host(sdiodev->func[1]); return err; } @@ -3580,10 +3558,11 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus) if (!bus->dpc_triggered) { u8 devpend; - sdio_claim_host(bus->sdiodev->func1); - devpend = brcmf_sdiod_func0_rb(bus->sdiodev, - SDIO_CCCR_INTx, NULL); - sdio_release_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); + devpend = brcmf_sdiod_regrb(bus->sdiodev, + SDIO_CCCR_INTx, + NULL); + sdio_release_host(bus->sdiodev->func[1]); intstatus = devpend & (INTR_STATUS_FUNC1 | INTR_STATUS_FUNC2); } @@ -3609,13 +3588,13 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus) bus->console.count += jiffies_to_msecs(BRCMF_WD_POLL); if (bus->console.count >= bus->console_interval) { bus->console.count -= bus->console_interval; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); /* Make sure backplane clock is on */ brcmf_sdio_bus_sleep(bus, false, false); if (brcmf_sdio_readconsole(bus) < 0) /* stop on error */ bus->console_interval = 0; - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); } } #endif /* DEBUG */ @@ -3628,11 +3607,11 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus) bus->idlecount++; if (bus->idlecount > bus->idletime) { brcmf_dbg(SDIO, "idle\n"); - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_wd_timer(bus, false); bus->idlecount = 0; brcmf_sdio_bus_sleep(bus, true, false); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); } } else { bus->idlecount = 0; @@ -3649,7 +3628,7 @@ static void brcmf_sdio_dataworker(struct work_struct *work) bus->dpc_running = true; wmb(); - while (READ_ONCE(bus->dpc_triggered)) { + while (ACCESS_ONCE(bus->dpc_triggered)) { bus->dpc_triggered = false; brcmf_sdio_dpc(bus); bus->idlecount = 0; @@ -3720,12 +3699,12 @@ brcmf_sdio_drivestrengthinit(struct brcmf_sdio_dev *sdiodev, } } addr = CORE_CC_REG(pmu->base, chipcontrol_addr); - brcmf_sdiod_writel(sdiodev, addr, 1, NULL); - cc_data_temp = brcmf_sdiod_readl(sdiodev, addr, NULL); + brcmf_sdiod_regwl(sdiodev, addr, 1, NULL); + cc_data_temp = brcmf_sdiod_regrl(sdiodev, addr, NULL); cc_data_temp &= ~str_mask; drivestrength_sel <<= str_shift; cc_data_temp |= drivestrength_sel; - brcmf_sdiod_writel(sdiodev, addr, cc_data_temp, NULL); + brcmf_sdiod_regwl(sdiodev, addr, cc_data_temp, NULL); brcmf_dbg(INFO, "SDIO: %d mA (req=%d mA) drive strength selected, set to 0x%08x\n", str_tab[i].strength, drivestrength, cc_data_temp); @@ -3740,7 +3719,7 @@ static int brcmf_sdio_buscoreprep(void *ctx) /* Try forcing SDIO core to do ALPAvail request only */ clkset = SBSDIO_FORCE_HW_CLKREQ_OFF | SBSDIO_ALP_AVAIL_REQ; - brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err); if (err) { brcmf_err("error writing for HT off\n"); return err; @@ -3748,7 +3727,8 @@ static int brcmf_sdio_buscoreprep(void *ctx) /* If register supported, wait for ALPAvail and then force ALP */ /* This may take up to 15 milliseconds */ - clkval = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, NULL); + clkval = brcmf_sdiod_regrb(sdiodev, + SBSDIO_FUNC1_CHIPCLKCSR, NULL); if ((clkval & ~SBSDIO_AVBITS) != clkset) { brcmf_err("ChipClkCSR access: wrote 0x%02x read 0x%02x\n", @@ -3756,11 +3736,10 @@ static int brcmf_sdio_buscoreprep(void *ctx) return -EACCES; } - SPINWAIT(((clkval = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - NULL)), - !SBSDIO_ALPAV(clkval)), - PMU_MAX_TRANSITION_DLY); - + SPINWAIT(((clkval = brcmf_sdiod_regrb(sdiodev, + SBSDIO_FUNC1_CHIPCLKCSR, NULL)), + !SBSDIO_ALPAV(clkval)), + PMU_MAX_TRANSITION_DLY); if (!SBSDIO_ALPAV(clkval)) { brcmf_err("timeout on ALPAV wait, clkval 0x%02x\n", clkval); @@ -3768,11 +3747,11 @@ static int brcmf_sdio_buscoreprep(void *ctx) } clkset = SBSDIO_FORCE_HW_CLKREQ_OFF | SBSDIO_FORCE_ALP; - brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, clkset, &err); udelay(65); /* Also, disable the extra SDIO pull-ups */ - brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_SDIOPULLUP, 0, NULL); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_SDIOPULLUP, 0, NULL); return 0; } @@ -3781,12 +3760,13 @@ static void brcmf_sdio_buscore_activate(void *ctx, struct brcmf_chip *chip, u32 rstvec) { struct brcmf_sdio_dev *sdiodev = ctx; - struct brcmf_core *core = sdiodev->bus->sdio_core; + struct brcmf_core *core; u32 reg_addr; /* clear all interrupts */ - reg_addr = core->base + SD_REG(intstatus); - brcmf_sdiod_writel(sdiodev, reg_addr, 0xFFFFFFFF, NULL); + core = brcmf_chip_get_core(chip, BCMA_CORE_SDIO_DEV); + reg_addr = core->base + offsetof(struct sdpcmd_regs, intstatus); + brcmf_sdiod_regwl(sdiodev, reg_addr, 0xFFFFFFFF, NULL); if (rstvec) /* Write reset vector to address 0 */ @@ -3799,25 +3779,16 @@ static u32 brcmf_sdio_buscore_read32(void *ctx, u32 addr) struct brcmf_sdio_dev *sdiodev = ctx; u32 val, rev; - val = brcmf_sdiod_readl(sdiodev, addr, NULL); - - /* - * this is a bit of special handling if reading the chipcommon chipid - * register. The 4339 is a next-gen of the 4335. It uses the same - * SDIO device id as 4335 and the chipid register returns 4335 as well. - * It can be identified as 4339 by looking at the chip revision. It - * is corrected here so the chip.c module has the right info. - */ - if (addr == CORE_CC_REG(SI_ENUM_BASE, chipid) && - (sdiodev->func1->device == SDIO_DEVICE_ID_BROADCOM_4339 || - sdiodev->func1->device == SDIO_DEVICE_ID_BROADCOM_4335_4339)) { + val = brcmf_sdiod_regrl(sdiodev, addr, NULL); + if ((sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4335_4339 || + sdiodev->func[0]->device == SDIO_DEVICE_ID_BROADCOM_4339) && + addr == CORE_CC_REG(SI_ENUM_BASE, chipid)) { rev = (val & CID_REV_MASK) >> CID_REV_SHIFT; if (rev >= 2) { val &= ~CID_ID_MASK; val |= BRCM_CC_4339_CHIP_ID; } } - return val; } @@ -3825,7 +3796,7 @@ static void brcmf_sdio_buscore_write32(void *ctx, u32 addr, u32 val) { struct brcmf_sdio_dev *sdiodev = ctx; - brcmf_sdiod_writel(sdiodev, addr, val, NULL); + brcmf_sdiod_regwl(sdiodev, addr, val, NULL); } static const struct brcmf_buscore_ops brcmf_sdio_buscore_ops = { @@ -3846,21 +3817,21 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus) u32 drivestrength; sdiodev = bus->sdiodev; - sdio_claim_host(sdiodev->func1); + sdio_claim_host(sdiodev->func[1]); pr_debug("F1 signature read @0x18000000=0x%4x\n", - brcmf_sdiod_readl(sdiodev, SI_ENUM_BASE, NULL)); + brcmf_sdiod_regrl(sdiodev, SI_ENUM_BASE, NULL)); /* * Force PLL off until brcmf_chip_attach() * programs PLL control regs */ - brcmf_sdiod_writeb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, BRCMF_INIT_CLKCTL1, - &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + BRCMF_INIT_CLKCTL1, &err); if (!err) - clkctl = brcmf_sdiod_readb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - &err); + clkctl = brcmf_sdiod_regrb(sdiodev, + SBSDIO_FUNC1_CHIPCLKCSR, &err); if (err || ((clkctl & ~SBSDIO_AVBITS) != BRCMF_INIT_CLKCTL1)) { brcmf_err("ChipClkCSR access: err %d wrote 0x%02x read 0x%02x\n", @@ -3874,17 +3845,6 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus) bus->ci = NULL; goto fail; } - - /* Pick up the SDIO core info struct from chip.c */ - bus->sdio_core = brcmf_chip_get_core(bus->ci, BCMA_CORE_SDIO_DEV); - if (!bus->sdio_core) - goto fail; - - /* Pick up the CHIPCOMMON core info struct, for bulk IO in bcmsdh.c */ - sdiodev->cc_core = brcmf_chip_get_core(bus->ci, BCMA_CORE_CHIPCOMMON); - if (!sdiodev->cc_core) - goto fail; - sdiodev->settings = brcmf_get_module_param(sdiodev->dev, BRCMF_BUSTYPE_SDIO, bus->ci->chip, @@ -3913,8 +3873,8 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus) /* wowl can be supported when KEEP_POWER is true and (WAKE_SDIO_IRQ * is true or when platform data OOB irq is true). */ - if ((sdio_get_host_pm_caps(sdiodev->func1) & MMC_PM_KEEP_POWER) && - ((sdio_get_host_pm_caps(sdiodev->func1) & MMC_PM_WAKE_SDIO_IRQ) || + if ((sdio_get_host_pm_caps(sdiodev->func[1]) & MMC_PM_KEEP_POWER) && + ((sdio_get_host_pm_caps(sdiodev->func[1]) & MMC_PM_WAKE_SDIO_IRQ) || (sdiodev->settings->bus.sdio.oob_irq_supported))) sdiodev->bus_if->wowl_supported = true; #endif @@ -3931,29 +3891,29 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus) brcmf_sdio_drivestrengthinit(sdiodev, bus->ci, drivestrength); /* Set card control so an SDIO card reset does a WLAN backplane reset */ - reg_val = brcmf_sdiod_func0_rb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, &err); + reg_val = brcmf_sdiod_regrb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, &err); if (err) goto fail; reg_val |= SDIO_CCCR_BRCM_CARDCTRL_WLANRESET; - brcmf_sdiod_func0_wb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, reg_val, &err); + brcmf_sdiod_regwb(sdiodev, SDIO_CCCR_BRCM_CARDCTRL, reg_val, &err); if (err) goto fail; /* set PMUControl so a backplane reset does PMU state reload */ reg_addr = CORE_CC_REG(brcmf_chip_get_pmu(bus->ci)->base, pmucontrol); - reg_val = brcmf_sdiod_readl(sdiodev, reg_addr, &err); + reg_val = brcmf_sdiod_regrl(sdiodev, reg_addr, &err); if (err) goto fail; reg_val |= (BCMA_CC_PMU_CTL_RES_RELOAD << BCMA_CC_PMU_CTL_RES_SHIFT); - brcmf_sdiod_writel(sdiodev, reg_addr, reg_val, &err); + brcmf_sdiod_regwl(sdiodev, reg_addr, reg_val, &err); if (err) goto fail; - sdio_release_host(sdiodev->func1); + sdio_release_host(sdiodev->func[1]); brcmu_pktq_init(&bus->txq, (PRIOMASK + 1), TXQLEN); @@ -3974,7 +3934,7 @@ brcmf_sdio_probe_attach(struct brcmf_sdio *bus) return true; fail: - sdio_release_host(sdiodev->func1); + sdio_release_host(sdiodev->func[1]); return false; } @@ -4006,9 +3966,9 @@ brcmf_sdio_watchdog_thread(void *data) } static void -brcmf_sdio_watchdog(struct timer_list *t) +brcmf_sdio_watchdog(unsigned long data) { - struct brcmf_sdio *bus = from_timer(bus, t, timer); + struct brcmf_sdio *bus = (struct brcmf_sdio *)data; if (bus->watchdog_tsk) { complete(&bus->watchdog_wait); @@ -4019,26 +3979,6 @@ brcmf_sdio_watchdog(struct timer_list *t) } } -static -int brcmf_sdio_get_fwname(struct device *dev, const char *ext, u8 *fw_name) -{ - struct brcmf_bus *bus_if = dev_get_drvdata(dev); - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ext, fw_name }, - }; - - fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev, - brcmf_sdio_fwnames, - ARRAY_SIZE(brcmf_sdio_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return -ENOMEM; - - kfree(fwreq); - return 0; -} - static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .stop = brcmf_sdio_bus_stop, .preinit = brcmf_sdio_bus_preinit, @@ -4049,34 +3989,27 @@ static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .wowl_config = brcmf_sdio_wowl_config, .get_ramsize = brcmf_sdio_bus_get_ramsize, .get_memdump = brcmf_sdio_bus_get_memdump, - .get_fwname = brcmf_sdio_get_fwname, }; -#define BRCMF_SDIO_FW_CODE 0 -#define BRCMF_SDIO_FW_NVRAM 1 - static void brcmf_sdio_firmware_callback(struct device *dev, int err, - struct brcmf_fw_request *fwreq) + const struct firmware *code, + void *nvram, u32 nvram_len) { - struct brcmf_bus *bus_if = dev_get_drvdata(dev); - struct brcmf_sdio_dev *sdiod = bus_if->bus_priv.sdio; - struct brcmf_sdio *bus = sdiod->bus; - struct brcmf_core *core = bus->sdio_core; - const struct firmware *code; - void *nvram; - u32 nvram_len; + struct brcmf_bus *bus_if; + struct brcmf_sdio_dev *sdiodev; + struct brcmf_sdio *bus; u8 saveclk; - u8 devctl; brcmf_dbg(TRACE, "Enter: dev=%s, err=%d\n", dev_name(dev), err); - + bus_if = dev_get_drvdata(dev); + sdiodev = bus_if->bus_priv.sdio; if (err) goto fail; - code = fwreq->items[BRCMF_SDIO_FW_CODE].binary; - nvram = fwreq->items[BRCMF_SDIO_FW_NVRAM].nv_data.data; - nvram_len = fwreq->items[BRCMF_SDIO_FW_NVRAM].nv_data.len; - kfree(fwreq); + if (!bus_if->drvr) + return; + + bus = sdiodev->bus; /* try to download image and nvram to the dongle */ bus->alp_only = true; @@ -4089,7 +4022,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, bus->sdcnt.tickcnt = 0; brcmf_sdio_wd_timer(bus, true); - sdio_claim_host(sdiod->func1); + sdio_claim_host(sdiodev->func[1]); /* Make sure backplane clock is on, needed to generate F2 interrupt */ brcmf_sdio_clkctl(bus, CLK_AVAIL, false); @@ -4097,10 +4030,10 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, goto release; /* Force clocks on backplane to be sure F2 interrupt propagates */ - saveclk = brcmf_sdiod_readb(sdiod, SBSDIO_FUNC1_CHIPCLKCSR, &err); + saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); if (!err) { - brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_CHIPCLKCSR, - (saveclk | SBSDIO_FORCE_HT), &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | SBSDIO_FORCE_HT), &err); } if (err) { brcmf_err("Failed to force clock for F2: err %d\n", err); @@ -4108,10 +4041,10 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, } /* Enable function 2 (frame transfers) */ - brcmf_sdiod_writel(sdiod, core->base + SD_REG(tosbmailboxdata), - SDPCM_PROT_VERSION << SMB_DATA_VERSION_SHIFT, NULL); + w_sdreg32(bus, SDPCM_PROT_VERSION << SMB_DATA_VERSION_SHIFT, + offsetof(struct sdpcmd_regs, tosbmailboxdata)); + err = sdio_enable_func(sdiodev->func[SDIO_FUNC_2]); - err = sdio_enable_func(sdiod->func2); brcmf_dbg(INFO, "enable F2: err=%d\n", err); @@ -4119,32 +4052,13 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, if (!err) { /* Set up the interrupt mask and enable interrupts */ bus->hostintmask = HOSTINTMASK; - brcmf_sdiod_writel(sdiod, core->base + SD_REG(hostintmask), - bus->hostintmask, NULL); - - switch (sdiod->func1->device) { - case SDIO_DEVICE_ID_CYPRESS_4373: - brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", - CY_4373_F2_WATERMARK); - brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, - CY_4373_F2_WATERMARK, &err); - devctl = brcmf_sdiod_readb(sdiod, SBSDIO_DEVICE_CTL, - &err); - devctl |= SBSDIO_DEVCTL_F2WM_ENAB; - brcmf_sdiod_writeb(sdiod, SBSDIO_DEVICE_CTL, devctl, - &err); - brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_MESBUSYCTRL, - CY_4373_F2_WATERMARK | - SBSDIO_MESBUSYCTRL_ENAB, &err); - break; - default: - brcmf_sdiod_writeb(sdiod, SBSDIO_WATERMARK, - DEFAULT_F2_WATERMARK, &err); - break; - } + w_sdreg32(bus, bus->hostintmask, + offsetof(struct sdpcmd_regs, hostintmask)); + + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, 8, &err); } else { /* Disable F2 again */ - sdio_disable_func(sdiod->func2); + sdio_disable_func(sdiodev->func[SDIO_FUNC_2]); goto release; } @@ -4152,15 +4066,15 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_sdio_sr_init(bus); } else { /* Restore previous clock setting */ - brcmf_sdiod_writeb(sdiod, SBSDIO_FUNC1_CHIPCLKCSR, - saveclk, &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + saveclk, &err); } if (err == 0) { /* Allow full data communication using DPC from now on. */ brcmf_sdiod_change_state(bus->sdiodev, BRCMF_SDIOD_DATA); - err = brcmf_sdiod_intr_register(sdiod); + err = brcmf_sdiod_intr_register(sdiodev); if (err != 0) brcmf_err("intr register failed:%d\n", err); } @@ -4169,60 +4083,28 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, if (err != 0) brcmf_sdio_clkctl(bus, CLK_NONE, false); - sdio_release_host(sdiod->func1); + sdio_release_host(sdiodev->func[1]); - /* Assign bus interface call back */ - sdiod->bus_if->dev = sdiod->dev; - sdiod->bus_if->ops = &brcmf_sdio_bus_ops; - sdiod->bus_if->chip = bus->ci->chip; - sdiod->bus_if->chiprev = bus->ci->chiprev; - - /* Attach to the common layer, reserve hdr space */ - err = brcmf_attach(sdiod->dev, sdiod->settings); + err = brcmf_bus_started(dev); if (err != 0) { - brcmf_err("brcmf_attach failed\n"); + brcmf_err("dongle is not responding\n"); goto fail; } - - /* ready */ return; release: - sdio_release_host(sdiod->func1); + sdio_release_host(sdiodev->func[1]); fail: brcmf_dbg(TRACE, "failed: dev=%s, err=%d\n", dev_name(dev), err); - device_release_driver(&sdiod->func2->dev); + device_release_driver(&sdiodev->func[2]->dev); device_release_driver(dev); } -static struct brcmf_fw_request * -brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus) -{ - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ".bin", bus->sdiodev->fw_name }, - { ".txt", bus->sdiodev->nvram_name }, - }; - - fwreq = brcmf_fw_alloc_request(bus->ci->chip, bus->ci->chiprev, - brcmf_sdio_fwnames, - ARRAY_SIZE(brcmf_sdio_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return NULL; - - fwreq->items[BRCMF_SDIO_FW_CODE].type = BRCMF_FW_TYPE_BINARY; - fwreq->items[BRCMF_SDIO_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM; - - return fwreq; -} - struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) { int ret; struct brcmf_sdio *bus; struct workqueue_struct *wq; - struct brcmf_fw_request *fwreq; brcmf_dbg(TRACE, "Enter\n"); @@ -4241,7 +4123,7 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) /* single-threaded workqueue */ wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM, - dev_name(&sdiodev->func1->dev)); + dev_name(&sdiodev->func[1]->dev)); if (!wq) { brcmf_err("insufficient memory to create txworkqueue\n"); goto fail; @@ -4262,12 +4144,15 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) init_waitqueue_head(&bus->dcmd_resp_wait); /* Set up the watchdog timer */ - timer_setup(&bus->timer, brcmf_sdio_watchdog, 0); + init_timer(&bus->timer); + bus->timer.data = (unsigned long)bus; + bus->timer.function = brcmf_sdio_watchdog; + /* Initialize watchdog thread */ init_completion(&bus->watchdog_wait); bus->watchdog_tsk = kthread_run(brcmf_sdio_watchdog_thread, bus, "brcmf_wdog/%s", - dev_name(&sdiodev->func1->dev)); + dev_name(&sdiodev->func[1]->dev)); if (IS_ERR(bus->watchdog_tsk)) { pr_warn("brcmf_watchdog thread failed to start\n"); bus->watchdog_tsk = NULL; @@ -4276,24 +4161,50 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) bus->dpc_triggered = false; bus->dpc_running = false; + /* Assign bus interface call back */ + bus->sdiodev->bus_if->dev = bus->sdiodev->dev; + bus->sdiodev->bus_if->ops = &brcmf_sdio_bus_ops; + bus->sdiodev->bus_if->chip = bus->ci->chip; + bus->sdiodev->bus_if->chiprev = bus->ci->chiprev; + /* default sdio bus header length for tx packet */ bus->tx_hdrlen = SDPCM_HWHDR_LEN + SDPCM_SWHDR_LEN; + /* Attach to the common layer, reserve hdr space */ + ret = brcmf_attach(bus->sdiodev->dev, bus->sdiodev->settings); + if (ret != 0) { + brcmf_err("brcmf_attach failed\n"); + goto fail; + } + /* Query the F2 block size, set roundup accordingly */ - bus->blocksize = bus->sdiodev->func2->cur_blksize; + bus->blocksize = bus->sdiodev->func[2]->cur_blksize; bus->roundup = min(max_roundup, bus->blocksize); - sdio_claim_host(bus->sdiodev->func1); + /* Allocate buffers */ + if (bus->sdiodev->bus_if->maxctl) { + bus->sdiodev->bus_if->maxctl += bus->roundup; + bus->rxblen = + roundup((bus->sdiodev->bus_if->maxctl + SDPCM_HDRLEN), + ALIGNMENT) + bus->head_align; + bus->rxbuf = kmalloc(bus->rxblen, GFP_ATOMIC); + if (!(bus->rxbuf)) { + brcmf_err("rxbuf allocation failed\n"); + goto fail; + } + } + + sdio_claim_host(bus->sdiodev->func[1]); /* Disable F2 to clear any intermediate frame state on the dongle */ - sdio_disable_func(bus->sdiodev->func2); + sdio_disable_func(bus->sdiodev->func[SDIO_FUNC_2]); bus->rxflow = false; /* Done with backplane-dependent accesses, can drop clock... */ - brcmf_sdiod_writeb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, 0, NULL); + brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, 0, NULL); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); /* ...and initialize clock/power states */ bus->clkstate = CLK_SDONLY; @@ -4303,19 +4214,21 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) /* SR state */ bus->sr_enabled = false; + brcmf_sdio_debugfs_create(bus); brcmf_dbg(INFO, "completed!!\n"); - fwreq = brcmf_sdio_prepare_fw_request(bus); - if (!fwreq) { - ret = -ENOMEM; + ret = brcmf_fw_map_chip_to_name(bus->ci->chip, bus->ci->chiprev, + brcmf_sdio_fwnames, + ARRAY_SIZE(brcmf_sdio_fwnames), + sdiodev->fw_name, sdiodev->nvram_name); + if (ret) goto fail; - } - ret = brcmf_fw_get_firmwares(sdiodev->dev, fwreq, + ret = brcmf_fw_get_firmwares(sdiodev->dev, BRCMF_FW_REQUEST_NVRAM, + sdiodev->fw_name, sdiodev->nvram_name, brcmf_sdio_firmware_callback); if (ret != 0) { brcmf_err("async firmware request failed: %d\n", ret); - kfree(fwreq); goto fail; } @@ -4350,7 +4263,7 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) if (bus->ci) { if (bus->sdiodev->state != BRCMF_SDIOD_NOMEDIUM) { - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); brcmf_sdio_wd_timer(bus, false); brcmf_sdio_clkctl(bus, CLK_AVAIL, false); /* Leave the device in state where it is @@ -4360,7 +4273,7 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) msleep(20); brcmf_chip_set_passive(bus->ci); brcmf_sdio_clkctl(bus, CLK_NONE, false); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); } brcmf_chip_detach(bus->ci); } @@ -4407,9 +4320,9 @@ int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep) { int ret; - sdio_claim_host(bus->sdiodev->func1); + sdio_claim_host(bus->sdiodev->func[1]); ret = brcmf_sdio_bus_sleep(bus, sleep, false); - sdio_release_host(bus->sdiodev->func1); + sdio_release_host(bus->sdiodev->func[1]); return ret; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 34b031154da93..f3da32fc6360b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -21,6 +21,10 @@ #include #include "firmware.h" +#define SDIO_FUNC_0 0 +#define SDIO_FUNC_1 1 +#define SDIO_FUNC_2 2 + #define SDIOD_FBR_SIZE 0x100 /* io_en */ @@ -35,29 +39,28 @@ #define INTR_STATUS_FUNC1 0x2 #define INTR_STATUS_FUNC2 0x4 +/* Maximum number of I/O funcs */ +#define SDIOD_MAX_IOFUNCS 7 + /* mask of register map */ #define REG_F0_REG_MASK 0x7FF #define REG_F1_MISC_MASK 0x1FFFF -/* function 0 vendor specific CCCR registers */ +/* as of sdiod rev 0, supports 3 functions */ +#define SBSDIO_NUM_FUNCTION 3 +/* function 0 vendor specific CCCR registers */ #define SDIO_CCCR_BRCM_CARDCAP 0xf0 -#define SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT BIT(1) -#define SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT BIT(2) -#define SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC BIT(3) - -/* Interrupt enable bits for each function */ -#define SDIO_CCCR_IEN_FUNC0 BIT(0) -#define SDIO_CCCR_IEN_FUNC1 BIT(1) -#define SDIO_CCCR_IEN_FUNC2 BIT(2) - -#define SDIO_CCCR_BRCM_CARDCTRL 0xf1 -#define SDIO_CCCR_BRCM_CARDCTRL_WLANRESET BIT(1) - +#define SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT 0x02 +#define SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT 0x04 +#define SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC 0x08 +#define SDIO_CCCR_BRCM_CARDCTRL 0xf1 +#define SDIO_CCCR_BRCM_CARDCTRL_WLANRESET 0x02 #define SDIO_CCCR_BRCM_SEPINT 0xf2 -#define SDIO_CCCR_BRCM_SEPINT_MASK BIT(0) -#define SDIO_CCCR_BRCM_SEPINT_OE BIT(1) -#define SDIO_CCCR_BRCM_SEPINT_ACT_HI BIT(2) + +#define SDIO_SEPINT_MASK 0x01 +#define SDIO_SEPINT_OE 0x02 +#define SDIO_SEPINT_ACT_HI 0x04 /* function 1 miscellaneous registers */ @@ -77,7 +80,7 @@ #define SBSDIO_GPIO_OUT 0x10006 /* gpio enable */ #define SBSDIO_GPIO_EN 0x10007 -/* rev < 7, watermark for sdio device TX path */ +/* rev < 7, watermark for sdio device */ #define SBSDIO_WATERMARK 0x10008 /* control busy signal generation */ #define SBSDIO_DEVICE_CTL 0x10009 @@ -104,13 +107,6 @@ #define SBSDIO_FUNC1_RFRAMEBCHI 0x1001C /* MesBusyCtl (rev 11) */ #define SBSDIO_FUNC1_MESBUSYCTRL 0x1001D -/* Watermark for sdio device RX path */ -#define SBSDIO_MESBUSY_RXFIFO_WM_MASK 0x7F -#define SBSDIO_MESBUSY_RXFIFO_WM_SHIFT 0 -/* Enable busy capability for MES access */ -#define SBSDIO_MESBUSYCTRL_ENAB 0x80 -#define SBSDIO_MESBUSYCTRL_ENAB_SHIFT 7 - /* Sdio Core Rev 12 */ #define SBSDIO_FUNC1_WAKEUPCTRL 0x1001E #define SBSDIO_FUNC1_WCTRL_ALPWAIT_MASK 0x1 @@ -135,6 +131,11 @@ /* with b15, maps to 32-bit SB access */ #define SBSDIO_SB_ACCESS_2_4B_FLAG 0x08000 +/* valid bits in SBSDIO_FUNC1_SBADDRxxx regs */ + +#define SBSDIO_SBADDRLOW_MASK 0x80 /* Valid bits in SBADDRLOW */ +#define SBSDIO_SBADDRMID_MASK 0xff /* Valid bits in SBADDRMID */ +#define SBSDIO_SBADDRHIGH_MASK 0xffU /* Valid bits in SBADDRHIGH */ /* Address bits from SBADDR regs */ #define SBSDIO_SBWINDOW_MASK 0xffff8000 @@ -177,10 +178,9 @@ struct brcmf_sdio; struct brcmf_sdiod_freezer; struct brcmf_sdio_dev { - struct sdio_func *func1; - struct sdio_func *func2; + struct sdio_func *func[SDIO_MAX_FUNCS]; + u8 num_funcs; /* Supported funcs on client */ u32 sbwad; /* Save backplane window address */ - struct brcmf_core *cc_core; /* chipcommon core info struct */ struct brcmf_sdio *bus; struct device *dev; struct brcmf_bus *bus_if; @@ -296,24 +296,13 @@ struct sdpcmd_regs { int brcmf_sdiod_intr_register(struct brcmf_sdio_dev *sdiodev); void brcmf_sdiod_intr_unregister(struct brcmf_sdio_dev *sdiodev); -/* SDIO device register access interface */ -/* Accessors for SDIO Function 0 */ -#define brcmf_sdiod_func0_rb(sdiodev, addr, r) \ - sdio_f0_readb((sdiodev)->func1, (addr), (r)) - -#define brcmf_sdiod_func0_wb(sdiodev, addr, v, ret) \ - sdio_f0_writeb((sdiodev)->func1, (v), (addr), (ret)) - -/* Accessors for SDIO Function 1 */ -#define brcmf_sdiod_readb(sdiodev, addr, r) \ - sdio_readb((sdiodev)->func1, (addr), (r)) - -#define brcmf_sdiod_writeb(sdiodev, addr, v, ret) \ - sdio_writeb((sdiodev)->func1, (v), (addr), (ret)) - -u32 brcmf_sdiod_readl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret); -void brcmf_sdiod_writel(struct brcmf_sdio_dev *sdiodev, u32 addr, u32 data, - int *ret); +/* sdio device register access interface */ +u8 brcmf_sdiod_regrb(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret); +u32 brcmf_sdiod_regrl(struct brcmf_sdio_dev *sdiodev, u32 addr, int *ret); +void brcmf_sdiod_regwb(struct brcmf_sdio_dev *sdiodev, u32 addr, u8 data, + int *ret); +void brcmf_sdiod_regwl(struct brcmf_sdio_dev *sdiodev, u32 addr, u32 data, + int *ret); /* Buffer transfer to/from device (client) core via cmd53. * fn: function number @@ -353,8 +342,7 @@ int brcmf_sdiod_ramrw(struct brcmf_sdio_dev *sdiodev, bool write, u32 address, u8 *data, uint size); /* Issue an abort to the specified function */ -int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, struct sdio_func *func); - +int brcmf_sdiod_abort(struct brcmf_sdio_dev *sdiodev, uint fn); void brcmf_sdiod_sgtable_alloc(struct brcmf_sdio_dev *sdiodev); void brcmf_sdiod_change_state(struct brcmf_sdio_dev *sdiodev, enum brcmf_sdiod_state state); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 44ead0fea7c61..11ffaa01599eb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -46,13 +46,13 @@ #define BRCMF_USB_CBCTL_READ 1 #define BRCMF_USB_MAX_PKT_SIZE 1600 -BRCMF_FW_DEF(43143, "brcmfmac43143"); -BRCMF_FW_DEF(43236B, "brcmfmac43236b"); -BRCMF_FW_DEF(43242A, "brcmfmac43242a"); -BRCMF_FW_DEF(43569, "brcmfmac43569"); -BRCMF_FW_DEF(4373, "brcmfmac4373"); +BRCMF_FW_DEF(43143, "brcmfmac43143.bin"); +BRCMF_FW_DEF(43236B, "brcmfmac43236b.bin"); +BRCMF_FW_DEF(43242A, "brcmfmac43242a.bin"); +BRCMF_FW_DEF(43569, "brcmfmac43569.bin"); +BRCMF_FW_DEF(4373, "brcmfmac4373.bin"); -static const struct brcmf_firmware_mapping brcmf_usb_fwnames[] = { +static struct brcmf_firmware_mapping brcmf_usb_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143), BRCMF_FW_ENTRY(BRCM_CC_43235_CHIP_ID, 0x00000008, 43236B), BRCMF_FW_ENTRY(BRCM_CC_43236_CHIP_ID, 0x00000008, 43236B), @@ -160,7 +160,7 @@ struct brcmf_usbdev_info { struct usb_device *usbdev; struct device *dev; - struct completion dev_init_done; + struct mutex dev_init_lock; int ctl_in_pipe, ctl_out_pipe; struct urb *ctl_urb; /* URB for control endpoint */ @@ -684,18 +684,12 @@ static int brcmf_usb_up(struct device *dev) static void brcmf_cancel_all_urbs(struct brcmf_usbdev_info *devinfo) { - int i; - if (devinfo->ctl_urb) usb_kill_urb(devinfo->ctl_urb); if (devinfo->bulk_urb) usb_kill_urb(devinfo->bulk_urb); - if (devinfo->tx_reqs) - for (i = 0; i < devinfo->bus_pub.ntxq; i++) - usb_kill_urb(devinfo->tx_reqs[i].urb); - if (devinfo->rx_reqs) - for (i = 0; i < devinfo->bus_pub.nrxq; i++) - usb_kill_urb(devinfo->rx_reqs[i].urb); + brcmf_usb_free_q(&devinfo->tx_postq, true); + brcmf_usb_free_q(&devinfo->rx_postq, true); } static void brcmf_usb_down(struct device *dev) @@ -1134,53 +1128,51 @@ static void brcmf_usb_wowl_config(struct device *dev, bool enabled) device_set_wakeup_enable(devinfo->dev, false); } -static -int brcmf_usb_get_fwname(struct device *dev, const char *ext, u8 *fw_name) -{ - struct brcmf_bus *bus = dev_get_drvdata(dev); - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ext, fw_name }, - }; - - fwreq = brcmf_fw_alloc_request(bus->chip, bus->chiprev, - brcmf_usb_fwnames, - ARRAY_SIZE(brcmf_usb_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return -ENOMEM; - - kfree(fwreq); - return 0; -} - static const struct brcmf_bus_ops brcmf_usb_bus_ops = { - .preinit = brcmf_usb_up, - .stop = brcmf_usb_down, .txdata = brcmf_usb_tx, + .stop = brcmf_usb_down, .txctl = brcmf_usb_tx_ctlpkt, .rxctl = brcmf_usb_rx_ctlpkt, .wowl_config = brcmf_usb_wowl_config, - .get_fwname = brcmf_usb_get_fwname, }; -#define BRCMF_USB_FW_CODE 0 +static int brcmf_usb_bus_setup(struct brcmf_usbdev_info *devinfo) +{ + int ret; + + /* Attach to the common driver interface */ + ret = brcmf_attach(devinfo->dev, devinfo->settings); + if (ret) { + brcmf_err("brcmf_attach failed\n"); + return ret; + } + + ret = brcmf_usb_up(devinfo->dev); + if (ret) + goto fail; + + ret = brcmf_bus_started(devinfo->dev); + if (ret) + goto fail; + + return 0; +fail: + brcmf_detach(devinfo->dev); + return ret; +} static void brcmf_usb_probe_phase2(struct device *dev, int ret, - struct brcmf_fw_request *fwreq) + const struct firmware *fw, + void *nvram, u32 nvlen) { struct brcmf_bus *bus = dev_get_drvdata(dev); struct brcmf_usbdev_info *devinfo = bus->bus_priv.usb->devinfo; - const struct firmware *fw; if (ret) goto error; brcmf_dbg(USB, "Start fw downloading\n"); - fw = fwreq->items[BRCMF_USB_FW_CODE].binary; - kfree(fwreq); - ret = check_file(fw->data); if (ret < 0) { brcmf_err("invalid firmware\n"); @@ -1196,46 +1188,23 @@ static void brcmf_usb_probe_phase2(struct device *dev, int ret, if (ret) goto error; - /* Attach to the common driver interface */ - ret = brcmf_attach(devinfo->dev, devinfo->settings); + ret = brcmf_usb_bus_setup(devinfo); if (ret) goto error; - complete(&devinfo->dev_init_done); + mutex_unlock(&devinfo->dev_init_lock); return; error: brcmf_dbg(TRACE, "failed: dev=%s, err=%d\n", dev_name(dev), ret); - complete(&devinfo->dev_init_done); + mutex_unlock(&devinfo->dev_init_lock); device_release_driver(dev); } -static struct brcmf_fw_request * -brcmf_usb_prepare_fw_request(struct brcmf_usbdev_info *devinfo) -{ - struct brcmf_fw_request *fwreq; - struct brcmf_fw_name fwnames[] = { - { ".bin", devinfo->fw_name }, - }; - - fwreq = brcmf_fw_alloc_request(devinfo->bus_pub.devid, - devinfo->bus_pub.chiprev, - brcmf_usb_fwnames, - ARRAY_SIZE(brcmf_usb_fwnames), - fwnames, ARRAY_SIZE(fwnames)); - if (!fwreq) - return NULL; - - fwreq->items[BRCMF_USB_FW_CODE].type = BRCMF_FW_TYPE_BINARY; - - return fwreq; -} - static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) { struct brcmf_bus *bus = NULL; struct brcmf_usbdev *bus_pub = NULL; struct device *dev = devinfo->dev; - struct brcmf_fw_request *fwreq; int ret; brcmf_dbg(USB, "Enter\n"); @@ -1269,27 +1238,28 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) } if (!brcmf_usb_dlneeded(devinfo)) { - ret = brcmf_attach(devinfo->dev, devinfo->settings); + ret = brcmf_usb_bus_setup(devinfo); if (ret) goto fail; /* we are done */ - complete(&devinfo->dev_init_done); + mutex_unlock(&devinfo->dev_init_lock); return 0; } bus->chip = bus_pub->devid; bus->chiprev = bus_pub->chiprev; - fwreq = brcmf_usb_prepare_fw_request(devinfo); - if (!fwreq) { - ret = -ENOMEM; + ret = brcmf_fw_map_chip_to_name(bus_pub->devid, bus_pub->chiprev, + brcmf_usb_fwnames, + ARRAY_SIZE(brcmf_usb_fwnames), + devinfo->fw_name, NULL); + if (ret) goto fail; - } /* request firmware here */ - ret = brcmf_fw_get_firmwares(dev, fwreq, brcmf_usb_probe_phase2); + ret = brcmf_fw_get_firmwares(dev, 0, devinfo->fw_name, NULL, + brcmf_usb_probe_phase2); if (ret) { brcmf_err("firmware request failed: %d\n", ret); - kfree(fwreq); goto fail; } @@ -1333,10 +1303,11 @@ brcmf_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) devinfo->usbdev = usb; devinfo->dev = &usb->dev; - /* Init completion, to protect for disconnect while still loading. + /* Take an init lock, to protect for disconnect while still loading. * Necessary because of the asynchronous firmware load construction */ - init_completion(&devinfo->dev_init_done); + mutex_init(&devinfo->dev_init_lock); + mutex_lock(&devinfo->dev_init_lock); usb_set_intfdata(intf, devinfo); @@ -1414,7 +1385,7 @@ brcmf_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) return 0; fail: - complete(&devinfo->dev_init_done); + mutex_unlock(&devinfo->dev_init_lock); kfree(devinfo); usb_set_intfdata(intf, NULL); return ret; @@ -1429,7 +1400,7 @@ brcmf_usb_disconnect(struct usb_interface *intf) devinfo = (struct brcmf_usbdev_info *)usb_get_intfdata(intf); if (devinfo) { - wait_for_completion(&devinfo->dev_init_done); + mutex_lock(&devinfo->dev_init_lock); /* Make sure that devinfo still exists. Firmware probe routines * may have released the device and cleared the intfdata. */ @@ -1470,7 +1441,7 @@ static int brcmf_usb_resume(struct usb_interface *intf) brcmf_dbg(USB, "Enter\n"); if (!devinfo->wowl_enabled) - return brcmf_attach(devinfo->dev, devinfo->settings); + return brcmf_usb_bus_setup(devinfo); devinfo->bus_pub.state = BRCMFMAC_USB_STATE_UP; brcmf_usb_rx_fill_all(devinfo); @@ -1481,20 +1452,11 @@ static int brcmf_usb_reset_resume(struct usb_interface *intf) { struct usb_device *usb = interface_to_usbdev(intf); struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev); - struct brcmf_fw_request *fwreq; - int ret; brcmf_dbg(USB, "Enter\n"); - fwreq = brcmf_usb_prepare_fw_request(devinfo); - if (!fwreq) - return -ENOMEM; - - ret = brcmf_fw_get_firmwares(&usb->dev, fwreq, brcmf_usb_probe_phase2); - if (ret < 0) - kfree(fwreq); - - return ret; + return brcmf_fw_get_firmwares(&usb->dev, 0, devinfo->fw_name, NULL, + brcmf_usb_probe_phase2); } #define BRCMF_USB_DEVICE(dev_id) \ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d493021f60318..8eff2753abade 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -35,10 +35,9 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, struct brcmf_if *ifp; const struct brcmf_vndr_dcmd_hdr *cmdhdr = data; struct sk_buff *reply; - unsigned int payload, ret_len; + int ret, payload, ret_len; void *dcmd_buf = NULL, *wr_pointer; u16 msglen, maxmsglen = PAGE_SIZE - 0x100; - int ret; if (len < sizeof(*cmdhdr)) { brcmf_err("vendor command too short: %d\n", len); @@ -66,7 +65,7 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, brcmf_err("oversize return buffer %d\n", ret_len); ret_len = BRCMF_DCMD_MAXLEN; } - payload = max_t(unsigned int, ret_len, len) + 1; + payload = max(ret_len, len) + 1; dcmd_buf = vzalloc(payload); if (NULL == dcmd_buf) return -ENOMEM; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c index db783e94f929e..3a03287fa9122 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/channel.c @@ -652,6 +652,7 @@ static void brcms_reg_apply_radar_flags(struct wiphy *wiphy) */ if (!(ch->flags & IEEE80211_CHAN_DISABLED)) ch->flags |= IEEE80211_CHAN_RADAR | + IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_NO_IR; } } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/debug.c index 2fe1f68632782..7a1fbb2e3a717 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/debug.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/debug.c @@ -214,7 +214,7 @@ brcms_debugfs_add_entry(struct brcms_pub *drvr, const char *fn, entry->read = read_fn; entry->drvr = drvr; - dentry = debugfs_create_file(fn, 0444, dentry, entry, + dentry = debugfs_create_file(fn, S_IRUGO, dentry, entry, &brcms_debugfs_def_ops); return PTR_ERR_OR_ZERO(dentry); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c index 6188275b17e5a..ddfdfe177e245 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/mac80211_if.c @@ -108,7 +108,7 @@ MODULE_DEVICE_TABLE(bcma, brcms_coreid_table); * flags are specified by the BRCM_DL_* macros in * drivers/net/wireless/brcm80211/include/defs.h. */ -module_param_named(debug, brcm_msg_level, uint, 0644); +module_param_named(debug, brcm_msg_level, uint, S_IRUGO | S_IWUSR); #endif static struct ieee80211_channel brcms_2ghz_chantable[] = { @@ -502,7 +502,6 @@ brcms_ops_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif) } spin_lock_bh(&wl->lock); - wl->wlc->vif = vif; wl->mute_tx = false; brcms_c_mute(wl->wlc, false); if (vif->type == NL80211_IFTYPE_STATION) @@ -520,11 +519,6 @@ brcms_ops_add_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif) static void brcms_ops_remove_interface(struct ieee80211_hw *hw, struct ieee80211_vif *vif) { - struct brcms_info *wl = hw->priv; - - spin_lock_bh(&wl->lock); - wl->wlc->vif = NULL; - spin_unlock_bh(&wl->lock); } static int brcms_ops_config(struct ieee80211_hw *hw, u32 changed) @@ -846,8 +840,8 @@ brcms_ops_ampdu_action(struct ieee80211_hw *hw, status = brcms_c_aggregatable(wl->wlc, tid); spin_unlock_bh(&wl->lock); if (!status) { - brcms_dbg_ht(wl->wlc->hw->d11core, - "START: tid %d is not agg\'able\n", tid); + brcms_err(wl->wlc->hw->d11core, + "START: tid %d is not agg\'able\n", tid); return -EINVAL; } ieee80211_start_tx_ba_cb_irqsafe(vif, sta->addr, tid); @@ -943,25 +937,6 @@ static void brcms_ops_set_tsf(struct ieee80211_hw *hw, spin_unlock_bh(&wl->lock); } -static int brcms_ops_beacon_set_tim(struct ieee80211_hw *hw, - struct ieee80211_sta *sta, bool set) -{ - struct brcms_info *wl = hw->priv; - struct sk_buff *beacon = NULL; - u16 tim_offset = 0; - - spin_lock_bh(&wl->lock); - if (wl->wlc->vif) - beacon = ieee80211_beacon_get_tim(hw, wl->wlc->vif, - &tim_offset, NULL); - if (beacon) - brcms_c_set_new_beacon(wl->wlc, beacon, tim_offset, - wl->wlc->vif->bss_conf.dtim_period); - spin_unlock_bh(&wl->lock); - - return 0; -} - static const struct ieee80211_ops brcms_ops = { .tx = brcms_ops_tx, .start = brcms_ops_start, @@ -980,7 +955,6 @@ static const struct ieee80211_ops brcms_ops = { .flush = brcms_ops_flush, .get_tsf = brcms_ops_get_tsf, .set_tsf = brcms_ops_set_tsf, - .set_tim = brcms_ops_beacon_set_tim, }; void brcms_dpc(unsigned long data) @@ -1589,7 +1563,7 @@ void brcms_free_timer(struct brcms_timer *t) } /* - * precondition: no locking required + * precondition: perimeter lock has been acquired */ int brcms_ucode_init_buf(struct brcms_info *wl, void **pbuf, u32 idx) { @@ -1604,10 +1578,10 @@ int brcms_ucode_init_buf(struct brcms_info *wl, void **pbuf, u32 idx) if (le32_to_cpu(hdr->idx) == idx) { pdata = wl->fw.fw_bin[i]->data + le32_to_cpu(hdr->offset); - *pbuf = kvmalloc(len, GFP_KERNEL); + *pbuf = kmemdup(pdata, len, GFP_ATOMIC); if (*pbuf == NULL) goto fail; - memcpy(*pbuf, pdata, len); + return 0; } } @@ -1655,7 +1629,7 @@ int brcms_ucode_init_uint(struct brcms_info *wl, size_t *n_bytes, u32 idx) */ void brcms_ucode_free_buf(void *p) { - kvfree(p); + kfree(p); } /* diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c index 7d4e8f589fdc4..0a14942b82167 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.c @@ -507,7 +507,7 @@ brcms_c_attach_malloc(uint unit, uint *err, uint devid) wlc->hw->wlc = wlc; wlc->hw->bandstate[0] = - kcalloc(MAXBANDS, sizeof(struct brcms_hw_band), GFP_ATOMIC); + kzalloc(sizeof(struct brcms_hw_band) * MAXBANDS, GFP_ATOMIC); if (wlc->hw->bandstate[0] == NULL) { *err = 1006; goto fail; @@ -521,8 +521,7 @@ brcms_c_attach_malloc(uint unit, uint *err, uint devid) } wlc->modulecb = - kcalloc(BRCMS_MAXMODULES, sizeof(struct modulecb), - GFP_ATOMIC); + kzalloc(sizeof(struct modulecb) * BRCMS_MAXMODULES, GFP_ATOMIC); if (wlc->modulecb == NULL) { *err = 1009; goto fail; @@ -554,7 +553,7 @@ brcms_c_attach_malloc(uint unit, uint *err, uint devid) } wlc->bandstate[0] = - kcalloc(MAXBANDS, sizeof(struct brcms_band), GFP_ATOMIC); + kzalloc(sizeof(struct brcms_band)*MAXBANDS, GFP_ATOMIC); if (wlc->bandstate[0] == NULL) { *err = 1025; goto fail; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.h b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.h index 9f76b880814e8..c4d135cff04ad 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/main.h @@ -563,7 +563,6 @@ struct brcms_c_info { struct wiphy *wiphy; struct scb pri_scb; - struct ieee80211_vif *vif; struct sk_buff *beacon; u16 beacon_tim_offset; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c index 35e3b101e5cf0..1c4e9dd57960f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_cmn.c @@ -159,7 +159,7 @@ u16 read_radio_reg(struct brcms_phy *pi, u16 addr) { u16 data; - if (addr == RADIO_IDCODE) + if ((addr == RADIO_IDCODE)) return 0xffff; switch (pi->pubpi.phy_type) { @@ -1916,7 +1916,7 @@ void wlc_phy_txpower_update_shm(struct brcms_phy *pi) pi->hwpwr_txcur); for (j = TXP_FIRST_OFDM; j <= TXP_LAST_OFDM; j++) { - static const u8 ucode_ofdm_rates[] = { + const u8 ucode_ofdm_rates[] = { 0x0c, 0x12, 0x18, 0x24, 0x30, 0x48, 0x60, 0x6c }; offset = wlapi_bmac_rate_shm_offset( diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c index 9fb0d9fbd9395..93d4cde0eb313 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c @@ -1387,7 +1387,7 @@ wlc_lcnphy_rx_iq_cal(struct brcms_phy *pi, s16 *ptr; struct brcms_phy_lcnphy *pi_lcn = pi->u.pi_lcnphy; - ptr = kmalloc_array(131, sizeof(s16), GFP_ATOMIC); + ptr = kmalloc(sizeof(s16) * 131, GFP_ATOMIC); if (NULL == ptr) return false; if (module == 2) { @@ -2670,7 +2670,7 @@ wlc_lcnphy_tx_iqlo_cal(struct brcms_phy *pi, u16 *values_to_save; struct brcms_phy_lcnphy *pi_lcn = pi->u.pi_lcnphy; - values_to_save = kmalloc_array(20, sizeof(u16), GFP_ATOMIC); + values_to_save = kmalloc(sizeof(u16) * 20, GFP_ATOMIC); if (NULL == values_to_save) return; @@ -3388,8 +3388,13 @@ void wlc_lcnphy_deaf_mode(struct brcms_phy *pi, bool mode) u8 phybw40; phybw40 = CHSPEC_IS40(pi->radio_chanspec); - mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5); - mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9); + if (LCNREV_LT(pi->pubpi.phy_rev, 2)) { + mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5); + mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9); + } else { + mod_phy_reg(pi, 0x4b0, (0x1 << 5), (mode) << 5); + mod_phy_reg(pi, 0x4b1, (0x1 << 9), 0 << 9); + } if (phybw40 == 0) { mod_phy_reg((pi), 0x410, @@ -3678,11 +3683,11 @@ wlc_lcnphy_a1(struct brcms_phy *pi, int cal_type, int num_levels, u16 *phy_c32; phy_c21 = 0; phy_c10 = phy_c13 = phy_c14 = phy_c8 = 0; - ptr = kmalloc_array(131, sizeof(s16), GFP_ATOMIC); + ptr = kmalloc(sizeof(s16) * 131, GFP_ATOMIC); if (NULL == ptr) return; - phy_c32 = kmalloc_array(20, sizeof(u16), GFP_ATOMIC); + phy_c32 = kmalloc(sizeof(u16) * 20, GFP_ATOMIC); if (NULL == phy_c32) { kfree(ptr); return; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c index bedec1606caa4..ef685465f80ad 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_n.c @@ -16049,7 +16049,8 @@ static void wlc_phy_workarounds_nphy_gainctrl(struct brcms_phy *pi) wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_UPDATEGAINU, rfseq_updategainu_events, rfseq_updategainu_dlys, - ARRAY_SIZE(rfseq_updategainu_events)); + sizeof(rfseq_updategainu_events) / + sizeof(rfseq_updategainu_events[0])); mod_phy_reg(pi, 0x153, (0xff << 8), (90 << 8)); @@ -16060,8 +16061,52 @@ static void wlc_phy_workarounds_nphy_gainctrl(struct brcms_phy *pi) } } -static void wlc_phy_workarounds_nphy_rev7(struct brcms_phy *pi) +static void wlc_phy_workarounds_nphy(struct brcms_phy *pi) { + static const u8 rfseq_rx2tx_events[] = { + NPHY_RFSEQ_CMD_NOP, + NPHY_RFSEQ_CMD_RXG_FBW, + NPHY_RFSEQ_CMD_TR_SWITCH, + NPHY_RFSEQ_CMD_CLR_HIQ_DIS, + NPHY_RFSEQ_CMD_RXPD_TXPD, + NPHY_RFSEQ_CMD_TX_GAIN, + NPHY_RFSEQ_CMD_EXT_PA + }; + u8 rfseq_rx2tx_dlys[] = { 8, 6, 6, 2, 4, 60, 1 }; + static const u8 rfseq_tx2rx_events[] = { + NPHY_RFSEQ_CMD_NOP, + NPHY_RFSEQ_CMD_EXT_PA, + NPHY_RFSEQ_CMD_TX_GAIN, + NPHY_RFSEQ_CMD_RXPD_TXPD, + NPHY_RFSEQ_CMD_TR_SWITCH, + NPHY_RFSEQ_CMD_RXG_FBW, + NPHY_RFSEQ_CMD_CLR_HIQ_DIS + }; + static const u8 rfseq_tx2rx_dlys[] = { 8, 6, 2, 4, 4, 6, 1 }; + static const u8 rfseq_tx2rx_events_rev3[] = { + NPHY_REV3_RFSEQ_CMD_EXT_PA, + NPHY_REV3_RFSEQ_CMD_INT_PA_PU, + NPHY_REV3_RFSEQ_CMD_TX_GAIN, + NPHY_REV3_RFSEQ_CMD_RXPD_TXPD, + NPHY_REV3_RFSEQ_CMD_TR_SWITCH, + NPHY_REV3_RFSEQ_CMD_RXG_FBW, + NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS, + NPHY_REV3_RFSEQ_CMD_END + }; + static const u8 rfseq_tx2rx_dlys_rev3[] = { 8, 4, 2, 2, 4, 4, 6, 1 }; + u8 rfseq_rx2tx_events_rev3[] = { + NPHY_REV3_RFSEQ_CMD_NOP, + NPHY_REV3_RFSEQ_CMD_RXG_FBW, + NPHY_REV3_RFSEQ_CMD_TR_SWITCH, + NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS, + NPHY_REV3_RFSEQ_CMD_RXPD_TXPD, + NPHY_REV3_RFSEQ_CMD_TX_GAIN, + NPHY_REV3_RFSEQ_CMD_INT_PA_PU, + NPHY_REV3_RFSEQ_CMD_EXT_PA, + NPHY_REV3_RFSEQ_CMD_END + }; + u8 rfseq_rx2tx_dlys_rev3[] = { 8, 6, 6, 4, 4, 18, 42, 1, 1 }; + static const u8 rfseq_rx2tx_events_rev3_ipa[] = { NPHY_REV3_RFSEQ_CMD_NOP, NPHY_REV3_RFSEQ_CMD_RXG_FBW, @@ -16073,18 +16118,31 @@ static void wlc_phy_workarounds_nphy_rev7(struct brcms_phy *pi) NPHY_REV3_RFSEQ_CMD_INT_PA_PU, NPHY_REV3_RFSEQ_CMD_END }; - static const u8 rfseq_rx2tx_dlys_rev3_ipa[] = - { 8, 6, 6, 4, 4, 16, 43, 1, 1 }; + static const u8 rfseq_rx2tx_dlys_rev3_ipa[] = { 8, 6, 6, 4, 4, 16, 43, 1, 1 }; static const u16 rfseq_rx2tx_dacbufpu_rev7[] = { 0x10f, 0x10f }; - u32 leg_data_weights; + + s16 alpha0, alpha1, alpha2; + s16 beta0, beta1, beta2; + u32 leg_data_weights, ht_data_weights, nss1_data_weights, + stbc_data_weights; u8 chan_freq_range = 0; static const u16 dac_control = 0x0002; u16 aux_adc_vmid_rev7_core0[] = { 0x8e, 0x96, 0x96, 0x96 }; u16 aux_adc_vmid_rev7_core1[] = { 0x8f, 0x9f, 0x9f, 0x96 }; + u16 aux_adc_vmid_rev4[] = { 0xa2, 0xb4, 0xb4, 0x89 }; + u16 aux_adc_vmid_rev3[] = { 0xa2, 0xb4, 0xb4, 0x89 }; + u16 *aux_adc_vmid; u16 aux_adc_gain_rev7[] = { 0x02, 0x02, 0x02, 0x02 }; + u16 aux_adc_gain_rev4[] = { 0x02, 0x02, 0x02, 0x00 }; + u16 aux_adc_gain_rev3[] = { 0x02, 0x02, 0x02, 0x00 }; + u16 *aux_adc_gain; + static const u16 sk_adc_vmid[] = { 0xb4, 0xb4, 0xb4, 0x24 }; + static const u16 sk_adc_gain[] = { 0x02, 0x02, 0x02, 0x02 }; s32 min_nvar_val = 0x18d; s32 min_nvar_offset_6mbps = 20; u8 pdetrange; + u8 triso; + u16 regval; u16 afectrl_adc_ctrl1_rev7 = 0x20; u16 afectrl_adc_ctrl2_rev7 = 0x0; u16 rfseq_rx2tx_lpf_h_hpc_rev7 = 0x77; @@ -16113,938 +16171,964 @@ static void wlc_phy_workarounds_nphy_rev7(struct brcms_phy *pi) u16 freq; int coreNum; + if (CHSPEC_IS5G(pi->radio_chanspec)) + wlc_phy_classifier_nphy(pi, NPHY_ClassifierCtrl_cck_en, 0); + else + wlc_phy_classifier_nphy(pi, NPHY_ClassifierCtrl_cck_en, 1); - if (NREV_IS(pi->pubpi.phy_rev, 7)) { - mod_phy_reg(pi, 0x221, (0x1 << 4), (1 << 4)); - - mod_phy_reg(pi, 0x160, (0x7f << 0), (32 << 0)); - mod_phy_reg(pi, 0x160, (0x7f << 8), (39 << 8)); - mod_phy_reg(pi, 0x161, (0x7f << 0), (46 << 0)); - mod_phy_reg(pi, 0x161, (0x7f << 8), (51 << 8)); - mod_phy_reg(pi, 0x162, (0x7f << 0), (55 << 0)); - mod_phy_reg(pi, 0x162, (0x7f << 8), (58 << 8)); - mod_phy_reg(pi, 0x163, (0x7f << 0), (60 << 0)); - mod_phy_reg(pi, 0x163, (0x7f << 8), (62 << 8)); - mod_phy_reg(pi, 0x164, (0x7f << 0), (62 << 0)); - mod_phy_reg(pi, 0x164, (0x7f << 8), (63 << 8)); - mod_phy_reg(pi, 0x165, (0x7f << 0), (63 << 0)); - mod_phy_reg(pi, 0x165, (0x7f << 8), (64 << 8)); - mod_phy_reg(pi, 0x166, (0x7f << 0), (64 << 0)); - mod_phy_reg(pi, 0x166, (0x7f << 8), (64 << 8)); - mod_phy_reg(pi, 0x167, (0x7f << 0), (64 << 0)); - mod_phy_reg(pi, 0x167, (0x7f << 8), (64 << 8)); - } + if (pi->phyhang_avoid) + wlc_phy_stay_in_carriersearch_nphy(pi, true); - if (NREV_LE(pi->pubpi.phy_rev, 8)) { - write_phy_reg(pi, 0x23f, 0x1b0); - write_phy_reg(pi, 0x240, 0x1b0); - } + or_phy_reg(pi, 0xb1, NPHY_IQFlip_ADC1 | NPHY_IQFlip_ADC2); - if (NREV_GE(pi->pubpi.phy_rev, 8)) - mod_phy_reg(pi, 0xbd, (0xff << 0), (114 << 0)); + if (NREV_GE(pi->pubpi.phy_rev, 7)) { - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x00, 16, - &dac_control); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x10, 16, - &dac_control); + if (NREV_IS(pi->pubpi.phy_rev, 7)) { + mod_phy_reg(pi, 0x221, (0x1 << 4), (1 << 4)); + + mod_phy_reg(pi, 0x160, (0x7f << 0), (32 << 0)); + mod_phy_reg(pi, 0x160, (0x7f << 8), (39 << 8)); + mod_phy_reg(pi, 0x161, (0x7f << 0), (46 << 0)); + mod_phy_reg(pi, 0x161, (0x7f << 8), (51 << 8)); + mod_phy_reg(pi, 0x162, (0x7f << 0), (55 << 0)); + mod_phy_reg(pi, 0x162, (0x7f << 8), (58 << 8)); + mod_phy_reg(pi, 0x163, (0x7f << 0), (60 << 0)); + mod_phy_reg(pi, 0x163, (0x7f << 8), (62 << 8)); + mod_phy_reg(pi, 0x164, (0x7f << 0), (62 << 0)); + mod_phy_reg(pi, 0x164, (0x7f << 8), (63 << 8)); + mod_phy_reg(pi, 0x165, (0x7f << 0), (63 << 0)); + mod_phy_reg(pi, 0x165, (0x7f << 8), (64 << 8)); + mod_phy_reg(pi, 0x166, (0x7f << 0), (64 << 0)); + mod_phy_reg(pi, 0x166, (0x7f << 8), (64 << 8)); + mod_phy_reg(pi, 0x167, (0x7f << 0), (64 << 0)); + mod_phy_reg(pi, 0x167, (0x7f << 8), (64 << 8)); + } - wlc_phy_table_read_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, - 1, 0, 32, &leg_data_weights); - leg_data_weights = leg_data_weights & 0xffffff; - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, - 1, 0, 32, &leg_data_weights); + if (NREV_LE(pi->pubpi.phy_rev, 8)) { + write_phy_reg(pi, 0x23f, 0x1b0); + write_phy_reg(pi, 0x240, 0x1b0); + } - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, - 2, 0x15e, 16, rfseq_rx2tx_dacbufpu_rev7); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x16e, 16, - rfseq_rx2tx_dacbufpu_rev7); + if (NREV_GE(pi->pubpi.phy_rev, 8)) + mod_phy_reg(pi, 0xbd, (0xff << 0), (114 << 0)); - if (PHY_IPA(pi)) - wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, - rfseq_rx2tx_events_rev3_ipa, - rfseq_rx2tx_dlys_rev3_ipa, - ARRAY_SIZE - (rfseq_rx2tx_events_rev3_ipa)); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x00, 16, + &dac_control); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x10, 16, + &dac_control); - mod_phy_reg(pi, 0x299, (0x3 << 14), (0x1 << 14)); - mod_phy_reg(pi, 0x29d, (0x3 << 14), (0x1 << 14)); + wlc_phy_table_read_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, + 1, 0, 32, &leg_data_weights); + leg_data_weights = leg_data_weights & 0xffffff; + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, + 1, 0, 32, &leg_data_weights); - tx_lpf_bw_ofdm_20mhz = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x154); - tx_lpf_bw_ofdm_40mhz = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x159); - tx_lpf_bw_11b = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x152); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, + 2, 0x15e, 16, + rfseq_rx2tx_dacbufpu_rev7); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x16e, 16, + rfseq_rx2tx_dacbufpu_rev7); - if (PHY_IPA(pi)) { + if (PHY_IPA(pi)) + wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, + rfseq_rx2tx_events_rev3_ipa, + rfseq_rx2tx_dlys_rev3_ipa, + ARRAY_SIZE(rfseq_rx2tx_events_rev3_ipa)); - if (((pi->pubpi.radiorev == 5) - && (CHSPEC_IS40(pi->radio_chanspec) == 1)) - || (pi->pubpi.radiorev == 7) - || (pi->pubpi.radiorev == 8)) { + mod_phy_reg(pi, 0x299, (0x3 << 14), (0x1 << 14)); + mod_phy_reg(pi, 0x29d, (0x3 << 14), (0x1 << 14)); - rccal_bcap_val = - read_radio_reg(pi, RADIO_2057_RCCAL_BCAP_VAL); - rccal_scap_val = - read_radio_reg(pi, RADIO_2057_RCCAL_SCAP_VAL); + tx_lpf_bw_ofdm_20mhz = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x154); + tx_lpf_bw_ofdm_40mhz = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x159); + tx_lpf_bw_11b = wlc_phy_read_lpf_bw_ctl_nphy(pi, 0x152); - rccal_tx20_11b_bcap = rccal_bcap_val; - rccal_tx20_11b_scap = rccal_scap_val; + if (PHY_IPA(pi)) { - if ((pi->pubpi.radiorev == 5) && - (CHSPEC_IS40(pi->radio_chanspec) == 1)) { + if (((pi->pubpi.radiorev == 5) + && (CHSPEC_IS40(pi->radio_chanspec) == 1)) + || (pi->pubpi.radiorev == 7) + || (pi->pubpi.radiorev == 8)) { - rccal_tx20_11n_bcap = rccal_bcap_val; - rccal_tx20_11n_scap = rccal_scap_val; - rccal_tx40_11n_bcap = 0xc; - rccal_tx40_11n_scap = 0xc; + rccal_bcap_val = + read_radio_reg( + pi, + RADIO_2057_RCCAL_BCAP_VAL); + rccal_scap_val = + read_radio_reg( + pi, + RADIO_2057_RCCAL_SCAP_VAL); - rccal_ovrd = true; + rccal_tx20_11b_bcap = rccal_bcap_val; + rccal_tx20_11b_scap = rccal_scap_val; - } else if ((pi->pubpi.radiorev == 7) - || (pi->pubpi.radiorev == 8)) { + if ((pi->pubpi.radiorev == 5) && + (CHSPEC_IS40(pi->radio_chanspec) == 1)) { - tx_lpf_bw_ofdm_20mhz = 4; - tx_lpf_bw_11b = 1; + rccal_tx20_11n_bcap = rccal_bcap_val; + rccal_tx20_11n_scap = rccal_scap_val; + rccal_tx40_11n_bcap = 0xc; + rccal_tx40_11n_scap = 0xc; - if (CHSPEC_IS2G(pi->radio_chanspec)) { - rccal_tx20_11n_bcap = 0xc; - rccal_tx20_11n_scap = 0xc; - rccal_tx40_11n_bcap = 0xa; - rccal_tx40_11n_scap = 0xa; - } else { - rccal_tx20_11n_bcap = 0x14; - rccal_tx20_11n_scap = 0x14; - rccal_tx40_11n_bcap = 0xf; - rccal_tx40_11n_scap = 0xf; - } + rccal_ovrd = true; - rccal_ovrd = true; - } - } + } else if ((pi->pubpi.radiorev == 7) + || (pi->pubpi.radiorev == 8)) { - } else { + tx_lpf_bw_ofdm_20mhz = 4; + tx_lpf_bw_11b = 1; - if (pi->pubpi.radiorev == 5) { + if (CHSPEC_IS2G(pi->radio_chanspec)) { + rccal_tx20_11n_bcap = 0xc; + rccal_tx20_11n_scap = 0xc; + rccal_tx40_11n_bcap = 0xa; + rccal_tx40_11n_scap = 0xa; + } else { + rccal_tx20_11n_bcap = 0x14; + rccal_tx20_11n_scap = 0x14; + rccal_tx40_11n_bcap = 0xf; + rccal_tx40_11n_scap = 0xf; + } - tx_lpf_bw_ofdm_20mhz = 1; - tx_lpf_bw_ofdm_40mhz = 3; + rccal_ovrd = true; + } + } - rccal_bcap_val = - read_radio_reg(pi, RADIO_2057_RCCAL_BCAP_VAL); - rccal_scap_val = - read_radio_reg(pi, RADIO_2057_RCCAL_SCAP_VAL); + } else { - rccal_tx20_11b_bcap = rccal_bcap_val; - rccal_tx20_11b_scap = rccal_scap_val; + if (pi->pubpi.radiorev == 5) { - rccal_tx20_11n_bcap = 0x13; - rccal_tx20_11n_scap = 0x11; - rccal_tx40_11n_bcap = 0x13; - rccal_tx40_11n_scap = 0x11; + tx_lpf_bw_ofdm_20mhz = 1; + tx_lpf_bw_ofdm_40mhz = 3; - rccal_ovrd = true; - } - } + rccal_bcap_val = + read_radio_reg( + pi, + RADIO_2057_RCCAL_BCAP_VAL); + rccal_scap_val = + read_radio_reg( + pi, + RADIO_2057_RCCAL_SCAP_VAL); - if (rccal_ovrd) { + rccal_tx20_11b_bcap = rccal_bcap_val; + rccal_tx20_11b_scap = rccal_scap_val; - rx2tx_lpf_rc_lut_tx20_11b = - (rccal_tx20_11b_bcap << 8) | - (rccal_tx20_11b_scap << 3) | tx_lpf_bw_11b; - rx2tx_lpf_rc_lut_tx20_11n = - (rccal_tx20_11n_bcap << 8) | - (rccal_tx20_11n_scap << 3) | tx_lpf_bw_ofdm_20mhz; - rx2tx_lpf_rc_lut_tx40_11n = - (rccal_tx40_11n_bcap << 8) | - (rccal_tx40_11n_scap << 3) | tx_lpf_bw_ofdm_40mhz; + rccal_tx20_11n_bcap = 0x13; + rccal_tx20_11n_scap = 0x11; + rccal_tx40_11n_bcap = 0x13; + rccal_tx40_11n_scap = 0x11; - for (coreNum = 0; coreNum <= 1; coreNum++) { - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, - 0x152 + coreNum * 0x10, 16, - &rx2tx_lpf_rc_lut_tx20_11b); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, - 0x153 + coreNum * 0x10, 16, - &rx2tx_lpf_rc_lut_tx20_11n); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, - 0x154 + coreNum * 0x10, 16, - &rx2tx_lpf_rc_lut_tx20_11n); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, - 0x155 + coreNum * 0x10, 16, - &rx2tx_lpf_rc_lut_tx40_11n); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, - 0x156 + coreNum * 0x10, 16, - &rx2tx_lpf_rc_lut_tx40_11n); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, - 0x157 + coreNum * 0x10, 16, - &rx2tx_lpf_rc_lut_tx40_11n); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, - 0x158 + coreNum * 0x10, 16, - &rx2tx_lpf_rc_lut_tx40_11n); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, - 0x159 + coreNum * 0x10, 16, - &rx2tx_lpf_rc_lut_tx40_11n); + rccal_ovrd = true; + } } - wlc_phy_rfctrl_override_nphy_rev7(pi, (0x1 << 4), 1, 0x3, 0, - NPHY_REV7_RFCTRLOVERRIDE_ID2); - } - - write_phy_reg(pi, 0x32f, 0x3); - - if ((pi->pubpi.radiorev == 4) || (pi->pubpi.radiorev == 6)) - wlc_phy_rfctrl_override_nphy_rev7(pi, (0x1 << 2), 1, 0x3, 0, - NPHY_REV7_RFCTRLOVERRIDE_ID0); + if (rccal_ovrd) { + + rx2tx_lpf_rc_lut_tx20_11b = + (rccal_tx20_11b_bcap << 8) | + (rccal_tx20_11b_scap << 3) | + tx_lpf_bw_11b; + rx2tx_lpf_rc_lut_tx20_11n = + (rccal_tx20_11n_bcap << 8) | + (rccal_tx20_11n_scap << 3) | + tx_lpf_bw_ofdm_20mhz; + rx2tx_lpf_rc_lut_tx40_11n = + (rccal_tx40_11n_bcap << 8) | + (rccal_tx40_11n_scap << 3) | + tx_lpf_bw_ofdm_40mhz; - if ((pi->pubpi.radiorev == 3) || (pi->pubpi.radiorev == 4) || - (pi->pubpi.radiorev == 6)) { - if ((pi->sh->sromrev >= 8) - && (pi->sh->boardflags2 & BFL2_IPALVLSHIFT_3P3)) - ipalvlshift_3p3_war_en = 1; + for (coreNum = 0; coreNum <= 1; coreNum++) { + wlc_phy_table_write_nphy( + pi, NPHY_TBL_ID_RFSEQ, + 1, + 0x152 + coreNum * 0x10, + 16, + &rx2tx_lpf_rc_lut_tx20_11b); + wlc_phy_table_write_nphy( + pi, NPHY_TBL_ID_RFSEQ, + 1, + 0x153 + coreNum * 0x10, + 16, + &rx2tx_lpf_rc_lut_tx20_11n); + wlc_phy_table_write_nphy( + pi, NPHY_TBL_ID_RFSEQ, + 1, + 0x154 + coreNum * 0x10, + 16, + &rx2tx_lpf_rc_lut_tx20_11n); + wlc_phy_table_write_nphy( + pi, NPHY_TBL_ID_RFSEQ, + 1, + 0x155 + coreNum * 0x10, + 16, + &rx2tx_lpf_rc_lut_tx40_11n); + wlc_phy_table_write_nphy( + pi, NPHY_TBL_ID_RFSEQ, + 1, + 0x156 + coreNum * 0x10, + 16, + &rx2tx_lpf_rc_lut_tx40_11n); + wlc_phy_table_write_nphy( + pi, NPHY_TBL_ID_RFSEQ, + 1, + 0x157 + coreNum * 0x10, + 16, + &rx2tx_lpf_rc_lut_tx40_11n); + wlc_phy_table_write_nphy( + pi, NPHY_TBL_ID_RFSEQ, + 1, + 0x158 + coreNum * 0x10, + 16, + &rx2tx_lpf_rc_lut_tx40_11n); + wlc_phy_table_write_nphy( + pi, NPHY_TBL_ID_RFSEQ, + 1, + 0x159 + coreNum * 0x10, + 16, + &rx2tx_lpf_rc_lut_tx40_11n); + } - if (ipalvlshift_3p3_war_en) { - write_radio_reg(pi, RADIO_2057_GPAIO_CONFIG, 0x5); - write_radio_reg(pi, RADIO_2057_GPAIO_SEL1, 0x30); - write_radio_reg(pi, RADIO_2057_GPAIO_SEL0, 0x0); - or_radio_reg(pi, RADIO_2057_RXTXBIAS_CONFIG_CORE0, 0x1); - or_radio_reg(pi, RADIO_2057_RXTXBIAS_CONFIG_CORE1, 0x1); + wlc_phy_rfctrl_override_nphy_rev7( + pi, (0x1 << 4), + 1, 0x3, 0, + NPHY_REV7_RFCTRLOVERRIDE_ID2); + } - ipa2g_mainbias = 0x1f; + write_phy_reg(pi, 0x32f, 0x3); - ipa2g_casconv = 0x6f; + if ((pi->pubpi.radiorev == 4) || (pi->pubpi.radiorev == 6)) + wlc_phy_rfctrl_override_nphy_rev7( + pi, (0x1 << 2), + 1, 0x3, 0, + NPHY_REV7_RFCTRLOVERRIDE_ID0); - ipa2g_biasfilt = 0xaa; - } else { + if ((pi->pubpi.radiorev == 3) || (pi->pubpi.radiorev == 4) || + (pi->pubpi.radiorev == 6)) { + if ((pi->sh->sromrev >= 8) + && (pi->sh->boardflags2 & BFL2_IPALVLSHIFT_3P3)) + ipalvlshift_3p3_war_en = 1; + + if (ipalvlshift_3p3_war_en) { + write_radio_reg(pi, RADIO_2057_GPAIO_CONFIG, + 0x5); + write_radio_reg(pi, RADIO_2057_GPAIO_SEL1, + 0x30); + write_radio_reg(pi, RADIO_2057_GPAIO_SEL0, 0x0); + or_radio_reg(pi, + RADIO_2057_RXTXBIAS_CONFIG_CORE0, + 0x1); + or_radio_reg(pi, + RADIO_2057_RXTXBIAS_CONFIG_CORE1, + 0x1); + + ipa2g_mainbias = 0x1f; + + ipa2g_casconv = 0x6f; + + ipa2g_biasfilt = 0xaa; + } else { - ipa2g_mainbias = 0x2b; + ipa2g_mainbias = 0x2b; - ipa2g_casconv = 0x7f; + ipa2g_casconv = 0x7f; - ipa2g_biasfilt = 0xee; - } + ipa2g_biasfilt = 0xee; + } - if (CHSPEC_IS2G(pi->radio_chanspec)) { - for (coreNum = 0; coreNum <= 1; coreNum++) { - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - coreNum, IPA2G_IMAIN, - ipa2g_mainbias); - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - coreNum, IPA2G_CASCONV, - ipa2g_casconv); - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - coreNum, - IPA2G_BIAS_FILTER, - ipa2g_biasfilt); + if (CHSPEC_IS2G(pi->radio_chanspec)) { + for (coreNum = 0; coreNum <= 1; coreNum++) { + WRITE_RADIO_REG4(pi, RADIO_2057, CORE, + coreNum, IPA2G_IMAIN, + ipa2g_mainbias); + WRITE_RADIO_REG4(pi, RADIO_2057, CORE, + coreNum, IPA2G_CASCONV, + ipa2g_casconv); + WRITE_RADIO_REG4(pi, RADIO_2057, CORE, + coreNum, + IPA2G_BIAS_FILTER, + ipa2g_biasfilt); + } } } - } - - if (PHY_IPA(pi)) { - if (CHSPEC_IS2G(pi->radio_chanspec)) { - if ((pi->pubpi.radiorev == 3) - || (pi->pubpi.radiorev == 4) - || (pi->pubpi.radiorev == 6)) - txgm_idac_bleed = 0x7f; - for (coreNum = 0; coreNum <= 1; coreNum++) { - if (txgm_idac_bleed != 0) - WRITE_RADIO_REG4(pi, RADIO_2057, - CORE, coreNum, - TXGM_IDAC_BLEED, - txgm_idac_bleed); - } + if (PHY_IPA(pi)) { + if (CHSPEC_IS2G(pi->radio_chanspec)) { + if ((pi->pubpi.radiorev == 3) + || (pi->pubpi.radiorev == 4) + || (pi->pubpi.radiorev == 6)) + txgm_idac_bleed = 0x7f; - if (pi->pubpi.radiorev == 5) { for (coreNum = 0; coreNum <= 1; coreNum++) { - WRITE_RADIO_REG4(pi, RADIO_2057, - CORE, coreNum, - IPA2G_CASCONV, - 0x13); - WRITE_RADIO_REG4(pi, RADIO_2057, - CORE, coreNum, - IPA2G_IMAIN, - 0x1f); - WRITE_RADIO_REG4(pi, RADIO_2057, - CORE, coreNum, - IPA2G_BIAS_FILTER, - 0xee); - WRITE_RADIO_REG4(pi, RADIO_2057, - CORE, coreNum, - PAD2G_IDACS, - 0x8a); - WRITE_RADIO_REG4(pi, RADIO_2057, - CORE, coreNum, - PAD_BIAS_FILTER_BWS, - 0x3e); + if (txgm_idac_bleed != 0) + WRITE_RADIO_REG4( + pi, RADIO_2057, + CORE, coreNum, + TXGM_IDAC_BLEED, + txgm_idac_bleed); } - } else if ((pi->pubpi.radiorev == 7) || - (pi->pubpi.radiorev == 8)) { - if (CHSPEC_IS40(pi->radio_chanspec) == 0) { - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - 0, IPA2G_IMAIN, 0x14); - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - 1, IPA2G_IMAIN, 0x12); - } else { + if (pi->pubpi.radiorev == 5) { + + for (coreNum = 0; coreNum <= 1; + coreNum++) { + WRITE_RADIO_REG4(pi, RADIO_2057, + CORE, coreNum, + IPA2G_CASCONV, + 0x13); + WRITE_RADIO_REG4(pi, RADIO_2057, + CORE, coreNum, + IPA2G_IMAIN, + 0x1f); + WRITE_RADIO_REG4( + pi, RADIO_2057, + CORE, coreNum, + IPA2G_BIAS_FILTER, + 0xee); + WRITE_RADIO_REG4(pi, RADIO_2057, + CORE, coreNum, + PAD2G_IDACS, + 0x8a); + WRITE_RADIO_REG4( + pi, RADIO_2057, + CORE, coreNum, + PAD_BIAS_FILTER_BWS, + 0x3e); + } + + } else if ((pi->pubpi.radiorev == 7) + || (pi->pubpi.radiorev == 8)) { + + if (CHSPEC_IS40(pi->radio_chanspec) == + 0) { + WRITE_RADIO_REG4(pi, RADIO_2057, + CORE, 0, + IPA2G_IMAIN, + 0x14); + WRITE_RADIO_REG4(pi, RADIO_2057, + CORE, 1, + IPA2G_IMAIN, + 0x12); + } else { + WRITE_RADIO_REG4(pi, RADIO_2057, + CORE, 0, + IPA2G_IMAIN, + 0x16); + WRITE_RADIO_REG4(pi, RADIO_2057, + CORE, 1, + IPA2G_IMAIN, + 0x16); + } + } + + } else { + freq = CHAN5G_FREQ(CHSPEC_CHANNEL( + pi->radio_chanspec)); + if (((freq >= 5180) && (freq <= 5230)) + || ((freq >= 5745) && (freq <= 5805))) { WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - 0, IPA2G_IMAIN, 0x16); + 0, IPA5G_BIAS_FILTER, + 0xff); WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - 1, IPA2G_IMAIN, 0x16); + 1, IPA5G_BIAS_FILTER, + 0xff); } } - } else { - freq = - CHAN5G_FREQ(CHSPEC_CHANNEL - (pi->radio_chanspec)); - if (((freq >= 5180) && (freq <= 5230)) - || ((freq >= 5745) && (freq <= 5805))) { - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - 0, IPA5G_BIAS_FILTER, 0xff); - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, - 1, IPA5G_BIAS_FILTER, 0xff); + + if (pi->pubpi.radiorev != 5) { + for (coreNum = 0; coreNum <= 1; coreNum++) { + WRITE_RADIO_REG4(pi, RADIO_2057, CORE, + coreNum, + TXMIX2G_TUNE_BOOST_PU, + 0x61); + WRITE_RADIO_REG4(pi, RADIO_2057, CORE, + coreNum, + TXGM_IDAC_BLEED, 0x70); + } } } - } else { - if (pi->pubpi.radiorev != 5) { + if (pi->pubpi.radiorev == 4) { + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, + 0x05, 16, + &afectrl_adc_ctrl1_rev7); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, + 0x15, 16, + &afectrl_adc_ctrl1_rev7); + for (coreNum = 0; coreNum <= 1; coreNum++) { WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum, - TXMIX2G_TUNE_BOOST_PU, 0x61); + AFE_VCM_CAL_MASTER, 0x0); + WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum, + AFE_SET_VCM_I, 0x3f); WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum, - TXGM_IDAC_BLEED, 0x70); + AFE_SET_VCM_Q, 0x3f); } + } else { + mod_phy_reg(pi, 0xa6, (0x1 << 2), (0x1 << 2)); + mod_phy_reg(pi, 0x8f, (0x1 << 2), (0x1 << 2)); + mod_phy_reg(pi, 0xa7, (0x1 << 2), (0x1 << 2)); + mod_phy_reg(pi, 0xa5, (0x1 << 2), (0x1 << 2)); + + mod_phy_reg(pi, 0xa6, (0x1 << 0), 0); + mod_phy_reg(pi, 0x8f, (0x1 << 0), (0x1 << 0)); + mod_phy_reg(pi, 0xa7, (0x1 << 0), 0); + mod_phy_reg(pi, 0xa5, (0x1 << 0), (0x1 << 0)); + + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, + 0x05, 16, + &afectrl_adc_ctrl2_rev7); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, + 0x15, 16, + &afectrl_adc_ctrl2_rev7); + + mod_phy_reg(pi, 0xa6, (0x1 << 2), 0); + mod_phy_reg(pi, 0x8f, (0x1 << 2), 0); + mod_phy_reg(pi, 0xa7, (0x1 << 2), 0); + mod_phy_reg(pi, 0xa5, (0x1 << 2), 0); } - } - if (pi->pubpi.radiorev == 4) { - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x05, 16, - &afectrl_adc_ctrl1_rev7); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x15, 16, - &afectrl_adc_ctrl1_rev7); - - for (coreNum = 0; coreNum <= 1; coreNum++) { - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum, - AFE_VCM_CAL_MASTER, 0x0); - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum, - AFE_SET_VCM_I, 0x3f); - WRITE_RADIO_REG4(pi, RADIO_2057, CORE, coreNum, - AFE_SET_VCM_Q, 0x3f); - } - } else { - mod_phy_reg(pi, 0xa6, (0x1 << 2), (0x1 << 2)); - mod_phy_reg(pi, 0x8f, (0x1 << 2), (0x1 << 2)); - mod_phy_reg(pi, 0xa7, (0x1 << 2), (0x1 << 2)); - mod_phy_reg(pi, 0xa5, (0x1 << 2), (0x1 << 2)); + write_phy_reg(pi, 0x6a, 0x2); - mod_phy_reg(pi, 0xa6, (0x1 << 0), 0); - mod_phy_reg(pi, 0x8f, (0x1 << 0), (0x1 << 0)); - mod_phy_reg(pi, 0xa7, (0x1 << 0), 0); - mod_phy_reg(pi, 0xa5, (0x1 << 0), (0x1 << 0)); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 256, 32, + &min_nvar_offset_6mbps); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x05, 16, - &afectrl_adc_ctrl2_rev7); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x15, 16, - &afectrl_adc_ctrl2_rev7); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x138, 16, + &rfseq_pktgn_lpf_hpc_rev7); - mod_phy_reg(pi, 0xa6, (0x1 << 2), 0); - mod_phy_reg(pi, 0x8f, (0x1 << 2), 0); - mod_phy_reg(pi, 0xa7, (0x1 << 2), 0); - mod_phy_reg(pi, 0xa5, (0x1 << 2), 0); - } + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x141, 16, + &rfseq_pktgn_lpf_h_hpc_rev7); - write_phy_reg(pi, 0x6a, 0x2); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 3, 0x133, 16, + &rfseq_htpktgn_lpf_hpc_rev7); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 256, 32, - &min_nvar_offset_6mbps); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x146, 16, + &rfseq_cckpktgn_lpf_hpc_rev7); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x138, 16, - &rfseq_pktgn_lpf_hpc_rev7); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x123, 16, + &rfseq_tx2rx_lpf_h_hpc_rev7); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x141, 16, - &rfseq_pktgn_lpf_h_hpc_rev7); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x12A, 16, + &rfseq_rx2tx_lpf_h_hpc_rev7); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 3, 0x133, 16, - &rfseq_htpktgn_lpf_hpc_rev7); + if (CHSPEC_IS40(pi->radio_chanspec) == 0) { + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3, + 32, &min_nvar_val); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, + 127, 32, &min_nvar_val); + } else { + min_nvar_val = noise_var_tbl_rev7[3]; + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3, + 32, &min_nvar_val); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 2, 0x146, 16, - &rfseq_cckpktgn_lpf_hpc_rev7); + min_nvar_val = noise_var_tbl_rev7[127]; + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, + 127, 32, &min_nvar_val); + } - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x123, 16, - &rfseq_tx2rx_lpf_h_hpc_rev7); + wlc_phy_workarounds_nphy_gainctrl(pi); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_RFSEQ, 1, 0x12A, 16, - &rfseq_rx2tx_lpf_h_hpc_rev7); + pdetrange = + (CHSPEC_IS5G(pi->radio_chanspec)) ? pi->srom_fem5g. + pdetrange : pi->srom_fem2g.pdetrange; - if (CHSPEC_IS40(pi->radio_chanspec) == 0) { - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3, - 32, &min_nvar_val); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, - 127, 32, &min_nvar_val); - } else { - min_nvar_val = noise_var_tbl_rev7[3]; - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3, - 32, &min_nvar_val); - - min_nvar_val = noise_var_tbl_rev7[127]; - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, - 127, 32, &min_nvar_val); - } + if (pdetrange == 0) { + chan_freq_range = + wlc_phy_get_chan_freq_range_nphy(pi, 0); + if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { + aux_adc_vmid_rev7_core0[3] = 0x70; + aux_adc_vmid_rev7_core1[3] = 0x70; + aux_adc_gain_rev7[3] = 2; + } else { + aux_adc_vmid_rev7_core0[3] = 0x80; + aux_adc_vmid_rev7_core1[3] = 0x80; + aux_adc_gain_rev7[3] = 3; + } + } else if (pdetrange == 1) { + if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { + aux_adc_vmid_rev7_core0[3] = 0x7c; + aux_adc_vmid_rev7_core1[3] = 0x7c; + aux_adc_gain_rev7[3] = 2; + } else { + aux_adc_vmid_rev7_core0[3] = 0x8c; + aux_adc_vmid_rev7_core1[3] = 0x8c; + aux_adc_gain_rev7[3] = 1; + } + } else if (pdetrange == 2) { + if (pi->pubpi.radioid == BCM2057_ID) { + if ((pi->pubpi.radiorev == 5) + || (pi->pubpi.radiorev == 7) + || (pi->pubpi.radiorev == 8)) { + if (chan_freq_range == + WL_CHAN_FREQ_RANGE_2G) { + aux_adc_vmid_rev7_core0[3] = + 0x8c; + aux_adc_vmid_rev7_core1[3] = + 0x8c; + aux_adc_gain_rev7[3] = 0; + } else { + aux_adc_vmid_rev7_core0[3] = + 0x96; + aux_adc_vmid_rev7_core1[3] = + 0x96; + aux_adc_gain_rev7[3] = 0; + } + } + } - wlc_phy_workarounds_nphy_gainctrl(pi); + } else if (pdetrange == 3) { + if (chan_freq_range == WL_CHAN_FREQ_RANGE_2G) { + aux_adc_vmid_rev7_core0[3] = 0x89; + aux_adc_vmid_rev7_core1[3] = 0x89; + aux_adc_gain_rev7[3] = 0; + } - pdetrange = (CHSPEC_IS5G(pi->radio_chanspec)) ? - pi->srom_fem5g.pdetrange : pi->srom_fem2g.pdetrange; + } else if (pdetrange == 5) { - if (pdetrange == 0) { - chan_freq_range = wlc_phy_get_chan_freq_range_nphy(pi, 0); - if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { - aux_adc_vmid_rev7_core0[3] = 0x70; - aux_adc_vmid_rev7_core1[3] = 0x70; - aux_adc_gain_rev7[3] = 2; - } else { - aux_adc_vmid_rev7_core0[3] = 0x80; - aux_adc_vmid_rev7_core1[3] = 0x80; - aux_adc_gain_rev7[3] = 3; - } - } else if (pdetrange == 1) { - if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { - aux_adc_vmid_rev7_core0[3] = 0x7c; - aux_adc_vmid_rev7_core1[3] = 0x7c; - aux_adc_gain_rev7[3] = 2; - } else { - aux_adc_vmid_rev7_core0[3] = 0x8c; - aux_adc_vmid_rev7_core1[3] = 0x8c; - aux_adc_gain_rev7[3] = 1; - } - } else if (pdetrange == 2) { - if (pi->pubpi.radioid == BCM2057_ID) { - if ((pi->pubpi.radiorev == 5) - || (pi->pubpi.radiorev == 7) - || (pi->pubpi.radiorev == 8)) { - if (chan_freq_range == - WL_CHAN_FREQ_RANGE_2G) { - aux_adc_vmid_rev7_core0[3] = 0x8c; - aux_adc_vmid_rev7_core1[3] = 0x8c; - aux_adc_gain_rev7[3] = 0; - } else { - aux_adc_vmid_rev7_core0[3] = 0x96; - aux_adc_vmid_rev7_core1[3] = 0x96; - aux_adc_gain_rev7[3] = 0; - } + if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { + aux_adc_vmid_rev7_core0[3] = 0x80; + aux_adc_vmid_rev7_core1[3] = 0x80; + aux_adc_gain_rev7[3] = 3; + } else { + aux_adc_vmid_rev7_core0[3] = 0x70; + aux_adc_vmid_rev7_core1[3] = 0x70; + aux_adc_gain_rev7[3] = 2; } } - } else if (pdetrange == 3) { - if (chan_freq_range == WL_CHAN_FREQ_RANGE_2G) { - aux_adc_vmid_rev7_core0[3] = 0x89; - aux_adc_vmid_rev7_core1[3] = 0x89; - aux_adc_gain_rev7[3] = 0; - } + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x08, 16, + &aux_adc_vmid_rev7_core0); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x18, 16, + &aux_adc_vmid_rev7_core1); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x0c, 16, + &aux_adc_gain_rev7); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x1c, 16, + &aux_adc_gain_rev7); - } else if (pdetrange == 5) { + } else if (NREV_GE(pi->pubpi.phy_rev, 3)) { - if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { - aux_adc_vmid_rev7_core0[3] = 0x80; - aux_adc_vmid_rev7_core1[3] = 0x80; - aux_adc_gain_rev7[3] = 3; - } else { - aux_adc_vmid_rev7_core0[3] = 0x70; - aux_adc_vmid_rev7_core1[3] = 0x70; - aux_adc_gain_rev7[3] = 2; - } - } + write_phy_reg(pi, 0x23f, 0x1f8); + write_phy_reg(pi, 0x240, 0x1f8); + + wlc_phy_table_read_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, + 1, 0, 32, &leg_data_weights); + leg_data_weights = leg_data_weights & 0xffffff; + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, + 1, 0, 32, &leg_data_weights); + + alpha0 = 293; + alpha1 = 435; + alpha2 = 261; + beta0 = 366; + beta1 = 205; + beta2 = 32; + write_phy_reg(pi, 0x145, alpha0); + write_phy_reg(pi, 0x146, alpha1); + write_phy_reg(pi, 0x147, alpha2); + write_phy_reg(pi, 0x148, beta0); + write_phy_reg(pi, 0x149, beta1); + write_phy_reg(pi, 0x14a, beta2); + + write_phy_reg(pi, 0x38, 0xC); + write_phy_reg(pi, 0x2ae, 0xC); + + wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_TX2RX, + rfseq_tx2rx_events_rev3, + rfseq_tx2rx_dlys_rev3, + ARRAY_SIZE(rfseq_tx2rx_events_rev3)); + + if (PHY_IPA(pi)) + wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, + rfseq_rx2tx_events_rev3_ipa, + rfseq_rx2tx_dlys_rev3_ipa, + ARRAY_SIZE(rfseq_rx2tx_events_rev3_ipa)); + + if ((pi->sh->hw_phyrxchain != 0x3) && + (pi->sh->hw_phyrxchain != pi->sh->hw_phytxchain)) { - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x08, 16, - &aux_adc_vmid_rev7_core0); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x18, 16, - &aux_adc_vmid_rev7_core1); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x0c, 16, - &aux_adc_gain_rev7); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, 0x1c, 16, - &aux_adc_gain_rev7); -} + if (PHY_IPA(pi)) { + rfseq_rx2tx_dlys_rev3[5] = 59; + rfseq_rx2tx_dlys_rev3[6] = 1; + rfseq_rx2tx_events_rev3[7] = + NPHY_REV3_RFSEQ_CMD_END; + } -static void wlc_phy_workarounds_nphy_rev3(struct brcms_phy *pi) -{ - static const u8 rfseq_tx2rx_events_rev3[] = { - NPHY_REV3_RFSEQ_CMD_EXT_PA, - NPHY_REV3_RFSEQ_CMD_INT_PA_PU, - NPHY_REV3_RFSEQ_CMD_TX_GAIN, - NPHY_REV3_RFSEQ_CMD_RXPD_TXPD, - NPHY_REV3_RFSEQ_CMD_TR_SWITCH, - NPHY_REV3_RFSEQ_CMD_RXG_FBW, - NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS, - NPHY_REV3_RFSEQ_CMD_END - }; - static const u8 rfseq_tx2rx_dlys_rev3[] = { 8, 4, 2, 2, 4, 4, 6, 1 }; - u8 rfseq_rx2tx_events_rev3[] = { - NPHY_REV3_RFSEQ_CMD_NOP, - NPHY_REV3_RFSEQ_CMD_RXG_FBW, - NPHY_REV3_RFSEQ_CMD_TR_SWITCH, - NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS, - NPHY_REV3_RFSEQ_CMD_RXPD_TXPD, - NPHY_REV3_RFSEQ_CMD_TX_GAIN, - NPHY_REV3_RFSEQ_CMD_INT_PA_PU, - NPHY_REV3_RFSEQ_CMD_EXT_PA, - NPHY_REV3_RFSEQ_CMD_END - }; - u8 rfseq_rx2tx_dlys_rev3[] = { 8, 6, 6, 4, 4, 18, 42, 1, 1 }; - static const u8 rfseq_rx2tx_events_rev3_ipa[] = { - NPHY_REV3_RFSEQ_CMD_NOP, - NPHY_REV3_RFSEQ_CMD_RXG_FBW, - NPHY_REV3_RFSEQ_CMD_TR_SWITCH, - NPHY_REV3_RFSEQ_CMD_CLR_HIQ_DIS, - NPHY_REV3_RFSEQ_CMD_RXPD_TXPD, - NPHY_REV3_RFSEQ_CMD_TX_GAIN, - NPHY_REV3_RFSEQ_CMD_CLR_RXRX_BIAS, - NPHY_REV3_RFSEQ_CMD_INT_PA_PU, - NPHY_REV3_RFSEQ_CMD_END - }; - static const u8 rfseq_rx2tx_dlys_rev3_ipa[] = - { 8, 6, 6, 4, 4, 16, 43, 1, 1 }; - s16 alpha0, alpha1, alpha2; - s16 beta0, beta1, beta2; - u32 leg_data_weights, ht_data_weights, nss1_data_weights, - stbc_data_weights; - u8 chan_freq_range = 0; - static const u16 dac_control = 0x0002; - u16 aux_adc_vmid_rev4[] = { 0xa2, 0xb4, 0xb4, 0x89 }; - u16 aux_adc_vmid_rev3[] = { 0xa2, 0xb4, 0xb4, 0x89 }; - u16 *aux_adc_vmid; - u16 aux_adc_gain_rev4[] = { 0x02, 0x02, 0x02, 0x00 }; - u16 aux_adc_gain_rev3[] = { 0x02, 0x02, 0x02, 0x00 }; - u16 *aux_adc_gain; - static const u16 sk_adc_vmid[] = { 0xb4, 0xb4, 0xb4, 0x24 }; - static const u16 sk_adc_gain[] = { 0x02, 0x02, 0x02, 0x02 }; - s32 min_nvar_val = 0x18d; - u8 pdetrange; - u8 triso; + wlc_phy_set_rfseq_nphy( + pi, NPHY_RFSEQ_RX2TX, + rfseq_rx2tx_events_rev3, + rfseq_rx2tx_dlys_rev3, + ARRAY_SIZE(rfseq_rx2tx_events_rev3)); + } - write_phy_reg(pi, 0x23f, 0x1f8); - write_phy_reg(pi, 0x240, 0x1f8); - - wlc_phy_table_read_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, - 1, 0, 32, &leg_data_weights); - leg_data_weights = leg_data_weights & 0xffffff; - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, - 1, 0, 32, &leg_data_weights); - - alpha0 = 293; - alpha1 = 435; - alpha2 = 261; - beta0 = 366; - beta1 = 205; - beta2 = 32; - write_phy_reg(pi, 0x145, alpha0); - write_phy_reg(pi, 0x146, alpha1); - write_phy_reg(pi, 0x147, alpha2); - write_phy_reg(pi, 0x148, beta0); - write_phy_reg(pi, 0x149, beta1); - write_phy_reg(pi, 0x14a, beta2); - - write_phy_reg(pi, 0x38, 0xC); - write_phy_reg(pi, 0x2ae, 0xC); - - wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_TX2RX, - rfseq_tx2rx_events_rev3, - rfseq_tx2rx_dlys_rev3, - ARRAY_SIZE(rfseq_tx2rx_events_rev3)); + if (CHSPEC_IS2G(pi->radio_chanspec)) + write_phy_reg(pi, 0x6a, 0x2); + else + write_phy_reg(pi, 0x6a, 0x9c40); - if (PHY_IPA(pi)) - wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, - rfseq_rx2tx_events_rev3_ipa, - rfseq_rx2tx_dlys_rev3_ipa, - ARRAY_SIZE (rfseq_rx2tx_events_rev3_ipa)); + mod_phy_reg(pi, 0x294, (0xf << 8), (7 << 8)); - if ((pi->sh->hw_phyrxchain != 0x3) && - (pi->sh->hw_phyrxchain != pi->sh->hw_phytxchain)) { + if (CHSPEC_IS40(pi->radio_chanspec) == 0) { + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3, + 32, &min_nvar_val); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, + 127, 32, &min_nvar_val); + } else { + min_nvar_val = noise_var_tbl_rev3[3]; + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3, + 32, &min_nvar_val); - if (PHY_IPA(pi)) { - rfseq_rx2tx_dlys_rev3[5] = 59; - rfseq_rx2tx_dlys_rev3[6] = 1; - rfseq_rx2tx_events_rev3[7] = NPHY_REV3_RFSEQ_CMD_END; + min_nvar_val = noise_var_tbl_rev3[127]; + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, + 127, 32, &min_nvar_val); } - wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, - rfseq_rx2tx_events_rev3, - rfseq_rx2tx_dlys_rev3, - ARRAY_SIZE (rfseq_rx2tx_events_rev3)); - } + wlc_phy_workarounds_nphy_gainctrl(pi); - if (CHSPEC_IS2G(pi->radio_chanspec)) - write_phy_reg(pi, 0x6a, 0x2); - else - write_phy_reg(pi, 0x6a, 0x9c40); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x00, 16, + &dac_control); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x10, 16, + &dac_control); - mod_phy_reg(pi, 0x294, (0xf << 8), (7 << 8)); + pdetrange = + (CHSPEC_IS5G(pi->radio_chanspec)) ? pi->srom_fem5g. + pdetrange : pi->srom_fem2g.pdetrange; - if (CHSPEC_IS40(pi->radio_chanspec) == 0) { - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3, - 32, &min_nvar_val); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, - 127, 32, &min_nvar_val); - } else { - min_nvar_val = noise_var_tbl_rev3[3]; - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, 3, - 32, &min_nvar_val); + if (pdetrange == 0) { + if (NREV_GE(pi->pubpi.phy_rev, 4)) { + aux_adc_vmid = aux_adc_vmid_rev4; + aux_adc_gain = aux_adc_gain_rev4; + } else { + aux_adc_vmid = aux_adc_vmid_rev3; + aux_adc_gain = aux_adc_gain_rev3; + } + chan_freq_range = + wlc_phy_get_chan_freq_range_nphy(pi, 0); + if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { + switch (chan_freq_range) { + case WL_CHAN_FREQ_RANGE_5GL: + aux_adc_vmid[3] = 0x89; + aux_adc_gain[3] = 0; + break; + case WL_CHAN_FREQ_RANGE_5GM: + aux_adc_vmid[3] = 0x89; + aux_adc_gain[3] = 0; + break; + case WL_CHAN_FREQ_RANGE_5GH: + aux_adc_vmid[3] = 0x89; + aux_adc_gain[3] = 0; + break; + default: + break; + } + } + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x08, 16, aux_adc_vmid); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x18, 16, aux_adc_vmid); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x0c, 16, aux_adc_gain); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x1c, 16, aux_adc_gain); + } else if (pdetrange == 1) { + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x08, 16, sk_adc_vmid); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x18, 16, sk_adc_vmid); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x0c, 16, sk_adc_gain); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x1c, 16, sk_adc_gain); + } else if (pdetrange == 2) { - min_nvar_val = noise_var_tbl_rev3[127]; - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_NOISEVAR, 1, - 127, 32, &min_nvar_val); - } + u16 bcm_adc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x74 }; + u16 bcm_adc_gain[] = { 0x02, 0x02, 0x02, 0x04 }; - wlc_phy_workarounds_nphy_gainctrl(pi); + if (NREV_GE(pi->pubpi.phy_rev, 6)) { + chan_freq_range = + wlc_phy_get_chan_freq_range_nphy(pi, 0); + if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { + bcm_adc_vmid[3] = 0x8e; + bcm_adc_gain[3] = 0x03; + } else { + bcm_adc_vmid[3] = 0x94; + bcm_adc_gain[3] = 0x03; + } + } else if (NREV_IS(pi->pubpi.phy_rev, 5)) { + bcm_adc_vmid[3] = 0x84; + bcm_adc_gain[3] = 0x02; + } - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x00, 16, - &dac_control); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 1, 0x10, 16, - &dac_control); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x08, 16, bcm_adc_vmid); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x18, 16, bcm_adc_vmid); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x0c, 16, bcm_adc_gain); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, + 0x1c, 16, bcm_adc_gain); + } else if (pdetrange == 3) { + chan_freq_range = + wlc_phy_get_chan_freq_range_nphy(pi, 0); + if ((NREV_GE(pi->pubpi.phy_rev, 4)) + && (chan_freq_range == WL_CHAN_FREQ_RANGE_2G)) { - pdetrange = (CHSPEC_IS5G(pi->radio_chanspec)) ? - pi->srom_fem5g.pdetrange : pi->srom_fem2g.pdetrange; + u16 auxadc_vmid[] = { + 0xa2, 0xb4, 0xb4, 0x270 + }; + u16 auxadc_gain[] = { + 0x02, 0x02, 0x02, 0x00 + }; - if (pdetrange == 0) { - if (NREV_GE(pi->pubpi.phy_rev, 4)) { - aux_adc_vmid = aux_adc_vmid_rev4; - aux_adc_gain = aux_adc_gain_rev4; - } else { - aux_adc_vmid = aux_adc_vmid_rev3; - aux_adc_gain = aux_adc_gain_rev3; - } - chan_freq_range = wlc_phy_get_chan_freq_range_nphy(pi, 0); - if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { - switch (chan_freq_range) { - case WL_CHAN_FREQ_RANGE_5GL: - aux_adc_vmid[3] = 0x89; - aux_adc_gain[3] = 0; - break; - case WL_CHAN_FREQ_RANGE_5GM: - aux_adc_vmid[3] = 0x89; - aux_adc_gain[3] = 0; - break; - case WL_CHAN_FREQ_RANGE_5GH: - aux_adc_vmid[3] = 0x89; - aux_adc_gain[3] = 0; - break; - default: - break; + wlc_phy_table_write_nphy(pi, + NPHY_TBL_ID_AFECTRL, 4, + 0x08, 16, auxadc_vmid); + wlc_phy_table_write_nphy(pi, + NPHY_TBL_ID_AFECTRL, 4, + 0x18, 16, auxadc_vmid); + wlc_phy_table_write_nphy(pi, + NPHY_TBL_ID_AFECTRL, 4, + 0x0c, 16, auxadc_gain); + wlc_phy_table_write_nphy(pi, + NPHY_TBL_ID_AFECTRL, 4, + 0x1c, 16, auxadc_gain); } - } - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x08, 16, aux_adc_vmid); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x18, 16, aux_adc_vmid); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x0c, 16, aux_adc_gain); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x1c, 16, aux_adc_gain); - } else if (pdetrange == 1) { - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x08, 16, sk_adc_vmid); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x18, 16, sk_adc_vmid); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x0c, 16, sk_adc_gain); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x1c, 16, sk_adc_gain); - } else if (pdetrange == 2) { - - u16 bcm_adc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x74 }; - u16 bcm_adc_gain[] = { 0x02, 0x02, 0x02, 0x04 }; + } else if ((pdetrange == 4) || (pdetrange == 5)) { + u16 bcm_adc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x0 }; + u16 bcm_adc_gain[] = { 0x02, 0x02, 0x02, 0x0 }; + u16 Vmid[2], Av[2]; - if (NREV_GE(pi->pubpi.phy_rev, 6)) { chan_freq_range = - wlc_phy_get_chan_freq_range_nphy(pi, 0); + wlc_phy_get_chan_freq_range_nphy(pi, 0); if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { - bcm_adc_vmid[3] = 0x8e; - bcm_adc_gain[3] = 0x03; + Vmid[0] = (pdetrange == 4) ? 0x8e : 0x89; + Vmid[1] = (pdetrange == 4) ? 0x96 : 0x89; + Av[0] = (pdetrange == 4) ? 2 : 0; + Av[1] = (pdetrange == 4) ? 2 : 0; } else { - bcm_adc_vmid[3] = 0x94; - bcm_adc_gain[3] = 0x03; + Vmid[0] = (pdetrange == 4) ? 0x89 : 0x74; + Vmid[1] = (pdetrange == 4) ? 0x8b : 0x70; + Av[0] = (pdetrange == 4) ? 2 : 0; + Av[1] = (pdetrange == 4) ? 2 : 0; } - } else if (NREV_IS(pi->pubpi.phy_rev, 5)) { - bcm_adc_vmid[3] = 0x84; - bcm_adc_gain[3] = 0x02; - } - - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x08, 16, bcm_adc_vmid); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x18, 16, bcm_adc_vmid); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x0c, 16, bcm_adc_gain); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x1c, 16, bcm_adc_gain); - } else if (pdetrange == 3) { - chan_freq_range = wlc_phy_get_chan_freq_range_nphy(pi, 0); - if ((NREV_GE(pi->pubpi.phy_rev, 4)) && - (chan_freq_range == WL_CHAN_FREQ_RANGE_2G)) { - u16 auxadc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x270 }; - u16 auxadc_gain[] = { 0x02, 0x02, 0x02, 0x00 }; + bcm_adc_vmid[3] = Vmid[0]; + bcm_adc_gain[3] = Av[0]; wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x08, 16, auxadc_vmid); + 0x08, 16, bcm_adc_vmid); wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x18, 16, auxadc_vmid); + 0x0c, 16, bcm_adc_gain); + + bcm_adc_vmid[3] = Vmid[1]; + bcm_adc_gain[3] = Av[1]; wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x0c, 16, auxadc_gain); + 0x18, 16, bcm_adc_vmid); wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x1c, 16, auxadc_gain); + 0x1c, 16, bcm_adc_gain); } - } else if ((pdetrange == 4) || (pdetrange == 5)) { - u16 bcm_adc_vmid[] = { 0xa2, 0xb4, 0xb4, 0x0 }; - u16 bcm_adc_gain[] = { 0x02, 0x02, 0x02, 0x0 }; - u16 Vmid[2], Av[2]; - chan_freq_range = wlc_phy_get_chan_freq_range_nphy(pi, 0); - if (chan_freq_range != WL_CHAN_FREQ_RANGE_2G) { - Vmid[0] = (pdetrange == 4) ? 0x8e : 0x89; - Vmid[1] = (pdetrange == 4) ? 0x96 : 0x89; - Av[0] = (pdetrange == 4) ? 2 : 0; - Av[1] = (pdetrange == 4) ? 2 : 0; - } else { - Vmid[0] = (pdetrange == 4) ? 0x89 : 0x74; - Vmid[1] = (pdetrange == 4) ? 0x8b : 0x70; - Av[0] = (pdetrange == 4) ? 2 : 0; - Av[1] = (pdetrange == 4) ? 2 : 0; - } - - bcm_adc_vmid[3] = Vmid[0]; - bcm_adc_gain[3] = Av[0]; - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x08, 16, bcm_adc_vmid); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x0c, 16, bcm_adc_gain); - - bcm_adc_vmid[3] = Vmid[1]; - bcm_adc_gain[3] = Av[1]; - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x18, 16, bcm_adc_vmid); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_AFECTRL, 4, - 0x1c, 16, bcm_adc_gain); - } - - write_radio_reg(pi, (RADIO_2056_RX_MIXA_MAST_BIAS | RADIO_2056_RX0), 0x0); - write_radio_reg(pi, (RADIO_2056_RX_MIXA_MAST_BIAS | RADIO_2056_RX1), 0x0); - - write_radio_reg(pi, (RADIO_2056_RX_MIXA_BIAS_MAIN | RADIO_2056_RX0), 0x6); - write_radio_reg(pi, (RADIO_2056_RX_MIXA_BIAS_MAIN | RADIO_2056_RX1), 0x6); - - write_radio_reg(pi, (RADIO_2056_RX_MIXA_BIAS_AUX | RADIO_2056_RX0), 0x7); - write_radio_reg(pi, (RADIO_2056_RX_MIXA_BIAS_AUX | RADIO_2056_RX1), 0x7); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_MAST_BIAS | RADIO_2056_RX0), + 0x0); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_MAST_BIAS | RADIO_2056_RX1), + 0x0); - write_radio_reg(pi, (RADIO_2056_RX_MIXA_LOB_BIAS | RADIO_2056_RX0), 0x88); - write_radio_reg(pi, (RADIO_2056_RX_MIXA_LOB_BIAS | RADIO_2056_RX1), 0x88); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_BIAS_MAIN | RADIO_2056_RX0), + 0x6); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_BIAS_MAIN | RADIO_2056_RX1), + 0x6); - write_radio_reg(pi, (RADIO_2056_RX_MIXA_CMFB_IDAC | RADIO_2056_RX0), 0x0); - write_radio_reg(pi, (RADIO_2056_RX_MIXA_CMFB_IDAC | RADIO_2056_RX1), 0x0); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_BIAS_AUX | RADIO_2056_RX0), + 0x7); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_BIAS_AUX | RADIO_2056_RX1), + 0x7); - write_radio_reg(pi, (RADIO_2056_RX_MIXG_CMFB_IDAC | RADIO_2056_RX0), 0x0); - write_radio_reg(pi, (RADIO_2056_RX_MIXG_CMFB_IDAC | RADIO_2056_RX1), 0x0); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_LOB_BIAS | RADIO_2056_RX0), + 0x88); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_LOB_BIAS | RADIO_2056_RX1), + 0x88); - triso = (CHSPEC_IS5G(pi->radio_chanspec)) ? - pi->srom_fem5g.triso : pi->srom_fem2g.triso; - if (triso == 7) { - wlc_phy_war_force_trsw_to_R_cliplo_nphy(pi, PHY_CORE_0); - wlc_phy_war_force_trsw_to_R_cliplo_nphy(pi, PHY_CORE_1); - } + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_CMFB_IDAC | RADIO_2056_RX0), + 0x0); + write_radio_reg(pi, + (RADIO_2056_RX_MIXA_CMFB_IDAC | RADIO_2056_RX1), + 0x0); - wlc_phy_war_txchain_upd_nphy(pi, pi->sh->hw_phytxchain); + write_radio_reg(pi, + (RADIO_2056_RX_MIXG_CMFB_IDAC | RADIO_2056_RX0), + 0x0); + write_radio_reg(pi, + (RADIO_2056_RX_MIXG_CMFB_IDAC | RADIO_2056_RX1), + 0x0); - if (((pi->sh->boardflags2 & BFL2_APLL_WAR) && - (CHSPEC_IS5G(pi->radio_chanspec))) || - (((pi->sh->boardflags2 & BFL2_GPLL_WAR) || - (pi->sh->boardflags2 & BFL2_GPLL_WAR2)) && - (CHSPEC_IS2G(pi->radio_chanspec)))) { - nss1_data_weights = 0x00088888; - ht_data_weights = 0x00088888; - stbc_data_weights = 0x00088888; - } else { - nss1_data_weights = 0x88888888; - ht_data_weights = 0x88888888; - stbc_data_weights = 0x88888888; - } - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, - 1, 1, 32, &nss1_data_weights); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, - 1, 2, 32, &ht_data_weights); - wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, - 1, 3, 32, &stbc_data_weights); - - if (NREV_IS(pi->pubpi.phy_rev, 4)) { - if (CHSPEC_IS5G(pi->radio_chanspec)) { - write_radio_reg(pi, - RADIO_2056_TX_GMBB_IDAC | - RADIO_2056_TX0, 0x70); - write_radio_reg(pi, - RADIO_2056_TX_GMBB_IDAC | - RADIO_2056_TX1, 0x70); + triso = + (CHSPEC_IS5G(pi->radio_chanspec)) ? pi->srom_fem5g. + triso : pi->srom_fem2g.triso; + if (triso == 7) { + wlc_phy_war_force_trsw_to_R_cliplo_nphy(pi, PHY_CORE_0); + wlc_phy_war_force_trsw_to_R_cliplo_nphy(pi, PHY_CORE_1); } - } - if (!pi->edcrs_threshold_lock) { - write_phy_reg(pi, 0x224, 0x3eb); - write_phy_reg(pi, 0x225, 0x3eb); - write_phy_reg(pi, 0x226, 0x341); - write_phy_reg(pi, 0x227, 0x341); - write_phy_reg(pi, 0x228, 0x42b); - write_phy_reg(pi, 0x229, 0x42b); - write_phy_reg(pi, 0x22a, 0x381); - write_phy_reg(pi, 0x22b, 0x381); - write_phy_reg(pi, 0x22c, 0x42b); - write_phy_reg(pi, 0x22d, 0x42b); - write_phy_reg(pi, 0x22e, 0x381); - write_phy_reg(pi, 0x22f, 0x381); - } + wlc_phy_war_txchain_upd_nphy(pi, pi->sh->hw_phytxchain); - if (NREV_GE(pi->pubpi.phy_rev, 6)) { - - if (pi->sh->boardflags2 & BFL2_SINGLEANT_CCK) - wlapi_bmac_mhf(pi->sh->physhim, MHF4, - MHF4_BPHY_TXCORE0, - MHF4_BPHY_TXCORE0, BRCM_BAND_ALL); - } -} + if (((pi->sh->boardflags2 & BFL2_APLL_WAR) && + (CHSPEC_IS5G(pi->radio_chanspec))) || + (((pi->sh->boardflags2 & BFL2_GPLL_WAR) || + (pi->sh->boardflags2 & BFL2_GPLL_WAR2)) && + (CHSPEC_IS2G(pi->radio_chanspec)))) { + nss1_data_weights = 0x00088888; + ht_data_weights = 0x00088888; + stbc_data_weights = 0x00088888; + } else { + nss1_data_weights = 0x88888888; + ht_data_weights = 0x88888888; + stbc_data_weights = 0x88888888; + } + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, + 1, 1, 32, &nss1_data_weights); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, + 1, 2, 32, &ht_data_weights); + wlc_phy_table_write_nphy(pi, NPHY_TBL_ID_CMPMETRICDATAWEIGHTTBL, + 1, 3, 32, &stbc_data_weights); + + if (NREV_IS(pi->pubpi.phy_rev, 4)) { + if (CHSPEC_IS5G(pi->radio_chanspec)) { + write_radio_reg(pi, + RADIO_2056_TX_GMBB_IDAC | + RADIO_2056_TX0, 0x70); + write_radio_reg(pi, + RADIO_2056_TX_GMBB_IDAC | + RADIO_2056_TX1, 0x70); + } + } -static void wlc_phy_workarounds_nphy_rev1(struct brcms_phy *pi) -{ - static const u8 rfseq_rx2tx_events[] = { - NPHY_RFSEQ_CMD_NOP, - NPHY_RFSEQ_CMD_RXG_FBW, - NPHY_RFSEQ_CMD_TR_SWITCH, - NPHY_RFSEQ_CMD_CLR_HIQ_DIS, - NPHY_RFSEQ_CMD_RXPD_TXPD, - NPHY_RFSEQ_CMD_TX_GAIN, - NPHY_RFSEQ_CMD_EXT_PA - }; - u8 rfseq_rx2tx_dlys[] = { 8, 6, 6, 2, 4, 60, 1 }; - static const u8 rfseq_tx2rx_events[] = { - NPHY_RFSEQ_CMD_NOP, - NPHY_RFSEQ_CMD_EXT_PA, - NPHY_RFSEQ_CMD_TX_GAIN, - NPHY_RFSEQ_CMD_RXPD_TXPD, - NPHY_RFSEQ_CMD_TR_SWITCH, - NPHY_RFSEQ_CMD_RXG_FBW, - NPHY_RFSEQ_CMD_CLR_HIQ_DIS - }; - static const u8 rfseq_tx2rx_dlys[] = { 8, 6, 2, 4, 4, 6, 1 }; - s16 alpha0, alpha1, alpha2; - s16 beta0, beta1, beta2; - u16 regval; + if (!pi->edcrs_threshold_lock) { + write_phy_reg(pi, 0x224, 0x3eb); + write_phy_reg(pi, 0x225, 0x3eb); + write_phy_reg(pi, 0x226, 0x341); + write_phy_reg(pi, 0x227, 0x341); + write_phy_reg(pi, 0x228, 0x42b); + write_phy_reg(pi, 0x229, 0x42b); + write_phy_reg(pi, 0x22a, 0x381); + write_phy_reg(pi, 0x22b, 0x381); + write_phy_reg(pi, 0x22c, 0x42b); + write_phy_reg(pi, 0x22d, 0x42b); + write_phy_reg(pi, 0x22e, 0x381); + write_phy_reg(pi, 0x22f, 0x381); + } - if (pi->sh->boardflags2 & BFL2_SKWRKFEM_BRD || - (pi->sh->boardtype == 0x8b)) { - uint i; - u8 war_dlys[] = { 1, 6, 6, 2, 4, 20, 1 }; - for (i = 0; i < ARRAY_SIZE(rfseq_rx2tx_dlys); i++) - rfseq_rx2tx_dlys[i] = war_dlys[i]; - } + if (NREV_GE(pi->pubpi.phy_rev, 6)) { - if (CHSPEC_IS5G(pi->radio_chanspec) && pi->phy_5g_pwrgain) { - and_radio_reg(pi, RADIO_2055_CORE1_TX_RF_SPARE, 0xf7); - and_radio_reg(pi, RADIO_2055_CORE2_TX_RF_SPARE, 0xf7); + if (pi->sh->boardflags2 & BFL2_SINGLEANT_CCK) + wlapi_bmac_mhf(pi->sh->physhim, MHF4, + MHF4_BPHY_TXCORE0, + MHF4_BPHY_TXCORE0, BRCM_BAND_ALL); + } } else { - or_radio_reg(pi, RADIO_2055_CORE1_TX_RF_SPARE, 0x8); - or_radio_reg(pi, RADIO_2055_CORE2_TX_RF_SPARE, 0x8); - } - - regval = 0x000a; - wlc_phy_table_write_nphy(pi, 8, 1, 0, 16, ®val); - wlc_phy_table_write_nphy(pi, 8, 1, 0x10, 16, ®val); - - if (NREV_LT(pi->pubpi.phy_rev, 3)) { - regval = 0xcdaa; - wlc_phy_table_write_nphy(pi, 8, 1, 0x02, 16, ®val); - wlc_phy_table_write_nphy(pi, 8, 1, 0x12, 16, ®val); - } - if (NREV_LT(pi->pubpi.phy_rev, 2)) { - regval = 0x0000; - wlc_phy_table_write_nphy(pi, 8, 1, 0x08, 16, ®val); - wlc_phy_table_write_nphy(pi, 8, 1, 0x18, 16, ®val); + if (pi->sh->boardflags2 & BFL2_SKWRKFEM_BRD || + (pi->sh->boardtype == 0x8b)) { + uint i; + u8 war_dlys[] = { 1, 6, 6, 2, 4, 20, 1 }; + for (i = 0; i < ARRAY_SIZE(rfseq_rx2tx_dlys); i++) + rfseq_rx2tx_dlys[i] = war_dlys[i]; + } - regval = 0x7aab; - wlc_phy_table_write_nphy(pi, 8, 1, 0x07, 16, ®val); - wlc_phy_table_write_nphy(pi, 8, 1, 0x17, 16, ®val); + if (CHSPEC_IS5G(pi->radio_chanspec) && pi->phy_5g_pwrgain) { + and_radio_reg(pi, RADIO_2055_CORE1_TX_RF_SPARE, 0xf7); + and_radio_reg(pi, RADIO_2055_CORE2_TX_RF_SPARE, 0xf7); + } else { + or_radio_reg(pi, RADIO_2055_CORE1_TX_RF_SPARE, 0x8); + or_radio_reg(pi, RADIO_2055_CORE2_TX_RF_SPARE, 0x8); + } - regval = 0x0800; - wlc_phy_table_write_nphy(pi, 8, 1, 0x06, 16, ®val); - wlc_phy_table_write_nphy(pi, 8, 1, 0x16, 16, ®val); - } + regval = 0x000a; + wlc_phy_table_write_nphy(pi, 8, 1, 0, 16, ®val); + wlc_phy_table_write_nphy(pi, 8, 1, 0x10, 16, ®val); - write_phy_reg(pi, 0xf8, 0x02d8); - write_phy_reg(pi, 0xf9, 0x0301); - write_phy_reg(pi, 0xfa, 0x02d8); - write_phy_reg(pi, 0xfb, 0x0301); + if (NREV_LT(pi->pubpi.phy_rev, 3)) { + regval = 0xcdaa; + wlc_phy_table_write_nphy(pi, 8, 1, 0x02, 16, ®val); + wlc_phy_table_write_nphy(pi, 8, 1, 0x12, 16, ®val); + } - wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, rfseq_rx2tx_events, - rfseq_rx2tx_dlys, - ARRAY_SIZE(rfseq_rx2tx_events)); + if (NREV_LT(pi->pubpi.phy_rev, 2)) { + regval = 0x0000; + wlc_phy_table_write_nphy(pi, 8, 1, 0x08, 16, ®val); + wlc_phy_table_write_nphy(pi, 8, 1, 0x18, 16, ®val); - wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_TX2RX, rfseq_tx2rx_events, - rfseq_tx2rx_dlys, - ARRAY_SIZE(rfseq_tx2rx_events)); + regval = 0x7aab; + wlc_phy_table_write_nphy(pi, 8, 1, 0x07, 16, ®val); + wlc_phy_table_write_nphy(pi, 8, 1, 0x17, 16, ®val); - wlc_phy_workarounds_nphy_gainctrl(pi); + regval = 0x0800; + wlc_phy_table_write_nphy(pi, 8, 1, 0x06, 16, ®val); + wlc_phy_table_write_nphy(pi, 8, 1, 0x16, 16, ®val); + } - if (NREV_LT(pi->pubpi.phy_rev, 2)) { + write_phy_reg(pi, 0xf8, 0x02d8); + write_phy_reg(pi, 0xf9, 0x0301); + write_phy_reg(pi, 0xfa, 0x02d8); + write_phy_reg(pi, 0xfb, 0x0301); - if (read_phy_reg(pi, 0xa0) & NPHY_MLenable) - wlapi_bmac_mhf(pi->sh->physhim, MHF3, - MHF3_NPHY_MLADV_WAR, - MHF3_NPHY_MLADV_WAR, BRCM_BAND_ALL); + wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_RX2TX, rfseq_rx2tx_events, + rfseq_rx2tx_dlys, + ARRAY_SIZE(rfseq_rx2tx_events)); - } else if (NREV_IS(pi->pubpi.phy_rev, 2)) { - write_phy_reg(pi, 0x1e3, 0x0); - write_phy_reg(pi, 0x1e4, 0x0); - } + wlc_phy_set_rfseq_nphy(pi, NPHY_RFSEQ_TX2RX, rfseq_tx2rx_events, + rfseq_tx2rx_dlys, + ARRAY_SIZE(rfseq_tx2rx_events)); - if (NREV_LT(pi->pubpi.phy_rev, 2)) - mod_phy_reg(pi, 0x90, (0x1 << 7), 0); - - alpha0 = 293; - alpha1 = 435; - alpha2 = 261; - beta0 = 366; - beta1 = 205; - beta2 = 32; - write_phy_reg(pi, 0x145, alpha0); - write_phy_reg(pi, 0x146, alpha1); - write_phy_reg(pi, 0x147, alpha2); - write_phy_reg(pi, 0x148, beta0); - write_phy_reg(pi, 0x149, beta1); - write_phy_reg(pi, 0x14a, beta2); - - if (NREV_LT(pi->pubpi.phy_rev, 3)) { - mod_phy_reg(pi, 0x142, (0xf << 12), 0); - - write_phy_reg(pi, 0x192, 0xb5); - write_phy_reg(pi, 0x193, 0xa4); - write_phy_reg(pi, 0x194, 0x0); - } + wlc_phy_workarounds_nphy_gainctrl(pi); - if (NREV_IS(pi->pubpi.phy_rev, 2)) - mod_phy_reg(pi, 0x221, - NPHY_FORCESIG_DECODEGATEDCLKS, - NPHY_FORCESIG_DECODEGATEDCLKS); -} + if (NREV_LT(pi->pubpi.phy_rev, 2)) { -static void wlc_phy_workarounds_nphy(struct brcms_phy *pi) -{ - if (CHSPEC_IS5G(pi->radio_chanspec)) - wlc_phy_classifier_nphy(pi, NPHY_ClassifierCtrl_cck_en, 0); - else - wlc_phy_classifier_nphy(pi, NPHY_ClassifierCtrl_cck_en, 1); + if (read_phy_reg(pi, 0xa0) & NPHY_MLenable) + wlapi_bmac_mhf(pi->sh->physhim, MHF3, + MHF3_NPHY_MLADV_WAR, + MHF3_NPHY_MLADV_WAR, + BRCM_BAND_ALL); - if (pi->phyhang_avoid) - wlc_phy_stay_in_carriersearch_nphy(pi, true); + } else if (NREV_IS(pi->pubpi.phy_rev, 2)) { + write_phy_reg(pi, 0x1e3, 0x0); + write_phy_reg(pi, 0x1e4, 0x0); + } - or_phy_reg(pi, 0xb1, NPHY_IQFlip_ADC1 | NPHY_IQFlip_ADC2); + if (NREV_LT(pi->pubpi.phy_rev, 2)) + mod_phy_reg(pi, 0x90, (0x1 << 7), 0); + + alpha0 = 293; + alpha1 = 435; + alpha2 = 261; + beta0 = 366; + beta1 = 205; + beta2 = 32; + write_phy_reg(pi, 0x145, alpha0); + write_phy_reg(pi, 0x146, alpha1); + write_phy_reg(pi, 0x147, alpha2); + write_phy_reg(pi, 0x148, beta0); + write_phy_reg(pi, 0x149, beta1); + write_phy_reg(pi, 0x14a, beta2); + + if (NREV_LT(pi->pubpi.phy_rev, 3)) { + mod_phy_reg(pi, 0x142, (0xf << 12), 0); + + write_phy_reg(pi, 0x192, 0xb5); + write_phy_reg(pi, 0x193, 0xa4); + write_phy_reg(pi, 0x194, 0x0); + } - if (NREV_GE(pi->pubpi.phy_rev, 7)) - wlc_phy_workarounds_nphy_rev7(pi); - else if (NREV_GE(pi->pubpi.phy_rev, 3)) - wlc_phy_workarounds_nphy_rev3(pi); - else - wlc_phy_workarounds_nphy_rev1(pi); + if (NREV_IS(pi->pubpi.phy_rev, 2)) + mod_phy_reg(pi, 0x221, + NPHY_FORCESIG_DECODEGATEDCLKS, + NPHY_FORCESIG_DECODEGATEDCLKS); + } if (pi->phyhang_avoid) wlc_phy_stay_in_carriersearch_nphy(pi, false); @@ -23032,7 +23116,7 @@ wlc_phy_loadsampletable_nphy(struct brcms_phy *pi, struct cordic_iq *tone_buf, u16 t; u32 *data_buf = NULL; - data_buf = kmalloc_array(num_samps, sizeof(u32), GFP_ATOMIC); + data_buf = kmalloc(sizeof(u32) * num_samps, GFP_ATOMIC); if (data_buf == NULL) return; @@ -23074,8 +23158,7 @@ wlc_phy_gen_load_samples_nphy(struct brcms_phy *pi, u32 f_kHz, u16 max_val, tbl_len = (phy_bw << 1); } - tone_buf = kmalloc_array(tbl_len, sizeof(struct cordic_iq), - GFP_ATOMIC); + tone_buf = kmalloc(sizeof(struct cordic_iq) * tbl_len, GFP_ATOMIC); if (tone_buf == NULL) return 0; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c index 533bd4b0277ea..dbf50ef6cd758 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phytbl_n.c @@ -14,7 +14,6 @@ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include #include #include "phytbl_n.h" @@ -4438,39 +4437,109 @@ static const u16 loft_lut_core1_rev0[] = { }; const struct phytbl_info mimophytbl_info_rev0_volatile[] = { - {&bdi_tbl_rev0, ARRAY_SIZE(bdi_tbl_rev0), 21, 0, 16}, - {&pltlut_tbl_rev0, ARRAY_SIZE(pltlut_tbl_rev0), 20, 0, 32}, - {&gainctrl_lut_core0_rev0, ARRAY_SIZE(gainctrl_lut_core0_rev0), 26, 192, 32}, - {&gainctrl_lut_core1_rev0, ARRAY_SIZE(gainctrl_lut_core1_rev0), 27, 192, 32}, - {&est_pwr_lut_core0_rev0, ARRAY_SIZE(est_pwr_lut_core0_rev0), 26, 0, 8}, - {&est_pwr_lut_core1_rev0, ARRAY_SIZE(est_pwr_lut_core1_rev0), 27, 0, 8}, - {&adj_pwr_lut_core0_rev0, ARRAY_SIZE(adj_pwr_lut_core0_rev0), 26, 64, 8}, - {&adj_pwr_lut_core1_rev0, ARRAY_SIZE(adj_pwr_lut_core1_rev0), 27, 64, 8}, - {&iq_lut_core0_rev0, ARRAY_SIZE(iq_lut_core0_rev0), 26, 320, 32}, - {&iq_lut_core1_rev0, ARRAY_SIZE(iq_lut_core1_rev0), 27, 320, 32}, - {&loft_lut_core0_rev0, ARRAY_SIZE(loft_lut_core0_rev0), 26, 448, 16}, - {&loft_lut_core1_rev0, ARRAY_SIZE(loft_lut_core1_rev0), 27, 448, 16}, + {&bdi_tbl_rev0, sizeof(bdi_tbl_rev0) / sizeof(bdi_tbl_rev0[0]), 21, 0, + 16} + , + {&pltlut_tbl_rev0, sizeof(pltlut_tbl_rev0) / sizeof(pltlut_tbl_rev0[0]), + 20, 0, 32} + , + {&gainctrl_lut_core0_rev0, + sizeof(gainctrl_lut_core0_rev0) / sizeof(gainctrl_lut_core0_rev0[0]), + 26, 192, 32} + , + {&gainctrl_lut_core1_rev0, + sizeof(gainctrl_lut_core1_rev0) / sizeof(gainctrl_lut_core1_rev0[0]), + 27, 192, 32} + , + + {&est_pwr_lut_core0_rev0, + sizeof(est_pwr_lut_core0_rev0) / sizeof(est_pwr_lut_core0_rev0[0]), 26, + 0, 8} + , + {&est_pwr_lut_core1_rev0, + sizeof(est_pwr_lut_core1_rev0) / sizeof(est_pwr_lut_core1_rev0[0]), 27, + 0, 8} + , + {&adj_pwr_lut_core0_rev0, + sizeof(adj_pwr_lut_core0_rev0) / sizeof(adj_pwr_lut_core0_rev0[0]), 26, + 64, 8} + , + {&adj_pwr_lut_core1_rev0, + sizeof(adj_pwr_lut_core1_rev0) / sizeof(adj_pwr_lut_core1_rev0[0]), 27, + 64, 8} + , + {&iq_lut_core0_rev0, + sizeof(iq_lut_core0_rev0) / sizeof(iq_lut_core0_rev0[0]), 26, 320, 32} + , + {&iq_lut_core1_rev0, + sizeof(iq_lut_core1_rev0) / sizeof(iq_lut_core1_rev0[0]), 27, 320, 32} + , + {&loft_lut_core0_rev0, + sizeof(loft_lut_core0_rev0) / sizeof(loft_lut_core0_rev0[0]), 26, 448, + 16} + , + {&loft_lut_core1_rev0, + sizeof(loft_lut_core1_rev0) / sizeof(loft_lut_core1_rev0[0]), 27, 448, + 16} + , }; const struct phytbl_info mimophytbl_info_rev0[] = { - {&frame_struct_rev0, ARRAY_SIZE(frame_struct_rev0), 10, 0, 32}, - {&frame_lut_rev0, ARRAY_SIZE(frame_lut_rev0), 24, 0, 8}, - {&tmap_tbl_rev0, ARRAY_SIZE(tmap_tbl_rev0), 12, 0, 32}, - {&tdtrn_tbl_rev0, ARRAY_SIZE(tdtrn_tbl_rev0), 14, 0, 32}, - {&intlv_tbl_rev0, ARRAY_SIZE(intlv_tbl_rev0), 13, 0, 32}, - {&pilot_tbl_rev0, ARRAY_SIZE(pilot_tbl_rev0), 11, 0, 16}, - {&tdi_tbl20_ant0_rev0, ARRAY_SIZE(tdi_tbl20_ant0_rev0), 19, 128, 32}, - {&tdi_tbl20_ant1_rev0, ARRAY_SIZE(tdi_tbl20_ant1_rev0), 19, 256, 32}, - {&tdi_tbl40_ant0_rev0, ARRAY_SIZE(tdi_tbl40_ant0_rev0), 19, 640, 32}, - {&tdi_tbl40_ant1_rev0, ARRAY_SIZE(tdi_tbl40_ant1_rev0), 19, 768, 32}, - {&chanest_tbl_rev0, ARRAY_SIZE(chanest_tbl_rev0), 22, 0, 32}, - {&mcs_tbl_rev0, ARRAY_SIZE(mcs_tbl_rev0), 18, 0, 8}, - {&noise_var_tbl0_rev0, ARRAY_SIZE(noise_var_tbl0_rev0), 16, 0, 32}, - {&noise_var_tbl1_rev0, ARRAY_SIZE(noise_var_tbl1_rev0), 16, 128, 32}, + {&frame_struct_rev0, + sizeof(frame_struct_rev0) / sizeof(frame_struct_rev0[0]), 10, 0, 32} + , + {&frame_lut_rev0, sizeof(frame_lut_rev0) / sizeof(frame_lut_rev0[0]), + 24, 0, 8} + , + {&tmap_tbl_rev0, sizeof(tmap_tbl_rev0) / sizeof(tmap_tbl_rev0[0]), 12, + 0, 32} + , + {&tdtrn_tbl_rev0, sizeof(tdtrn_tbl_rev0) / sizeof(tdtrn_tbl_rev0[0]), + 14, 0, 32} + , + {&intlv_tbl_rev0, sizeof(intlv_tbl_rev0) / sizeof(intlv_tbl_rev0[0]), + 13, 0, 32} + , + {&pilot_tbl_rev0, sizeof(pilot_tbl_rev0) / sizeof(pilot_tbl_rev0[0]), + 11, 0, 16} + , + {&tdi_tbl20_ant0_rev0, + sizeof(tdi_tbl20_ant0_rev0) / sizeof(tdi_tbl20_ant0_rev0[0]), 19, 128, + 32} + , + {&tdi_tbl20_ant1_rev0, + sizeof(tdi_tbl20_ant1_rev0) / sizeof(tdi_tbl20_ant1_rev0[0]), 19, 256, + 32} + , + {&tdi_tbl40_ant0_rev0, + sizeof(tdi_tbl40_ant0_rev0) / sizeof(tdi_tbl40_ant0_rev0[0]), 19, 640, + 32} + , + {&tdi_tbl40_ant1_rev0, + sizeof(tdi_tbl40_ant1_rev0) / sizeof(tdi_tbl40_ant1_rev0[0]), 19, 768, + 32} + , + {&chanest_tbl_rev0, + sizeof(chanest_tbl_rev0) / sizeof(chanest_tbl_rev0[0]), 22, 0, 32} + , + {&mcs_tbl_rev0, sizeof(mcs_tbl_rev0) / sizeof(mcs_tbl_rev0[0]), 18, 0, + 8} + , + {&noise_var_tbl0_rev0, + sizeof(noise_var_tbl0_rev0) / sizeof(noise_var_tbl0_rev0[0]), 16, 0, + 32} + , + {&noise_var_tbl1_rev0, + sizeof(noise_var_tbl1_rev0) / sizeof(noise_var_tbl1_rev0[0]), 16, 128, + 32} + , }; -const u32 mimophytbl_info_sz_rev0 = ARRAY_SIZE(mimophytbl_info_rev0); -const u32 mimophytbl_info_sz_rev0_volatile = ARRAY_SIZE(mimophytbl_info_rev0_volatile); +const u32 mimophytbl_info_sz_rev0 = + sizeof(mimophytbl_info_rev0) / sizeof(mimophytbl_info_rev0[0]); +const u32 mimophytbl_info_sz_rev0_volatile = + sizeof(mimophytbl_info_rev0_volatile) / + sizeof(mimophytbl_info_rev0_volatile[0]); static const u16 ant_swctrl_tbl_rev3[] = { 0x0082, @@ -9294,53 +9363,132 @@ static const u32 papd_cal_scalars_tbl_core1_rev3[] = { }; const struct phytbl_info mimophytbl_info_rev3_volatile[] = { - {&ant_swctrl_tbl_rev3, ARRAY_SIZE(ant_swctrl_tbl_rev3), 9, 0, 16}, + {&ant_swctrl_tbl_rev3, + sizeof(ant_swctrl_tbl_rev3) / sizeof(ant_swctrl_tbl_rev3[0]), 9, 0, 16} + , }; const struct phytbl_info mimophytbl_info_rev3_volatile1[] = { - {&ant_swctrl_tbl_rev3_1, ARRAY_SIZE(ant_swctrl_tbl_rev3_1), 9, 0, 16}, + {&ant_swctrl_tbl_rev3_1, + sizeof(ant_swctrl_tbl_rev3_1) / sizeof(ant_swctrl_tbl_rev3_1[0]), 9, 0, + 16} + , }; const struct phytbl_info mimophytbl_info_rev3_volatile2[] = { - {&ant_swctrl_tbl_rev3_2, ARRAY_SIZE(ant_swctrl_tbl_rev3_2), 9, 0, 16}, + {&ant_swctrl_tbl_rev3_2, + sizeof(ant_swctrl_tbl_rev3_2) / sizeof(ant_swctrl_tbl_rev3_2[0]), 9, 0, + 16} + , }; const struct phytbl_info mimophytbl_info_rev3_volatile3[] = { - {&ant_swctrl_tbl_rev3_3, ARRAY_SIZE(ant_swctrl_tbl_rev3_3), 9, 0, 16}, + {&ant_swctrl_tbl_rev3_3, + sizeof(ant_swctrl_tbl_rev3_3) / sizeof(ant_swctrl_tbl_rev3_3[0]), 9, 0, + 16} + , }; const struct phytbl_info mimophytbl_info_rev3[] = { - {&frame_struct_rev3, ARRAY_SIZE(frame_struct_rev3), 10, 0, 32}, - {&pilot_tbl_rev3, ARRAY_SIZE(pilot_tbl_rev3), 11, 0, 16}, - {&tmap_tbl_rev3, ARRAY_SIZE(tmap_tbl_rev3), 12, 0, 32}, - {&intlv_tbl_rev3, ARRAY_SIZE(intlv_tbl_rev3), 13, 0, 32}, - {&tdtrn_tbl_rev3, ARRAY_SIZE(tdtrn_tbl_rev3), 14, 0, 32}, - {&noise_var_tbl_rev3, ARRAY_SIZE(noise_var_tbl_rev3), 16, 0, 32}, - {&mcs_tbl_rev3, ARRAY_SIZE(mcs_tbl_rev3), 18, 0, 16}, - {&tdi_tbl20_ant0_rev3, ARRAY_SIZE(tdi_tbl20_ant0_rev3), 19, 128, 32}, - {&tdi_tbl20_ant1_rev3, ARRAY_SIZE(tdi_tbl20_ant1_rev3), 19, 256, 32}, - {&tdi_tbl40_ant0_rev3, ARRAY_SIZE(tdi_tbl40_ant0_rev3), 19, 640, 32}, - {&tdi_tbl40_ant1_rev3, ARRAY_SIZE(tdi_tbl40_ant1_rev3), 19, 768, 32}, - {&pltlut_tbl_rev3, ARRAY_SIZE(pltlut_tbl_rev3), 20, 0, 32}, - {&chanest_tbl_rev3, ARRAY_SIZE(chanest_tbl_rev3), 22, 0, 32}, - {&frame_lut_rev3, ARRAY_SIZE(frame_lut_rev3), 24, 0, 8}, - {&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8}, - {&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8}, - {&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8}, - {&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8}, - {&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32}, - {&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32}, - {&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32}, - {&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32}, - {&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16}, - {&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16} + {&frame_struct_rev3, + sizeof(frame_struct_rev3) / sizeof(frame_struct_rev3[0]), 10, 0, 32} + , + {&pilot_tbl_rev3, sizeof(pilot_tbl_rev3) / sizeof(pilot_tbl_rev3[0]), + 11, 0, 16} + , + {&tmap_tbl_rev3, sizeof(tmap_tbl_rev3) / sizeof(tmap_tbl_rev3[0]), 12, + 0, 32} + , + {&intlv_tbl_rev3, sizeof(intlv_tbl_rev3) / sizeof(intlv_tbl_rev3[0]), + 13, 0, 32} + , + {&tdtrn_tbl_rev3, sizeof(tdtrn_tbl_rev3) / sizeof(tdtrn_tbl_rev3[0]), + 14, 0, 32} + , + {&noise_var_tbl_rev3, + sizeof(noise_var_tbl_rev3) / sizeof(noise_var_tbl_rev3[0]), 16, 0, 32} + , + {&mcs_tbl_rev3, sizeof(mcs_tbl_rev3) / sizeof(mcs_tbl_rev3[0]), 18, 0, + 16} + , + {&tdi_tbl20_ant0_rev3, + sizeof(tdi_tbl20_ant0_rev3) / sizeof(tdi_tbl20_ant0_rev3[0]), 19, 128, + 32} + , + {&tdi_tbl20_ant1_rev3, + sizeof(tdi_tbl20_ant1_rev3) / sizeof(tdi_tbl20_ant1_rev3[0]), 19, 256, + 32} + , + {&tdi_tbl40_ant0_rev3, + sizeof(tdi_tbl40_ant0_rev3) / sizeof(tdi_tbl40_ant0_rev3[0]), 19, 640, + 32} + , + {&tdi_tbl40_ant1_rev3, + sizeof(tdi_tbl40_ant1_rev3) / sizeof(tdi_tbl40_ant1_rev3[0]), 19, 768, + 32} + , + {&pltlut_tbl_rev3, sizeof(pltlut_tbl_rev3) / sizeof(pltlut_tbl_rev3[0]), + 20, 0, 32} + , + {&chanest_tbl_rev3, + sizeof(chanest_tbl_rev3) / sizeof(chanest_tbl_rev3[0]), 22, 0, 32} + , + {&frame_lut_rev3, sizeof(frame_lut_rev3) / sizeof(frame_lut_rev3[0]), + 24, 0, 8} + , + {&est_pwr_lut_core0_rev3, + sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26, + 0, 8} + , + {&est_pwr_lut_core1_rev3, + sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27, + 0, 8} + , + {&adj_pwr_lut_core0_rev3, + sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26, + 64, 8} + , + {&adj_pwr_lut_core1_rev3, + sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27, + 64, 8} + , + {&gainctrl_lut_core0_rev3, + sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]), + 26, 192, 32} + , + {&gainctrl_lut_core1_rev3, + sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]), + 27, 192, 32} + , + {&iq_lut_core0_rev3, + sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32} + , + {&iq_lut_core1_rev3, + sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32} + , + {&loft_lut_core0_rev3, + sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448, + 16} + , + {&loft_lut_core1_rev3, + sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448, + 16} }; -const u32 mimophytbl_info_sz_rev3 = ARRAY_SIZE(mimophytbl_info_rev3); -const u32 mimophytbl_info_sz_rev3_volatile = ARRAY_SIZE(mimophytbl_info_rev3_volatile); -const u32 mimophytbl_info_sz_rev3_volatile1 = ARRAY_SIZE(mimophytbl_info_rev3_volatile1); -const u32 mimophytbl_info_sz_rev3_volatile2 = ARRAY_SIZE(mimophytbl_info_rev3_volatile2); -const u32 mimophytbl_info_sz_rev3_volatile3 = ARRAY_SIZE(mimophytbl_info_rev3_volatile3); +const u32 mimophytbl_info_sz_rev3 = + sizeof(mimophytbl_info_rev3) / sizeof(mimophytbl_info_rev3[0]); +const u32 mimophytbl_info_sz_rev3_volatile = + sizeof(mimophytbl_info_rev3_volatile) / + sizeof(mimophytbl_info_rev3_volatile[0]); +const u32 mimophytbl_info_sz_rev3_volatile1 = + sizeof(mimophytbl_info_rev3_volatile1) / + sizeof(mimophytbl_info_rev3_volatile1[0]); +const u32 mimophytbl_info_sz_rev3_volatile2 = + sizeof(mimophytbl_info_rev3_volatile2) / + sizeof(mimophytbl_info_rev3_volatile2[0]); +const u32 mimophytbl_info_sz_rev3_volatile3 = + sizeof(mimophytbl_info_rev3_volatile3) / + sizeof(mimophytbl_info_rev3_volatile3[0]); static const u32 tmap_tbl_rev7[] = { 0x8a88aa80, @@ -10321,58 +10469,162 @@ static const u32 papd_cal_scalars_tbl_core1_rev7[] = { }; const struct phytbl_info mimophytbl_info_rev7[] = { - {&frame_struct_rev3, ARRAY_SIZE(frame_struct_rev3), 10, 0, 32}, - {&pilot_tbl_rev3, ARRAY_SIZE(pilot_tbl_rev3), 11, 0, 16}, - {&tmap_tbl_rev7, ARRAY_SIZE(tmap_tbl_rev7), 12, 0, 32}, - {&intlv_tbl_rev3, ARRAY_SIZE(intlv_tbl_rev3), 13, 0, 32}, - {&tdtrn_tbl_rev3, ARRAY_SIZE(tdtrn_tbl_rev3), 14, 0, 32}, - {&noise_var_tbl_rev7, ARRAY_SIZE(noise_var_tbl_rev7), 16, 0, 32}, - {&mcs_tbl_rev3, ARRAY_SIZE(mcs_tbl_rev3), 18, 0, 16}, - {&tdi_tbl20_ant0_rev3, ARRAY_SIZE(tdi_tbl20_ant0_rev3), 19, 128, 32}, - {&tdi_tbl20_ant1_rev3, ARRAY_SIZE(tdi_tbl20_ant1_rev3), 19, 256, 32}, - {&tdi_tbl40_ant0_rev3, ARRAY_SIZE(tdi_tbl40_ant0_rev3), 19, 640, 32}, - {&tdi_tbl40_ant1_rev3, ARRAY_SIZE(tdi_tbl40_ant1_rev3), 19, 768, 32}, - {&pltlut_tbl_rev3, ARRAY_SIZE(pltlut_tbl_rev3), 20, 0, 32}, - {&chanest_tbl_rev3, ARRAY_SIZE(chanest_tbl_rev3), 22, 0, 32}, - {&frame_lut_rev3, ARRAY_SIZE(frame_lut_rev3), 24, 0, 8}, - {&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8}, - {&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8}, - {&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8}, - {&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8}, - {&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32}, - {&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32}, - {&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32}, - {&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32}, - {&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16}, - {&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16}, + {&frame_struct_rev3, + sizeof(frame_struct_rev3) / sizeof(frame_struct_rev3[0]), 10, 0, 32} + , + {&pilot_tbl_rev3, sizeof(pilot_tbl_rev3) / sizeof(pilot_tbl_rev3[0]), + 11, 0, 16} + , + {&tmap_tbl_rev7, sizeof(tmap_tbl_rev7) / sizeof(tmap_tbl_rev7[0]), 12, + 0, 32} + , + {&intlv_tbl_rev3, sizeof(intlv_tbl_rev3) / sizeof(intlv_tbl_rev3[0]), + 13, 0, 32} + , + {&tdtrn_tbl_rev3, sizeof(tdtrn_tbl_rev3) / sizeof(tdtrn_tbl_rev3[0]), + 14, 0, 32} + , + {&noise_var_tbl_rev7, + sizeof(noise_var_tbl_rev7) / sizeof(noise_var_tbl_rev7[0]), 16, 0, 32} + , + {&mcs_tbl_rev3, sizeof(mcs_tbl_rev3) / sizeof(mcs_tbl_rev3[0]), 18, 0, + 16} + , + {&tdi_tbl20_ant0_rev3, + sizeof(tdi_tbl20_ant0_rev3) / sizeof(tdi_tbl20_ant0_rev3[0]), 19, 128, + 32} + , + {&tdi_tbl20_ant1_rev3, + sizeof(tdi_tbl20_ant1_rev3) / sizeof(tdi_tbl20_ant1_rev3[0]), 19, 256, + 32} + , + {&tdi_tbl40_ant0_rev3, + sizeof(tdi_tbl40_ant0_rev3) / sizeof(tdi_tbl40_ant0_rev3[0]), 19, 640, + 32} + , + {&tdi_tbl40_ant1_rev3, + sizeof(tdi_tbl40_ant1_rev3) / sizeof(tdi_tbl40_ant1_rev3[0]), 19, 768, + 32} + , + {&pltlut_tbl_rev3, sizeof(pltlut_tbl_rev3) / sizeof(pltlut_tbl_rev3[0]), + 20, 0, 32} + , + {&chanest_tbl_rev3, + sizeof(chanest_tbl_rev3) / sizeof(chanest_tbl_rev3[0]), 22, 0, 32} + , + {&frame_lut_rev3, sizeof(frame_lut_rev3) / sizeof(frame_lut_rev3[0]), + 24, 0, 8} + , + {&est_pwr_lut_core0_rev3, + sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26, + 0, 8} + , + {&est_pwr_lut_core1_rev3, + sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27, + 0, 8} + , + {&adj_pwr_lut_core0_rev3, + sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26, + 64, 8} + , + {&adj_pwr_lut_core1_rev3, + sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27, + 64, 8} + , + {&gainctrl_lut_core0_rev3, + sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]), + 26, 192, 32} + , + {&gainctrl_lut_core1_rev3, + sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]), + 27, 192, 32} + , + {&iq_lut_core0_rev3, + sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32} + , + {&iq_lut_core1_rev3, + sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32} + , + {&loft_lut_core0_rev3, + sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448, + 16} + , + {&loft_lut_core1_rev3, + sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448, + 16} + , {&papd_comp_rfpwr_tbl_core0_rev3, - ARRAY_SIZE(papd_comp_rfpwr_tbl_core0_rev3), 26, 576, 16}, + sizeof(papd_comp_rfpwr_tbl_core0_rev3) / + sizeof(papd_comp_rfpwr_tbl_core0_rev3[0]), 26, 576, 16} + , {&papd_comp_rfpwr_tbl_core1_rev3, - ARRAY_SIZE(papd_comp_rfpwr_tbl_core1_rev3), 27, 576, 16}, + sizeof(papd_comp_rfpwr_tbl_core1_rev3) / + sizeof(papd_comp_rfpwr_tbl_core1_rev3[0]), 27, 576, 16} + , {&papd_comp_epsilon_tbl_core0_rev7, - ARRAY_SIZE(papd_comp_epsilon_tbl_core0_rev7), 31, 0, 32}, + sizeof(papd_comp_epsilon_tbl_core0_rev7) / + sizeof(papd_comp_epsilon_tbl_core0_rev7[0]), 31, 0, 32} + , {&papd_cal_scalars_tbl_core0_rev7, - ARRAY_SIZE(papd_cal_scalars_tbl_core0_rev7), 32, 0, 32}, + sizeof(papd_cal_scalars_tbl_core0_rev7) / + sizeof(papd_cal_scalars_tbl_core0_rev7[0]), 32, 0, 32} + , {&papd_comp_epsilon_tbl_core1_rev7, - ARRAY_SIZE(papd_comp_epsilon_tbl_core1_rev7), 33, 0, 32}, + sizeof(papd_comp_epsilon_tbl_core1_rev7) / + sizeof(papd_comp_epsilon_tbl_core1_rev7[0]), 33, 0, 32} + , {&papd_cal_scalars_tbl_core1_rev7, - ARRAY_SIZE(papd_cal_scalars_tbl_core1_rev7), 34, 0, 32}, + sizeof(papd_cal_scalars_tbl_core1_rev7) / + sizeof(papd_cal_scalars_tbl_core1_rev7[0]), 34, 0, 32} + , }; -const u32 mimophytbl_info_sz_rev7 = ARRAY_SIZE(mimophytbl_info_rev7); +const u32 mimophytbl_info_sz_rev7 = + sizeof(mimophytbl_info_rev7) / sizeof(mimophytbl_info_rev7[0]); const struct phytbl_info mimophytbl_info_rev16[] = { - {&noise_var_tbl_rev7, ARRAY_SIZE(noise_var_tbl_rev7), 16, 0, 32}, - {&est_pwr_lut_core0_rev3, ARRAY_SIZE(est_pwr_lut_core0_rev3), 26, 0, 8}, - {&est_pwr_lut_core1_rev3, ARRAY_SIZE(est_pwr_lut_core1_rev3), 27, 0, 8}, - {&adj_pwr_lut_core0_rev3, ARRAY_SIZE(adj_pwr_lut_core0_rev3), 26, 64, 8}, - {&adj_pwr_lut_core1_rev3, ARRAY_SIZE(adj_pwr_lut_core1_rev3), 27, 64, 8}, - {&gainctrl_lut_core0_rev3, ARRAY_SIZE(gainctrl_lut_core0_rev3), 26, 192, 32}, - {&gainctrl_lut_core1_rev3, ARRAY_SIZE(gainctrl_lut_core1_rev3), 27, 192, 32}, - {&iq_lut_core0_rev3, ARRAY_SIZE(iq_lut_core0_rev3), 26, 320, 32}, - {&iq_lut_core1_rev3, ARRAY_SIZE(iq_lut_core1_rev3), 27, 320, 32}, - {&loft_lut_core0_rev3, ARRAY_SIZE(loft_lut_core0_rev3), 26, 448, 16}, - {&loft_lut_core1_rev3, ARRAY_SIZE(loft_lut_core1_rev3), 27, 448, 16}, + {&noise_var_tbl_rev7, + sizeof(noise_var_tbl_rev7) / sizeof(noise_var_tbl_rev7[0]), 16, 0, 32} + , + {&est_pwr_lut_core0_rev3, + sizeof(est_pwr_lut_core0_rev3) / sizeof(est_pwr_lut_core0_rev3[0]), 26, + 0, 8} + , + {&est_pwr_lut_core1_rev3, + sizeof(est_pwr_lut_core1_rev3) / sizeof(est_pwr_lut_core1_rev3[0]), 27, + 0, 8} + , + {&adj_pwr_lut_core0_rev3, + sizeof(adj_pwr_lut_core0_rev3) / sizeof(adj_pwr_lut_core0_rev3[0]), 26, + 64, 8} + , + {&adj_pwr_lut_core1_rev3, + sizeof(adj_pwr_lut_core1_rev3) / sizeof(adj_pwr_lut_core1_rev3[0]), 27, + 64, 8} + , + {&gainctrl_lut_core0_rev3, + sizeof(gainctrl_lut_core0_rev3) / sizeof(gainctrl_lut_core0_rev3[0]), + 26, 192, 32} + , + {&gainctrl_lut_core1_rev3, + sizeof(gainctrl_lut_core1_rev3) / sizeof(gainctrl_lut_core1_rev3[0]), + 27, 192, 32} + , + {&iq_lut_core0_rev3, + sizeof(iq_lut_core0_rev3) / sizeof(iq_lut_core0_rev3[0]), 26, 320, 32} + , + {&iq_lut_core1_rev3, + sizeof(iq_lut_core1_rev3) / sizeof(iq_lut_core1_rev3[0]), 27, 320, 32} + , + {&loft_lut_core0_rev3, + sizeof(loft_lut_core0_rev3) / sizeof(loft_lut_core0_rev3[0]), 26, 448, + 16} + , + {&loft_lut_core1_rev3, + sizeof(loft_lut_core1_rev3) / sizeof(loft_lut_core1_rev3[0]), 27, 448, + 16} + , }; -const u32 mimophytbl_info_sz_rev16 = ARRAY_SIZE(mimophytbl_info_rev16); +const u32 mimophytbl_info_sz_rev16 = + sizeof(mimophytbl_info_rev16) / sizeof(mimophytbl_info_rev16[0]); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c index eb5db94f57453..d8b79cb72b58d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c @@ -77,8 +77,6 @@ static u16 d11ac_bw(enum brcmu_chan_bw bw) return BRCMU_CHSPEC_D11AC_BW_40; case BRCMU_CHAN_BW_80: return BRCMU_CHSPEC_D11AC_BW_80; - case BRCMU_CHAN_BW_160: - return BRCMU_CHSPEC_D11AC_BW_160; default: WARN_ON(1); } @@ -192,41 +190,8 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) break; } break; - case BRCMU_CHSPEC_D11AC_BW_160: - ch->bw = BRCMU_CHAN_BW_160; - ch->sb = brcmu_maskget16(ch->chspec, BRCMU_CHSPEC_D11AC_SB_MASK, - BRCMU_CHSPEC_D11AC_SB_SHIFT); - switch (ch->sb) { - case BRCMU_CHAN_SB_LLL: - ch->control_ch_num -= CH_70MHZ_APART; - break; - case BRCMU_CHAN_SB_LLU: - ch->control_ch_num -= CH_50MHZ_APART; - break; - case BRCMU_CHAN_SB_LUL: - ch->control_ch_num -= CH_30MHZ_APART; - break; - case BRCMU_CHAN_SB_LUU: - ch->control_ch_num -= CH_10MHZ_APART; - break; - case BRCMU_CHAN_SB_ULL: - ch->control_ch_num += CH_10MHZ_APART; - break; - case BRCMU_CHAN_SB_ULU: - ch->control_ch_num += CH_30MHZ_APART; - break; - case BRCMU_CHAN_SB_UUL: - ch->control_ch_num += CH_50MHZ_APART; - break; - case BRCMU_CHAN_SB_UUU: - ch->control_ch_num += CH_70MHZ_APART; - break; - default: - WARN_ON_ONCE(1); - break; - } - break; case BRCMU_CHSPEC_D11AC_BW_8080: + case BRCMU_CHSPEC_D11AC_BW_160: default: WARN_ON_ONCE(1); break; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 686f7a85a0453..57544a3a3ce4a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -57,7 +57,6 @@ #define BRCM_CC_43602_CHIP_ID 43602 #define BRCM_CC_4365_CHIP_ID 0x4365 #define BRCM_CC_4366_CHIP_ID 0x4366 -#define BRCM_CC_43664_CHIP_ID 43664 #define BRCM_CC_4371_CHIP_ID 0x4371 #define CY_CC_4373_CHIP_ID 0x4373 diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 75b2a0438cfa7..7b9a77981df16 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -29,8 +29,6 @@ #define CH_UPPER_SB 0x01 #define CH_LOWER_SB 0x02 #define CH_EWA_VALID 0x04 -#define CH_70MHZ_APART 14 -#define CH_50MHZ_APART 10 #define CH_30MHZ_APART 6 #define CH_20MHZ_APART 4 #define CH_10MHZ_APART 2 From c2ee7383594c69eba98539c3148eabbdc9119c7c Mon Sep 17 00:00:00 2001 From: Allen Pais Date: Thu, 21 Sep 2017 22:34:25 +0530 Subject: [PATCH 016/213] drivers: net: brcm80211: use setup_timer() helper. Use setup_timer function instead of initializing timer with the function and data fields. Signed-off-by: Allen Pais Signed-off-by: David S. Miller --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 083e5ce7eac73..136eb438d133c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3261,9 +3261,8 @@ static void brcmf_init_escan(struct brcmf_cfg80211_info *cfg) brcmf_cfg80211_escan_handler); cfg->escan_info.escan_state = WL_ESCAN_STATE_IDLE; /* Init scan_timeout timer */ - init_timer(&cfg->escan_timeout); - cfg->escan_timeout.data = (unsigned long) cfg; - cfg->escan_timeout.function = brcmf_escan_timeout; + setup_timer(&cfg->escan_timeout, brcmf_escan_timeout, + (unsigned long)cfg); INIT_WORK(&cfg->escan_timeout_work, brcmf_cfg80211_escan_timeout_worker); } From f9840f583ae3c9024d49aecb01953197c5136f4d Mon Sep 17 00:00:00 2001 From: Allen Pais Date: Thu, 21 Sep 2017 18:13:06 +0530 Subject: [PATCH 017/213] brcmfmac: use setup_timer() helper Use setup_timer function instead of initializing timer with the function and data fields. Signed-off-by: Allen Pais Signed-off-by: Kalle Valo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 4c28b04ea6053..b2f2a3d4106f4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4144,10 +4144,8 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) init_waitqueue_head(&bus->dcmd_resp_wait); /* Set up the watchdog timer */ - init_timer(&bus->timer); - bus->timer.data = (unsigned long)bus; - bus->timer.function = brcmf_sdio_watchdog; - + setup_timer(&bus->timer, brcmf_sdio_watchdog, + (unsigned long)bus); /* Initialize watchdog thread */ init_completion(&bus->watchdog_wait); bus->watchdog_tsk = kthread_run(brcmf_sdio_watchdog_thread, From 1e37ae5497278f9550a3216146b6611d784a04ab Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 16 Oct 2017 14:43:17 -0700 Subject: [PATCH 018/213] treewide: setup_timer() -> timer_setup() This converts all remaining cases of the old setup_timer() API into using timer_setup(), where the callback argument is the structure already holding the struct timer_list. These should have no behavioral changes, since they just change which pointer is passed into the callback with the same available pointers after conversion. It handles the following examples, in addition to some other variations. Casting from unsigned long: void my_callback(unsigned long data) { struct something *ptr = (struct something *)data; ... } ... setup_timer(&ptr->my_timer, my_callback, ptr); and forced object casts: void my_callback(struct something *ptr) { ... } ... setup_timer(&ptr->my_timer, my_callback, (unsigned long)ptr); become: void my_callback(struct timer_list *t) { struct something *ptr = from_timer(ptr, t, my_timer); ... } ... timer_setup(&ptr->my_timer, my_callback, 0); Direct function assignments: void my_callback(unsigned long data) { struct something *ptr = (struct something *)data; ... } ... ptr->my_timer.function = my_callback; have a temporary cast added, along with converting the args: void my_callback(struct timer_list *t) { struct something *ptr = from_timer(ptr, t, my_timer); ... } ... ptr->my_timer.function = (TIMER_FUNC_TYPE)my_callback; And finally, callbacks without a data assignment: void my_callback(unsigned long data) { ... } ... setup_timer(&ptr->my_timer, my_callback, 0); have their argument renamed to verify they're unused during conversion: void my_callback(struct timer_list *unused) { ... } ... timer_setup(&ptr->my_timer, my_callback, 0); The conversion is done with the following Coccinelle script: spatch --very-quiet --all-includes --include-headers \ -I ./arch/x86/include -I ./arch/x86/include/generated \ -I ./include -I ./arch/x86/include/uapi \ -I ./arch/x86/include/generated/uapi -I ./include/uapi \ -I ./include/generated/uapi --include ./include/linux/kconfig.h \ --dir . \ --cocci-file ~/src/data/timer_setup.cocci @fix_address_of@ expression e; @@ setup_timer( -&(e) +&e , ...) // Update any raw setup_timer() usages that have a NULL callback, but // would otherwise match change_timer_function_usage, since the latter // will update all function assignments done in the face of a NULL // function initialization in setup_timer(). @change_timer_function_usage_NULL@ expression _E; identifier _timer; type _cast_data; @@ ( -setup_timer(&_E->_timer, NULL, _E); +timer_setup(&_E->_timer, NULL, 0); | -setup_timer(&_E->_timer, NULL, (_cast_data)_E); +timer_setup(&_E->_timer, NULL, 0); | -setup_timer(&_E._timer, NULL, &_E); +timer_setup(&_E._timer, NULL, 0); | -setup_timer(&_E._timer, NULL, (_cast_data)&_E); +timer_setup(&_E._timer, NULL, 0); ) @change_timer_function_usage@ expression _E; identifier _timer; struct timer_list _stl; identifier _callback; type _cast_func, _cast_data; @@ ( -setup_timer(&_E->_timer, _callback, _E); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, &_callback, _E); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, _callback, (_cast_data)_E); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, &_callback, (_cast_data)_E); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, (_cast_func)_callback, _E); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, (_cast_func)&_callback, _E); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, (_cast_func)_callback, (_cast_data)_E); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, (_cast_func)&_callback, (_cast_data)_E); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E._timer, _callback, (_cast_data)_E); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, _callback, (_cast_data)&_E); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, &_callback, (_cast_data)_E); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, &_callback, (_cast_data)&_E); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, (_cast_func)_callback, (_cast_data)_E); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, (_cast_func)_callback, (_cast_data)&_E); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, (_cast_func)&_callback, (_cast_data)_E); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, (_cast_func)&_callback, (_cast_data)&_E); +timer_setup(&_E._timer, _callback, 0); | _E->_timer@_stl.function = _callback; | _E->_timer@_stl.function = &_callback; | _E->_timer@_stl.function = (_cast_func)_callback; | _E->_timer@_stl.function = (_cast_func)&_callback; | _E._timer@_stl.function = _callback; | _E._timer@_stl.function = &_callback; | _E._timer@_stl.function = (_cast_func)_callback; | _E._timer@_stl.function = (_cast_func)&_callback; ) // callback(unsigned long arg) @change_callback_handle_cast depends on change_timer_function_usage@ identifier change_timer_function_usage._callback; identifier change_timer_function_usage._timer; type _origtype; identifier _origarg; type _handletype; identifier _handle; @@ void _callback( -_origtype _origarg +struct timer_list *t ) { ( ... when != _origarg _handletype *_handle = -(_handletype *)_origarg; +from_timer(_handle, t, _timer); ... when != _origarg | ... when != _origarg _handletype *_handle = -(void *)_origarg; +from_timer(_handle, t, _timer); ... when != _origarg | ... when != _origarg _handletype *_handle; ... when != _handle _handle = -(_handletype *)_origarg; +from_timer(_handle, t, _timer); ... when != _origarg | ... when != _origarg _handletype *_handle; ... when != _handle _handle = -(void *)_origarg; +from_timer(_handle, t, _timer); ... when != _origarg ) } // callback(unsigned long arg) without existing variable @change_callback_handle_cast_no_arg depends on change_timer_function_usage && !change_callback_handle_cast@ identifier change_timer_function_usage._callback; identifier change_timer_function_usage._timer; type _origtype; identifier _origarg; type _handletype; @@ void _callback( -_origtype _origarg +struct timer_list *t ) { + _handletype *_origarg = from_timer(_origarg, t, _timer); + ... when != _origarg - (_handletype *)_origarg + _origarg ... when != _origarg } // Avoid already converted callbacks. @match_callback_converted depends on change_timer_function_usage && !change_callback_handle_cast && !change_callback_handle_cast_no_arg@ identifier change_timer_function_usage._callback; identifier t; @@ void _callback(struct timer_list *t) { ... } // callback(struct something *handle) @change_callback_handle_arg depends on change_timer_function_usage && !match_callback_converted && !change_callback_handle_cast && !change_callback_handle_cast_no_arg@ identifier change_timer_function_usage._callback; identifier change_timer_function_usage._timer; type _handletype; identifier _handle; @@ void _callback( -_handletype *_handle +struct timer_list *t ) { + _handletype *_handle = from_timer(_handle, t, _timer); ... } // If change_callback_handle_arg ran on an empty function, remove // the added handler. @unchange_callback_handle_arg depends on change_timer_function_usage && change_callback_handle_arg@ identifier change_timer_function_usage._callback; identifier change_timer_function_usage._timer; type _handletype; identifier _handle; identifier t; @@ void _callback(struct timer_list *t) { - _handletype *_handle = from_timer(_handle, t, _timer); } // We only want to refactor the setup_timer() data argument if we've found // the matching callback. This undoes changes in change_timer_function_usage. @unchange_timer_function_usage depends on change_timer_function_usage && !change_callback_handle_cast && !change_callback_handle_cast_no_arg && !change_callback_handle_arg@ expression change_timer_function_usage._E; identifier change_timer_function_usage._timer; identifier change_timer_function_usage._callback; type change_timer_function_usage._cast_data; @@ ( -timer_setup(&_E->_timer, _callback, 0); +setup_timer(&_E->_timer, _callback, (_cast_data)_E); | -timer_setup(&_E._timer, _callback, 0); +setup_timer(&_E._timer, _callback, (_cast_data)&_E); ) // If we fixed a callback from a .function assignment, fix the // assignment cast now. @change_timer_function_assignment depends on change_timer_function_usage && (change_callback_handle_cast || change_callback_handle_cast_no_arg || change_callback_handle_arg)@ expression change_timer_function_usage._E; identifier change_timer_function_usage._timer; identifier change_timer_function_usage._callback; type _cast_func; typedef TIMER_FUNC_TYPE; @@ ( _E->_timer.function = -_callback +(TIMER_FUNC_TYPE)_callback ; | _E->_timer.function = -&_callback +(TIMER_FUNC_TYPE)_callback ; | _E->_timer.function = -(_cast_func)_callback; +(TIMER_FUNC_TYPE)_callback ; | _E->_timer.function = -(_cast_func)&_callback +(TIMER_FUNC_TYPE)_callback ; | _E._timer.function = -_callback +(TIMER_FUNC_TYPE)_callback ; | _E._timer.function = -&_callback; +(TIMER_FUNC_TYPE)_callback ; | _E._timer.function = -(_cast_func)_callback +(TIMER_FUNC_TYPE)_callback ; | _E._timer.function = -(_cast_func)&_callback +(TIMER_FUNC_TYPE)_callback ; ) // Sometimes timer functions are called directly. Replace matched args. @change_timer_function_calls depends on change_timer_function_usage && (change_callback_handle_cast || change_callback_handle_cast_no_arg || change_callback_handle_arg)@ expression _E; identifier change_timer_function_usage._timer; identifier change_timer_function_usage._callback; type _cast_data; @@ _callback( ( -(_cast_data)_E +&_E->_timer | -(_cast_data)&_E +&_E._timer | -_E +&_E->_timer ) ) // If a timer has been configured without a data argument, it can be // converted without regard to the callback argument, since it is unused. @match_timer_function_unused_data@ expression _E; identifier _timer; identifier _callback; @@ ( -setup_timer(&_E->_timer, _callback, 0); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, _callback, 0L); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E->_timer, _callback, 0UL); +timer_setup(&_E->_timer, _callback, 0); | -setup_timer(&_E._timer, _callback, 0); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, _callback, 0L); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_E._timer, _callback, 0UL); +timer_setup(&_E._timer, _callback, 0); | -setup_timer(&_timer, _callback, 0); +timer_setup(&_timer, _callback, 0); | -setup_timer(&_timer, _callback, 0L); +timer_setup(&_timer, _callback, 0); | -setup_timer(&_timer, _callback, 0UL); +timer_setup(&_timer, _callback, 0); | -setup_timer(_timer, _callback, 0); +timer_setup(_timer, _callback, 0); | -setup_timer(_timer, _callback, 0L); +timer_setup(_timer, _callback, 0); | -setup_timer(_timer, _callback, 0UL); +timer_setup(_timer, _callback, 0); ) @change_callback_unused_data depends on match_timer_function_unused_data@ identifier match_timer_function_unused_data._callback; type _origtype; identifier _origarg; @@ void _callback( -_origtype _origarg +struct timer_list *unused ) { ... when != _origarg } Signed-off-by: Kees Cook --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c | 6 +++--- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 7 +++---- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 7 +++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c index 3559fb5b8fb03..03aae6bc18388 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/btcoex.c @@ -280,9 +280,9 @@ static void brcmf_btcoex_restore_part1(struct brcmf_btcoex_info *btci) /** * brcmf_btcoex_timerfunc() - BT coex timer callback */ -static void brcmf_btcoex_timerfunc(ulong data) +static void brcmf_btcoex_timerfunc(struct timer_list *t) { - struct brcmf_btcoex_info *bt_local = (struct brcmf_btcoex_info *)data; + struct brcmf_btcoex_info *bt_local = from_timer(bt_local, t, timer); brcmf_dbg(TRACE, "enter\n"); bt_local->timer_on = false; @@ -380,7 +380,7 @@ int brcmf_btcoex_attach(struct brcmf_cfg80211_info *cfg) /* Set up timer for BT */ btci->timer_on = false; btci->timeout = BRCMF_BTCOEX_OPPR_WIN_TIME; - setup_timer(&btci->timer, brcmf_btcoex_timerfunc, (ulong)btci); + timer_setup(&btci->timer, brcmf_btcoex_timerfunc, 0); btci->cfg = cfg; btci->saved_regs_part1 = false; btci->saved_regs_part2 = false; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 136eb438d133c..d984fb55fba77 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3094,10 +3094,10 @@ static void brcmf_cfg80211_escan_timeout_worker(struct work_struct *work) brcmf_notify_escan_complete(cfg, cfg->escan_info.ifp, true, true); } -static void brcmf_escan_timeout(unsigned long data) +static void brcmf_escan_timeout(struct timer_list *t) { struct brcmf_cfg80211_info *cfg = - (struct brcmf_cfg80211_info *)data; + from_timer(cfg, t, escan_timeout); if (cfg->int_escan_map || cfg->scan_request) { brcmf_err("timer expired\n"); @@ -3261,8 +3261,7 @@ static void brcmf_init_escan(struct brcmf_cfg80211_info *cfg) brcmf_cfg80211_escan_handler); cfg->escan_info.escan_state = WL_ESCAN_STATE_IDLE; /* Init scan_timeout timer */ - setup_timer(&cfg->escan_timeout, brcmf_escan_timeout, - (unsigned long)cfg); + timer_setup(&cfg->escan_timeout, brcmf_escan_timeout, 0); INIT_WORK(&cfg->escan_timeout_work, brcmf_cfg80211_escan_timeout_worker); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index b2f2a3d4106f4..1ccea3e921d60 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3966,9 +3966,9 @@ brcmf_sdio_watchdog_thread(void *data) } static void -brcmf_sdio_watchdog(unsigned long data) +brcmf_sdio_watchdog(struct timer_list *t) { - struct brcmf_sdio *bus = (struct brcmf_sdio *)data; + struct brcmf_sdio *bus = from_timer(bus, t, timer); if (bus->watchdog_tsk) { complete(&bus->watchdog_wait); @@ -4144,8 +4144,7 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) init_waitqueue_head(&bus->dcmd_resp_wait); /* Set up the watchdog timer */ - setup_timer(&bus->timer, brcmf_sdio_watchdog, - (unsigned long)bus); + timer_setup(&bus->timer, brcmf_sdio_watchdog, 0); /* Initialize watchdog thread */ init_completion(&bus->watchdog_wait); bus->watchdog_tsk = kthread_run(brcmf_sdio_watchdog_thread, From 4810c3af7e096a5618d97f5fd310bf690a01f73a Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Wed, 4 Sep 2019 16:17:27 -0500 Subject: [PATCH 019/213] Revert "compiler.h: Remove ACCESS_ONCE()" This reverts commit b899a850431e2dd0943205a63a68573f3e312d0d. --- include/linux/compiler.h | 47 ++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index bb22908c79e83..7c876739dd937 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -228,21 +228,21 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s /* * Prevent the compiler from merging or refetching reads or writes. The * compiler is also forbidden from reordering successive instances of - * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some - * particular ordering. One way to make the compiler aware of ordering is to - * put the two invocations of READ_ONCE or WRITE_ONCE in different C - * statements. + * READ_ONCE, WRITE_ONCE and ACCESS_ONCE (see below), but only when the + * compiler is aware of some particular ordering. One way to make the + * compiler aware of ordering is to put the two invocations of READ_ONCE, + * WRITE_ONCE or ACCESS_ONCE() in different C statements. * - * These two macros will also work on aggregate data types like structs or - * unions. If the size of the accessed data type exceeds the word size of - * the machine (e.g., 32 bits or 64 bits) READ_ONCE() and WRITE_ONCE() will - * fall back to memcpy(). There's at least two memcpy()s: one for the - * __builtin_memcpy() and then one for the macro doing the copy of variable - * - '__u' allocated on the stack. + * In contrast to ACCESS_ONCE these two macros will also work on aggregate + * data types like structs or unions. If the size of the accessed data + * type exceeds the word size of the machine (e.g., 32 bits or 64 bits) + * READ_ONCE() and WRITE_ONCE() will fall back to memcpy(). There's at + * least two memcpy()s: one for the __builtin_memcpy() and then one for + * the macro doing the copy of variable - '__u' allocated on the stack. * * Their two major use cases are: (1) Mediating communication between * process-level code and irq/NMI handlers, all running on the same CPU, - * and (2) Ensuring that the compiler does not fold, spindle, or otherwise + * and (2) Ensuring that the compiler does not fold, spindle, or otherwise * mutilate accesses that either do not require ordering or that interact * with an explicit memory barrier or atomic instruction that provides the * required ordering. @@ -351,4 +351,29 @@ static inline void *offset_to_ptr(const int *off) compiletime_assert(__native_word(t), \ "Need native word sized stores/loads for atomicity.") +/* + * Prevent the compiler from merging or refetching accesses. The compiler + * is also forbidden from reordering successive instances of ACCESS_ONCE(), + * but only when the compiler is aware of some particular ordering. One way + * to make the compiler aware of ordering is to put the two invocations of + * ACCESS_ONCE() in different C statements. + * + * ACCESS_ONCE will only work on scalar types. For union types, ACCESS_ONCE + * on a union member will work as long as the size of the member matches the + * size of the union and the size is smaller than word size. + * + * The major use cases of ACCESS_ONCE used to be (1) Mediating communication + * between process-level code and irq/NMI handlers, all running on the same CPU, + * and (2) Ensuring that the compiler does not fold, spindle, or otherwise + * mutilate accesses that either do not require ordering or that interact + * with an explicit memory barrier or atomic instruction that provides the + * required ordering. + * + * If possible use READ_ONCE()/WRITE_ONCE() instead. + */ +#define __ACCESS_ONCE(x) ({ \ + __maybe_unused typeof(x) __var = (__force typeof(x)) 0; \ + (volatile typeof(x) *)&(x); }) +#define ACCESS_ONCE(x) (*__ACCESS_ONCE(x)) + #endif /* __LINUX_COMPILER_H */ From f8d81b0c94048591eb865daa261af3249710d753 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Fri, 10 Nov 2017 17:27:15 +0800 Subject: [PATCH 020/213] brcmfmac: add CLM download support The firmware for brcmfmac devices includes information regarding regulatory constraints. For certain devices this information is kept separately in a binary form that needs to be downloaded to the device. This patch adds support to download this so-called CLM blob file. It uses the same naming scheme as the other firmware files with extension of .clm_blob. The CLM blob file is optional. If the file does not exist, the download process will be bypassed. It will not affect the driver loading. Reviewed-by: Arend van Spriel Signed-off-by: Chung-Hsien Hsu Signed-off-by: Kalle Valo --- .../broadcom/brcm80211/brcmfmac/bus.h | 10 ++ .../broadcom/brcm80211/brcmfmac/common.c | 157 ++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/core.c | 2 + .../broadcom/brcm80211/brcmfmac/core.h | 2 + .../broadcom/brcm80211/brcmfmac/fwil_types.h | 31 ++++ .../broadcom/brcm80211/brcmfmac/pcie.c | 19 +++ .../broadcom/brcm80211/brcmfmac/sdio.c | 19 +++ .../broadcom/brcm80211/brcmfmac/usb.c | 18 ++ 8 files changed, 258 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 163ddc49f9511..0b76a615708e1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -71,6 +71,7 @@ struct brcmf_bus_dcmd { * @wowl_config: specify if dongle is configured for wowl when going to suspend * @get_ramsize: obtain size of device memory. * @get_memdump: obtain device memory dump in provided buffer. + * @get_fwname: obtain firmware name. * * This structure provides an abstract interface towards the * bus specific driver. For control messages to common driver @@ -87,6 +88,8 @@ struct brcmf_bus_ops { void (*wowl_config)(struct device *dev, bool enabled); size_t (*get_ramsize)(struct device *dev); int (*get_memdump)(struct device *dev, void *data, size_t len); + int (*get_fwname)(struct device *dev, uint chip, uint chiprev, + unsigned char *fw_name); }; @@ -224,6 +227,13 @@ int brcmf_bus_get_memdump(struct brcmf_bus *bus, void *data, size_t len) return bus->ops->get_memdump(bus->dev, data, len); } +static inline +int brcmf_bus_get_fwname(struct brcmf_bus *bus, uint chip, uint chiprev, + unsigned char *fw_name) +{ + return bus->ops->get_fwname(bus->dev, chip, chiprev, fw_name); +} + /* * interface functions from common layer */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 7a2b49587b4d3..6a59d0609d300 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include "core.h" @@ -28,6 +29,7 @@ #include "tracepoint.h" #include "common.h" #include "of.h" +#include "firmware.h" MODULE_AUTHOR("Broadcom Corporation"); MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver."); @@ -104,12 +106,140 @@ void brcmf_c_set_joinpref_default(struct brcmf_if *ifp) brcmf_err("Set join_pref error (%d)\n", err); } +static int brcmf_c_download(struct brcmf_if *ifp, u16 flag, + struct brcmf_dload_data_le *dload_buf, + u32 len) +{ + s32 err; + + flag |= (DLOAD_HANDLER_VER << DLOAD_FLAG_VER_SHIFT); + dload_buf->flag = cpu_to_le16(flag); + dload_buf->dload_type = cpu_to_le16(DL_TYPE_CLM); + dload_buf->len = cpu_to_le32(len); + dload_buf->crc = cpu_to_le32(0); + len = sizeof(*dload_buf) + len - 1; + + err = brcmf_fil_iovar_data_set(ifp, "clmload", dload_buf, len); + + return err; +} + +static int brcmf_c_get_clm_name(struct brcmf_if *ifp, u8 *clm_name) +{ + struct brcmf_bus *bus = ifp->drvr->bus_if; + struct brcmf_rev_info *ri = &ifp->drvr->revinfo; + u8 fw_name[BRCMF_FW_NAME_LEN]; + u8 *ptr; + size_t len; + s32 err; + + memset(fw_name, 0, BRCMF_FW_NAME_LEN); + err = brcmf_bus_get_fwname(bus, ri->chipnum, ri->chiprev, fw_name); + if (err) { + brcmf_err("get firmware name failed (%d)\n", err); + goto done; + } + + /* generate CLM blob file name */ + ptr = strrchr(fw_name, '.'); + if (!ptr) { + err = -ENOENT; + goto done; + } + + len = ptr - fw_name + 1; + if (len + strlen(".clm_blob") > BRCMF_FW_NAME_LEN) { + err = -E2BIG; + } else { + strlcpy(clm_name, fw_name, len); + strlcat(clm_name, ".clm_blob", BRCMF_FW_NAME_LEN); + } +done: + return err; +} + +static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) +{ + struct device *dev = ifp->drvr->bus_if->dev; + struct brcmf_dload_data_le *chunk_buf; + const struct firmware *clm = NULL; + u8 clm_name[BRCMF_FW_NAME_LEN]; + u32 chunk_len; + u32 datalen; + u32 cumulative_len; + u16 dl_flag = DL_BEGIN; + u32 status; + s32 err; + + brcmf_dbg(TRACE, "Enter\n"); + + memset(clm_name, 0, BRCMF_FW_NAME_LEN); + err = brcmf_c_get_clm_name(ifp, clm_name); + if (err) { + brcmf_err("get CLM blob file name failed (%d)\n", err); + return err; + } + + err = request_firmware(&clm, clm_name, dev); + if (err) { + if (err == -ENOENT) { + brcmf_dbg(INFO, "continue with CLM data currently present in firmware\n"); + return 0; + } + brcmf_err("request CLM blob file failed (%d)\n", err); + return err; + } + + chunk_buf = kzalloc(sizeof(*chunk_buf) + MAX_CHUNK_LEN - 1, GFP_KERNEL); + if (!chunk_buf) { + err = -ENOMEM; + goto done; + } + + datalen = clm->size; + cumulative_len = 0; + do { + if (datalen > MAX_CHUNK_LEN) { + chunk_len = MAX_CHUNK_LEN; + } else { + chunk_len = datalen; + dl_flag |= DL_END; + } + memcpy(chunk_buf->data, clm->data + cumulative_len, chunk_len); + + err = brcmf_c_download(ifp, dl_flag, chunk_buf, chunk_len); + + dl_flag &= ~DL_BEGIN; + + cumulative_len += chunk_len; + datalen -= chunk_len; + } while ((datalen > 0) && (err == 0)); + + if (err) { + brcmf_err("clmload (%zu byte file) failed (%d); ", + clm->size, err); + /* Retrieve clmload_status and print */ + err = brcmf_fil_iovar_int_get(ifp, "clmload_status", &status); + if (err) + brcmf_err("get clmload_status failed (%d)\n", err); + else + brcmf_dbg(INFO, "clmload_status=%d\n", status); + err = -EIO; + } + + kfree(chunk_buf); +done: + release_firmware(clm); + return err; +} + int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) { s8 eventmask[BRCMF_EVENTING_MASK_LEN]; u8 buf[BRCMF_DCMD_SMLEN]; struct brcmf_rev_info_le revinfo; struct brcmf_rev_info *ri; + char *clmver; char *ptr; s32 err; @@ -148,6 +278,13 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) } ri->result = err; + /* Do any CLM downloading */ + err = brcmf_c_process_clm_blob(ifp); + if (err < 0) { + brcmf_err("download CLM blob file failed, %d\n", err); + goto done; + } + /* query for 'ver' to get version info from firmware */ memset(buf, 0, sizeof(buf)); strcpy(buf, "ver"); @@ -167,6 +304,26 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) ptr = strrchr(buf, ' ') + 1; strlcpy(ifp->drvr->fwver, ptr, sizeof(ifp->drvr->fwver)); + /* Query for 'clmver' to get CLM version info from firmware */ + memset(buf, 0, sizeof(buf)); + err = brcmf_fil_iovar_data_get(ifp, "clmver", buf, sizeof(buf)); + if (err) { + brcmf_dbg(TRACE, "retrieving clmver failed, %d\n", err); + } else { + clmver = (char *)buf; + /* store CLM version for adding it to revinfo debugfs file */ + memcpy(ifp->drvr->clmver, clmver, sizeof(ifp->drvr->clmver)); + + /* Replace all newline/linefeed characters with space + * character + */ + ptr = clmver; + while ((ptr = strnchr(ptr, '\n', sizeof(buf))) != NULL) + *ptr = ' '; + + brcmf_dbg(INFO, "CLM version = %s\n", clmver); + } + /* set mpc */ err = brcmf_fil_iovar_int_set(ifp, "mpc", 1); if (err) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 5cc3a07dda9e6..e6b87a73599b1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -950,6 +950,8 @@ static int brcmf_revinfo_read(struct seq_file *s, void *data) seq_printf(s, "anarev: %u\n", ri->anarev); seq_printf(s, "nvramrev: %08x\n", ri->nvramrev); + seq_printf(s, "clmver: %s\n", bus_if->drvr->clmver); + return 0; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index a4dd313140f37..ae39128406cb3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -141,6 +141,8 @@ struct brcmf_pub { struct notifier_block inetaddr_notifier; struct notifier_block inet6addr_notifier; struct brcmf_mp_device *settings; + + u8 clmver[BRCMF_DCMD_SMLEN]; }; /* forward declarations */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index e0d22fedb2b45..4b290705e3e64 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -155,6 +155,21 @@ #define BRCMF_MFP_CAPABLE 1 #define BRCMF_MFP_REQUIRED 2 +/* MAX_CHUNK_LEN is the maximum length for data passing to firmware in each + * ioctl. It is relatively small because firmware has small maximum size input + * playload restriction for ioctls. + */ +#define MAX_CHUNK_LEN 1400 + +#define DLOAD_HANDLER_VER 1 /* Downloader version */ +#define DLOAD_FLAG_VER_MASK 0xf000 /* Downloader version mask */ +#define DLOAD_FLAG_VER_SHIFT 12 /* Downloader version shift */ + +#define DL_BEGIN 0x0002 +#define DL_END 0x0004 + +#define DL_TYPE_CLM 2 + /* join preference types for join_pref iovar */ enum brcmf_join_pref_types { BRCMF_JOIN_PREF_RSSI = 1, @@ -826,6 +841,22 @@ struct brcmf_pno_macaddr_le { u8 mac[ETH_ALEN]; }; +/** + * struct brcmf_dload_data_le - data passing to firmware for downloading + * @flag: flags related to download data. + * @dload_type: type of download data. + * @len: length in bytes of download data. + * @crc: crc of download data. + * @data: download data. + */ +struct brcmf_dload_data_le { + __le16 flag; + __le16 dload_type; + __le32 len; + __le32 crc; + u8 data[1]; +}; + /** * struct brcmf_pno_bssid_le - bssid configuration for PNO scan. * diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index e6e9b00b79d71..3c87157f5b851 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -1350,6 +1350,24 @@ static int brcmf_pcie_get_memdump(struct device *dev, void *data, size_t len) return 0; } +static int brcmf_pcie_get_fwname(struct device *dev, u32 chip, u32 chiprev, + u8 *fw_name) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + struct brcmf_pciedev_info *devinfo = buspub->devinfo; + int ret = 0; + + if (devinfo->fw_name[0] != '\0') + strlcpy(fw_name, devinfo->fw_name, BRCMF_FW_NAME_LEN); + else + ret = brcmf_fw_map_chip_to_name(chip, chiprev, + brcmf_pcie_fwnames, + ARRAY_SIZE(brcmf_pcie_fwnames), + fw_name, NULL); + + return ret; +} static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .txdata = brcmf_pcie_tx, @@ -1359,6 +1377,7 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .wowl_config = brcmf_pcie_wowl_config, .get_ramsize = brcmf_pcie_get_ramsize, .get_memdump = brcmf_pcie_get_memdump, + .get_fwname = brcmf_pcie_get_fwname, }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 1ccea3e921d60..8e9353f3fce9e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3979,6 +3979,24 @@ brcmf_sdio_watchdog(struct timer_list *t) } } +static int brcmf_sdio_get_fwname(struct device *dev, u32 chip, u32 chiprev, + u8 *fw_name) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + int ret = 0; + + if (sdiodev->fw_name[0] != '\0') + strlcpy(fw_name, sdiodev->fw_name, BRCMF_FW_NAME_LEN); + else + ret = brcmf_fw_map_chip_to_name(chip, chiprev, + brcmf_sdio_fwnames, + ARRAY_SIZE(brcmf_sdio_fwnames), + fw_name, NULL); + + return ret; +} + static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .stop = brcmf_sdio_bus_stop, .preinit = brcmf_sdio_bus_preinit, @@ -3989,6 +4007,7 @@ static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .wowl_config = brcmf_sdio_wowl_config, .get_ramsize = brcmf_sdio_bus_get_ramsize, .get_memdump = brcmf_sdio_bus_get_memdump, + .get_fwname = brcmf_sdio_get_fwname, }; static void brcmf_sdio_firmware_callback(struct device *dev, int err, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 11ffaa01599eb..b27170c12482d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -1128,12 +1128,30 @@ static void brcmf_usb_wowl_config(struct device *dev, bool enabled) device_set_wakeup_enable(devinfo->dev, false); } +static int brcmf_usb_get_fwname(struct device *dev, u32 chip, u32 chiprev, + u8 *fw_name) +{ + struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); + int ret = 0; + + if (devinfo->fw_name[0] != '\0') + strlcpy(fw_name, devinfo->fw_name, BRCMF_FW_NAME_LEN); + else + ret = brcmf_fw_map_chip_to_name(chip, chiprev, + brcmf_usb_fwnames, + ARRAY_SIZE(brcmf_usb_fwnames), + fw_name, NULL); + + return ret; +} + static const struct brcmf_bus_ops brcmf_usb_bus_ops = { .txdata = brcmf_usb_tx, .stop = brcmf_usb_down, .txctl = brcmf_usb_tx_ctlpkt, .rxctl = brcmf_usb_rx_ctlpkt, .wowl_config = brcmf_usb_wowl_config, + .get_fwname = brcmf_usb_get_fwname, }; static int brcmf_usb_bus_setup(struct brcmf_usbdev_info *devinfo) From a6a87799064e3f7ad314a91405713406bb6774ed Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 20 Jun 2017 05:11:24 -0500 Subject: [PATCH 021/213] brcmfmac: Set F2 blksz and Watermark to 256 for 4373 We got SDIO_CRC_ERROR with 4373 on SDR104 when doing bi-directional throughput test. Setting F2 blocksize and enable watermark to 256 to guarantee the operation stability. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 11 +++++++++- .../broadcom/brcm80211/brcmfmac/sdio.c | 21 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index dd6e27513cc1b..f4fa525afa6e6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -54,6 +54,7 @@ #define SDIO_FUNC1_BLOCKSIZE 64 #define SDIO_FUNC2_BLOCKSIZE 512 +#define SDIO_4373_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -1038,6 +1039,7 @@ static void brcmf_sdiod_host_fixup(struct mmc_host *host) static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) { int ret = 0; + unsigned int f2_blksz = SDIO_FUNC2_BLOCKSIZE; sdiodev->num_funcs = 2; @@ -1049,11 +1051,18 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) sdio_release_host(sdiodev->func[1]); goto out; } - ret = sdio_set_block_size(sdiodev->func[2], SDIO_FUNC2_BLOCKSIZE); + + if (sdiodev->func[0]->device == SDIO_DEVICE_ID_CYPRESS_4373) { + f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE; + } + + ret = sdio_set_block_size(sdiodev->func[2], f2_blksz); if (ret) { brcmf_err("Failed to set F2 blocksize\n"); sdio_release_host(sdiodev->func[1]); goto out; + } else { + brcmf_dbg(SDIO, "set F2 blocksize to %d\n", f2_blksz); } /* increase F2 timeout */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 8e9353f3fce9e..9a7e131db1ce1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -49,6 +49,9 @@ #define DCMD_RESP_TIMEOUT msecs_to_jiffies(2500) #define CTL_DONE_TIMEOUT msecs_to_jiffies(2500) +#define DEFAULT_F2_WATERMARK 0x8 +#define CY_4373_F2_WATERMARK 0x40 + #ifdef DEBUG #define BRCMF_TRAP_INFO_SIZE 80 @@ -138,6 +141,8 @@ struct rte_console { /* 1: isolate internal sdio signals, put external pads in tri-state; requires * sdio bus power cycle to clear (rev 9) */ #define SBSDIO_DEVCTL_PADS_ISO 0x08 +/* 1: enable F2 Watermark */ +#define SBSDIO_DEVCTL_F2WM_ENAB 0x10 /* Force SD->SB reset mapping (rev 11) */ #define SBSDIO_DEVCTL_SB_RST_CTL 0x30 /* Determined by CoreControl bit */ @@ -4018,6 +4023,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, struct brcmf_sdio_dev *sdiodev; struct brcmf_sdio *bus; u8 saveclk; + u8 devctl; brcmf_dbg(TRACE, "Enter: dev=%s, err=%d\n", dev_name(dev), err); bus_if = dev_get_drvdata(dev); @@ -4073,8 +4079,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, bus->hostintmask = HOSTINTMASK; w_sdreg32(bus, bus->hostintmask, offsetof(struct sdpcmd_regs, hostintmask)); - - brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, 8, &err); + switch (sdiodev->func[0]->device) { + case SDIO_DEVICE_ID_CYPRESS_4373: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 4373\n", + CY_4373_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, CY_4373_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, &err); + break; + default: + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); + break; + } } else { /* Disable F2 again */ sdio_disable_func(sdiodev->func[SDIO_FUNC_2]); From 70f80f71d91213808867119a04a62a0be3733d38 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Tue, 15 Aug 2017 00:25:14 -0500 Subject: [PATCH 022/213] brcmfmac: Add sg parameters dts parsing broken_sg_support, sd_head_align, and sd_sgentry_align are used in brcmfmac code but not configurable in dts file. Add the parsing logic. Now they can be configured like below in dts: brcm,broken_sg_support; brcm,sd_head_align = /bits/ 16 <4>; brcm,sd_sgentry_align = /bits/ 16 <4>; Signed-off-by: Chi-hsien Lin --- .../net/wireless/broadcom/brcm80211/brcmfmac/of.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c index aee6e5937c41c..fb19ab792f818 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c @@ -30,14 +30,22 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type, struct device_node *np = dev->of_node; int irq; u32 irqf; - u32 val; + u32 val32; + u16 val16; if (!np || bus_type != BRCMF_BUSTYPE_SDIO || !of_device_is_compatible(np, "brcm,bcm4329-fmac")) return; - if (of_property_read_u32(np, "brcm,drive-strength", &val) == 0) - sdio->drive_strength = val; + if (of_property_read_u32(np, "brcm,drive-strength", &val32) == 0) + sdio->drive_strength = val32; + + sdio->broken_sg_support = of_property_read_bool(np, + "brcm,broken_sg_support"); + if (of_property_read_u16(np, "brcm,sd_head_align", &val16) == 0) + sdio->sd_head_align = val16; + if (of_property_read_u16(np, "brcm,sd_sgentry_align", &val16) == 0) + sdio->sd_sgentry_align = val16; /* make sure there are interrupts defined in the node */ if (!of_find_property(np, "interrupts", NULL)) From 14dd6ebe8043b13e0d1dfb4fac337534f33eeb74 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Mon, 21 Aug 2017 22:36:04 -0500 Subject: [PATCH 023/213] brcmfmac: return -EPERM when getting error in vendor command handler Firmware returns proprietary error code when getting error in fil_cmd_data_set or fil_cmd_data_get. Sometimes the vendor tool or utilities which uses libnl may stuck in some commands when wl is down. For example, issue "scan" command after issuing "down" command, the "scan" command will be the blocking call and stuck as no response from firmware. It is caused by that firmware returns BCME_NOTUP(-4) when wl is down, but in Linux the -4 is -EINTR, so libnl catches the error and not pass to upper layer. Because of that, the driver should return Linux error code instead of the proprietary error code, and the tools or utilities need to get the real firmware error code by another command "bcmerrorstr" after receiving the error. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index 8eff2753abade..2b88ba132bc8f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -80,8 +80,12 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, else ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf, ret_len); - if (ret != 0) + + if (ret != 0) { + brcmf_dbg(INFO, "error(%d), return -EPERM\n", ret); + ret = -EPERM; goto exit; + } wr_pointer = dcmd_buf; while (ret_len > 0) { From fd4be58b3f156ab0f60b5e46413812be86365247 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Tue, 20 Jun 2017 02:20:23 -0500 Subject: [PATCH 024/213] brcmfmac: Add support for CYW43012 SDIO chipset CYW43012 is a 1x1 802.11a/b/g/n Dual-Band HT20, 256-QAM/Turbo QAM. It is an Ultra Low Power WLAN+BT combo chip. Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 1 + .../broadcom/brcm80211/brcmfmac/chip.c | 9 ++++- .../broadcom/brcm80211/brcmfmac/sdio.c | 40 ++++++++++++++++--- .../broadcom/brcm80211/include/brcm_hw_ids.h | 1 + include/linux/mmc/sdio_ids.h | 1 + 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index f4fa525afa6e6..88b7410851c6f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -1116,6 +1116,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = { BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4354), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4356), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_4373), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_43012), { /* end: all zeroes */ } }; MODULE_DEVICE_TABLE(sdio, brcmf_sdmmc_ids); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index c5d1a1cbf6019..4b1c8bab5d7d4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -165,6 +165,7 @@ struct sbconfig { #define SRCI_LSS_MASK 0x00f00000 #define SRCI_LSS_SHIFT 20 #define SRCI_SRNB_MASK 0xf0 +#define SRCI_SRNB_MASK_EXT 0x100 #define SRCI_SRNB_SHIFT 4 #define SRCI_SRBSZ_MASK 0xf #define SRCI_SRBSZ_SHIFT 0 @@ -592,7 +593,13 @@ static void brcmf_chip_socram_ramsize(struct brcmf_core_priv *sr, u32 *ramsize, if (lss != 0) *ramsize += (1 << ((lss - 1) + SR_BSZ_BASE)); } else { - nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT; + /* length of SRAM Banks increased for corerev greater than 23 */ + if (sr->pub.rev >= 23) { + nb = (coreinfo & (SRCI_SRNB_MASK | SRCI_SRNB_MASK_EXT)) + >> SRCI_SRNB_SHIFT; + } else { + nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT; + } for (i = 0; i < nb; i++) { retent = brcmf_chip_socram_banksize(sr, i, &banksize); *ramsize += banksize; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 9a7e131db1ce1..8d96f2287f89a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -624,6 +624,7 @@ BRCMF_FW_NVRAM_DEF(43455, "brcmfmac43455-sdio.bin", "brcmfmac43455-sdio.txt"); BRCMF_FW_NVRAM_DEF(4354, "brcmfmac4354-sdio.bin", "brcmfmac4354-sdio.txt"); BRCMF_FW_NVRAM_DEF(4356, "brcmfmac4356-sdio.bin", "brcmfmac4356-sdio.txt"); BRCMF_FW_NVRAM_DEF(4373, "brcmfmac4373-sdio.bin", "brcmfmac4373-sdio.txt"); +BRCMF_FW_NVRAM_DEF(43012, "brcmfmac43012-sdio.bin", "brcmfmac43012-sdio.txt"); static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43143_CHIP_ID, 0xFFFFFFFF, 43143), @@ -643,7 +644,8 @@ static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFFC0, 43455), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356), - BRCMF_FW_NVRAM_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373) + BRCMF_FW_NVRAM_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373), + BRCMF_FW_NVRAM_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012) }; static void pkt_align(struct sk_buff *p, int len, int align) @@ -704,6 +706,15 @@ brcmf_sdio_kso_control(struct brcmf_sdio *bus, bool on) brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_SLEEPCSR, wr_val, &err); + /* In case of 43012 chip, the chip could go down immediately after + * KSO bit is cleared. So the further reads of KSO register could + * fail. Thereby just bailing out immediately after clearing KSO + * bit, to avoid polling of KSO bit. + */ + if (!on && (bus->ci->chip == CY_CC_43012_CHIP_ID)) { + return err; + } + if (on) { /* device WAKEUP through KSO: * write bit 0 & read back until @@ -2447,9 +2458,20 @@ static void brcmf_sdio_bus_stop(struct device *dev) /* Force backplane clocks to assure F2 interrupt propagates */ saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); - if (!err) - brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - (saveclk | SBSDIO_FORCE_HT), &err); + if (!err) { + if (bus->ci->chip == CY_CC_43012_CHIP_ID) { + brcmf_sdiod_regwb(sdiodev, + SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | + SBSDIO_HT_AVAIL_REQ), + &err); + } else { + brcmf_sdiod_regwb(sdiodev, + SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | SBSDIO_FORCE_HT), + &err); + } + } if (err) brcmf_err("Failed to force clock for F2: err %d\n", err); @@ -4057,8 +4079,14 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, /* Force clocks on backplane to be sure F2 interrupt propagates */ saveclk = brcmf_sdiod_regrb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, &err); if (!err) { - brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - (saveclk | SBSDIO_FORCE_HT), &err); + if (bus->ci->chip == CY_CC_43012_CHIP_ID) { + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | SBSDIO_HT_AVAIL_REQ), + &err); + } else { + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, + (saveclk | SBSDIO_FORCE_HT), &err); + } } if (err) { brcmf_err("Failed to force clock for F2: err %d\n", err); diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 57544a3a3ce4a..37b0acdb111db 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -59,6 +59,7 @@ #define BRCM_CC_4366_CHIP_ID 0x4366 #define BRCM_CC_4371_CHIP_ID 0x4371 #define CY_CC_4373_CHIP_ID 0x4373 +#define CY_CC_43012_CHIP_ID 43012 /* USB Device IDs */ #define BRCM_USB_43143_DEVICE_ID 0xbd1e diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index 4224902a8e22a..4332199c71c25 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -42,6 +42,7 @@ #define SDIO_DEVICE_ID_BROADCOM_4354 0x4354 #define SDIO_DEVICE_ID_BROADCOM_4356 0x4356 #define SDIO_DEVICE_ID_CYPRESS_4373 0x4373 +#define SDIO_DEVICE_ID_CYPRESS_43012 43012 #define SDIO_VENDOR_ID_INTEL 0x0089 #define SDIO_DEVICE_ID_INTEL_IWMC3200WIMAX 0x1402 From bffd7fd585d01286a0c65d670196228f922d47fa Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Mon, 11 Sep 2017 02:48:57 -0500 Subject: [PATCH 025/213] brcmfmac: set apsta to 0 when AP starts on primary interface APSTA can work on two band concurrently with using VSDB(Virtual Simultaneous Dual-Band) or RSDB(Real Simultaneous Dual-Band) features. In this case, we have to keep apsta is 1 in firmware side. If we start wpa_supplicant on wlan0 and then start hostapd on wlan1, the apsta will be set to 0, and data will be stall on wlan0(station). Because that, we only set apsta to 0 when AP start on primary interface. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d984fb55fba77..fd416cc04c260 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4618,9 +4618,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, } } - if ((dev_role == NL80211_IFTYPE_AP) && - ((ifp->ifidx == 0) || - !brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB))) { + if ((dev_role == NL80211_IFTYPE_AP) && (ifp->ifidx == 0)) { err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); if (err < 0) { brcmf_err("BRCMF_C_DOWN error %d\n", err); From e7cc9c854d83016df0d48330f481b1e57f4dfd9c Mon Sep 17 00:00:00 2001 From: pucn Date: Thu, 14 Sep 2017 18:23:52 +0530 Subject: [PATCH 026/213] brcmfmac: Saverestore support changes for 43012 Saverestore register settings for 43012. Signed-off-by: Praveen Babu Chandran --- .../broadcom/brcm80211/brcmfmac/chip.c | 5 ++++ .../broadcom/brcm80211/brcmfmac/sdio.c | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 4b1c8bab5d7d4..ad54409395524 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -1361,6 +1361,11 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) addr = CORE_CC_REG(base, sr_control1); reg = chip->ops->read32(chip->ctx, addr); return reg != 0; + case CY_CC_43012_CHIP_ID: + addr = CORE_CC_REG(pmu->base, retention_ctl); + reg = chip->ops->read32(chip->ctx, addr); + return (reg & (PMU_RCTL_MACPHY_DISABLE_MASK | + PMU_RCTL_LOGIC_DISABLE_MASK)) == 0; default: addr = CORE_CC_REG(pmu->base, pmucapabilities_ext); reg = chip->ops->read32(chip->ctx, addr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 8d96f2287f89a..e83e0525aecef 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3365,34 +3365,43 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus) { int err = 0; u8 val; + u8 wakeupctrl; + u8 cardcap; + u8 chipclkcsr; brcmf_dbg(TRACE, "Enter\n"); + if (bus->ci->chip == CY_CC_43012_CHIP_ID) { + wakeupctrl = SBSDIO_FUNC1_WCTRL_ALPWAIT_SHIFT; + cardcap = SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC; + chipclkcsr = SBSDIO_HT_AVAIL_REQ; + } else { + wakeupctrl = SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT; + cardcap = (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | + SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT); + chipclkcsr = SBSDIO_FORCE_HT; + } + val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err); if (err) { brcmf_err("error reading SBSDIO_FUNC1_WAKEUPCTRL\n"); return; } - - val |= 1 << SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT; + val |= 1 << wakeupctrl; brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, val, &err); if (err) { brcmf_err("error writing SBSDIO_FUNC1_WAKEUPCTRL\n"); return; } - - /* Add CMD14 Support */ brcmf_sdiod_regwb(bus->sdiodev, SDIO_CCCR_BRCM_CARDCAP, - (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | - SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT), + cardcap, &err); if (err) { brcmf_err("error writing SDIO_CCCR_BRCM_CARDCAP\n"); return; } - brcmf_sdiod_regwb(bus->sdiodev, SBSDIO_FUNC1_CHIPCLKCSR, - SBSDIO_FORCE_HT, &err); + chipclkcsr, &err); if (err) { brcmf_err("error writing SBSDIO_FUNC1_CHIPCLKCSR\n"); return; @@ -4109,8 +4118,9 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, offsetof(struct sdpcmd_regs, hostintmask)); switch (sdiodev->func[0]->device) { case SDIO_DEVICE_ID_CYPRESS_4373: - brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 4373\n", - CY_4373_F2_WATERMARK); + case SDIO_DEVICE_ID_CYPRESS_43012: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", + CY_4373_F2_WATERMARK); brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, CY_4373_F2_WATERMARK, &err); devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, &err); devctl |= SBSDIO_DEVCTL_F2WM_ENAB; From 5c2852b7192002335b2f78f7e2572fc7221dd1d9 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Thu, 16 Nov 2017 00:21:10 -0600 Subject: [PATCH 027/213] brcmfmac: Support 43455 save-restore (SR) feature if FW include -sr This patch will add 43455 into the save-restore(SR) capable chip list, so the SR engine will be enabled with 43455 FW which built-in the -sr function. Signed-off-by: Lo-Hsiang Lo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index ad54409395524..01c6a4923aa1d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -1345,6 +1345,7 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) switch (pub->chip) { case BRCM_CC_4354_CHIP_ID: case BRCM_CC_4356_CHIP_ID: + case BRCM_CC_4345_CHIP_ID: /* explicitly check SR engine enable bit */ pmu_cc3_mask = BIT(2); /* fall-through */ From 201e62f9d6889c6d3e5e834e68b8b2b94d553956 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 22 Nov 2017 20:27:47 -0600 Subject: [PATCH 028/213] brcmfmac: fix CLM load error for legacy chips when user helper is enabled For legacy chips w/o CLM blob files, kernel with user helper function enabled returns -EAGAIN when we request_firmware() for blob file: "request_firmware() -> _request_firmware() -> fw_load_from_user_helper() -> _request_firmware_load() -> retval=-EAGAIN" We should do retries and also continue brcmf_c_process_clm_blob if getting -EAGAIN from request_firmware function. Signed-off-by: Wright Feng --- .../wireless/broadcom/brcm80211/brcmfmac/common.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 6a59d0609d300..56e2654d39604 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -45,6 +45,8 @@ const u8 ALLFFMAC[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; #define BRCMF_DEFAULT_TXGLOM_SIZE 32 /* max tx frames in glom chain */ +#define CLM_LOAD_RETRIES 1 /* # of retries to load clm_blob file */ + static int brcmf_sdiod_txglomsz = BRCMF_DEFAULT_TXGLOM_SIZE; module_param_named(txglomsz, brcmf_sdiod_txglomsz, int, 0); MODULE_PARM_DESC(txglomsz, "Maximum tx packet chain size [SDIO]"); @@ -170,6 +172,7 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) u16 dl_flag = DL_BEGIN; u32 status; s32 err; + uint retries = 0; brcmf_dbg(TRACE, "Enter\n"); @@ -180,11 +183,18 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) return err; } - err = request_firmware(&clm, clm_name, dev); + do { + err = request_firmware(&clm, clm_name, dev); + } while (err == -EAGAIN && retries++ < CLM_LOAD_RETRIES); if (err) { if (err == -ENOENT) { brcmf_dbg(INFO, "continue with CLM data currently present in firmware\n"); return 0; + } else if (err == -EAGAIN) { + brcmf_dbg(INFO, "reached maximum retries(%d)\n", + CLM_LOAD_RETRIES); + brcmf_dbg(INFO, "continue with CLM data in firmware\n"); + return 0; } brcmf_err("request CLM blob file failed (%d)\n", err); return err; From ea52cea6479d0419635ed12657e721c92766c609 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 8 Dec 2017 03:05:19 -0600 Subject: [PATCH 029/213] brcmfmac: enlarge buffer size of caps to 512 bytes The buffer size of return of cap iovar is greater than 256 bytes in some firmares. For instance, the return size of cap iovar is 271 bytes in 4373 13.10.246.79 firmare. It makes caps buffer is default value and cause the feature capability parsing failed. Because of that, we enlarge caps buffer size to 512 bytes and add the error print for cap iovar error. Signed-off-by: Wright Feng --- .../wireless/broadcom/brcm80211/brcmfmac/feature.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 53ae302599899..c7c3563f9a53c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -132,11 +132,16 @@ static void brcmf_feat_iovar_data_set(struct brcmf_if *ifp, static void brcmf_feat_firmware_capabilities(struct brcmf_if *ifp) { - char caps[256]; + char caps[512]; enum brcmf_feat_id id; - int i; + int i, err; + + err = brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps)); + if (err) { + brcmf_err("could not get firmware cap (%d)\n", err); + return; + } - brcmf_fil_iovar_data_get(ifp, "cap", caps, sizeof(caps)); brcmf_dbg(INFO, "[ %s]\n", caps); for (i = 0; i < ARRAY_SIZE(brcmf_fwcap_map); i++) { From 75b77ca1b7bcee80bd459ebe469946d4df35dfe8 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Mon, 1 Jan 2018 20:12:54 -0600 Subject: [PATCH 030/213] brcmfmac: calling skb_orphan before sending skb to SDIO bus Linux 3.6 introduces TSQ which has a per socket threshold for TCP Tx packet to reduce latency. In fcmode 1/2, host driver enqueues skb in hanger and TCP doesn't push new skb frees until host frees the skb when receiving fwstatus event. So using skb_orphan before sending skb to bus will make the skb removing the ownership of socket. With this patch, we got better throughput in fcmode 1/2. Tested 43455 TCP throughput in 20 MHz bandwidth with/without this patch. fcmode 0: 59.5 / 59.6 (Mbps) fcmode 1: 59.3 / 23.4 (Mbps) fcmode 2: 59.6 / 21.5 (Mbps) Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index e83e0525aecef..a27af29600237 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -2329,6 +2329,7 @@ static uint brcmf_sdio_sendfromq(struct brcmf_sdio *bus, uint maxframes) &prec_out); if (pkt == NULL) break; + skb_orphan(pkt); __skb_queue_tail(&pktq, pkt); } spin_unlock_bh(&bus->txq_lock); From 86f583df009dfe58f089c94b1c0969061a5adde2 Mon Sep 17 00:00:00 2001 From: Naveen Date: Thu, 4 Jan 2018 11:34:15 +0530 Subject: [PATCH 031/213] brcmfmac: 43012 Update F2 Watermark to 0x60 to fix DMA Error during UDP RX Traffic. The number of words that the read FIFO has to contain except the end of frame before sends data back to the host. Max watermark = (512B - 2* (BurstLength))/4 = (512 - 128)/4 = 384/4 = 0x60 so if burst length (i.e. BurstLength = 64) is increased, watermark has to be reduced. This is the optimal setting for this chip. Signed-off-by: Naveen Gupta --- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index a27af29600237..88c25aaf544a0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -51,6 +51,7 @@ #define DEFAULT_F2_WATERMARK 0x8 #define CY_4373_F2_WATERMARK 0x40 +#define CY_43012_F2_WATERMARK 0x60 #ifdef DEBUG @@ -4119,7 +4120,6 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, offsetof(struct sdpcmd_regs, hostintmask)); switch (sdiodev->func[0]->device) { case SDIO_DEVICE_ID_CYPRESS_4373: - case SDIO_DEVICE_ID_CYPRESS_43012: brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", CY_4373_F2_WATERMARK); brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, CY_4373_F2_WATERMARK, &err); @@ -4127,6 +4127,18 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, devctl |= SBSDIO_DEVCTL_F2WM_ENAB; brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, &err); break; + case SDIO_DEVICE_ID_CYPRESS_43012: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43012\n", + CY_43012_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, + SBSDIO_WATERMARK, + CY_43012_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, + SBSDIO_DEVICE_CTL, &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, + SBSDIO_DEVICE_CTL, devctl, &err); + break; default: brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); break; From f0be6a158b263c759a88b45466cf4535521ae891 Mon Sep 17 00:00:00 2001 From: Naveen Date: Tue, 9 Jan 2018 11:33:10 +0530 Subject: [PATCH 032/213] brcmfmac: DS1 Exit should re download the firmware. In Deep Sleep mode ARM is off and once Exit trigger comes than Mail Box Interrupt comes to Host and whole Re Initiation should be done in the ARM to start TX/RX. Signed-off-by: Naveen Gupta --- .../broadcom/brcm80211/brcmfmac/sdio.c | 175 +++++++++++++++++- .../broadcom/brcm80211/brcmfmac/sdio.h | 60 ++++++ .../broadcom/brcm80211/include/chipcommon.h | 4 +- 3 files changed, 234 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 88c25aaf544a0..c6e52d2eaab40 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -45,9 +45,11 @@ #include "core.h" #include "common.h" #include "bcdc.h" +#include "fwil.h" #define DCMD_RESP_TIMEOUT msecs_to_jiffies(2500) #define CTL_DONE_TIMEOUT msecs_to_jiffies(2500) +#define ULP_HUDI_PROC_DONE_TIME msecs_to_jiffies(2500) #define DEFAULT_F2_WATERMARK 0x8 #define CY_4373_F2_WATERMARK 0x40 @@ -322,6 +324,10 @@ struct rte_console { #define MAX_KSO_ATTEMPTS (PMU_MAX_TRANSITION_DLY/KSO_WAIT_US) #define BRCMF_SDIO_MAX_ACCESS_ERRORS 5 +static void brcmf_sdio_firmware_callback(struct device *dev, int err, + const struct firmware *code, + void *nvram, u32 nvram_len); + /* * Conversion of 802.1D priority to precedence level */ @@ -2550,6 +2556,154 @@ static int brcmf_sdio_intr_rstatus(struct brcmf_sdio *bus) return ret; } +/* This Function is used to retrieve important + * details from dongle related to ULP mode Mostly + * values/SHM details that will be vary depending + * on the firmware branches + */ +static void +brcmf_sdio_ulp_preinit(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + struct brcmf_if *ifp = bus_if->drvr->iflist[0]; + int err = 0; + char iovbuf[BRCMF_DCMD_SMLEN]; + + brcmf_dbg(TRACE, "Enter\n"); + memset(iovbuf, 0, sizeof(iovbuf)); + + /* Query ulp_sdioctrl iovar to get the ULP related SHM offsets */ + err = brcmf_fil_iovar_data_get(ifp, "ulp_sdioctrl", iovbuf, + sizeof(iovbuf)); + if (err) + brcmf_err("fail to get ulp_sdioctrl err:%d\n", err); + + sdiodev->ulp = false; + + /* Copy the data shared by dongle to FMAC structure */ + memcpy(&sdiodev->shm_ulp, iovbuf, sizeof(struct ulp_shm_info)); + + brcmf_dbg(TRACE, "m_ulp_ctrl_sdio[%x] m_ulp_wakeevt_ind [%x]\n", + M_DS1_CTRL_SDIO(sdiodev->shm_ulp), + M_WAKEEVENT_IND(sdiodev->shm_ulp)); + brcmf_dbg(TRACE, "m_ulp_wakeind [%x]\n", + M_ULP_WAKE_IND(sdiodev->shm_ulp)); +} + +/* Reinitialize ARM because In DS1 mode ARM got off */ +static int +brcmf_sdio_ulp_reinit_fw(struct brcmf_sdio *bus) +{ + struct brcmf_sdio_dev *sdiodev = bus->sdiodev; + int err = 0; + + /* After firmware redownload tx/rx seq are reset accordingly + * these values are reset on FMAC side tx_max is initially set to 4, + * which later is updated by FW. + */ + bus->tx_seq = 0; + bus->rx_seq = 0; + bus->tx_max = 4; + + err = brcmf_fw_get_firmwares(sdiodev->dev, BRCMF_FW_REQUEST_NVRAM, + sdiodev->fw_name, sdiodev->nvram_name, + brcmf_sdio_firmware_callback); + if (err != 0) + brcmf_err("async firmware request failed: %d\n", err); + return err; +} + +/* Check if device is in DS1 mode and handshake with ULP UCODE */ +static bool +brcmf_sdio_ulp_pre_redownload_check(struct brcmf_sdio *bus) +{ + int err = 0; + u32 value = 0; + u32 val32, ulp_wake_ind, wowl_wake_ind; + int reg_addr; + unsigned long timeout; + + value = brcmf_sdiod_regrb(bus->sdiodev, SDIO_CCCR_IOEx, &err); + + if (value == SDIO_FUNC_ENABLE_1) { + brcmf_dbg(SDIO, "GOT THE INTERRUPT FROM UCODE\n"); + bus->sdiodev->ulp = true; + ulp_wake_ind = D11SHM_RD(bus->sdiodev, M_ULP_WAKE_IND( + bus->sdiodev->shm_ulp), &err) >> 16; + wowl_wake_ind = D11SHM_RD(bus->sdiodev, M_WAKEEVENT_IND( + bus->sdiodev->shm_ulp), &err) >> 16; + + brcmf_dbg(SDIO, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x\n", + wowl_wake_ind, ulp_wake_ind); + + if (wowl_wake_ind || ulp_wake_ind) { + /* TX wake Don't do anything. + * Just bail out and re-download firmware. + */ + } else { + /* RX wake negotiate with MAC */ + brcmf_dbg(SDIO, "M_DS1_CTRL_SDIO: 0x%08x\n", + (u32)D11SHM_RD(bus->sdiodev, + M_DS1_CTRL_SDIO(bus->sdiodev->shm_ulp), + &err)); + D11SHM_WR(bus->sdiodev, M_DS1_CTRL_SDIO( + bus->sdiodev->shm_ulp), + C_DS1_CTRL_SDIO_DS1_EXIT | + C_DS1_CTRL_REQ_VALID, + &err); + val32 = D11REG_RD(bus->sdiodev, + D11_MACCONTROL_REG, &err); + val32 = val32 | D11_MACCONTROL_REG_WAKE; + D11REG_WR(bus->sdiodev, + D11_MACCONTROL_REG, val32, &err); + + /* Poll for PROC_DONE to be set by ucode */ + value = D11SHM_RD(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->shm_ulp), &err); + /* Wait here (polling) for C_DS1_CTRL_PROC_DONE */ + timeout = jiffies + ULP_HUDI_PROC_DONE_TIME; + while (!(value & C_DS1_CTRL_PROC_DONE)) { + value = D11SHM_RD(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->shm_ulp), &err); + if (time_after(jiffies, timeout)) + break; + usleep_range(1000, 2000); + } + brcmf_dbg(SDIO, "M_DS1_CTRL_SDIO: 0x%08x\n", + (u32)D11SHM_RD(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->shm_ulp), &err)); + value = D11SHM_RD(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->shm_ulp), &err); + if (!(value & C_DS1_CTRL_PROC_DONE)) { + brcmf_err("%s: timeout Failed to enter DS1 Exit state!\n", + __func__); + return false; + } + } + ulp_wake_ind = D11SHM_RD(bus->sdiodev, M_ULP_WAKE_IND( + bus->sdiodev->shm_ulp), &err) >> 16; + wowl_wake_ind = D11SHM_RD(bus->sdiodev, M_WAKEEVENT_IND( + bus->sdiodev->shm_ulp), &err) >> 16; + brcmf_dbg(SDIO, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x\n", + wowl_wake_ind, ulp_wake_ind); + reg_addr = CORE_CC_REG( + brcmf_chip_get_pmu(bus->ci)->base, min_res_mask); + brcmf_sdiod_regwl(bus->sdiodev, reg_addr, + DEFAULT_43012_MIN_RES_MASK, &err); + if (err) + brcmf_err("min_res_mask failed\n"); + + return true; + } + + return false; +} + static void brcmf_sdio_dpc(struct brcmf_sdio *bus) { u32 newstatus = 0; @@ -2622,6 +2776,8 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) if (intstatus & I_HMB_HOST_INT) { intstatus &= ~I_HMB_HOST_INT; intstatus |= brcmf_sdio_hostmail(bus); + if (brcmf_sdio_ulp_pre_redownload_check(bus)) + brcmf_sdio_ulp_reinit_fw(bus); } sdio_release_host(bus->sdiodev->func[1]); @@ -3477,6 +3633,10 @@ static int brcmf_sdio_bus_preinit(struct device *dev) if (err < 0) goto done; + /* initialize SHM address from firmware for DS1 */ + if (!bus->sdiodev->ulp) + brcmf_sdio_ulp_preinit(dev); + bus->tx_hdrlen = SDPCM_HWHDR_LEN + SDPCM_SWHDR_LEN; if (sdiodev->sg_support) { bus->txglom = false; @@ -4172,11 +4332,18 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, sdio_release_host(sdiodev->func[1]); - err = brcmf_bus_started(dev); - if (err != 0) { - brcmf_err("dongle is not responding\n"); - goto fail; + /* Waking up from deep sleep don't requirerd to reint the sdio bus + * as all sdiod core registers will get restored by Firmware using + * FCBS engine. + */ + if (!bus->sdiodev->ulp) { + err = brcmf_bus_started(dev); + if (err != 0) { + brcmf_err("dongle is not responding\n"); + goto fail; + } } + return; release: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index f3da32fc6360b..36c662be0cce8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -177,6 +177,17 @@ struct brcmf_sdreg { struct brcmf_sdio; struct brcmf_sdiod_freezer; +/* ULP SHM Offsets info */ +struct ulp_shm_info { + u32 m_ulp_ctrl_sdio; + u32 m_ulp_wakeevt_ind; + u32 m_ulp_wakeind; +}; + +struct fmac_ulp { + struct ulp_shm_info ulp_shm_offset; +}; + struct brcmf_sdio_dev { struct sdio_func *func[SDIO_MAX_FUNCS]; u8 num_funcs; /* Supported funcs on client */ @@ -201,6 +212,8 @@ struct brcmf_sdio_dev { bool wowl_enabled; enum brcmf_sdiod_state state; struct brcmf_sdiod_freezer *freezer; + struct fmac_ulp shm_ulp; + bool ulp; }; /* sdio core registers */ @@ -376,4 +389,51 @@ void brcmf_sdio_wowl_config(struct device *dev, bool enabled); int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep); void brcmf_sdio_trigger_dpc(struct brcmf_sdio *bus); +/* SHM offsets */ +#define M_DS1_CTRL_SDIO(ptr) ((ptr).ulp_shm_offset.m_ulp_ctrl_sdio) +#define M_WAKEEVENT_IND(ptr) ((ptr).ulp_shm_offset.m_ulp_wakeevt_ind) +#define M_ULP_WAKE_IND(ptr) ((ptr).ulp_shm_offset.m_ulp_wakeind) + +#define D11_BASE_ADDR 0x18001000 +#define D11_AXI_BASE_ADDR 0xE8000000 +#define D11_SHM_BASE_ADDR (D11_AXI_BASE_ADDR + 0x4000) + +#define D11REG_ADDR(offset) (D11_BASE_ADDR + (offset)) +#define D11IHR_ADDR(offset) (D11_AXI_BASE_ADDR + 0x400 + (2 * (offset))) +#define D11SHM_ADDR(offset) (D11_SHM_BASE_ADDR + (offset)) + +/* MacControl register */ +#define D11_MACCONTROL_REG D11REG_ADDR(0x120) +#define D11_MACCONTROL_REG_WAKE 0x4000000 + +/* Following are the offsets in M_DRVR_UCODE_IF_PTR block. Start address of + * M_DRVR_UCODE_IF_PTR block is present in M_DRVR_UCODE_IF_PTR. + */ + +/* M_ULP_WAKE_IND bits */ +#define ULP_WAKE_IND_WATCHDOG_EXP 0x1 +#define ULP_WAKE_IND_FCBS_ERROR 0x2 +#define ULP_WAKE_IND_RE_TRANSMIT_ERR 0x4 +#define ULP_WAKE_IND_HOST_WKUP 0x8 +#define ULP_WAKE_IND_INVALID_FCBS_BLK 0x10 + +#define C_DS1_CTRL_SDIO_DS1_SLEEP 0x1 +#define C_DS1_CTRL_SDIO_MAC_ON 0x2 +#define C_DS1_CTRL_SDIO_RADIO_PHY_ON 0x4 +#define C_DS1_CTRL_SDIO_DS1_EXIT 0x8 +#define C_DS1_CTRL_PROC_DONE 0x100 +#define C_DS1_CTRL_REQ_VALID 0x200 + +#define D11SHM_WR(sdh, offset, val, ret) \ + brcmf_sdiod_regwl(sdh, D11SHM_ADDR(offset), val, ret) + +#define D11SHM_RD(sdh, offset, ret) \ + brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) + +#define D11REG_WR(sdh, addr, val, ret) \ + brcmf_sdiod_regwl(sdh, addr, val, ret) + +#define D11REG_RD(sdh, addr, ret) \ + brcmf_sdiod_regrl(sdh, addr, ret) + #endif /* BRCMFMAC_SDIO_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h index e1fd499930a03..1f23092cdad99 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h @@ -298,6 +298,8 @@ struct chipcregs { * Maximum delay for the PMU state transition in us. * This is an upper bound intended for spinwaits etc. */ -#define PMU_MAX_TRANSITION_DLY 15000 +#define PMU_MAX_TRANSITION_DLY 15000 + +#define DEFAULT_43012_MIN_RES_MASK 0x0f8bfe77 #endif /* _SBCHIPC_H */ From 8251916d281bd39471d71f942346476ddd2d7a63 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 1 Feb 2018 11:50:33 +0800 Subject: [PATCH 033/213] brcmfmac: add FT-based AKMs in brcmf_set_key_mgmt() for FT support Add WLAN_AKM_SUITE_FT_8021X and WLAN_AKM_SUITE_FT_PSK in brcmf_set_key_mgmt() for FT support. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 6 ++++++ .../net/wireless/broadcom/brcm80211/include/brcmu_wifi.h | 1 + 2 files changed, 7 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index fd416cc04c260..d1d7266ea09aa 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1760,6 +1760,12 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) case WLAN_AKM_SUITE_PSK: val = WPA2_AUTH_PSK; break; + case WLAN_AKM_SUITE_FT_8021X: + val = WPA2_AUTH_UNSPECIFIED | WPA2_AUTH_FT; + break; + case WLAN_AKM_SUITE_FT_PSK: + val = WPA2_AUTH_PSK | WPA2_AUTH_FT; + break; default: brcmf_err("invalid cipher group (%d)\n", sme->crypto.cipher_group); diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 7b9a77981df16..91fca79e61583 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -237,6 +237,7 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_RESERVED4 0x0400 #define WPA2_AUTH_RESERVED5 0x0800 #define WPA2_AUTH_1X_SHA256 0x1000 /* 1X with SHA256 key derivation */ +#define WPA2_AUTH_FT 0x4000 /* Fast BSS Transition */ #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ #define DOT11_DEFAULT_RTS_LEN 2347 From 54bfcc7c5ba3197483cd6f3b1f399edb8298f8be Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 31 Jan 2018 04:07:12 -0600 Subject: [PATCH 034/213] brcmfmac: support AP isolation Hostap daemon has a parameter "ap_isolate which is used to prevent low-level bridging of frames between associated stations in the BSS. For driver side, we add cfg80211 ops method change_bss to support setting AP isolation from user space. Signed-off-by: Wright Feng Signed-off-by: Chi-hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d1d7266ea09aa..d93d7ac171cb1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5255,6 +5255,26 @@ static int brcmf_cfg80211_del_pmk(struct wiphy *wiphy, struct net_device *dev, return brcmf_set_pmk(ifp, NULL, 0); } +static int +brcmf_cfg80211_change_bss(struct wiphy *wiphy, struct net_device *dev, + struct bss_parameters *params) +{ + struct brcmf_if *ifp; + int ret = 0; + u32 ap_isolate; + + brcmf_dbg(TRACE, "Enter\n"); + ifp = netdev_priv(dev); + if (params->ap_isolate >= 0) { + ap_isolate = (u32)params->ap_isolate; + ret = brcmf_fil_iovar_int_set(ifp, "ap_isolate", ap_isolate); + if (ret < 0) + brcmf_err("ap_isolate iovar failed: ret=%d\n", ret); + } + + return ret; +} + static struct cfg80211_ops brcmf_cfg80211_ops = { .add_virtual_intf = brcmf_cfg80211_add_iface, .del_virtual_intf = brcmf_cfg80211_del_iface, @@ -5300,6 +5320,7 @@ static struct cfg80211_ops brcmf_cfg80211_ops = { .update_connect_params = brcmf_cfg80211_update_conn_params, .set_pmk = brcmf_cfg80211_set_pmk, .del_pmk = brcmf_cfg80211_del_pmk, + .change_bss = brcmf_cfg80211_change_bss, }; struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, From 81d401bda5a3f0b1d2f4d4bd513bf38741d85ceb Mon Sep 17 00:00:00 2001 From: Naveen Date: Tue, 6 Feb 2018 11:52:58 +0530 Subject: [PATCH 035/213] brcmfmac: do not print ulp_sdioctrl get error Don't print ulp_sdioctrl get error as errors are expected for non ulp cases. Signed-off-by: Naveen Gupta --- .../net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index c6e52d2eaab40..7683648aa2a11 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -2567,23 +2567,15 @@ brcmf_sdio_ulp_preinit(struct device *dev) struct brcmf_bus *bus_if = dev_get_drvdata(dev); struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; struct brcmf_if *ifp = bus_if->drvr->iflist[0]; - int err = 0; - char iovbuf[BRCMF_DCMD_SMLEN]; brcmf_dbg(TRACE, "Enter\n"); - memset(iovbuf, 0, sizeof(iovbuf)); /* Query ulp_sdioctrl iovar to get the ULP related SHM offsets */ - err = brcmf_fil_iovar_data_get(ifp, "ulp_sdioctrl", iovbuf, - sizeof(iovbuf)); - if (err) - brcmf_err("fail to get ulp_sdioctrl err:%d\n", err); + brcmf_fil_iovar_data_get(ifp, "ulp_sdioctrl", &sdiodev->shm_ulp, + sizeof(sdiodev->shm_ulp)); sdiodev->ulp = false; - /* Copy the data shared by dongle to FMAC structure */ - memcpy(&sdiodev->shm_ulp, iovbuf, sizeof(struct ulp_shm_info)); - brcmf_dbg(TRACE, "m_ulp_ctrl_sdio[%x] m_ulp_wakeevt_ind [%x]\n", M_DS1_CTRL_SDIO(sdiodev->shm_ulp), M_WAKEEVENT_IND(sdiodev->shm_ulp)); From 695053a7c89b1169a86950278d612f2cb9510a11 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Mon, 5 Feb 2018 04:33:49 -0600 Subject: [PATCH 036/213] brcmfmac: fix system warning message during wowl suspend There is a system warning message, warn_slowpath-fmt, during suspend while using supplicant join AP and enable wowl feature by IW command. It's cuased by brcmf_pno_remove_request path can't find the reqid. This fix will not go to remove pno request function if there is no pno scan. Signed-off-by: Lo-Hsiang Lo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c index ffa243e2e2d0f..55974a43796aa 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c @@ -496,6 +496,11 @@ int brcmf_pno_stop_sched_scan(struct brcmf_if *ifp, u64 reqid) brcmf_dbg(TRACE, "reqid=%llu\n", reqid); pi = ifp_to_pno(ifp); + + /* No PNO reqeuset */ + if (!pi->n_reqs) + return 0; + err = brcmf_pno_remove_request(pi, reqid); if (err) return err; From 3ac750db7877f72287a16f1517bf6d19628bbaff Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 1 Mar 2018 20:41:21 -0600 Subject: [PATCH 037/213] brcmfmac: add a module parameter to set scheduling priority of sdio_dpc To enhance RX throughput, we add a module parameter "sdio_dpc_prio" to let user can set scheduling priority for sdio_dpc. It can improve RX throughput by reducing the receiving time in sdio_dpc. Signed-off-by: Wright Feng --- .../net/wireless/broadcom/brcm80211/brcmfmac/common.c | 5 +++++ .../net/wireless/broadcom/brcm80211/brcmfmac/common.h | 1 + .../net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 10 ++++++++++ 3 files changed, 16 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 56e2654d39604..c73a67a7d54b2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -77,6 +77,10 @@ static int brcmf_roamoff; module_param_named(roamoff, brcmf_roamoff, int, S_IRUSR); MODULE_PARM_DESC(roamoff, "Do not use internal roaming engine"); +static int brcmf_sdio_dpc_prio; +module_param_named(sdio_dpc_prio, brcmf_sdio_dpc_prio, int, S_IRUSR); +MODULE_PARM_DESC(sdio_dpc_prio, "The scheduling priority of sdio_dpc thread"); + #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -454,6 +458,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->feature_disable = brcmf_feature_disable; settings->fcmode = brcmf_fcmode; settings->roamoff = !!brcmf_roamoff; + settings->sdio_dpc_prio = brcmf_sdio_dpc_prio; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index a62f8e70b3207..4b4dc16a8ba3d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -58,6 +58,7 @@ struct brcmf_mp_device { unsigned int feature_disable; int fcmode; bool roamoff; + int sdio_dpc_prio; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; union { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 7683648aa2a11..2325365169ca3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -2703,9 +2704,18 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) uint txlimit = bus->txbound; /* Tx frames to send before resched */ uint framecnt; /* Temporary counter of tx/rx frames */ int err = 0; + struct sched_param param; + struct brcmf_pub *drvr = bus->sdiodev->bus_if->drvr; + int sdio_dpc_prio = drvr->settings->sdio_dpc_prio; brcmf_dbg(TRACE, "Enter\n"); + if (sdio_dpc_prio > 0) { + param.sched_priority = (sdio_dpc_prio < MAX_RT_PRIO) ? + sdio_dpc_prio : (MAX_RT_PRIO - 1); + sched_setscheduler(current, SCHED_FIFO, ¶m); + } + sdio_claim_host(bus->sdiodev->func[1]); /* If waiting for HTAVAIL, check status */ From 3f0688de02afc3a2206790a7db0ce85bbfe522c7 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 2 Mar 2018 00:45:32 -0600 Subject: [PATCH 038/213] brcmfmac: make firmware eap_restrict a module parameter When eap_restrict is enabled, firmware will toss non-802.1x frames from tx/rx data path if station not yet authorized. Internal firmware eap_restrict is disabled by default. This patch makes it possible to enable firmware eap_restrict by specifying eap_restrict=1 as module parameter. Signed-off-by: Wright Feng --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 9 +++++++++ .../net/wireless/broadcom/brcm80211/brcmfmac/common.c | 5 +++++ .../net/wireless/broadcom/brcm80211/brcmfmac/common.h | 2 ++ 3 files changed, 16 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d93d7ac171cb1..56cbb9562ca13 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6722,6 +6722,7 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) struct wireless_dev *wdev; struct brcmf_if *ifp; s32 power_mode; + s32 eap_restrict; s32 err = 0; if (cfg->dongle_up) @@ -6746,6 +6747,14 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) err = brcmf_dongle_roam(ifp); if (err) goto default_conf_out; + + eap_restrict = ifp->drvr->settings->eap_restrict; + if (eap_restrict) { + err = brcmf_fil_iovar_int_set(ifp, "eap_restrict", + eap_restrict); + if (err) + brcmf_info("eap_restrict error (%d)\n", err); + } err = brcmf_cfg80211_change_iface(wdev->wiphy, ndev, wdev->iftype, NULL); if (err) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index c73a67a7d54b2..7328e44a0d726 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -77,6 +77,10 @@ static int brcmf_roamoff; module_param_named(roamoff, brcmf_roamoff, int, S_IRUSR); MODULE_PARM_DESC(roamoff, "Do not use internal roaming engine"); +static int brcmf_eap_restrict; +module_param_named(eap_restrict, brcmf_eap_restrict, int, 0400); +MODULE_PARM_DESC(eap_restrict, "Block non-802.1X frames until auth finished"); + static int brcmf_sdio_dpc_prio; module_param_named(sdio_dpc_prio, brcmf_sdio_dpc_prio, int, S_IRUSR); MODULE_PARM_DESC(sdio_dpc_prio, "The scheduling priority of sdio_dpc thread"); @@ -458,6 +462,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->feature_disable = brcmf_feature_disable; settings->fcmode = brcmf_fcmode; settings->roamoff = !!brcmf_roamoff; + settings->eap_restrict = !!brcmf_eap_restrict; settings->sdio_dpc_prio = brcmf_sdio_dpc_prio; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 4b4dc16a8ba3d..b1b738c09707f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -49,6 +49,7 @@ extern struct brcmf_mp_global_t brcmf_mp_global; * @feature_disable: Feature_disable bitmask. * @fcmode: FWS flow control. * @roamoff: Firmware roaming off? + * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds * @ignore_probe_fail: Ignore probe failure. * @country_codes: If available, pointer to struct for translating country codes * @bus: Bus specific platform data. Only SDIO at the mmoment. @@ -58,6 +59,7 @@ struct brcmf_mp_device { unsigned int feature_disable; int fcmode; bool roamoff; + bool eap_restrict; int sdio_dpc_prio; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; From 5255bc4cfe55df8458c18f5fe127fb9339818c45 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Tue, 6 Feb 2018 05:07:05 -0600 Subject: [PATCH 039/213] brcmfmac: Support wake on ping packet FMAC driver need to provide a dummy wowlan filter for kernel and provided the well configured wowlan stack. So the system will keep driver in connected state in suspend mode and can be wake up by ping packet. Enable unicast packet filter before system suspend and disable it after resume. Signed-off-by: Lo-Hsiang Lo --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 31 +++- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 7 + .../broadcom/brcm80211/brcmfmac/common.c | 12 +- .../broadcom/brcm80211/brcmfmac/core.c | 175 ++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/core.h | 9 +- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 13 ++ 6 files changed, 241 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 56cbb9562ca13..2db798fead3ba 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3704,6 +3704,10 @@ static s32 brcmf_cfg80211_resume(struct wiphy *wiphy) brcmf_notify_sched_scan_results); cfg->wowl.nd_enabled = false; } + + /* disable packet filters */ + brcmf_pktfilter_enable(ifp->ndev, false); + } return 0; } @@ -3762,6 +3766,9 @@ static void brcmf_configure_wowl(struct brcmf_cfg80211_info *cfg, brcmf_fil_iovar_int_set(ifp, "wowl_activate", 1); brcmf_bus_wowl_config(cfg->pub->bus_if, true); cfg->wowl.active = true; + + /* enable packet filters */ + brcmf_pktfilter_enable(ifp->ndev, true); } static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, @@ -3808,7 +3815,7 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, brcmf_set_mpc(ifp, 1); } else { - /* Configure WOWL paramaters */ + /* Configure WOWL parameters */ brcmf_configure_wowl(cfg, ifp, wowl); } @@ -6569,6 +6576,7 @@ static void brcmf_wiphy_wowl_params(struct wiphy *wiphy, struct brcmf_if *ifp) #ifdef CONFIG_PM struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct wiphy_wowlan_support *wowl; + struct cfg80211_wowlan *brcmf_wowlan_config = NULL; wowl = kmemdup(&brcmf_wowlan_support, sizeof(brcmf_wowlan_support), GFP_KERNEL); @@ -6591,6 +6599,27 @@ static void brcmf_wiphy_wowl_params(struct wiphy *wiphy, struct brcmf_if *ifp) } wiphy->wowlan = wowl; + + /* wowlan_config structure report for kernels */ + brcmf_wowlan_config = kzalloc(sizeof(*brcmf_wowlan_config), + GFP_KERNEL); + if (brcmf_wowlan_config) { + brcmf_wowlan_config->any = false; + brcmf_wowlan_config->disconnect = true; + brcmf_wowlan_config->eap_identity_req = true; + brcmf_wowlan_config->four_way_handshake = true; + brcmf_wowlan_config->rfkill_release = false; + brcmf_wowlan_config->patterns = NULL; + brcmf_wowlan_config->n_patterns = 0; + brcmf_wowlan_config->tcp = NULL; + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL_GTK)) + brcmf_wowlan_config->gtk_rekey_failure = true; + else + brcmf_wowlan_config->gtk_rekey_failure = false; + } else { + brcmf_err("Can not allocate memory for brcm_wowlan_config\n"); + } + wiphy->wowlan_config = brcmf_wowlan_config; #endif } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 7b2835e5e4344..606eaeb66fae6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -86,6 +86,13 @@ #define BRCMF_VIF_EVENT_TIMEOUT msecs_to_jiffies(1500) +/* cfg80211 wowlan definitions */ +#define WL_WOWLAN_MAX_PATTERNS 8 +#define WL_WOWLAN_MIN_PATTERN_LEN 1 +#define WL_WOWLAN_MAX_PATTERN_LEN 255 +#define WL_WOWLAN_PKT_FILTER_ID_FIRST 201 +#define WL_WOWLAN_PKT_FILTER_ID_LAST (WL_WOWLAN_PKT_FILTER_ID_FIRST + \ + WL_WOWLAN_MAX_PATTERNS - 1) /** * enum brcmf_scan_status - scan engine status * diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 7328e44a0d726..8038a66059102 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -261,7 +261,7 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) char *ptr; s32 err; - /* retreive mac address */ + /* retrieve mac addresses */ err = brcmf_fil_iovar_data_get(ifp, "cur_etheraddr", ifp->mac_addr, sizeof(ifp->mac_addr)); if (err < 0) { @@ -387,6 +387,12 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) /* Enable tx beamforming, errors can be ignored (not supported) */ (void)brcmf_fil_iovar_int_set(ifp, "txbf", 1); + /* add unicast packet filter */ + err = brcmf_pktfilter_add_remove(ifp->ndev, + BRCMF_UNICAST_FILTER_NUM, true); + if (err) + brcmf_info("Add unicast filter error (%d)\n", err); + /* do bus specific preinit here */ err = brcmf_bus_preinit(ifp->drvr->bus_if); done: @@ -457,7 +463,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, if (!settings) return NULL; - /* start by using the module paramaters */ + /* start by using the module parameters */ settings->p2p_enable = !!brcmf_p2p_enable; settings->feature_disable = brcmf_feature_disable; settings->fcmode = brcmf_fcmode; @@ -545,7 +551,7 @@ static int __init brcmfmac_module_init(void) if (err == -ENODEV) brcmf_dbg(INFO, "No platform data available.\n"); - /* Initialize global module paramaters */ + /* Initialize global module parameters */ brcmf_mp_attach(); /* Continue the initialization by registering the different busses */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index e6b87a73599b1..4778aeb11d51a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1198,3 +1198,178 @@ void __exit brcmf_core_exit(void) #endif } +int +brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num, bool add) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_pkt_filter_le *pkt_filter; + int filter_fixed_len = offsetof(struct brcmf_pkt_filter_le, u); + int pattern_fixed_len = offsetof(struct brcmf_pkt_filter_pattern_le, + mask_and_pattern); + u16 mask_and_pattern[MAX_PKTFILTER_PATTERN_SIZE]; + int buflen = 0; + int ret = 0; + + brcmf_dbg(INFO, "%s packet filter number %d\n", + (add ? "add" : "remove"), filter_num); + + pkt_filter = kzalloc(sizeof(*pkt_filter) + + (MAX_PKTFILTER_PATTERN_SIZE * 2), GFP_ATOMIC); + if (!pkt_filter) + return -ENOMEM; + + switch (filter_num) { + case BRCMF_UNICAST_FILTER_NUM: + pkt_filter->id = 100; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 1; + mask_and_pattern[0] = 0x0001; + break; + case BRCMF_BROADCAST_FILTER_NUM: + //filter_pattern = "101 0 0 0 0xFFFFFFFFFFFF 0xFFFFFFFFFFFF"; + pkt_filter->id = 101; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 6; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0xFFFF; + mask_and_pattern[4] = 0xFFFF; + mask_and_pattern[5] = 0xFFFF; + break; + case BRCMF_MULTICAST4_FILTER_NUM: + //filter_pattern = "102 0 0 0 0xFFFFFF 0x01005E"; + pkt_filter->id = 102; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 3; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x01FF; + mask_and_pattern[2] = 0x5E00; + break; + case BRCMF_MULTICAST6_FILTER_NUM: + //filter_pattern = "103 0 0 0 0xFFFF 0x3333"; + pkt_filter->id = 103; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 2; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x3333; + break; + case BRCMF_MDNS_FILTER_NUM: + //filter_pattern = "104 0 0 0 0xFFFFFFFFFFFF 0x01005E0000FB"; + pkt_filter->id = 104; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 6; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0x0001; + mask_and_pattern[4] = 0x005E; + mask_and_pattern[5] = 0xFB00; + break; + case BRCMF_ARP_FILTER_NUM: + //filter_pattern = "105 0 0 12 0xFFFF 0x0806"; + pkt_filter->id = 105; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 12; + pkt_filter->u.pattern.size_bytes = 2; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0x0608; + break; + case BRCMF_BROADCAST_ARP_FILTER_NUM: + //filter_pattern = "106 0 0 0 + //0xFFFFFFFFFFFF0000000000000806 + //0xFFFFFFFFFFFF0000000000000806"; + pkt_filter->id = 106; + pkt_filter->type = 0; + pkt_filter->negate_match = 0; + pkt_filter->u.pattern.offset = 0; + pkt_filter->u.pattern.size_bytes = 14; + mask_and_pattern[0] = 0xFFFF; + mask_and_pattern[1] = 0xFFFF; + mask_and_pattern[2] = 0xFFFF; + mask_and_pattern[3] = 0x0000; + mask_and_pattern[4] = 0x0000; + mask_and_pattern[5] = 0x0000; + mask_and_pattern[6] = 0x0608; + mask_and_pattern[7] = 0xFFFF; + mask_and_pattern[8] = 0xFFFF; + mask_and_pattern[9] = 0xFFFF; + mask_and_pattern[10] = 0x0000; + mask_and_pattern[11] = 0x0000; + mask_and_pattern[12] = 0x0000; + mask_and_pattern[13] = 0x0608; + break; + default: + ret = -EINVAL; + goto failed; + } + memcpy(pkt_filter->u.pattern.mask_and_pattern, mask_and_pattern, + pkt_filter->u.pattern.size_bytes * 2); + buflen = filter_fixed_len + pattern_fixed_len + + pkt_filter->u.pattern.size_bytes * 2; + + if (add) { + /* Add filter */ + ret = brcmf_fil_iovar_data_set(ifp, "pkt_filter_add", + pkt_filter, buflen); + if (ret) + goto failed; + drvr->pkt_filter[filter_num].id = pkt_filter->id; + drvr->pkt_filter[filter_num].enable = 0; + + } else { + /* Delete filter */ + ret = brcmf_fil_iovar_int_set(ifp, "pkt_filter_delete", + pkt_filter->id); + if (ret == -ENOENT) + ret = 0; + if (ret) + goto failed; + + drvr->pkt_filter[filter_num].id = 0; + drvr->pkt_filter[filter_num].enable = 0; + + } +failed: + if (ret) + brcmf_err("%s packet filter failed, ret=%d\n", + (add ? "add" : "remove"), ret); + + kfree(pkt_filter); + return ret; +} + +int brcmf_pktfilter_enable(struct net_device *ndev, bool enable) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + int ret = 0; + int idx = 0; + + for (idx = 0; idx < MAX_PKT_FILTER_COUNT; ++idx) { + if (drvr->pkt_filter[idx].id != 0) { + drvr->pkt_filter[idx].enable = enable; + ret = brcmf_fil_iovar_data_set(ifp, "pkt_filter_enable", + &drvr->pkt_filter[idx], + sizeof(struct brcmf_pkt_filter_enable_le)); + if (ret) { + brcmf_err("%s packet filter id(%d) failed, ret=%d\n", + (enable ? "enable" : "disable"), + drvr->pkt_filter[idx].id, ret); + } + } + } + return ret; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index ae39128406cb3..e38f78e4d92c8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -23,6 +23,7 @@ #include #include "fweh.h" +#include "fwil_types.h" #define TOE_TX_CSUM_OL 0x00000001 #define TOE_RX_CSUM_OL 0x00000002 @@ -36,7 +37,7 @@ #define BRCMF_DCMD_MEDLEN 1536 #define BRCMF_DCMD_MAXLEN 8192 -/* IOCTL from host to device are limited in lenght. A device can only handle +/* IOCTL from host to device are limited in length. A device can only handle * ethernet frame size. This limitation is to be applied by protocol layer. */ #define BRCMF_TX_IOCTL_MAX_MSG_SIZE (ETH_FRAME_LEN+ETH_FCS_LEN) @@ -143,6 +144,8 @@ struct brcmf_pub { struct brcmf_mp_device *settings; u8 clmver[BRCMF_DCMD_SMLEN]; + struct brcmf_pkt_filter_enable_le pkt_filter[MAX_PKT_FILTER_COUNT]; + }; /* forward declarations */ @@ -216,5 +219,7 @@ void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_net_setcarrier(struct brcmf_if *ifp, bool on); int __init brcmf_core_init(void); void __exit brcmf_core_exit(void); - +int brcmf_pktfilter_add_remove(struct net_device *ndev, int filter_num, + bool add); +int brcmf_pktfilter_enable(struct net_device *ndev, bool enable); #endif /* BRCMFMAC_CORE_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 4b290705e3e64..9616c94afb766 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -128,6 +128,19 @@ #define BRCMF_WOWL_MAXPATTERNS 8 #define BRCMF_WOWL_MAXPATTERNSIZE 128 +enum { + BRCMF_UNICAST_FILTER_NUM = 0, + BRCMF_BROADCAST_FILTER_NUM, + BRCMF_MULTICAST4_FILTER_NUM, + BRCMF_MULTICAST6_FILTER_NUM, + BRCMF_MDNS_FILTER_NUM, + BRCMF_ARP_FILTER_NUM, + BRCMF_BROADCAST_ARP_FILTER_NUM, + MAX_PKT_FILTER_COUNT +}; + +#define MAX_PKTFILTER_PATTERN_SIZE 16 + #define BRCMF_COUNTRY_BUF_SZ 4 #define BRCMF_ANT_MAX 4 From 43adc0a88b72dba34a0b605eeb7d4e3f3f1847b2 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Tue, 13 Mar 2018 03:22:20 -0500 Subject: [PATCH 040/213] brcmfmac: Remove WOWL configuration in disconnect state Set wowl configuration in disconnect state is redundant. Remove it to fix no scan result issue after resume. Signed-off-by: Lo-Hsiang Lo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 2db798fead3ba..42b7978f8214b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3795,7 +3795,8 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, if (test_bit(BRCMF_SCAN_STATUS_BUSY, &cfg->scan_status)) brcmf_abort_scanning(cfg); - if (wowl == NULL) { + if (!wowl || !test_bit(BRCMF_VIF_STATUS_CONNECTED, + &ifp->vif->sme_state)) { brcmf_bus_wowl_config(cfg->pub->bus_if, false); list_for_each_entry(vif, &cfg->vif_list, list) { if (!test_bit(BRCMF_VIF_STATUS_READY, &vif->sme_state)) From 1f9120309a59e0ff1fe6c2eb30efd378d891844a Mon Sep 17 00:00:00 2001 From: Saint Chuang Date: Fri, 4 May 2018 03:49:53 -0500 Subject: [PATCH 041/213] brcmfmac: add CYW89342 PCIE device CYW89342 is a 2x2 MIMO,802.11a/b/g/n/ac,SDIO 3.0 and PCIe 3.0 for WLAN. It is an automotive wireless chip. Signed-off-by: Saint Chuang --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 1 + drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 3c87157f5b851..e9fcbee61a7bc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -1972,6 +1972,7 @@ static const struct dev_pm_ops brcmf_pciedrvr_pm = { static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4350_DEVICE_ID), + BRCMF_PCIE_DEVICE_SUB(BRCM_PCIE_4355_DEVICE_ID,BRCM_PCIE_VENDOR_ID_BROADCOM,BRCM_PCIE_4355_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4356_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43567_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_DEVICE_ID), diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 37b0acdb111db..31580360081c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -74,6 +74,7 @@ /* PCIE Device IDs */ #define BRCM_PCIE_4350_DEVICE_ID 0x43a3 #define BRCM_PCIE_4354_DEVICE_ID 0x43df +#define BRCM_PCIE_4355_DEVICE_ID 0x4355 #define BRCM_PCIE_4356_DEVICE_ID 0x43ec #define BRCM_PCIE_43567_DEVICE_ID 0x43d3 #define BRCM_PCIE_43570_DEVICE_ID 0x43d9 From 4e23fa34345f63a0b2e60ff8a2ec5165ae8970a8 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 8 May 2018 15:29:35 +0800 Subject: [PATCH 042/213] brcmfmac: handle compressed tx status signal Firmware inform the driver about tx status by normal tx status signal or compressed tx status signal. This patch adds support to handle the compressed tx status signal. Signed-off-by: Chung-Hsien Hsu --- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 121 ++++++++++-------- 1 file changed, 71 insertions(+), 50 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index f59642b2c935a..e9678ba27f22e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -1451,9 +1451,10 @@ static int brcmf_fws_txstatus_suppressed(struct brcmf_fws_info *fws, int fifo, static int brcmf_fws_txs_process(struct brcmf_fws_info *fws, u8 flags, u32 hslot, - u32 genbit, u16 seq) + u32 genbit, u16 seq, u8 compcnt) { u32 fifo; + u8 cnt = 0; int ret; bool remove_from_hanger = true; struct sk_buff *skb; @@ -1464,60 +1465,71 @@ brcmf_fws_txs_process(struct brcmf_fws_info *fws, u8 flags, u32 hslot, brcmf_dbg(DATA, "flags %d\n", flags); if (flags == BRCMF_FWS_TXSTATUS_DISCARD) - fws->stats.txs_discard++; + fws->stats.txs_discard += compcnt; else if (flags == BRCMF_FWS_TXSTATUS_CORE_SUPPRESS) { - fws->stats.txs_supp_core++; + fws->stats.txs_supp_core += compcnt; remove_from_hanger = false; } else if (flags == BRCMF_FWS_TXSTATUS_FW_PS_SUPPRESS) { - fws->stats.txs_supp_ps++; + fws->stats.txs_supp_ps += compcnt; remove_from_hanger = false; } else if (flags == BRCMF_FWS_TXSTATUS_FW_TOSSED) - fws->stats.txs_tossed++; + fws->stats.txs_tossed += compcnt; else if (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED) - fws->stats.txs_host_tossed++; + fws->stats.txs_host_tossed += compcnt; else brcmf_err("unexpected txstatus\n"); - ret = brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb, - remove_from_hanger); - if (ret != 0) { - brcmf_err("no packet in hanger slot: hslot=%d\n", hslot); - return ret; - } + while (cnt < compcnt) { + ret = brcmf_fws_hanger_poppkt(&fws->hanger, hslot, &skb, + remove_from_hanger); + if (ret != 0) { + brcmf_err("no packet in hanger slot: hslot=%d\n", + hslot); + goto cont; + } - skcb = brcmf_skbcb(skb); - entry = skcb->mac; - if (WARN_ON(!entry)) { - brcmu_pkt_buf_free_skb(skb); - return -EINVAL; - } - entry->transit_count--; - if (entry->suppressed && entry->suppr_transit_count) - entry->suppr_transit_count--; + skcb = brcmf_skbcb(skb); + entry = skcb->mac; + if (WARN_ON(!entry)) { + brcmu_pkt_buf_free_skb(skb); + goto cont; + } + entry->transit_count--; + if (entry->suppressed && entry->suppr_transit_count) + entry->suppr_transit_count--; - brcmf_dbg(DATA, "%s flags %d htod %X seq %X\n", entry->name, flags, - skcb->htod, seq); + brcmf_dbg(DATA, "%s flags %d htod %X seq %X\n", entry->name, + flags, skcb->htod, seq); - /* pick up the implicit credit from this packet */ - fifo = brcmf_skb_htod_tag_get_field(skb, FIFO); - if ((fws->fcmode == BRCMF_FWS_FCMODE_IMPLIED_CREDIT) || - (brcmf_skb_if_flags_get_field(skb, REQ_CREDIT)) || - (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED)) { - brcmf_fws_return_credits(fws, fifo, 1); - brcmf_fws_schedule_deq(fws); - } - brcmf_fws_macdesc_return_req_credit(skb); + /* pick up the implicit credit from this packet */ + fifo = brcmf_skb_htod_tag_get_field(skb, FIFO); + if ((fws->fcmode == BRCMF_FWS_FCMODE_IMPLIED_CREDIT) || + (brcmf_skb_if_flags_get_field(skb, REQ_CREDIT)) || + (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED)) { + brcmf_fws_return_credits(fws, fifo, 1); + brcmf_fws_schedule_deq(fws); + } + brcmf_fws_macdesc_return_req_credit(skb); - ret = brcmf_proto_hdrpull(fws->drvr, false, skb, &ifp); - if (ret) { - brcmu_pkt_buf_free_skb(skb); - return -EINVAL; + ret = brcmf_proto_hdrpull(fws->drvr, false, skb, &ifp); + if (ret) { + brcmu_pkt_buf_free_skb(skb); + goto cont; + } + if (!remove_from_hanger) + ret = brcmf_fws_txstatus_suppressed(fws, fifo, skb, + genbit, seq); + if (remove_from_hanger || ret) + brcmf_txfinalize(ifp, skb, true); + +cont: + hslot = (hslot + 1) & (BRCMF_FWS_TXSTAT_HSLOT_MASK >> + BRCMF_FWS_TXSTAT_HSLOT_SHIFT); + if (BRCMF_FWS_MODE_GET_REUSESEQ(fws->mode)) + seq = (seq + 1) & BRCMF_SKB_HTOD_SEQ_NR_MASK; + + cnt++; } - if (!remove_from_hanger) - ret = brcmf_fws_txstatus_suppressed(fws, fifo, skb, - genbit, seq); - if (remove_from_hanger || ret) - brcmf_txfinalize(ifp, skb, true); return 0; } @@ -1543,7 +1555,8 @@ static int brcmf_fws_fifocreditback_indicate(struct brcmf_fws_info *fws, return BRCMF_FWS_RET_OK_SCHEDULE; } -static int brcmf_fws_txstatus_indicate(struct brcmf_fws_info *fws, u8 *data) +static int brcmf_fws_txstatus_indicate(struct brcmf_fws_info *fws, u8 type, + u8 *data) { __le32 status_le; __le16 seq_le; @@ -1552,23 +1565,31 @@ static int brcmf_fws_txstatus_indicate(struct brcmf_fws_info *fws, u8 *data) u32 genbit; u8 flags; u16 seq; + u8 compcnt; + u8 compcnt_offset = BRCMF_FWS_TYPE_TXSTATUS_LEN; - fws->stats.txs_indicate++; memcpy(&status_le, data, sizeof(status_le)); status = le32_to_cpu(status_le); flags = brcmf_txstatus_get_field(status, FLAGS); hslot = brcmf_txstatus_get_field(status, HSLOT); genbit = brcmf_txstatus_get_field(status, GENERATION); if (BRCMF_FWS_MODE_GET_REUSESEQ(fws->mode)) { - memcpy(&seq_le, &data[BRCMF_FWS_TYPE_PKTTAG_LEN], + memcpy(&seq_le, &data[BRCMF_FWS_TYPE_TXSTATUS_LEN], sizeof(seq_le)); seq = le16_to_cpu(seq_le); + compcnt_offset += BRCMF_FWS_TYPE_SEQ_LEN; } else { seq = 0; } + if (type == BRCMF_FWS_TYPE_COMP_TXSTATUS) + compcnt = data[compcnt_offset]; + else + compcnt = 1; + fws->stats.txs_indicate += compcnt; + brcmf_fws_lock(fws); - brcmf_fws_txs_process(fws, flags, hslot, genbit, seq); + brcmf_fws_txs_process(fws, flags, hslot, genbit, seq, compcnt); brcmf_fws_unlock(fws); return BRCMF_FWS_RET_OK_NOSCHEDULE; } @@ -1882,8 +1903,6 @@ void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb) err = BRCMF_FWS_RET_OK_NOSCHEDULE; switch (type) { - case BRCMF_FWS_TYPE_COMP_TXSTATUS: - break; case BRCMF_FWS_TYPE_HOST_REORDER_RXPKTS: rd = (struct brcmf_skb_reorder_data *)skb->cb; rd->reorder = data; @@ -1906,7 +1925,8 @@ void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb) err = brcmf_fws_request_indicate(fws, type, data); break; case BRCMF_FWS_TYPE_TXSTATUS: - brcmf_fws_txstatus_indicate(fws, data); + case BRCMF_FWS_TYPE_COMP_TXSTATUS: + brcmf_fws_txstatus_indicate(fws, type, data); break; case BRCMF_FWS_TYPE_FIFO_CREDITBACK: err = brcmf_fws_fifocreditback_indicate(fws, data); @@ -1995,7 +2015,7 @@ static void brcmf_fws_rollback_toq(struct brcmf_fws_info *fws, fws->stats.rollback_failed++; hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, - hslot, 0, 0); + hslot, 0, 0, 1); } else { fws->stats.rollback_success++; brcmf_fws_return_credits(fws, fifo, 1); @@ -2452,7 +2472,8 @@ void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb) } brcmf_fws_lock(fws); hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); - brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, 0, 0); + brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, 0, 0, + 1); brcmf_fws_unlock(fws); } From ca1087c022a769d542cc438e89d01125d34d2685 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 26 Apr 2018 01:35:53 -0500 Subject: [PATCH 043/213] revert: brcmfmac: add a module parameter to set scheduling priority of sdio_dpc will use WQ_HIGHPRI instead. Signed-off-by: Wright Feng --- .../net/wireless/broadcom/brcm80211/brcmfmac/common.c | 5 ----- .../net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 10 ---------- 2 files changed, 15 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 8038a66059102..f973bb332a777 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -81,10 +81,6 @@ static int brcmf_eap_restrict; module_param_named(eap_restrict, brcmf_eap_restrict, int, 0400); MODULE_PARM_DESC(eap_restrict, "Block non-802.1X frames until auth finished"); -static int brcmf_sdio_dpc_prio; -module_param_named(sdio_dpc_prio, brcmf_sdio_dpc_prio, int, S_IRUSR); -MODULE_PARM_DESC(sdio_dpc_prio, "The scheduling priority of sdio_dpc thread"); - #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -469,7 +465,6 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->fcmode = brcmf_fcmode; settings->roamoff = !!brcmf_roamoff; settings->eap_restrict = !!brcmf_eap_restrict; - settings->sdio_dpc_prio = brcmf_sdio_dpc_prio; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 2325365169ca3..7683648aa2a11 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include @@ -2704,18 +2703,9 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) uint txlimit = bus->txbound; /* Tx frames to send before resched */ uint framecnt; /* Temporary counter of tx/rx frames */ int err = 0; - struct sched_param param; - struct brcmf_pub *drvr = bus->sdiodev->bus_if->drvr; - int sdio_dpc_prio = drvr->settings->sdio_dpc_prio; brcmf_dbg(TRACE, "Enter\n"); - if (sdio_dpc_prio > 0) { - param.sched_priority = (sdio_dpc_prio < MAX_RT_PRIO) ? - sdio_dpc_prio : (MAX_RT_PRIO - 1); - sched_setscheduler(current, SCHED_FIFO, ¶m); - } - sdio_claim_host(bus->sdiodev->func[1]); /* If waiting for HTAVAIL, check status */ From 71319325c5d75a7ec2efdd3dd480229b7b3b9582 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Sun, 25 Mar 2018 21:53:42 -0500 Subject: [PATCH 044/213] brcmfmac: make setting SDIO workqueue WQ_HIGHPRI a module parameter With setting sdio_wq_highpri=1 in module parameters, tasks submitted to SDIO workqueue will put at the head of the queue and run immediately. This parameter is for getting higher TX/RX throughput with SDIO bus. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/common.c | 5 ++++ .../broadcom/brcm80211/brcmfmac/common.h | 2 ++ .../broadcom/brcm80211/brcmfmac/sdio.c | 23 ++++++++++++------- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index f973bb332a777..5eac84877b99e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -81,6 +81,10 @@ static int brcmf_eap_restrict; module_param_named(eap_restrict, brcmf_eap_restrict, int, 0400); MODULE_PARM_DESC(eap_restrict, "Block non-802.1X frames until auth finished"); +static int brcmf_sdio_wq_highpri; +module_param_named(sdio_wq_highpri, brcmf_sdio_wq_highpri, int, 0); +MODULE_PARM_DESC(sdio_wq_highpri, "SDIO workqueue is set to high priority"); + #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -465,6 +469,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->fcmode = brcmf_fcmode; settings->roamoff = !!brcmf_roamoff; settings->eap_restrict = !!brcmf_eap_restrict; + settings->sdio_wq_highpri = !!brcmf_sdio_wq_highpri; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index b1b738c09707f..89267da22bf27 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -50,6 +50,7 @@ extern struct brcmf_mp_global_t brcmf_mp_global; * @fcmode: FWS flow control. * @roamoff: Firmware roaming off? * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds + * @sdio_wq_highpri: Tasks submitted to SDIO workqueue will run immediately. * @ignore_probe_fail: Ignore probe failure. * @country_codes: If available, pointer to struct for translating country codes * @bus: Bus specific platform data. Only SDIO at the mmoment. @@ -61,6 +62,7 @@ struct brcmf_mp_device { bool roamoff; bool eap_restrict; int sdio_dpc_prio; + bool sdio_wq_highpri; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; union { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 7683648aa2a11..93e5371b4c6f2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4351,6 +4351,7 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) int ret; struct brcmf_sdio *bus; struct workqueue_struct *wq; + unsigned int wq_flags; brcmf_dbg(TRACE, "Enter\n"); @@ -4367,9 +4368,21 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) bus->txminmax = BRCMF_TXMINMAX; bus->tx_seq = SDPCM_SEQ_WRAP - 1; + /* attempt to attach to the dongle */ + if (!(brcmf_sdio_probe_attach(bus))) { + brcmf_err("brcmf_sdio_probe_attach failed\n"); + goto fail; + } + /* single-threaded workqueue */ - wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM, - dev_name(&sdiodev->func[1]->dev)); + if (sdiodev->settings->sdio_wq_highpri) { + wq = alloc_workqueue("brcmf_wq/%s", + WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, + 1, dev_name(&sdiodev->func[1]->dev)); + } else { + wq = alloc_ordered_workqueue("brcmf_wq/%s", WQ_MEM_RECLAIM, + dev_name(&sdiodev->func[1]->dev)); + } if (!wq) { brcmf_err("insufficient memory to create txworkqueue\n"); goto fail; @@ -4378,12 +4391,6 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) INIT_WORK(&bus->datawork, brcmf_sdio_dataworker); bus->brcmf_wq = wq; - /* attempt to attach to the dongle */ - if (!(brcmf_sdio_probe_attach(bus))) { - brcmf_err("brcmf_sdio_probe_attach failed\n"); - goto fail; - } - spin_lock_init(&bus->rxctl_lock); spin_lock_init(&bus->txq_lock); init_waitqueue_head(&bus->ctrl_wait); From 95360bcc1dce823a0b0b25ece6b31b663bbba2d6 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 17 Apr 2018 21:33:21 -0500 Subject: [PATCH 045/213] brcmfmac: add credit map updating support The credit numbers are static and tunable per chip in firmware side. However the credit number may be changed that is based on packet pool length and will send BRCMF_E_FIFO_CREDIT_MAP event to ask host driver updating the credit numbers during interface up. The purpose of this patch is making host driver has ability of updating the credit numbers when receiving the BRCMF_E_FIFO_CREDIT_MAP event. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index e9678ba27f22e..24536e487e5f9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -511,6 +511,7 @@ struct brcmf_fws_info { struct work_struct fws_dequeue_work; u32 fifo_enqpkt[BRCMF_FWS_FIFO_COUNT]; int fifo_credit[BRCMF_FWS_FIFO_COUNT]; + int init_fifo_credit[BRCMF_FWS_FIFO_COUNT]; int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1]; int deq_node_pos[BRCMF_FWS_FIFO_COUNT]; u32 fifo_credit_map; @@ -1237,6 +1238,9 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, } fws->fifo_credit[fifo] += credits; + if (fws->fifo_credit[fifo] > fws->init_fifo_credit[fifo]) + fws->fifo_credit[fifo] = fws->init_fifo_credit[fifo]; + } static void brcmf_fws_schedule_deq(struct brcmf_fws_info *fws) @@ -1616,19 +1620,21 @@ static int brcmf_fws_notify_credit_map(struct brcmf_if *ifp, brcmf_err("event payload too small (%d)\n", e->datalen); return -EINVAL; } - if (fws->creditmap_received) - return 0; fws->creditmap_received = true; brcmf_dbg(TRACE, "enter: credits %pM\n", credits); brcmf_fws_lock(fws); for (i = 0; i < ARRAY_SIZE(fws->fifo_credit); i++) { - if (*credits) + fws->fifo_credit[i] += credits[i] - fws->init_fifo_credit[i]; + fws->init_fifo_credit[i] = credits[i]; + if (fws->fifo_credit[i] > 0) fws->fifo_credit_map |= 1 << i; else fws->fifo_credit_map &= ~(1 << i); - fws->fifo_credit[i] = *credits++; + if (fws->fifo_credit[i] < 0) + brcmf_err("fifo_credit[%d] value is negative(%d)\n", + i, fws->fifo_credit[i]); } brcmf_fws_schedule_deq(fws); brcmf_fws_unlock(fws); @@ -2033,7 +2039,7 @@ static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws) } for (lender_ac = 0; lender_ac <= BRCMF_FWS_FIFO_AC_VO; lender_ac++) { - if (fws->fifo_credit[lender_ac]) { + if (fws->fifo_credit[lender_ac] > 0) { fws->credits_borrowed[lender_ac]++; fws->fifo_credit[lender_ac]--; if (fws->fifo_credit[lender_ac] == 0) @@ -2230,8 +2236,9 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) } continue; } - while ((fws->fifo_credit[fifo]) || ((!fws->bcmc_credit_check) && - (fifo == BRCMF_FWS_FIFO_BCMC))) { + while ((fws->fifo_credit[fifo] > 0) || + ((!fws->bcmc_credit_check) && + (fifo == BRCMF_FWS_FIFO_BCMC))) { skb = brcmf_fws_deq(fws, fifo); if (!skb) break; @@ -2242,7 +2249,7 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) break; } if ((fifo == BRCMF_FWS_FIFO_AC_BE) && - (fws->fifo_credit[fifo] == 0) && + (fws->fifo_credit[fifo] <= 0) && (!fws->bus_flow_blocked)) { while (brcmf_fws_borrow_credit(fws) == 0) { skb = brcmf_fws_deq(fws, fifo); From 278369f4061ad91e0535bf45f3cf4bd24229d6f8 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 24 May 2018 11:15:09 +0800 Subject: [PATCH 046/213] brcmfmac: add 4-way handshake offload detection for FT-802.1X Add 4-way handshake offload detection for FT with EAP authentication. Signed-off-by: Chung-Hsien Hsu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 42b7978f8214b..44b1cd95883ea 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1762,6 +1762,8 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) break; case WLAN_AKM_SUITE_FT_8021X: val = WPA2_AUTH_UNSPECIFIED | WPA2_AUTH_FT; + if (sme->want_1x) + profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; break; case WLAN_AKM_SUITE_FT_PSK: val = WPA2_AUTH_PSK | WPA2_AUTH_FT; From 82519bf563e46f94e80a06e0b7bac6752f986e15 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 25 May 2018 02:02:04 -0500 Subject: [PATCH 047/213] brcmfmac: remove "arp_hostip_clear" from "brcmf_netdev_stop" The firmware does not respond ARP request and causes ping failed with following steps. 1. Bring up inteface ifconfig wlan0 up or start wpa_supplicant 2. Set the IP address ifconfig wlan0 192.168.100.10 3. Bring down interface or ifconfig wlan0 down or kill wpa_supplicant 4. Bring up inteface again and set the same IP address 5. Connect to AP(192.168.100.1) and ping to AP will be failed. FMAC clears arp_hostip when bringing down the interface, but not set it back if setting the same IP address. We are able to see the IP address in interface info(inconfig wlan0) but the ping still cannot work because the firmware ARP offload does not respond the ARP request. Because of that, we remove "arp_hostip_clear" from function "brcmf_netdev_stop" Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 4778aeb11d51a..92f9db3768f5b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -413,8 +413,6 @@ static int brcmf_netdev_stop(struct net_device *ndev) brcmf_cfg80211_down(ndev); - brcmf_fil_iovar_data_set(ifp, "arp_hostip_clear", NULL, 0); - brcmf_net_setcarrier(ifp, false); return 0; From c2f04fe9f96ec0d5358c549bd1de7f8ce603d190 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 29 May 2018 02:52:58 -0500 Subject: [PATCH 048/213] brcmfmac: fix unused variable building warning message The variable "wq_flags" is not used anymore. Remove it. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 93e5371b4c6f2..add7a30bd961c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4351,7 +4351,6 @@ struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) int ret; struct brcmf_sdio *bus; struct workqueue_struct *wq; - unsigned int wq_flags; brcmf_dbg(TRACE, "Enter\n"); From d89bbd29df63c63a59b304679adc9bf626091841 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 30 May 2018 21:11:20 -0500 Subject: [PATCH 049/213] brcmfmac: disable command decode in sdio_aos for 4339/4345 Transaction between AOS and SDIOD is not protected, and if cmd 52 received in AOS and in the middle of response state changed from AOS to SDIOD, response is corrupted and it causes to SDIO Host controller to hang. Signed-off-by: Wright Feng --- .../net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index add7a30bd961c..e4b1f9db5378e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3523,15 +3523,21 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus) if (bus->ci->chip == CY_CC_43012_CHIP_ID) { wakeupctrl = SBSDIO_FUNC1_WCTRL_ALPWAIT_SHIFT; - cardcap = SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC; chipclkcsr = SBSDIO_HT_AVAIL_REQ; } else { wakeupctrl = SBSDIO_FUNC1_WCTRL_HTWAIT_SHIFT; - cardcap = (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | - SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT); chipclkcsr = SBSDIO_FORCE_HT; } + if (bus->ci->chip == CY_CC_43012_CHIP_ID || + bus->ci->chip == BRCM_CC_4339_CHIP_ID || + bus->ci->chip == BRCM_CC_4345_CHIP_ID) { + cardcap = SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC; + } else { + cardcap = (SDIO_CCCR_BRCM_CARDCAP_CMD14_SUPPORT | + SDIO_CCCR_BRCM_CARDCAP_CMD14_EXT); + } + val = brcmf_sdiod_regrb(bus->sdiodev, SBSDIO_FUNC1_WAKEUPCTRL, &err); if (err) { brcmf_err("error reading SBSDIO_FUNC1_WAKEUPCTRL\n"); From 128a854321951c89b538d702c4f27efca41724ee Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 1 Jun 2018 02:47:24 -0500 Subject: [PATCH 050/213] Revert "brcmfmac: fix CLM load error for legacy chips when user helper is enabled" This reverts commit b56e808f2e14c364ea907ce183be3ed7affb5b9c. Signed-off-by: Wright Feng --- .../wireless/broadcom/brcm80211/brcmfmac/common.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 5eac84877b99e..a0f1eb15f31d3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -45,8 +45,6 @@ const u8 ALLFFMAC[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; #define BRCMF_DEFAULT_TXGLOM_SIZE 32 /* max tx frames in glom chain */ -#define CLM_LOAD_RETRIES 1 /* # of retries to load clm_blob file */ - static int brcmf_sdiod_txglomsz = BRCMF_DEFAULT_TXGLOM_SIZE; module_param_named(txglomsz, brcmf_sdiod_txglomsz, int, 0); MODULE_PARM_DESC(txglomsz, "Maximum tx packet chain size [SDIO]"); @@ -180,7 +178,6 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) u16 dl_flag = DL_BEGIN; u32 status; s32 err; - uint retries = 0; brcmf_dbg(TRACE, "Enter\n"); @@ -191,18 +188,11 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) return err; } - do { - err = request_firmware(&clm, clm_name, dev); - } while (err == -EAGAIN && retries++ < CLM_LOAD_RETRIES); + err = request_firmware(&clm, clm_name, dev); if (err) { if (err == -ENOENT) { brcmf_dbg(INFO, "continue with CLM data currently present in firmware\n"); return 0; - } else if (err == -EAGAIN) { - brcmf_dbg(INFO, "reached maximum retries(%d)\n", - CLM_LOAD_RETRIES); - brcmf_dbg(INFO, "continue with CLM data in firmware\n"); - return 0; } brcmf_err("request CLM blob file failed (%d)\n", err); return err; From d100885643351d00f7824363bb0aa3e27e26c133 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 16 Jan 2018 17:26:50 +0800 Subject: [PATCH 051/213] brcmfmac: fix CLM load error for legacy chips when user helper is enabled commit cc124d5cc8d81985c3511892d7a6d546552ff754 upstream. For legacy chips without CLM blob files, kernel with user helper function returns -EAGAIN when we request_firmware(), and then driver got failed when bringing up legacy chips. We expect the CLM blob file for legacy chip is not existence in firmware path, but the -ENOENT error is transferred to -EAGAIN in firmware_class.c with user helper. Because of that, we continue with CLM data currently present in firmware if getting error from doing request_firmware(). Cc: stable@vger.kernel.org # v4.15.y Reviewed-by: Arend van Spriel Signed-off-by: Wright Feng Signed-off-by: Kalle Valo --- .../net/wireless/broadcom/brcm80211/brcmfmac/common.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index a0f1eb15f31d3..df13bb7cb51e5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -190,12 +190,9 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp) err = request_firmware(&clm, clm_name, dev); if (err) { - if (err == -ENOENT) { - brcmf_dbg(INFO, "continue with CLM data currently present in firmware\n"); - return 0; - } - brcmf_err("request CLM blob file failed (%d)\n", err); - return err; + brcmf_info("no clm_blob available(err=%d), device may have limited channels available\n", + err); + return 0; } chunk_buf = kzalloc(sizeof(*chunk_buf) + MAX_CHUNK_LEN - 1, GFP_KERNEL); From 6c8a2c5552dc54af8614bc2d3867c0c51b71703c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Thu, 10 May 2018 15:21:39 +0200 Subject: [PATCH 052/213] brcmfmac: set WIPHY_FLAG_HAVE_AP_SME flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit brcmfmac is a FullMAC driver and it implements/uses cfg80211 interface for stations management. At the same time it doesn't receive or pass up management frames. This flag indicates that authenticator doesn't have to subscribe to or handle management frames. Some authenticators (e.g. hostapd) were working with brcmfmac thanks to some extra assumptions. This commit clears up the situation. Signed-off-by: RafaÅ‚ MiÅ‚ecki Acked-by: Arend van Spriel Signed-off-by: Kalle Valo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 44b1cd95883ea..58a2720bd97b3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6674,6 +6674,7 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy->flags |= WIPHY_FLAG_NETNS_OK | WIPHY_FLAG_PS_ON_BY_DEFAULT | + WIPHY_FLAG_HAVE_AP_SME | WIPHY_FLAG_OFFCHAN_TX | WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_TDLS)) From 11dd83b111c7c2c1313799bf1da9c52093fc071c Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Wed, 23 May 2018 17:22:44 +0530 Subject: [PATCH 053/213] brcmfmac: P2P CERT 6.1.9-Support GOUT handling P2P Presence Request Send P2P Presence Response from the p2p interface address instead of the p2p device address. Signed-off-by: Madhan Mohan R --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 450f2216fac2d..79a51d8cdb375 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1497,6 +1497,7 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, struct brcmf_fil_af_params_le *af_params) { struct brcmf_cfg80211_vif *vif; + struct brcmf_p2p_action_frame *p2p_act_frame; s32 err = 0; s32 timeout = 0; @@ -1506,7 +1507,13 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, clear_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status); clear_bit(BRCMF_P2P_STATUS_ACTION_TX_NOACK, &p2p->status); - vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + /* check if it is a p2p_presence response */ + p2p_act_frame = (struct brcmf_p2p_action_frame *) af_params->action_frame.data; + if (p2p_act_frame->subtype == P2P_AF_PRESENCE_RSP) + vif = p2p->bss_idx[P2PAPI_BSSCFG_CONNECTION].vif; + else + vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe", af_params, sizeof(*af_params)); if (err) { From c9b3d5867336bd0c314f7d613a2bd4de1ea76c1b Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Fri, 1 Jun 2018 05:22:05 -0500 Subject: [PATCH 054/213] brcmfmac: only generate random p2p address when needed P2p spec mentioned that the p2p device address should be the globally administered address with locally administered bit set. Therefore, follow this guideline by default. When the primary interface is set to a locally administered address, the locally administered bit cannot be set again. Generate a random locally administered address for this case. Signed-off-by: Chi-Hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/p2p.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 79a51d8cdb375..92808540761f5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -467,10 +467,21 @@ static int brcmf_p2p_set_firmware(struct brcmf_if *ifp, u8 *p2p_mac) */ static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr) { + struct brcmf_if *pri_ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp; bool random_addr = false; + bool local_admin = false; - if (!dev_addr || is_zero_ether_addr(dev_addr)) - random_addr = true; + if (!dev_addr || is_zero_ether_addr(dev_addr)) { + /* If the primary interface address is already locally + * administered, create a new random address. + */ + if (pri_ifp->mac_addr[0] & 0x02) { + random_addr = true; + } else { + dev_addr = pri_ifp->mac_addr; + local_admin = true; + } + } /* Generate the P2P Device Address obtaining a random ethernet * address with the locally administered bit set. @@ -480,6 +491,9 @@ static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr) else memcpy(p2p->dev_addr, dev_addr, ETH_ALEN); + if (local_admin) + p2p->dev_addr[0] |= 0x02; + /* Generate the P2P Interface Address. If the discovery and connection * BSSCFGs need to simultaneously co-exist, then this address must be * different from the P2P Device Address, but also locally administered. From fabc0ec13e2a4dc30328ac523fb8606b91ab03e0 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Tue, 5 Jun 2018 21:39:58 -0500 Subject: [PATCH 055/213] brcmfmac: disable command decode in sdio_aos for 4354 Transaction between AOS and SDIOD is not protected, and if cmd 52 received in AOS and in the middle of response state changed from AOS to SDIOD, response is corrupted and it causes to SDIO Host controller to hang. Signed-off-by: Lo-Hsiang Lo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index e4b1f9db5378e..43b23cf9b4c40 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3531,6 +3531,7 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus) if (bus->ci->chip == CY_CC_43012_CHIP_ID || bus->ci->chip == BRCM_CC_4339_CHIP_ID || + bus->ci->chip == BRCM_CC_4354_CHIP_ID || bus->ci->chip == BRCM_CC_4345_CHIP_ID) { cardcap = SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC; } else { From c8e65aefa0ba8e01be91fcf54484f1ec3c0c8ae0 Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Thu, 24 May 2018 14:19:00 +0530 Subject: [PATCH 056/213] brcmfmac: increase max hanger slots from 1K to 3K in fws layer Will enable FMAC to push more packets to bus tx queue and help improve throughput when fws queuing is enabled. This change is required to tune the throughput for passing WMM CERT tests. Signed-off-by: Madhan Mohan R --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 24536e487e5f9..21720f5f6a8a2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -415,7 +415,7 @@ struct brcmf_fws_mac_descriptor { u8 traffic_lastreported_bmp; }; -#define BRCMF_FWS_HANGER_MAXITEMS 1024 +#define BRCMF_FWS_HANGER_MAXITEMS 3072 /** * enum brcmf_fws_hanger_item_state - state of hanger item. From 4cb3a392572db5bccc735eed9a9ef36e208975d5 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Fri, 8 Jun 2018 17:34:33 +0800 Subject: [PATCH 057/213] brcmfmac: reduce timeout for action frame scan Finding a common channel to send an action frame out is required for some action types. Since a loop with several scan retry is used to find the channel, a short wait time could be considered for each attempt. This patch reduces the wait time from 1500 to 450 msec for each action frame scan. Signed-off-by: Chung-Hsien Hsu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 92808540761f5..2f5b361fc969f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -74,7 +74,7 @@ #define P2P_AF_MAX_WAIT_TIME msecs_to_jiffies(2000) #define P2P_INVALID_CHANNEL -1 #define P2P_CHANNEL_SYNC_RETRY 5 -#define P2P_AF_FRM_SCAN_MAX_WAIT msecs_to_jiffies(1500) +#define P2P_AF_FRM_SCAN_MAX_WAIT msecs_to_jiffies(450) #define P2P_DEFAULT_SLEEP_TIME_VSDB 200 /* WiFi P2P Public Action Frame OUI Subtypes */ @@ -1153,7 +1153,6 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) { struct afx_hdl *afx_hdl = &p2p->afx_hdl; struct brcmf_cfg80211_vif *pri_vif; - unsigned long duration; s32 retry; brcmf_dbg(TRACE, "Enter\n"); @@ -1169,7 +1168,6 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) * pending action frame tx is cancelled. */ retry = 0; - duration = msecs_to_jiffies(P2P_AF_FRM_SCAN_MAX_WAIT); while ((retry < P2P_CHANNEL_SYNC_RETRY) && (afx_hdl->peer_chan == P2P_INVALID_CHANNEL)) { afx_hdl->is_listen = false; @@ -1177,7 +1175,8 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) retry); /* search peer on peer's listen channel */ schedule_work(&afx_hdl->afx_work); - wait_for_completion_timeout(&afx_hdl->act_frm_scan, duration); + wait_for_completion_timeout(&afx_hdl->act_frm_scan, + P2P_AF_FRM_SCAN_MAX_WAIT); if ((afx_hdl->peer_chan != P2P_INVALID_CHANNEL) || (!test_bit(BRCMF_P2P_STATUS_FINDING_COMMON_CHANNEL, &p2p->status))) @@ -1190,7 +1189,7 @@ static s32 brcmf_p2p_af_searching_channel(struct brcmf_p2p_info *p2p) afx_hdl->is_listen = true; schedule_work(&afx_hdl->afx_work); wait_for_completion_timeout(&afx_hdl->act_frm_scan, - duration); + P2P_AF_FRM_SCAN_MAX_WAIT); } if ((afx_hdl->peer_chan != P2P_INVALID_CHANNEL) || (!test_bit(BRCMF_P2P_STATUS_FINDING_COMMON_CHANNEL, From d5b159359e298a3b49a1bb8336521e62b3c62cc7 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Mon, 11 Jun 2018 17:32:21 +0800 Subject: [PATCH 058/213] brcmfmac: fix full timeout waiting for action frame on-channel tx The driver sends an action frame down and waits for a completion signal triggered by the received BRCMF_E_ACTION_FRAME_OFF_CHAN_COMPLETE event to continue the process. However, the action frame could be transmitted either on the current channel or on an off channel. For the on-channel case, only BRCMF_E_ACTION_FRAME_COMPLETE event will be received when the frame is transmitted, which make the driver always wait a full timeout duration. This patch has the completion signal be triggered by receiving the BRCMF_E_ACTION_FRAME_COMPLETE event for the on-channel case. Signed-off-by: Chung-Hsien Hsu --- .../wireless/broadcom/brcm80211/brcmfmac/p2p.c | 17 +++++++++++++++-- .../wireless/broadcom/brcm80211/brcmfmac/p2p.h | 2 ++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 2f5b361fc969f..52016373dce7a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1476,10 +1476,12 @@ int brcmf_p2p_notify_action_tx_complete(struct brcmf_if *ifp, return 0; if (e->event_code == BRCMF_E_ACTION_FRAME_COMPLETE) { - if (e->status == BRCMF_E_STATUS_SUCCESS) + if (e->status == BRCMF_E_STATUS_SUCCESS) { set_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status); - else { + if (!p2p->wait_for_offchan_complete) + complete(&p2p->send_af_done); + } else { set_bit(BRCMF_P2P_STATUS_ACTION_TX_NOACK, &p2p->status); /* If there is no ack, we don't need to wait for * WLC_E_ACTION_FRAME_OFFCHAN_COMPLETE event @@ -1537,6 +1539,17 @@ static s32 brcmf_p2p_tx_action_frame(struct brcmf_p2p_info *p2p, p2p->af_sent_channel = le32_to_cpu(af_params->channel); p2p->af_tx_sent_jiffies = jiffies; + if (test_bit(BRCMF_P2P_STATUS_DISCOVER_LISTEN, &p2p->status) && + p2p->af_sent_channel == + ieee80211_frequency_to_channel(p2p->remain_on_channel.center_freq)) + p2p->wait_for_offchan_complete = false; + else + p2p->wait_for_offchan_complete = true; + + brcmf_dbg(TRACE, "Waiting for %s tx completion event\n", + (p2p->wait_for_offchan_complete) ? + "off-channel" : "on-channel"); + timeout = wait_for_completion_timeout(&p2p->send_af_done, P2P_AF_MAX_WAIT_TIME); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h index 0e8b34d2d85cb..39f0d02180882 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h @@ -124,6 +124,7 @@ struct afx_hdl { * @gon_req_action: about to send go negotiation requets frame. * @block_gon_req_tx: drop tx go negotiation requets frame. * @p2pdev_dynamically: is p2p device if created by module param or supplicant. + * @wait_for_offchan_complete: wait for off-channel tx completion event. */ struct brcmf_p2p_info { struct brcmf_cfg80211_info *cfg; @@ -144,6 +145,7 @@ struct brcmf_p2p_info { bool gon_req_action; bool block_gon_req_tx; bool p2pdev_dynamically; + bool wait_for_offchan_complete; }; s32 brcmf_p2p_attach(struct brcmf_cfg80211_info *cfg, bool p2pdev_forced); From ea07fc9b870cf38c023076782cb46d4036848da3 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Tue, 12 Jun 2018 04:41:43 -0500 Subject: [PATCH 059/213] brcmfmac: 4373 save-restore support Use sr_eng_en bit to check 4373 sr support. Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 01c6a4923aa1d..ba1d279497ea3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -1329,7 +1329,7 @@ bool brcmf_chip_set_active(struct brcmf_chip *pub, u32 rstvec) bool brcmf_chip_sr_capable(struct brcmf_chip *pub) { - u32 base, addr, reg, pmu_cc3_mask = ~0; + u32 base, addr, reg, sr_eng_en, pmu_cc3_mask = ~0; struct brcmf_chip_priv *chip; struct brcmf_core *pmu = brcmf_chip_get_pmu(pub); @@ -1362,6 +1362,12 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) addr = CORE_CC_REG(base, sr_control1); reg = chip->ops->read32(chip->ctx, addr); return reg != 0; + case CY_CC_4373_CHIP_ID: + /* explicitly check SR engine enable bit */ + sr_eng_en = BIT(0); + addr = CORE_CC_REG(base, sr_control0); + reg = chip->ops->read32(chip->ctx, addr); + return (reg & sr_eng_en) != 0; case CY_CC_43012_CHIP_ID: addr = CORE_CC_REG(pmu->base, retention_ctl); reg = chip->ops->read32(chip->ctx, addr); From bfa58ea08fbaa86e392c6f260591970a4ef9e95c Mon Sep 17 00:00:00 2001 From: Saravanan Shanmugham Date: Wed, 13 Sep 2017 14:04:14 +0900 Subject: [PATCH 060/213] brcmfmac: map 802.1d priority to precedence level based on AP WMM params In WLAN, priority among various access categories of traffic is always set by the AP using WMM parameters and this may not always follow the standard 802.1d priority. In this change, priority is adjusted based on the AP WMM params received as part of the Assoc Response and the same is later used to map the priority of all incoming traffic. This change should fix the following 802.11n certification tests: * 5.2.31 ACM Bit Conformance test * 5.2.32 AC Parameter Modification test Signed-off-by: Saravanan Shanmugham --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 144 ++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 24 +++ .../broadcom/brcm80211/brcmfmac/common.h | 4 + .../broadcom/brcm80211/brcmfmac/fwsignal.c | 5 +- .../broadcom/brcm80211/brcmfmac/sdio.c | 17 +-- 5 files changed, 183 insertions(+), 11 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 58a2720bd97b3..7270cfbaf86b3 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5465,11 +5465,124 @@ static void brcmf_clear_assoc_ies(struct brcmf_cfg80211_info *cfg) conn_info->resp_ie_len = 0; } +u8 brcmf_map_prio_to_prec(void *config, u8 prio) +{ + struct brcmf_cfg80211_info *cfg = (struct brcmf_cfg80211_info *)config; + + if (!cfg) + return (prio == PRIO_8021D_NONE || prio == PRIO_8021D_BE) ? + (prio ^ 2) : prio; + + /* For those AC(s) with ACM flag set to 1, convert its 4-level priority + * to an 8-level precedence which is the same as BE's */ + if (prio > PRIO_8021D_EE && + cfg->ac_priority[prio] == cfg->ac_priority[PRIO_8021D_BE]) + return cfg->ac_priority[prio] * 2; + + /* Conversion of 4-level priority to 8-level precedence */ + if (prio == PRIO_8021D_BE || prio == PRIO_8021D_BK || + prio == PRIO_8021D_CL || prio == PRIO_8021D_VO) + return cfg->ac_priority[prio] * 2; + else + return cfg->ac_priority[prio] * 2 + 1; +} + +u8 brcmf_map_prio_to_aci(void *config, u8 prio) +{ + /* Prio here refers to the 802.1d priority in range of 0 to 7. + * ACI here refers to the WLAN AC Index in range of 0 to 3. + * This function will return ACI corresponding to input prio. + */ + struct brcmf_cfg80211_info *cfg = (struct brcmf_cfg80211_info *)config; + + if (cfg) + return cfg->ac_priority[prio]; + + return prio; +} + +static void brcmf_wifi_prioritize_acparams(const + struct brcmf_cfg80211_edcf_acparam *acp, u8 *priority) +{ + u8 aci; + u8 aifsn; + u8 ecwmin; + u8 ecwmax; + u8 acm; + u8 ranking_basis[EDCF_AC_COUNT]; + u8 aci_prio[EDCF_AC_COUNT]; /* AC_BE, AC_BK, AC_VI, AC_VO */ + u8 index; + + for (aci = 0; aci < EDCF_AC_COUNT; aci++, acp++) { + aifsn = acp->ACI & EDCF_AIFSN_MASK; + acm = (acp->ACI & EDCF_ACM_MASK) ? 1 : 0; + ecwmin = acp->ECW & EDCF_ECWMIN_MASK; + ecwmax = (acp->ECW & EDCF_ECWMAX_MASK) >> EDCF_ECWMAX_SHIFT; + brcmf_dbg(CONN, "ACI %d aifsn %d acm %d ecwmin %d ecwmax %d\n", + aci, aifsn, acm, ecwmin, ecwmax); + /* Default AC_VO will be the lowest ranking value */ + ranking_basis[aci] = aifsn + ecwmin + ecwmax; + /* Initialise priority starting at 0 (AC_BE) */ + aci_prio[aci] = 0; + + /* If ACM is set, STA can't use this AC as per 802.11. + * Change the ranking to BE + */ + if (aci != AC_BE && aci != AC_BK && acm == 1) + ranking_basis[aci] = ranking_basis[AC_BE]; + } + + /* Ranking method which works for AC priority + * swapping when values for cwmin, cwmax and aifsn are varied + * Compare each aci_prio against each other aci_prio + */ + for (aci = 0; aci < EDCF_AC_COUNT; aci++) { + for (index = 0; index < EDCF_AC_COUNT; index++) { + if (index != aci) { + /* Smaller ranking value has higher priority, + * so increment priority for each ACI which has + * a higher ranking value + */ + if (ranking_basis[aci] < ranking_basis[index]) + aci_prio[aci]++; + } + } + } + + /* By now, aci_prio[] will be in range of 0 to 3. + * Use ACI prio to get the new priority value for + * each 802.1d traffic type, in this range. + */ + + /* 802.1d 0,3 maps to BE */ + priority[0] = aci_prio[AC_BE]; + priority[3] = aci_prio[AC_BE]; + + /* 802.1d 1,2 maps to BK */ + priority[1] = aci_prio[AC_BK]; + priority[2] = aci_prio[AC_BK]; + + /* 802.1d 4,5 maps to VO */ + priority[4] = aci_prio[AC_VI]; + priority[5] = aci_prio[AC_VI]; + + /* 802.1d 6,7 maps to VO */ + priority[6] = aci_prio[AC_VO]; + priority[7] = aci_prio[AC_VO]; + + brcmf_dbg(CONN, "Adj prio BE 0->%d, BK 1->%d, BK 2->%d, BE 3->%d\n", + priority[0], priority[1], priority[2], priority[3]); + + brcmf_dbg(CONN, "Adj prio VI 4->%d, VI 5->%d, VO 6->%d, VO 7->%d\n", + priority[4], priority[5], priority[6], priority[7]); +} + static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp) { struct brcmf_cfg80211_assoc_ielen_le *assoc_info; struct brcmf_cfg80211_connect_info *conn_info = cfg_to_conn(cfg); + struct brcmf_cfg80211_edcf_acparam edcf_acparam_info[EDCF_AC_COUNT]; u32 req_len; u32 resp_len; s32 err = 0; @@ -5514,6 +5627,17 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, conn_info->resp_ie = kmemdup(cfg->extra_buf, conn_info->resp_ie_len, GFP_KERNEL); + + err = brcmf_fil_iovar_data_get(ifp, "wme_ac_sta", + edcf_acparam_info, + sizeof(edcf_acparam_info)); + if (err) { + brcmf_err("could not get wme_ac_sta (%d)\n", err); + return err; + } + + brcmf_wifi_prioritize_acparams(edcf_acparam_info, + cfg->ac_priority); } else { conn_info->resp_ie_len = 0; conn_info->resp_ie = NULL; @@ -5824,6 +5948,25 @@ static void brcmf_init_conf(struct brcmf_cfg80211_conf *conf) conf->retry_long = (u32)-1; } +static void brcmf_init_wmm_prio(u8 *priority) +{ + /* Initialize AC priority array to default + * 802.1d priority as per following table: + * 802.1d prio 0,3 maps to BE + * 802.1d prio 1,2 maps to BK + * 802.1d prio 4,5 maps to VI + * 802.1d prio 6,7 maps to VO + */ + priority[0] = AC_BE; + priority[3] = AC_BE; + priority[1] = AC_BK; + priority[2] = AC_BK; + priority[4] = AC_VI; + priority[5] = AC_VI; + priority[6] = AC_VO; + priority[7] = AC_VO; +} + static void brcmf_register_event_handlers(struct brcmf_cfg80211_info *cfg) { brcmf_fweh_register(cfg->pub, BRCMF_E_LINK, @@ -5919,6 +6062,7 @@ static s32 wl_init_priv(struct brcmf_cfg80211_info *cfg) mutex_init(&cfg->usr_sync); brcmf_init_escan(cfg); brcmf_init_conf(cfg->conf); + brcmf_init_wmm_prio(cfg->ac_priority); init_completion(&cfg->vif_disabled); return err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 606eaeb66fae6..5cedee086f4f6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -34,6 +34,23 @@ #define WL_ROAM_TRIGGER_LEVEL -75 #define WL_ROAM_DELTA 20 +/* WME Access Category Indices (ACIs) */ +#define AC_BE 0 /* Best Effort */ +#define AC_BK 1 /* Background */ +#define AC_VI 2 /* Video */ +#define AC_VO 3 /* Voice */ +#define EDCF_AC_COUNT 4 +#define MAX_8021D_PRIO 8 + +#define EDCF_ACI_MASK 0x60 +#define EDCF_ACI_SHIFT 5 +#define EDCF_ACM_MASK 0x10 +#define EDCF_ECWMIN_MASK 0x0f +#define EDCF_ECWMAX_SHIFT 4 +#define EDCF_AIFSN_MASK 0x0f +#define EDCF_AIFSN_MAX 15 +#define EDCF_ECWMAX_MASK 0xf0 + /* Keep BRCMF_ESCAN_BUF_SIZE below 64K (65536). Allocing over 64K can be * problematic on some systems and should be avoided. */ @@ -221,6 +238,12 @@ struct brcmf_cfg80211_assoc_ielen_le { __le32 resp_len; }; +struct brcmf_cfg80211_edcf_acparam { + u8 ACI; + u8 ECW; + u16 TXOP; /* stored in network order (ls octet first) */ +}; + /* dongle escan state */ enum wl_escan_state { WL_ESCAN_STATE_IDLE, @@ -342,6 +365,7 @@ struct brcmf_cfg80211_info { struct brcmf_assoclist_le assoclist; struct brcmf_cfg80211_wowl wowl; struct brcmf_pno_info *pno; + u8 ac_priority[MAX_8021D_PRIO]; }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 89267da22bf27..f415f208db40e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -80,4 +80,8 @@ void brcmf_release_module_param(struct brcmf_mp_device *module_param); /* Sets dongle media info (drv_version, mac address). */ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp); +u8 brcmf_map_prio_to_prec(void *cfg, u8 prio); + +u8 brcmf_map_prio_to_aci(void *cfg, u8 prio); + #endif /* BRCMFMAC_COMMON_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 21720f5f6a8a2..4725b2c4f4f2c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2118,6 +2118,7 @@ static int brcmf_fws_assign_htod(struct brcmf_fws_info *fws, struct sk_buff *p, int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) { + struct brcmf_pub *drvr = ifp->drvr; struct brcmf_fws_info *fws = drvr_to_fws(ifp->drvr); struct brcmf_skbuff_cb *skcb = brcmf_skbcb(skb); struct ethhdr *eh = (struct ethhdr *)(skb->data); @@ -2131,8 +2132,10 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) skcb->if_flags = 0; skcb->state = BRCMF_FWS_SKBSTATE_NEW; brcmf_skb_if_flags_set_field(skb, INDEX, ifp->ifidx); + + /* mapping from 802.1d priority to firmware fifo index */ if (!multicast) - fifo = brcmf_fws_prio2fifo[skb->priority]; + fifo = brcmf_map_prio_to_aci(drvr->config, skb->priority); brcmf_fws_lock(fws); if (fifo != BRCMF_FWS_FIFO_AC_BE && fifo < BRCMF_FWS_FIFO_BCMC) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 43b23cf9b4c40..37292badb0eeb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -328,15 +328,6 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, const struct firmware *code, void *nvram, u32 nvram_len); -/* - * Conversion of 802.1D priority to precedence level - */ -static uint prio2prec(u32 prio) -{ - return (prio == PRIO_8021D_NONE || prio == PRIO_8021D_BE) ? - (prio^2) : prio; -} - #ifdef DEBUG /* Device console log buffer state */ struct brcmf_console { @@ -2924,7 +2915,13 @@ static int brcmf_sdio_bus_txdata(struct device *dev, struct sk_buff *pkt) skb_push(pkt, bus->tx_hdrlen); /* precondition: IS_ALIGNED((unsigned long)(pkt->data), 2) */ - prec = prio2prec((pkt->priority & PRIOMASK)); + /* In WLAN, priority is always set by the AP using WMM parameters + * and this need not always follow the standard 802.1d priority. + * Based on AP WMM config, map from 802.1d priority to corresponding + * precedence level. + */ + prec = brcmf_map_prio_to_prec(bus_if->drvr->config, + (pkt->priority & PRIOMASK)); /* Check for existing queue, current flow-control, pending event, or pending clock */ From 1fede2092a7fe8d4b9ae916e73df0a73c9b4b943 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Tue, 17 Jul 2018 05:45:25 -0500 Subject: [PATCH 061/213] brcmfmac: allow GCI core enumuration GCI core is needed for ULP operation. Allow GCI core enumuration with below changes: - Allow GCI to be added to core list even when it doesn't have a wrapper. - Allow 8K address space size. - Don't overwrite the address value when an additional size descriptor is in place. Signed-off-by: Chi-Hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/chip.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index ba1d279497ea3..46200b657a788 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -785,7 +785,7 @@ static int brcmf_chip_dmp_get_regaddr(struct brcmf_chip_priv *ci, u32 *eromaddr, u32 *regbase, u32 *wrapbase) { u8 desc; - u32 val; + u32 val, szdesc; u8 mpnum = 0; u8 stype, sztype, wraptype; @@ -831,14 +831,15 @@ static int brcmf_chip_dmp_get_regaddr(struct brcmf_chip_priv *ci, u32 *eromaddr, /* next size descriptor can be skipped */ if (sztype == DMP_SLAVE_SIZE_DESC) { - val = brcmf_chip_dmp_get_desc(ci, eromaddr, NULL); + szdesc = brcmf_chip_dmp_get_desc(ci, eromaddr, NULL); /* skip upper size descriptor if present */ - if (val & DMP_DESC_ADDRSIZE_GT32) + if (szdesc & DMP_DESC_ADDRSIZE_GT32) brcmf_chip_dmp_get_desc(ci, eromaddr, NULL); } - /* only look for 4K register regions */ - if (sztype != DMP_SLAVE_SIZE_4K) + /* look for 4K or 8K register regions */ + if (sztype != DMP_SLAVE_SIZE_4K && + sztype != DMP_SLAVE_SIZE_8K) continue; stype = (val & DMP_SLAVE_TYPE) >> DMP_SLAVE_TYPE_S; @@ -895,7 +896,8 @@ int brcmf_chip_dmp_erom_scan(struct brcmf_chip_priv *ci) /* need core with ports */ if (nmw + nsw == 0 && - id != BCMA_CORE_PMU) + id != BCMA_CORE_PMU && + id != BCMA_CORE_GCI) continue; /* try to obtain register address info */ From 7310ad86a187f0610880ed4664f6862d6969d249 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Sun, 25 Mar 2018 22:39:50 -0500 Subject: [PATCH 062/213] brcmfmac: make firmware frameburst mode a module parameter With setting frameburst=1 in module parameters, firmware frameburst mode will be enabled. The feature can enable per-packet framebursting in firmware side and gets higher TX throughput in High Throughput(HT) mode. Signed-off-by: Wright Feng --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 7 +++++++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c | 5 +++++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h | 2 ++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h | 1 + 4 files changed, 15 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 7270cfbaf86b3..741b94faa9549 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6939,6 +6939,13 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) brcmf_configure_arp_nd_offload(ifp, true); + if (ifp->drvr->settings->frameburst) { + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_FAKEFRAG, 1); + if (err) + brcmf_info("setting frameburst mode failed\n"); + brcmf_dbg(INFO, "frameburst mode enabled\n"); + } + cfg->dongle_up = true; default_conf_out: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index df13bb7cb51e5..803264c500ece 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -83,6 +83,10 @@ static int brcmf_sdio_wq_highpri; module_param_named(sdio_wq_highpri, brcmf_sdio_wq_highpri, int, 0); MODULE_PARM_DESC(sdio_wq_highpri, "SDIO workqueue is set to high priority"); +static int brcmf_frameburst; +module_param_named(frameburst, brcmf_frameburst, int, 0); +MODULE_PARM_DESC(frameburst, "Enable firmware frameburst feature"); + #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -457,6 +461,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->roamoff = !!brcmf_roamoff; settings->eap_restrict = !!brcmf_eap_restrict; settings->sdio_wq_highpri = !!brcmf_sdio_wq_highpri; + settings->frameburst = !!brcmf_frameburst; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index f415f208db40e..544382623191c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -51,6 +51,7 @@ extern struct brcmf_mp_global_t brcmf_mp_global; * @roamoff: Firmware roaming off? * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds * @sdio_wq_highpri: Tasks submitted to SDIO workqueue will run immediately. + * @frameburst: Firmware frame burst mode. * @ignore_probe_fail: Ignore probe failure. * @country_codes: If available, pointer to struct for translating country codes * @bus: Bus specific platform data. Only SDIO at the mmoment. @@ -63,6 +64,7 @@ struct brcmf_mp_device { bool eap_restrict; int sdio_dpc_prio; bool sdio_wq_highpri; + bool frameburst; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; union { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h index 63b1287e2e6d4..b6b183b184135 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil.h @@ -80,6 +80,7 @@ #define BRCMF_C_SCB_DEAUTHENTICATE_FOR_REASON 201 #define BRCMF_C_SET_ASSOC_PREFER 205 #define BRCMF_C_GET_VALID_CHANNELS 217 +#define BRCMF_C_SET_FAKEFRAG 219 #define BRCMF_C_GET_KEY_PRIMARY 235 #define BRCMF_C_SET_KEY_PRIMARY 236 #define BRCMF_C_SET_SCAN_PASSIVE_TIME 258 From c91e5546c91e02e48b57327739fc2e3f4b599e07 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 24 Jul 2018 01:21:30 -0500 Subject: [PATCH 063/213] brcmfmac: set state of hanger slot to FREE when flushing PSQ When USB or SDIO device got abnormal bus disconnection, host driver tried to clean up the SKBs in PSQ and TXQ. The SKBs pointer in hanger linked to PSQ and TSQ, so we should set the state of skb hanger slot to BRCMF_FWS_HANGER_ITEM_STATE_FREE before freeing skb. In brcmf_fws_bus_txq_cleanup it already set BRCMF_FWS_HANGER_ITEM_STATE_FREE before freeing SKB, therefore we add this in brcmf_fws_psq_flush to avoid following warning message. [ 1580.012880] ------------ [ cut here ]------------ [ 1580.017550] WARNING: CPU: 3 PID: 3065 at drivers/net/wireless/broadcom/brcm80211/brcmutil/utils.c:49 brcmu_pkt_buf_free_skb+0x21/0x30 [brcmutil] [ 1580.184017] Call Trace: [ 1580.186514] brcmf_fws_cleanup+0x14e/0x190 [brcmfmac] [ 1580.191594] brcmf_fws_del_interface+0x70/0x90 [brcmfmac] [ 1580.197029] brcmf_proto_bcdc_del_if+0xe/0x10 [brcmfmac] [ 1580.202418] brcmf_remove_interface+0x69/0x190 [brcmfmac] [ 1580.207888] brcmf_detach+0x90/0xe0 [brcmfmac] [ 1580.212385] brcmf_usb_disconnect+0x76/0xb0 [brcmfmac] [ 1580.217557] usb_unbind_interface+0x72/0x260 [ 1580.221857] device_release_driver_internal+0x141/0x200 [ 1580.227152] device_release_driver+0x12/0x20 [ 1580.231460] bus_remove_device+0xfd/0x170 [ 1580.235504] device_del+0x1d9/0x300 [ 1580.239041] usb_disable_device+0x9e/0x270 [ 1580.243160] usb_disconnect+0x94/0x270 [ 1580.246980] hub_event+0x76d/0x13b0 [ 1580.250499] process_one_work+0x144/0x360 [ 1580.254564] worker_thread+0x4d/0x3c0 [ 1580.258247] kthread+0x109/0x140 [ 1580.261515] ? rescuer_thread+0x340/0x340 [ 1580.265543] ? kthread_park+0x60/0x60 [ 1580.269237] ? SyS_exit_group+0x14/0x20 [ 1580.273118] ret_from_fork+0x25/0x30 [ 1580.300446] ------------ [ cut here ]------------ Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 4725b2c4f4f2c..9abdbe6d91f11 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -583,15 +583,21 @@ static bool brcmf_fws_ifidx_match(struct sk_buff *skb, void *arg) static void brcmf_fws_psq_flush(struct brcmf_fws_info *fws, struct pktq *q, int ifidx) { + struct brcmf_fws_hanger_item *hi; bool (*matchfn)(struct sk_buff *, void *) = NULL; struct sk_buff *skb; int prec; + u32 hslot; if (ifidx != -1) matchfn = brcmf_fws_ifidx_match; for (prec = 0; prec < q->num_prec; prec++) { skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); while (skb) { + hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT); + hi = &fws->hanger.items[hslot]; + WARN_ON(skb != hi->pkt); + hi->state = BRCMF_FWS_HANGER_ITEM_STATE_FREE; brcmu_pkt_buf_free_skb(skb); skb = brcmu_pktq_pdeq_match(q, prec, matchfn, &ifidx); } From 8a6f9dd49216627a62586429af972c662f7688c1 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 26 Jun 2018 22:19:31 -0500 Subject: [PATCH 064/213] brcmfmac: add creating station interface support With RSDB device, it is able to control two station interfaces concurrently. So we add creating station interface support and allow user to create it via cfg80211. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 80 ++++++++++++++++--- .../broadcom/brcm80211/brcmfmac/core.h | 1 + 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 741b94faa9549..f8ad354a63d6f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -239,6 +239,19 @@ struct parsed_vndr_ies { struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT]; }; +#define WL_INTERFACE_MAC_DONT_USE 0x0 +#define WL_INTERFACE_MAC_USE 0x2 + +#define WL_INTERFACE_CREATE_STA 0x0 +#define WL_INTERFACE_CREATE_AP 0x1 + +struct wl_interface_create { + u16 ver; + u32 flags; + u8 mac_addr[ETH_ALEN]; + u8 pad[13]; +}; + static u8 nl80211_band_to_fwil(enum nl80211_band band) { switch (band) { @@ -547,6 +560,42 @@ static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr) return -ENOMEM; } +static void brcmf_set_sta_iface_macaddr(struct brcmf_if *ifp, + struct wl_interface_create *iface) +{ + u8 mac_idx = ifp->drvr->sta_mac_idx; + + /* set difference MAC address with locally administered bit */ + iface->flags |= WL_INTERFACE_MAC_USE; + memcpy(iface->mac_addr, ifp->mac_addr, ETH_ALEN); + iface->mac_addr[0] |= 0x02; + iface->mac_addr[3] ^= mac_idx ? 0xC0 : 0xA0; + mac_idx++; + mac_idx = mac_idx % 2; + ifp->drvr->sta_mac_idx = mac_idx; +} + +static int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) +{ + struct wl_interface_create iface; + int err; + + memset(&iface, 0, sizeof(iface)); + + iface.ver = 0; + iface.flags = WL_INTERFACE_CREATE_STA; + if (!is_zero_ether_addr(macaddr)) { + /* set MAC address in cfg80211 params */ + memcpy(iface.mac_addr, macaddr, ETH_ALEN); + } else { + brcmf_set_sta_iface_macaddr(ifp, &iface); + } + + err = brcmf_fil_iovar_data_get(ifp, "interface_create", &iface, + sizeof(iface)); + return err; +} + static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) { struct brcmf_mbss_ssid_le mbss_ssid_le; @@ -571,33 +620,40 @@ static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) } /** - * brcmf_ap_add_vif() - create a new AP virtual interface for multiple BSS + * brcmf_ap_add_vif() - create a new AP or STA virtual interface * * @wiphy: wiphy device of new interface. * @name: name of the new interface. - * @params: contains mac address for AP device. + * @params: contains mac address for AP or STA device. */ static -struct wireless_dev *brcmf_ap_add_vif(struct wiphy *wiphy, const char *name, - struct vif_params *params) +struct wireless_dev *brcmf_apsta_add_vif(struct wiphy *wiphy, const char *name, + struct vif_params *params, + enum nl80211_iftype type) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); struct brcmf_cfg80211_vif *vif; int err; + if (type != NL80211_IFTYPE_STATION && type != NL80211_IFTYPE_AP) + return ERR_PTR(-EINVAL); + if (brcmf_cfg80211_vif_event_armed(cfg)) return ERR_PTR(-EBUSY); brcmf_dbg(INFO, "Adding vif \"%s\"\n", name); - vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_AP); + vif = brcmf_alloc_vif(cfg, type); if (IS_ERR(vif)) return (struct wireless_dev *)vif; brcmf_cfg80211_arm_vif_event(cfg, vif); - err = brcmf_cfg80211_request_ap_if(ifp); + if (type == NL80211_IFTYPE_STATION) + err = brcmf_cfg80211_request_sta_if(ifp, params->macaddr); + else + err = brcmf_cfg80211_request_ap_if(ifp); if (err) { brcmf_cfg80211_arm_vif_event(cfg, NULL); goto fail; @@ -666,14 +722,14 @@ static struct wireless_dev *brcmf_cfg80211_add_iface(struct wiphy *wiphy, } switch (type) { case NL80211_IFTYPE_ADHOC: - case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_WDS: case NL80211_IFTYPE_MONITOR: case NL80211_IFTYPE_MESH_POINT: return ERR_PTR(-EOPNOTSUPP); + case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP: - wdev = brcmf_ap_add_vif(wiphy, name, params); + wdev = brcmf_apsta_add_vif(wiphy, name, params, type); break; case NL80211_IFTYPE_P2P_CLIENT: case NL80211_IFTYPE_P2P_GO: @@ -791,8 +847,8 @@ s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, return err; } -static int brcmf_cfg80211_del_ap_iface(struct wiphy *wiphy, - struct wireless_dev *wdev) +static int brcmf_cfg80211_del_apsta_iface(struct wiphy *wiphy, + struct wireless_dev *wdev) { struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); struct net_device *ndev = wdev->netdev; @@ -848,14 +904,14 @@ int brcmf_cfg80211_del_iface(struct wiphy *wiphy, struct wireless_dev *wdev) switch (wdev->iftype) { case NL80211_IFTYPE_ADHOC: - case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_WDS: case NL80211_IFTYPE_MONITOR: case NL80211_IFTYPE_MESH_POINT: return -EOPNOTSUPP; + case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP: - return brcmf_cfg80211_del_ap_iface(wiphy, wdev); + return brcmf_cfg80211_del_apsta_iface(wiphy, wdev); case NL80211_IFTYPE_P2P_CLIENT: case NL80211_IFTYPE_P2P_GO: case NL80211_IFTYPE_P2P_DEVICE: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index e38f78e4d92c8..c2ca3b3d2da7c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -145,6 +145,7 @@ struct brcmf_pub { u8 clmver[BRCMF_DCMD_SMLEN]; struct brcmf_pkt_filter_enable_le pkt_filter[MAX_PKT_FILTER_COUNT]; + u8 sta_mac_idx; }; From f492d18110c12fc9c8286ea63292849af75bb1d7 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 28 Jun 2018 02:47:14 -0500 Subject: [PATCH 065/213] brcmfmac: add RSDB condition when setting interface combinations With firmware RSDB feature 1. The maximum support interface is four. 2. The maximum difference channel is two. 3. The maximum interfaces of {station/p2p client/AP} are two. 4. The maximum interface of p2p device is one. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index f8ad354a63d6f..4adf6f64f0b3b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6655,6 +6655,9 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #STA <= 1, #AP <= 1, channels = 1, 2 total * #AP <= 4, matching BI, channels = 1, 4 total * + * no p2p and rsdb: + * #STA <= 2, #AP <= 2, channels = 2, 4 total + * * p2p, no mchan, and mbss: * * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 1, channels = 1, 3 total @@ -6666,6 +6669,10 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 1, channels = 2, 3 total * #STA <= 1, #P2P-DEV <= 1, #AP <= 1, #P2P-CL <= 1, channels = 1, 4 total * #AP <= 4, matching BI, channels = 1, 4 total + * + * p2p, rsdb, and no mbss: + * #STA <= 2, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, + * channels = 2, 4 total */ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) { @@ -6673,13 +6680,14 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) struct ieee80211_iface_limit *c0_limits = NULL; struct ieee80211_iface_limit *p2p_limits = NULL; struct ieee80211_iface_limit *mbss_limits = NULL; - bool mbss, p2p; + bool mbss, p2p, rsdb; int i, c, n_combos; mbss = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS); p2p = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_P2P); + rsdb = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB); - n_combos = 1 + !!p2p + !!mbss; + n_combos = 1 + !!(p2p && !rsdb) + !!mbss; combo = kcalloc(n_combos, sizeof(*combo), GFP_KERNEL); if (!combo) goto err; @@ -6690,16 +6698,36 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) c = 0; i = 0; - c0_limits = kcalloc(p2p ? 3 : 2, sizeof(*c0_limits), GFP_KERNEL); + if (p2p && rsdb) + c0_limits = kcalloc(4, sizeof(*c0_limits), GFP_KERNEL); + else if (p2p) + c0_limits = kcalloc(3, sizeof(*c0_limits), GFP_KERNEL); + else + c0_limits = kcalloc(2, sizeof(*c0_limits), GFP_KERNEL); if (!c0_limits) goto err; - c0_limits[i].max = 1; - c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); - if (p2p) { + if (p2p && rsdb) { + combo[c].num_different_channels = 2; + wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO) | + BIT(NL80211_IFTYPE_P2P_DEVICE); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_DEVICE); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) | + BIT(NL80211_IFTYPE_P2P_GO); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = 5; + } else if (p2p) { if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MCHAN)) combo[c].num_different_channels = 2; else combo[c].num_different_channels = 1; + c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO) | BIT(NL80211_IFTYPE_P2P_DEVICE); @@ -6708,16 +6736,26 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO); + combo[c].max_interfaces = i; + } else if (rsdb) { + combo[c].num_different_channels = 2; + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 2; + c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = 3; } else { combo[c].num_different_channels = 1; c0_limits[i].max = 1; + c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); + c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); + combo[c].max_interfaces = i; } - combo[c].max_interfaces = i; combo[c].n_limits = i; combo[c].limits = c0_limits; - if (p2p) { + if (p2p && !rsdb) { c++; i = 0; p2p_limits = kcalloc(4, sizeof(*p2p_limits), GFP_KERNEL); From e6da116bf28f077988befe9c647aaf828960fe6a Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Mon, 2 Jul 2018 05:04:23 -0500 Subject: [PATCH 066/213] brcmfmac: not set mbss in vif if firmware does not support MBSS With RSDB mode, FMAC and firmware are able to create 2 or more AP, so we should not set mbss in vif structure if firmware does not support MBSS feature. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 4adf6f64f0b3b..187c12d03ec5e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5395,6 +5395,7 @@ struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif_walk; struct brcmf_cfg80211_vif *vif; bool mbss; + struct brcmf_if *ifp = brcmf_get_ifp(cfg->pub, 0); brcmf_dbg(TRACE, "allocating virtual interface (size=%zu)\n", sizeof(*vif)); @@ -5407,7 +5408,8 @@ struct brcmf_cfg80211_vif *brcmf_alloc_vif(struct brcmf_cfg80211_info *cfg, brcmf_init_prof(&vif->profile); - if (type == NL80211_IFTYPE_AP) { + if (type == NL80211_IFTYPE_AP && + brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) { mbss = false; list_for_each_entry(vif_walk, &cfg->vif_list, list) { if (vif_walk->wdev.iftype == NL80211_IFTYPE_AP) { From 88062255b7e64227a2f46359e7592e2f461cd610 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 29 Jun 2018 02:07:02 -0500 Subject: [PATCH 067/213] brcmfmac: support the second p2p connection With RSDB feature, firmware is able to support two P2P-AGO or two P2P-GC at the same time. So we add the second p2p connection type to maps to driver's second P2P connection bsscfg. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/p2p.c | 56 ++++++++++++++++--- .../broadcom/brcm80211/brcmfmac/p2p.h | 9 ++- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index 52016373dce7a..c8ad5c297468c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -28,6 +28,7 @@ #include "fwil_types.h" #include "p2p.h" #include "cfg80211.h" +#include "feature.h" /* parameters used for p2p escan */ #define P2PAPI_SCAN_NPROBES 1 @@ -498,9 +499,13 @@ static void brcmf_p2p_generate_bss_mac(struct brcmf_p2p_info *p2p, u8 *dev_addr) * BSSCFGs need to simultaneously co-exist, then this address must be * different from the P2P Device Address, but also locally administered. */ - memcpy(p2p->int_addr, p2p->dev_addr, ETH_ALEN); - p2p->int_addr[0] |= 0x02; - p2p->int_addr[4] ^= 0x80; + memcpy(p2p->conn_int_addr, p2p->dev_addr, ETH_ALEN); + p2p->conn_int_addr[0] |= 0x02; + p2p->conn_int_addr[4] ^= 0x80; + + memcpy(p2p->conn2_int_addr, p2p->dev_addr, ETH_ALEN); + p2p->conn2_int_addr[0] |= 0x02; + p2p->conn2_int_addr[4] ^= 0x90; } /** @@ -2019,7 +2024,7 @@ int brcmf_p2p_ifchange(struct brcmf_cfg80211_info *cfg, if_request.type = cpu_to_le16((u16)if_type); if_request.chspec = cpu_to_le16(chanspec); - memcpy(if_request.addr, p2p->int_addr, sizeof(if_request.addr)); + memcpy(if_request.addr, p2p->conn_int_addr, sizeof(if_request.addr)); brcmf_cfg80211_arm_vif_event(cfg, vif); err = brcmf_fil_iovar_data_set(vif->ifp, "p2p_ifupd", &if_request, @@ -2165,6 +2170,27 @@ static struct wireless_dev *brcmf_p2p_create_p2pdev(struct brcmf_p2p_info *p2p, return ERR_PTR(err); } +int brcmf_p2p_get_conn_idx(struct brcmf_cfg80211_info *cfg) +{ + int i; + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); + + if (!ifp) + return -ENODEV; + + for (i = P2PAPI_BSSCFG_CONNECTION; i < P2PAPI_BSSCFG_MAX; i++) { + if (!cfg->p2p.bss_idx[i].vif) { + if (i == P2PAPI_BSSCFG_CONNECTION2 && + !(brcmf_feat_is_enabled(ifp, BRCMF_FEAT_RSDB))) { + brcmf_err("Multi p2p not supported"); + return -EIO; + } + return i; + } + } + return -EIO; +} + /** * brcmf_p2p_add_vif() - create a new P2P virtual interface. * @@ -2183,7 +2209,9 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); struct brcmf_cfg80211_vif *vif; enum brcmf_fil_p2p_if_types iftype; - int err; + int err = 0; + int connidx; + u8 *p2p_intf_addr; if (brcmf_cfg80211_vif_event_armed(cfg)) return ERR_PTR(-EBUSY); @@ -2209,9 +2237,21 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, return (struct wireless_dev *)vif; brcmf_cfg80211_arm_vif_event(cfg, vif); - err = brcmf_p2p_request_p2p_if(&cfg->p2p, ifp, cfg->p2p.int_addr, - iftype); + connidx = brcmf_p2p_get_conn_idx(cfg); + + if (connidx == P2PAPI_BSSCFG_CONNECTION) + p2p_intf_addr = cfg->p2p.conn_int_addr; + else if (connidx == P2PAPI_BSSCFG_CONNECTION2) + p2p_intf_addr = cfg->p2p.conn2_int_addr; + else + err = -EINVAL; + + if (!err) + err = brcmf_p2p_request_p2p_if(&cfg->p2p, ifp, + p2p_intf_addr, iftype); + if (err) { + brcmf_err("request p2p interface failed\n"); brcmf_cfg80211_arm_vif_event(cfg, NULL); goto fail; } @@ -2243,7 +2283,7 @@ struct wireless_dev *brcmf_p2p_add_vif(struct wiphy *wiphy, const char *name, goto fail; } - cfg->p2p.bss_idx[P2PAPI_BSSCFG_CONNECTION].vif = vif; + cfg->p2p.bss_idx[connidx].vif = vif; /* Disable firmware roaming for P2P interface */ brcmf_fil_iovar_int_set(ifp, "roam_off", 1); if (iftype == BRCMF_FIL_P2P_IF_GO) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h index 39f0d02180882..fe83962eead30 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h @@ -25,13 +25,15 @@ struct brcmf_cfg80211_info; * * @P2PAPI_BSSCFG_PRIMARY: maps to driver's primary bsscfg. * @P2PAPI_BSSCFG_DEVICE: maps to driver's P2P device discovery bsscfg. - * @P2PAPI_BSSCFG_CONNECTION: maps to driver's P2P connection bsscfg. + * @P2PAPI_BSSCFG_CONNECTION: maps to driver's 1st P2P connection bsscfg. + * @P2PAPI_BSSCFG_CONNECTION2: maps to driver's 2nd P2P connection bsscfg. * @P2PAPI_BSSCFG_MAX: used for range checking. */ enum p2p_bss_type { P2PAPI_BSSCFG_PRIMARY, /* maps to driver's primary bsscfg */ P2PAPI_BSSCFG_DEVICE, /* maps to driver's P2P device discovery bsscfg */ - P2PAPI_BSSCFG_CONNECTION, /* maps to driver's P2P connection bsscfg */ + P2PAPI_BSSCFG_CONNECTION, /* driver's 1st P2P connection bsscfg */ + P2PAPI_BSSCFG_CONNECTION2, /* driver's 2nd P2P connection bsscfg */ P2PAPI_BSSCFG_MAX }; @@ -130,7 +132,8 @@ struct brcmf_p2p_info { struct brcmf_cfg80211_info *cfg; unsigned long status; u8 dev_addr[ETH_ALEN]; - u8 int_addr[ETH_ALEN]; + u8 conn_int_addr[ETH_ALEN]; + u8 conn2_int_addr[ETH_ALEN]; struct p2p_bss bss_idx[P2PAPI_BSSCFG_MAX]; struct timer_list listen_timer; u8 listen_channel; From cb4822ccd79b9ddabbc5b3fedb07375d2ee0a844 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 2 Aug 2018 21:22:38 -0500 Subject: [PATCH 068/213] brcmfmac: Add support for BCM4359 SDIO chipset BCM4359 is a 2x2 802.11 abgn+ac Dual-Band HT80 combo chip and it supports Real Simultaneous Dual Band feature. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 1 + drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 1 + drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 2 ++ include/linux/mmc/sdio_ids.h | 1 + 4 files changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 88b7410851c6f..ee918f8f65118 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -1115,6 +1115,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = { BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_43455), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4354), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4356), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4359), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_4373), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_43012), { /* end: all zeroes */ } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 46200b657a788..dda2c995f16cf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -1370,6 +1370,7 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) addr = CORE_CC_REG(base, sr_control0); reg = chip->ops->read32(chip->ctx, addr); return (reg & sr_eng_en) != 0; + case BRCM_CC_4359_CHIP_ID: case CY_CC_43012_CHIP_ID: addr = CORE_CC_REG(pmu->base, retention_ctl); reg = chip->ops->read32(chip->ctx, addr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 37292badb0eeb..33c92b113eda9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -621,6 +621,7 @@ BRCMF_FW_NVRAM_DEF(43430A1, "brcmfmac43430-sdio.bin", "brcmfmac43430-sdio.txt"); BRCMF_FW_NVRAM_DEF(43455, "brcmfmac43455-sdio.bin", "brcmfmac43455-sdio.txt"); BRCMF_FW_NVRAM_DEF(4354, "brcmfmac4354-sdio.bin", "brcmfmac4354-sdio.txt"); BRCMF_FW_NVRAM_DEF(4356, "brcmfmac4356-sdio.bin", "brcmfmac4356-sdio.txt"); +BRCMF_FW_NVRAM_DEF(4359, "brcmfmac4359-sdio.bin", "brcmfmac4359-sdio.txt"); BRCMF_FW_NVRAM_DEF(4373, "brcmfmac4373-sdio.bin", "brcmfmac4373-sdio.txt"); BRCMF_FW_NVRAM_DEF(43012, "brcmfmac43012-sdio.bin", "brcmfmac43012-sdio.txt"); @@ -642,6 +643,7 @@ static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4345_CHIP_ID, 0xFFFFFFC0, 43455), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4354_CHIP_ID, 0xFFFFFFFF, 4354), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4356_CHIP_ID, 0xFFFFFFFF, 4356), + BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359), BRCMF_FW_NVRAM_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373), BRCMF_FW_NVRAM_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012) }; diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index 4332199c71c25..ae4ba5d6244e9 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -41,6 +41,7 @@ #define SDIO_DEVICE_ID_BROADCOM_43455 0xa9bf #define SDIO_DEVICE_ID_BROADCOM_4354 0x4354 #define SDIO_DEVICE_ID_BROADCOM_4356 0x4356 +#define SDIO_DEVICE_ID_BROADCOM_4359 0x4355 #define SDIO_DEVICE_ID_CYPRESS_4373 0x4373 #define SDIO_DEVICE_ID_CYPRESS_43012 43012 From 3194f2ab66617cceda5d86423083397bac480b9b Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 2 Aug 2018 15:13:13 +0800 Subject: [PATCH 069/213] nl80211: add NL80211_ATTR_IFINDEX to port authorized event Add NL80211_ATTR_IFINDEX attribute to port authorized event to indicate the operating interface of the device. Signed-off-by: Chung-Hsien Hsu --- net/wireless/nl80211.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 5075fd293febb..c149f3ffcd2bf 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -14560,7 +14560,8 @@ void nl80211_send_port_authorized(struct cfg80211_registered_device *rdev, return; } - if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, bssid)) + if (nla_put_u32(msg, NL80211_ATTR_IFINDEX, netdev->ifindex) || + nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, bssid)) goto nla_put_failure; genlmsg_end(msg, hdr); From 75b74f791ea52c5f3e56109196a460f7e8f4ea3c Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 2 Aug 2018 15:35:03 +0800 Subject: [PATCH 070/213] brcmfmac: send port authorized event for 802.1X 4-way handshake offload With 4-way handshake offload for 802.1X authentication, a port authorized event should be sent to user space after the completion of 4-way handshake. It is used to indicate that a connection is authorized and 802.1X authentication is no longer required. Signed-off-by: Chung-Hsien Hsu --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 187c12d03ec5e..eba9fdd0936f7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5448,10 +5448,13 @@ static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, u32 event = e->event_code; u32 status = e->status; - if (vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_PSK && - event == BRCMF_E_PSK_SUP && - status == BRCMF_E_STATUS_FWSUP_COMPLETED) + if (event == BRCMF_E_PSK_SUP && + status == BRCMF_E_STATUS_FWSUP_COMPLETED) { set_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state); + if (vif->profile.use_fwsup == BRCMF_PROFILE_FWSUP_1X) + return true; + } + if (event == BRCMF_E_SET_SSID && status == BRCMF_E_STATUS_SUCCESS) { brcmf_dbg(CONN, "Processing set ssid\n"); memcpy(vif->profile.bssid, e->addr, ETH_ALEN); @@ -5462,11 +5465,10 @@ static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, } if (test_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state) && - test_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state)) { - clear_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, &vif->sme_state); - clear_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state); + test_and_clear_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, + &vif->sme_state)) return true; - } + return false; } @@ -5807,6 +5809,13 @@ brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg, brcmf_dbg(CONN, "Report connect result - connection %s\n", completed ? "succeeded" : "failed"); } + + if (test_and_clear_bit(BRCMF_VIF_STATUS_EAP_SUCCESS, + &ifp->vif->sme_state) && + profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) { + cfg80211_port_authorized(ndev, profile->bssid, GFP_KERNEL); + brcmf_dbg(CONN, "Report port authorized\n"); + } brcmf_dbg(TRACE, "Exit\n"); return 0; } From 85c39dc96cffd0c6e0c759306821d8c31cb23774 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 2 Aug 2018 16:05:39 +0800 Subject: [PATCH 071/213] brcmfmac: send port authorized event for FT-802.1X With FT-802.1X, driver should send a port authorized event right after sending a roamed event. It is used to indicate that a new AP is already authorized so 802.1X is not required. Signed-off-by: Chung-Hsien Hsu --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 8 ++++++++ .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h | 1 + 2 files changed, 9 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index eba9fdd0936f7..6d8f83a227601 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1774,6 +1774,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) u16 count; profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; + profile->is_ft = false; if (!sme->crypto.n_akm_suites) return 0; @@ -1818,11 +1819,13 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) break; case WLAN_AKM_SUITE_FT_8021X: val = WPA2_AUTH_UNSPECIFIED | WPA2_AUTH_FT; + profile->is_ft = true; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; break; case WLAN_AKM_SUITE_FT_PSK: val = WPA2_AUTH_PSK | WPA2_AUTH_FT; + profile->is_ft = true; break; default: brcmf_err("invalid cipher group (%d)\n", @@ -5771,6 +5774,11 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); brcmf_dbg(CONN, "Report roaming result\n"); + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) { + cfg80211_port_authorized(ndev, profile->bssid, GFP_KERNEL); + brcmf_dbg(CONN, "Report port authorized\n"); + } + set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); brcmf_dbg(TRACE, "Exit\n"); return err; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 5cedee086f4f6..3ead7e1898377 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -157,6 +157,7 @@ struct brcmf_cfg80211_profile { struct brcmf_cfg80211_security sec; struct brcmf_wsec_key key[BRCMF_MAX_DEFAULT_KEYS]; enum brcmf_profile_fwsup use_fwsup; + bool is_ft; }; /** From 2d5fd72db5b3ff7408ab574364a919ca60099ce5 Mon Sep 17 00:00:00 2001 From: Praveen Babu C Date: Mon, 6 Aug 2018 16:15:23 +0530 Subject: [PATCH 072/213] brcmfmac: Support DS1 TX Exit in FMAC List of issues fixed: 1. Sent Tx Control frame only after firmware redownload complete (check F2 Ready before sending Tx Control frame to Firmware) 2. intermittent High DS1 TX Exit latency time (almost 3sec) ==> This is fixed by skipping host Mailbox interrupt Multiple times (ulp state mechanism) 3. RX GlOM save/restore in Firmware 4. Add ULP event enable & event_msgs_ext iovar configuration in FMAC 5. Add ULP_EVENT_RECV state machine for sbwad support 6. Support 2 Byte Shared memory read for DS1 Exit HUDI implementation Signed-off-by: Praveen Babu Chandran --- .../broadcom/brcm80211/brcmfmac/common.c | 40 ++++ .../broadcom/brcm80211/brcmfmac/debug.h | 1 + .../broadcom/brcm80211/brcmfmac/fweh.h | 27 ++- .../broadcom/brcm80211/brcmfmac/sdio.c | 182 +++++++++++++----- .../broadcom/brcm80211/brcmfmac/sdio.h | 80 ++++++-- 5 files changed, 265 insertions(+), 65 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 803264c500ece..f65f2a5df97de 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -30,6 +30,8 @@ #include "common.h" #include "of.h" #include "firmware.h" +#include "fweh.h" +#include MODULE_AUTHOR("Broadcom Corporation"); MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver."); @@ -251,6 +253,9 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) char *clmver; char *ptr; s32 err; + struct eventmsgs_ext *eventmask_msg = NULL; + u8 msglen; + struct brcmf_bus *bus = ifp->drvr->bus_if; /* retrieve mac addresses */ err = brcmf_fil_iovar_data_get(ifp, "cur_etheraddr", ifp->mac_addr, @@ -357,6 +362,41 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) goto done; } + /* Enable event_msg_ext specific to 43012 chip */ + if (bus->chip == CY_CC_43012_CHIP_ID) { + /* Program event_msg_ext to support event larger than 128 */ + msglen = (roundup(BRCMF_E_LAST, NBBY) / NBBY) + + EVENTMSGS_EXT_STRUCT_SIZE; + /* Allocate buffer for eventmask_msg */ + eventmask_msg = kzalloc(msglen, GFP_KERNEL); + if (!eventmask_msg) { + err = -ENOMEM; + goto done; + } + + /* Read the current programmed event_msgs_ext */ + eventmask_msg->ver = EVENTMSGS_VER; + eventmask_msg->len = roundup(BRCMF_E_LAST, NBBY) / NBBY; + err = brcmf_fil_iovar_data_get(ifp, "event_msgs_ext", + eventmask_msg, + msglen); + + /* Enable ULP event */ + brcmf_dbg(EVENT, "enable event ULP\n"); + setbit(eventmask_msg->mask, BRCMF_E_ULP); + + /* Write updated Event mask */ + eventmask_msg->ver = EVENTMSGS_VER; + eventmask_msg->command = EVENTMSGS_SET_MASK; + eventmask_msg->len = (roundup(BRCMF_E_LAST, NBBY) / NBBY); + + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmask_msg, msglen); + if (err) { + brcmf_err("Set event_msgs_ext error (%d)\n", err); + goto done; + } + } /* Setup default scan channel time */ err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_SCAN_CHANNEL_TIME, BRCMF_DEFAULT_SCAN_CHANNEL_TIME); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 35919d9e8e139..23a290f13a889 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -40,6 +40,7 @@ #define BRCMF_MSGBUF_VAL 0x00040000 #define BRCMF_PCIE_VAL 0x00080000 #define BRCMF_FWCON_VAL 0x00100000 +#define BRCMF_ULP_VAL 0x00200000 /* set default print format */ #undef pr_fmt diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index 816f80ea925b1..47bd34e901a79 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -101,7 +101,8 @@ struct brcmf_cfg80211_info; BRCMF_ENUM_DEF(FIFO_CREDIT_MAP, 74) \ BRCMF_ENUM_DEF(ACTION_FRAME_RX, 75) \ BRCMF_ENUM_DEF(TDLS_PEER_EVENT, 92) \ - BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) + BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) \ + BRCMF_ENUM_DEF(ULP, 146) #define BRCMF_ENUM_DEF(id, val) \ BRCMF_E_##id = (val), @@ -113,7 +114,7 @@ enum brcmf_fweh_event_code { * minimum length check in device firmware so it is * hard-coded here. */ - BRCMF_E_LAST = 139 + BRCMF_E_LAST = 147 }; #undef BRCMF_ENUM_DEF @@ -294,6 +295,28 @@ struct brcmf_if_event { u8 role; }; +enum event_msgs_ext_command { + EVENTMSGS_NONE = 0, + EVENTMSGS_SET_BIT = 1, + EVENTMSGS_RESET_BIT = 2, + EVENTMSGS_SET_MASK = 3 +}; + +#define EVENTMSGS_VER 1 +#define EVENTMSGS_EXT_STRUCT_SIZE offsetof(struct eventmsgs_ext, mask[0]) + +/* len- for SET it would be mask size from the application to the firmware */ +/* for GET it would be actual firmware mask size */ +/* maxgetsize - is only used for GET. indicate max mask size that the */ +/* application can read from the firmware */ +struct eventmsgs_ext { + u8 ver; + u8 command; + u8 len; + u8 maxgetsize; + u8 mask[1]; +}; + typedef int (*brcmf_fweh_handler_t)(struct brcmf_if *ifp, const struct brcmf_event_msg *evtmsg, void *data); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 33c92b113eda9..886a66978bdec 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -327,6 +327,10 @@ struct rte_console { static void brcmf_sdio_firmware_callback(struct device *dev, int err, const struct firmware *code, void *nvram, u32 nvram_len); +static int brcmf_sdio_f2_ready(struct brcmf_sdio *bus); +static int brcmf_ulp_event_notify(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *data); #ifdef DEBUG /* Device console log buffer state */ @@ -1092,7 +1096,7 @@ static void brcmf_sdio_get_console_addr(struct brcmf_sdio *bus) } #endif /* DEBUG */ -static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) +static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus, u32 *hmbd) { u32 intstatus = 0; u32 hmb_data; @@ -1170,6 +1174,9 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus) HMB_DATA_FCDATA_MASK | HMB_DATA_VERSION_MASK)) brcmf_err("Unknown mailbox data content: 0x%02x\n", hmb_data); + /* Populate hmb_data if argument is passed for DS1 check later */ + if (hmbd) + *hmbd = hmb_data; return intstatus; } @@ -2561,19 +2568,20 @@ brcmf_sdio_ulp_preinit(struct device *dev) struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; struct brcmf_if *ifp = bus_if->drvr->iflist[0]; - brcmf_dbg(TRACE, "Enter\n"); + brcmf_dbg(ULP, "Enter\n"); /* Query ulp_sdioctrl iovar to get the ULP related SHM offsets */ - brcmf_fil_iovar_data_get(ifp, "ulp_sdioctrl", &sdiodev->shm_ulp, - sizeof(sdiodev->shm_ulp)); + brcmf_fil_iovar_data_get(ifp, "ulp_sdioctrl", + &sdiodev->fmac_ulp.ulp_shm_offset, + sizeof(sdiodev->fmac_ulp.ulp_shm_offset)); sdiodev->ulp = false; - brcmf_dbg(TRACE, "m_ulp_ctrl_sdio[%x] m_ulp_wakeevt_ind [%x]\n", - M_DS1_CTRL_SDIO(sdiodev->shm_ulp), - M_WAKEEVENT_IND(sdiodev->shm_ulp)); - brcmf_dbg(TRACE, "m_ulp_wakeind [%x]\n", - M_ULP_WAKE_IND(sdiodev->shm_ulp)); + brcmf_dbg(ULP, "m_ulp_ctrl_sdio[%x] m_ulp_wakeevt_ind [%x]\n", + M_DS1_CTRL_SDIO(sdiodev->fmac_ulp), + M_WAKEEVENT_IND(sdiodev->fmac_ulp)); + brcmf_dbg(ULP, "m_ulp_wakeind [%x]\n", + M_ULP_WAKE_IND(sdiodev->fmac_ulp)); } /* Reinitialize ARM because In DS1 mode ARM got off */ @@ -2601,41 +2609,68 @@ brcmf_sdio_ulp_reinit_fw(struct brcmf_sdio *bus) /* Check if device is in DS1 mode and handshake with ULP UCODE */ static bool -brcmf_sdio_ulp_pre_redownload_check(struct brcmf_sdio *bus) +brcmf_sdio_ulp_pre_redownload_check(struct brcmf_sdio *bus, u32 hmb_data) { int err = 0; u32 value = 0; u32 val32, ulp_wake_ind, wowl_wake_ind; int reg_addr; unsigned long timeout; + struct brcmf_ulp *fmac_ulp = &bus->sdiodev->fmac_ulp; + int i = 0; + + /* If any host mail box data is present, ignore DS1 exit sequence */ + if (hmb_data) + return false; + /* Skip if DS1 Exit is already in progress + * This can happen if firmware download is taking more time + */ + if (fmac_ulp->ulp_state == FMAC_ULP_TRIGGERED) + return false; value = brcmf_sdiod_regrb(bus->sdiodev, SDIO_CCCR_IOEx, &err); if (value == SDIO_FUNC_ENABLE_1) { - brcmf_dbg(SDIO, "GOT THE INTERRUPT FROM UCODE\n"); + brcmf_dbg(ULP, "GOT THE INTERRUPT FROM UCODE\n"); bus->sdiodev->ulp = true; - ulp_wake_ind = D11SHM_RD(bus->sdiodev, M_ULP_WAKE_IND( - bus->sdiodev->shm_ulp), &err) >> 16; - wowl_wake_ind = D11SHM_RD(bus->sdiodev, M_WAKEEVENT_IND( - bus->sdiodev->shm_ulp), &err) >> 16; + fmac_ulp->ulp_state = FMAC_ULP_TRIGGERED; + ulp_wake_ind = D11SHM_RDW(bus->sdiodev, M_ULP_WAKE_IND( + bus->sdiodev->fmac_ulp), &err); + wowl_wake_ind = D11SHM_RDW(bus->sdiodev, M_WAKEEVENT_IND( + bus->sdiodev->fmac_ulp), &err); - brcmf_dbg(SDIO, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x\n", - wowl_wake_ind, ulp_wake_ind); + brcmf_dbg(ULP, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x state %s\n", + wowl_wake_ind, ulp_wake_ind, (fmac_ulp->ulp_state) ? + "DS1 Exit Triggered" : "IDLE State"); if (wowl_wake_ind || ulp_wake_ind) { - /* TX wake Don't do anything. + /* RX wake Don't do anything. * Just bail out and re-download firmware. */ + /* Print out PHY TX error block when bit 9 set */ + if ((ulp_wake_ind & C_DS1_PHY_TXERR) && + M_DS1_PHYTX_ERR_BLK(bus->sdiodev->fmac_ulp)) { + brcmf_err("Dump PHY TX Error SHM Locations\n"); + for (i = 0; i < PHYTX_ERR_BLK_SIZE; i++) { + pr_err("0x%x", D11SHM_RDW(bus->sdiodev, + (M_DS1_PHYTX_ERR_BLK + (bus->sdiodev->fmac_ulp) + + (i * 2)), &err)); + } + brcmf_err("\n"); + } } else { - /* RX wake negotiate with MAC */ - brcmf_dbg(SDIO, "M_DS1_CTRL_SDIO: 0x%08x\n", - (u32)D11SHM_RD(bus->sdiodev, - M_DS1_CTRL_SDIO(bus->sdiodev->shm_ulp), + /* TX wake negotiate with MAC */ + brcmf_dbg(ULP, "M_DS1_CTRL_SDIO: 0x%08x\n", + (u32)D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO(bus->sdiodev->fmac_ulp), &err)); + val32 = D11SHM_RD(bus->sdiodev, M_DS1_CTRL_SDIO( + bus->sdiodev->fmac_ulp), &err); D11SHM_WR(bus->sdiodev, M_DS1_CTRL_SDIO( - bus->sdiodev->shm_ulp), - C_DS1_CTRL_SDIO_DS1_EXIT | - C_DS1_CTRL_REQ_VALID, + bus->sdiodev->fmac_ulp), val32, + (C_DS1_CTRL_SDIO_DS1_EXIT | + C_DS1_CTRL_REQ_VALID), &err); val32 = D11REG_RD(bus->sdiodev, D11_MACCONTROL_REG, &err); @@ -2644,37 +2679,38 @@ brcmf_sdio_ulp_pre_redownload_check(struct brcmf_sdio *bus) D11_MACCONTROL_REG, val32, &err); /* Poll for PROC_DONE to be set by ucode */ - value = D11SHM_RD(bus->sdiodev, - M_DS1_CTRL_SDIO( - bus->sdiodev->shm_ulp), &err); + value = D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->fmac_ulp), &err); /* Wait here (polling) for C_DS1_CTRL_PROC_DONE */ timeout = jiffies + ULP_HUDI_PROC_DONE_TIME; while (!(value & C_DS1_CTRL_PROC_DONE)) { - value = D11SHM_RD(bus->sdiodev, - M_DS1_CTRL_SDIO( - bus->sdiodev->shm_ulp), &err); + value = D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO + (bus->sdiodev->fmac_ulp), + &err); if (time_after(jiffies, timeout)) break; usleep_range(1000, 2000); } - brcmf_dbg(SDIO, "M_DS1_CTRL_SDIO: 0x%08x\n", - (u32)D11SHM_RD(bus->sdiodev, + brcmf_dbg(ULP, "M_DS1_CTRL_SDIO: 0x%08x\n", + (u32)D11SHM_RDW(bus->sdiodev, M_DS1_CTRL_SDIO( - bus->sdiodev->shm_ulp), &err)); - value = D11SHM_RD(bus->sdiodev, - M_DS1_CTRL_SDIO( - bus->sdiodev->shm_ulp), &err); + bus->sdiodev->fmac_ulp), &err)); + value = D11SHM_RDW(bus->sdiodev, + M_DS1_CTRL_SDIO( + bus->sdiodev->fmac_ulp), &err); if (!(value & C_DS1_CTRL_PROC_DONE)) { brcmf_err("%s: timeout Failed to enter DS1 Exit state!\n", __func__); return false; } } - ulp_wake_ind = D11SHM_RD(bus->sdiodev, M_ULP_WAKE_IND( - bus->sdiodev->shm_ulp), &err) >> 16; - wowl_wake_ind = D11SHM_RD(bus->sdiodev, M_WAKEEVENT_IND( - bus->sdiodev->shm_ulp), &err) >> 16; - brcmf_dbg(SDIO, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x\n", + ulp_wake_ind = D11SHM_RDW(bus->sdiodev, M_ULP_WAKE_IND( + bus->sdiodev->fmac_ulp), &err); + wowl_wake_ind = D11SHM_RDW(bus->sdiodev, M_WAKEEVENT_IND( + bus->sdiodev->fmac_ulp), &err); + brcmf_dbg(ULP, "wowl_wake_ind: 0x%08x, ulp_wake_ind: 0x%08x\n", wowl_wake_ind, ulp_wake_ind); reg_addr = CORE_CC_REG( brcmf_chip_get_pmu(bus->ci)->base, min_res_mask); @@ -2759,9 +2795,10 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) /* Handle host mailbox indication */ if (intstatus & I_HMB_HOST_INT) { + u32 hmb_data = 0; intstatus &= ~I_HMB_HOST_INT; - intstatus |= brcmf_sdio_hostmail(bus); - if (brcmf_sdio_ulp_pre_redownload_check(bus)) + intstatus |= brcmf_sdio_hostmail(bus, &hmb_data); + if (brcmf_sdio_ulp_pre_redownload_check(bus, hmb_data)) brcmf_sdio_ulp_reinit_fw(bus); } @@ -2807,13 +2844,14 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) brcmf_sdio_clrintr(bus); if (bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL) && - data_ok(bus)) { + data_ok(bus) && brcmf_sdio_f2_ready(bus)) { sdio_claim_host(bus->sdiodev->func[1]); if (bus->ctrl_frame_stat) { - err = brcmf_sdio_tx_ctrlframe(bus, bus->ctrl_frame_buf, + err = brcmf_sdio_tx_ctrlframe(bus, + bus->ctrl_frame_buf, bus->ctrl_frame_len); bus->ctrl_frame_err = err; - wmb(); + wmb(); /* drain writebuffer */ bus->ctrl_frame_stat = false; } sdio_release_host(bus->sdiodev->func[1]); @@ -4216,7 +4254,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, u8 saveclk; u8 devctl; - brcmf_dbg(TRACE, "Enter: dev=%s, err=%d\n", dev_name(dev), err); + brcmf_dbg(ULP, "Enter: dev=%s, err=%d\n", dev_name(dev), err); bus_if = dev_get_drvdata(dev); sdiodev = bus_if->bus_priv.sdio; if (err) @@ -4296,6 +4334,10 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, devctl |= SBSDIO_DEVCTL_F2WM_ENAB; brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, &err); + /* Register for ULP events */ + brcmf_fweh_register(bus_if->drvr, BRCMF_E_ULP, + brcmf_ulp_event_notify); + break; default: brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); @@ -4340,6 +4382,12 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_err("dongle is not responding\n"); goto fail; } + } else { + /* For ULP, after firmware redownload complete + * set ULP state to IDLE + */ + if (bus->sdiodev->fmac_ulp.ulp_state == FMAC_ULP_TRIGGERED) + bus->sdiodev->fmac_ulp.ulp_state = FMAC_ULP_IDLE; } return; @@ -4582,3 +4630,41 @@ int brcmf_sdio_sleep(struct brcmf_sdio *bus, bool sleep) return ret; } +/* Check F2 Ready bit before sending data to Firmware */ +static int +brcmf_sdio_f2_ready(struct brcmf_sdio *bus) +{ + int ret = -1; + int iordy_status = 0; + + sdio_claim_host(bus->sdiodev->func[1]); + /* Read the status of IOR2 */ + iordy_status = brcmf_sdiod_regrb(bus->sdiodev, + SDIO_CCCR_IORx, + NULL); + + sdio_release_host(bus->sdiodev->func[1]); + ret = iordy_status & SDIO_FUNC_ENABLE_2; + return ret; +} + +static int brcmf_ulp_event_notify(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *data) +{ + int err = 0; + struct brcmf_bus *bus_if = ifp->drvr->bus_if; + struct brcmf_sdio_dev *sdiodev; + struct brcmf_sdio *bus; + struct brcmf_ulp_event *ulp_event = (struct brcmf_ulp_event *)data; + + sdiodev = bus_if->bus_priv.sdio; + bus = sdiodev->bus; + + brcmf_dbg(ULP, "Chip went to DS1 state : action %d\n", + ulp_event->ulp_dongle_action); + if (ulp_event->ulp_dongle_action == FMAC_ULP_ENTRY) + bus->sdiodev->fmac_ulp.ulp_state = FMAC_ULP_ENTRY_RECV; + + return err; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 36c662be0cce8..8b9fd24820a9d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -182,12 +182,30 @@ struct ulp_shm_info { u32 m_ulp_ctrl_sdio; u32 m_ulp_wakeevt_ind; u32 m_ulp_wakeind; + u32 m_ulp_phytxblk; }; -struct fmac_ulp { +/* FMAC ULP state machine */ +#define FMAC_ULP_IDLE (0) +#define FMAC_ULP_ENTRY_RECV (1) +#define FMAC_ULP_TRIGGERED (2) + +/* BRCMF_E_ULP event data */ +#define FMAC_ULP_EVENT_VERSION 1 +#define FMAC_ULP_DISABLE_CONSOLE 1 /* Disable console */ +#define FMAC_ULP_UCODE_DOWNLOAD 2 /* Download ULP ucode file */ +#define FMAC_ULP_ENTRY 3 /* Inform ulp entry to Host */ + +struct brcmf_ulp { + uint ulp_state; struct ulp_shm_info ulp_shm_offset; }; +struct brcmf_ulp_event { + u16 version; + u16 ulp_dongle_action; +}; + struct brcmf_sdio_dev { struct sdio_func *func[SDIO_MAX_FUNCS]; u8 num_funcs; /* Supported funcs on client */ @@ -212,7 +230,7 @@ struct brcmf_sdio_dev { bool wowl_enabled; enum brcmf_sdiod_state state; struct brcmf_sdiod_freezer *freezer; - struct fmac_ulp shm_ulp; + struct brcmf_ulp fmac_ulp; bool ulp; }; @@ -393,6 +411,7 @@ void brcmf_sdio_trigger_dpc(struct brcmf_sdio *bus); #define M_DS1_CTRL_SDIO(ptr) ((ptr).ulp_shm_offset.m_ulp_ctrl_sdio) #define M_WAKEEVENT_IND(ptr) ((ptr).ulp_shm_offset.m_ulp_wakeevt_ind) #define M_ULP_WAKE_IND(ptr) ((ptr).ulp_shm_offset.m_ulp_wakeind) +#define M_DS1_PHYTX_ERR_BLK(ptr) ((ptr).ulp_shm_offset.m_ulp_phytxblk) #define D11_BASE_ADDR 0x18001000 #define D11_AXI_BASE_ADDR 0xE8000000 @@ -406,17 +425,7 @@ void brcmf_sdio_trigger_dpc(struct brcmf_sdio *bus); #define D11_MACCONTROL_REG D11REG_ADDR(0x120) #define D11_MACCONTROL_REG_WAKE 0x4000000 -/* Following are the offsets in M_DRVR_UCODE_IF_PTR block. Start address of - * M_DRVR_UCODE_IF_PTR block is present in M_DRVR_UCODE_IF_PTR. - */ - -/* M_ULP_WAKE_IND bits */ -#define ULP_WAKE_IND_WATCHDOG_EXP 0x1 -#define ULP_WAKE_IND_FCBS_ERROR 0x2 -#define ULP_WAKE_IND_RE_TRANSMIT_ERR 0x4 -#define ULP_WAKE_IND_HOST_WKUP 0x8 -#define ULP_WAKE_IND_INVALID_FCBS_BLK 0x10 - +/* HUDI Sequence SHM bits */ #define C_DS1_CTRL_SDIO_DS1_SLEEP 0x1 #define C_DS1_CTRL_SDIO_MAC_ON 0x2 #define C_DS1_CTRL_SDIO_RADIO_PHY_ON 0x4 @@ -424,12 +433,53 @@ void brcmf_sdio_trigger_dpc(struct brcmf_sdio *bus); #define C_DS1_CTRL_PROC_DONE 0x100 #define C_DS1_CTRL_REQ_VALID 0x200 -#define D11SHM_WR(sdh, offset, val, ret) \ - brcmf_sdiod_regwl(sdh, D11SHM_ADDR(offset), val, ret) +/* M_ULP_WAKEIND bits */ +#define C_WATCHDOG_EXPIRY BIT(0) +#define C_FCBS_ERROR BIT(1) +#define C_RETX_FAILURE BIT(2) +#define C_HOST_WAKEUP BIT(3) +#define C_INVALID_FCBS_BLOCK BIT(4) +#define C_HUDI_DS1_EXIT BIT(5) +#define C_LOB_SLEEP BIT(6) +#define C_DS1_PHY_TXERR BIT(9) +#define C_DS1_WAKE_TIMER BIT(10) + +#define PHYTX_ERR_BLK_SIZE 18 +#define D11SHM_FIRST2BYTE_MASK 0xFFFF0000 +#define D11SHM_SECOND2BYTE_MASK 0x0000FFFF +#define D11SHM_2BYTE_SHIFT 16 #define D11SHM_RD(sdh, offset, ret) \ brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) +/* SHM Read is motified based on SHM 4 byte alignment as SHM size is 2 bytes and + * 2 byte is currently not working on FMAC + * If SHM address is not 4 byte aligned, then right shift by 16 + * otherwise, mask the first two MSB bytes + * Suppose data in address 7260 is 0x440002 and it is 4 byte aligned + * Correct SHM value is 0x2 for this SHM offset and next SHM value is 0x44 + */ +#define D11SHM_RDW(sdh, offset, ret) \ + ((offset % 4) ? \ + (brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) \ + >> D11SHM_2BYTE_SHIFT) : \ + (brcmf_sdiod_regrl(sdh, D11SHM_ADDR(offset), ret) \ + & D11SHM_SECOND2BYTE_MASK)) + +/* SHM is of size 2 bytes, 4 bytes write will overwrite other SHM's + * First read 4 bytes and then clear the required two bytes based on + * 4 byte alignment, then update the required value and write the + * 4 byte value now + */ +#define D11SHM_WR(sdh, offset, val, mask, ret) \ + do { \ + if ((offset) % 4) \ + val = (val & D11SHM_SECOND2BYTE_MASK) | \ + ((mask) << D11SHM_2BYTE_SHIFT); \ + else \ + val = (mask) | (val & D11SHM_FIRST2BYTE_MASK); \ + brcmf_sdiod_regwl(sdh, D11SHM_ADDR(offset), val, ret); \ + } while (0) #define D11REG_WR(sdh, addr, val, ret) \ brcmf_sdiod_regwl(sdh, addr, val, ret) From 41ff5613c7a48306681703234dc75a61d778f8bb Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Wed, 15 Aug 2018 17:02:26 +0530 Subject: [PATCH 073/213] brcmfmac: disable command decode in sdio_aos for 4373 By disabling command decode, sdiod_aos module supports the detection of sdio command line toggle only and generates a wakeup request to PMU and to sdiod core. It does not decode any sdio command and generates no response to any command. Signed-off-by: Madhan Mohan R --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 886a66978bdec..1c04b518d240e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3567,6 +3567,7 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus) } if (bus->ci->chip == CY_CC_43012_CHIP_ID || + bus->ci->chip == CY_CC_4373_CHIP_ID || bus->ci->chip == BRCM_CC_4339_CHIP_ID || bus->ci->chip == BRCM_CC_4354_CHIP_ID || bus->ci->chip == BRCM_CC_4345_CHIP_ID) { From 95a505589bf79b4259bf38a20180296a2f2c3e6d Mon Sep 17 00:00:00 2001 From: Ryohei Kondo Date: Mon, 20 Aug 2018 16:27:41 +0900 Subject: [PATCH 074/213] brcmfmac: add vendor ie for association responses Miracast Certification clause 6.1.2 may fail if there is no WFD IE in P2P assoc response. This change allows WFD IE to be added to P2P assoc response. Related WFA certification. 6.1.2 P-SnUT operating as a Group Owner accepts a WFD Session with a Reference Source Signed-off-by: Ryohei Kondo --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 14 ++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 6d8f83a227601..fae50c7b91e3a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4417,6 +4417,11 @@ s32 brcmf_vif_set_mgmt_ie(struct brcmf_cfg80211_vif *vif, s32 pktflag, mgmt_ie_len = &saved_ie->assoc_req_ie_len; mgmt_ie_buf_len = sizeof(saved_ie->assoc_req_ie); break; + case BRCMF_VNDR_IE_ASSOCRSP_FLAG: + mgmt_ie_buf = saved_ie->assoc_res_ie; + mgmt_ie_len = &saved_ie->assoc_res_ie_len; + mgmt_ie_buf_len = sizeof(saved_ie->assoc_res_ie); + break; default: err = -EPERM; brcmf_err("not suitable type\n"); @@ -4566,6 +4571,15 @@ brcmf_config_ap_mgmt_ie(struct brcmf_cfg80211_vif *vif, else brcmf_dbg(TRACE, "Applied Vndr IEs for Probe Resp\n"); + /* Set Assoc Response IEs to FW */ + err = brcmf_vif_set_mgmt_ie(vif, BRCMF_VNDR_IE_ASSOCRSP_FLAG, + beacon->assocresp_ies, + beacon->assocresp_ies_len); + if (err) + brcmf_err("Set Assoc Resp IE Failed\n"); + else + brcmf_dbg(TRACE, "Applied Vndr IEs for Assoc Resp\n"); + return err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 3ead7e1898377..8d514a851ded2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -187,19 +187,23 @@ enum brcmf_vif_status { * @probe_req_ie: IE info for probe request. * @probe_res_ie: IE info for probe response. * @beacon_ie: IE info for beacon frame. + * @assoc_res_ie: IE info for association response frame. * @probe_req_ie_len: IE info length for probe request. * @probe_res_ie_len: IE info length for probe response. * @beacon_ie_len: IE info length for beacon frame. + * @assoc_res_ie_len: IE info length for association response frame. */ struct vif_saved_ie { u8 probe_req_ie[IE_MAX_LEN]; u8 probe_res_ie[IE_MAX_LEN]; u8 beacon_ie[IE_MAX_LEN]; u8 assoc_req_ie[IE_MAX_LEN]; + u8 assoc_res_ie[IE_MAX_LEN]; u32 probe_req_ie_len; u32 probe_res_ie_len; u32 beacon_ie_len; u32 assoc_req_ie_len; + u32 assoc_res_ie_len; }; /** From d21bb877f3661922ce572bb61409031c387aa2c7 Mon Sep 17 00:00:00 2001 From: Nitin Bhaskar Date: Tue, 21 Aug 2018 16:03:10 +0530 Subject: [PATCH 075/213] brcmfmac: fix 43012 insmod-after-rmmod in DS1 failure. After entering ULP, issuing rmmod does not put chip in sane state because of which next insmod fails. Fix includes writing into few LHL, PMU registers. Signed-off-by: Nitin Bhaskar --- .../broadcom/brcm80211/brcmfmac/chip.c | 155 ++++++++++++++ .../broadcom/brcm80211/brcmfmac/chip.h | 7 + .../broadcom/brcm80211/brcmfmac/sdio.c | 14 +- .../broadcom/brcm80211/include/chipcommon.h | 193 +++++++++++++++++- 4 files changed, 366 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index dda2c995f16cf..faa5bc1376044 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -224,6 +224,18 @@ struct sbsocramregs { #define ARMCR4_BSZ_MASK 0x3f #define ARMCR4_BSZ_MULT 8192 +/* Minimum PMU resource mask for 43012C0 */ +#define CY_43012_PMU_MIN_RES_MASK 0xF8BFE77 + +/* PMU STATUS mask for 43012C0 */ +#define CY_43012_PMU_STATUS_MASK 0x1AC + +/* PMU CONTROL EXT mask for 43012C0 */ +#define CY_43012_PMU_CONTROL_EXT_MASK 0x11 + +/* PMU CONTROL EXT mask for 43012C0 */ +#define CY_43012_PMU_WATCHDOG_TICK_VAL 0x04 + struct brcmf_core_priv { struct brcmf_core pub; u32 wrapbase; @@ -1163,6 +1175,14 @@ struct brcmf_core *brcmf_chip_get_pmu(struct brcmf_chip *pub) return cc; } +struct brcmf_core *brcmf_chip_get_gci(struct brcmf_chip *pub) +{ + struct brcmf_core *gci; + + gci = brcmf_chip_get_core(pub, BCMA_CORE_GCI); + return gci; +} + bool brcmf_chip_iscoreup(struct brcmf_core *pub) { struct brcmf_core_priv *core; @@ -1388,3 +1408,138 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) PMU_RCTL_LOGIC_DISABLE_MASK)) == 0; } } + +void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 addr; + u32 base; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + /* SW scratch */ + addr = CORE_CC_REG(base, swscratch); + chip->ops->write32(chip->ctx, addr, 0); + + /* PMU status */ + addr = CORE_CC_REG(base, pmustatus); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_STATUS_MASK); + + /* PMU control ext */ + addr = CORE_CC_REG(base, pmucontrol_ext); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_CONTROL_EXT_MASK); + + /* PMU watchdog */ + addr = CORE_CC_REG(base, pmuwatchdog); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_WATCHDOG_TICK_VAL); + break; + + default: + brcmf_err("Unsupported chip id\n"); + break; + } +} + +void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 addr; + u32 base; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); + break; + + default: + brcmf_err("Unsupported chip id\n"); + break; + } +} + +void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 base; + u32 addr; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_gci(pub)->base; + + /* LHL Top Level Power Sequence Control */ + addr = CORE_GCI_REG(base, lhl_top_pwrseq_ctl_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* GPIO Interrupt Enable0 */ + addr = CORE_GCI_REG(base, gpio_int_en_port_adr[0]); + chip->ops->write32(chip->ctx, addr, 0); + + /* GPIO Interrupt Status0 */ + addr = CORE_GCI_REG(base, gpio_int_st_port_adr[0]); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL ARM Timer0 Interrupt Mask */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_intrp_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL ARM Timer0 Interrupt Status */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_st_adr); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL ARM Timer */ + addr = CORE_GCI_REG(base, lhl_wl_armtim0_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL MAC Timer0 Interrupt Mask */ + addr = CORE_GCI_REG(base, lhl_wl_mactim0_intrp_adr); + chip->ops->write32(chip->ctx, addr, 0); + + /* WL MAC Timer0 Interrupt Status */ + addr = CORE_GCI_REG(base, lhl_wl_mactim0_st_adr); + chip->ops->write32(chip->ctx, addr, ~0); + + /* WL MAC TimerInt0 */ + addr = CORE_GCI_REG(base, lhl_wl_mactim_int0_adr); + chip->ops->write32(chip->ctx, addr, 0x0); +} + +void brcmf_chip_reset_watchdog(struct brcmf_chip *pub) +{ + struct brcmf_chip_priv *chip; + u32 base; + u32 addr; + + brcmf_dbg(TRACE, "Enter\n"); + + chip = container_of(pub, struct brcmf_chip_priv, pub); + base = brcmf_chip_get_pmu(pub)->base; + + switch (pub->chip) { + case CY_CC_43012_CHIP_ID: + /* Watchdog res mask */ + addr = CORE_CC_REG(base, watchdog_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); + break; + + default: + break; + } +} + diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h index dd0ec3eba6a98..c7783dd056ee7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h @@ -21,6 +21,9 @@ #define CORE_CC_REG(base, field) \ (base + offsetof(struct chipcregs, field)) +#define CORE_GCI_REG(base, field) \ + (base + offsetof(struct chipgciregs, field)) + /** * struct brcmf_chip - chip level information. * @@ -93,5 +96,9 @@ void brcmf_chip_resetcore(struct brcmf_core *core, u32 prereset, u32 reset, void brcmf_chip_set_passive(struct brcmf_chip *ci); bool brcmf_chip_set_active(struct brcmf_chip *ci, u32 rstvec); bool brcmf_chip_sr_capable(struct brcmf_chip *pub); +void brcmf_chip_reset_watchdog(struct brcmf_chip *pub); +void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub); +void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub); +void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub); #endif /* BRCMF_AXIDMP_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 1c04b518d240e..09a26862052e5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4575,7 +4575,19 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) * necessary cores. */ msleep(20); - brcmf_chip_set_passive(bus->ci); + if (bus->sdiodev->fmac_ulp.ulp_state == + FMAC_ULP_ENTRY_RECV) { + brcmf_chip_ulp_reset_lhl_regs(bus->ci); + brcmf_chip_reset_pmu_regs(bus->ci); + brcmf_chip_set_default_min_res_mask( + bus->ci); + } else { + brcmf_chip_set_passive(bus->ci); + } + /* Reset the PMU, backplane and all the + * cores by using the PMUWatchdogCounter. + */ + brcmf_chip_reset_watchdog(bus->ci); brcmf_sdio_clkctl(bus, CLK_NONE, false); sdio_release_host(bus->sdiodev->func[1]); } diff --git a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h index 1f23092cdad99..f5265966bf181 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h @@ -225,8 +225,197 @@ struct chipcregs { u32 PAD[3]; u32 retention_grpidx; /* 0x680 */ u32 retention_grpctl; /* 0x684 */ - u32 PAD[94]; - u16 sromotp[768]; + u32 mac_res_req_timer; /* 0x688 */ + u32 mac_res_req_mask; /* 0x68c */ + u32 PAD[18]; + u32 pmucontrol_ext; /* 0x6d8 */ + u32 slowclkperiod; /* 0x6dc */ + u32 PAD[8]; + u32 pmuintmask0; /* 0x700 */ + u32 pmuintmask1; /* 0x704 */ + u32 PAD[14]; + u32 pmuintstatus; /* 0x740 */ + u32 extwakeupstatus; /* 0x744 */ + u32 watchdog_res_mask; /* 0x748 */ + u32 swscratch; /* 0x750 */ + u32 PAD[3]; + u32 extwakemask[2]; /* 0x760-0x764 */ + u32 PAD[2]; + u32 extwakereqmask[2]; /* 0x770-0x774 */ + u32 PAD[2]; + u32 pmuintctrl0; /* 0x780 */ + u32 pmuintctrl1; /* 0x784 */ + u32 PAD[2]; + u32 extwakectrl[2]; /* 0x790 */ +}; + +#define CHIPGCIREGOFFS(field) offsetof(struct chipgciregs, field) + +struct chipgciregs { + u32 gci_corecaps0; /* 0x000 */ + u32 gci_corecaps1; /* 0x004 */ + u32 gci_corecaps2; /* 0x008 */ + u32 gci_corectrl; /* 0x00c */ + u32 gci_corestat; /* 0x010 */ + u32 gci_intstat; /* 0x014 */ + u32 gci_intmask; /* 0x018 */ + u32 gci_wakemask; /* 0x01c */ + u32 gci_levelintstat; /* 0x020 */ + u32 gci_eventintstat; /* 0x024 */ + u32 gci_wakelevelintstat; /* 0x028 */ + u32 gci_wakeeventintstat; /* 0x02c */ + u32 semaphoreintstatus; /* 0x030 */ + u32 semaphoreintmask; /* 0x034 */ + u32 semaphorerequest; /* 0x038 */ + u32 semaphorereserve; /* 0x03c */ + u32 gci_indirect_addr; /* 0x040 */ + u32 gci_gpioctl; /* 0x044 */ + u32 gci_gpiostatus; /* 0x048 */ + u32 gci_gpiomask; /* 0x04c */ + u32 eventsummary; /* 0x050 */ + u32 gci_miscctl; /* 0x054 */ + u32 gci_gpiointmask; /* 0x058 */ + u32 gci_gpiowakemask; /* 0x05c */ + u32 gci_input[32]; /* 0x060 */ + u32 gci_event[32]; /* 0x0e0 */ + u32 gci_output[4]; /* 0x160 */ + u32 gci_control_0; /* 0x170 */ + u32 gci_control_1; /* 0x174 */ + u32 gci_intpolreg; /* 0x178 */ + u32 gci_levelintmask; /* 0x17c */ + u32 gci_eventintmask; /* 0x180 */ + u32 wakelevelintmask; /* 0x184 */ + u32 wakeeventintmask; /* 0x188 */ + u32 hwmask; /* 0x18c */ + u32 PAD; + u32 gci_inbandeventintmask; /* 0x194 */ + u32 PAD; + u32 gci_inbandeventstatus; /* 0x19c */ + u32 gci_seciauxtx; /* 0x1a0 */ + u32 gci_seciauxrx; /* 0x1a4 */ + u32 gci_secitx_datatag; /* 0x1a8 */ + u32 gci_secirx_datatag; /* 0x1ac */ + u32 gci_secitx_datamask; /* 0x1b0 */ + u32 gci_seciusef0tx_reg; /* 0x1b4 */ + u32 gci_secif0tx_offset; /* 0x1b8 */ + u32 gci_secif0rx_offset; /* 0x1bc */ + u32 gci_secif1tx_offset; /* 0x1c0 */ + u32 gci_rxfifo_common_ctrl; /* 0x1c4 */ + u32 gci_rxfifoctrl; /* 0x1c8 */ + u32 gci_hw_sema_status; /* 0x1cc */ + u32 gci_seciuartescval; /* 0x1d0 */ + u32 gic_seciuartautobaudctr; /* 0x1d4 */ + u32 gci_secififolevel; /* 0x1d8 */ + u32 gci_seciuartdata; /* 0x1dc */ + u32 gci_secibauddiv; /* 0x1e0 */ + u32 gci_secifcr; /* 0x1e4 */ + u32 gci_secilcr; /* 0x1e8 */ + u32 gci_secimcr; /* 0x1ec */ + u32 gci_secilsr; /* 0x1f0 */ + u32 gci_secimsr; /* 0x1f4 */ + u32 gci_baudadj; /* 0x1f8 */ + u32 gci_inbandintmask; /* 0x1fc */ + u32 gci_chipctrl; /* 0x200 */ + u32 gci_chipsts; /* 0x204 */ + u32 gci_gpioout; /* 0x208 */ + u32 gci_gpioout_read; /* 0x20C */ + u32 gci_mpwaketx; /* 0x210 */ + u32 gci_mpwakedetect; /* 0x214 */ + u32 gci_seciin_ctrl; /* 0x218 */ + u32 gci_seciout_ctrl; /* 0x21C */ + u32 gci_seciin_auxfifo_en; /* 0x220 */ + u32 gci_seciout_txen_txbr; /* 0x224 */ + u32 gci_seciin_rxbrstatus; /* 0x228 */ + u32 gci_seciin_rxerrstatus; /* 0x22C */ + u32 gci_seciin_fcstatus; /* 0x230 */ + u32 gci_seciout_txstatus; /* 0x234 */ + u32 gci_seciout_txbrstatus; /* 0x238 */ + u32 wlan_mem_info; /* 0x23C */ + u32 wlan_bankxinfo; /* 0x240 */ + u32 bt_smem_select; /* 0x244 */ + u32 bt_smem_stby; /* 0x248 */ + u32 bt_smem_status; /* 0x24C */ + u32 wlan_bankxactivepda; /* 0x250 */ + u32 wlan_bankxsleeppda; /* 0x254 */ + u32 wlan_bankxkill; /* 0x258 */ + u32 PAD[41]; + u32 gci_chipid; /* 0x300 */ + u32 PAD[3]; + u32 otpstatus; /* 0x310 */ + u32 otpcontrol; /* 0x314 */ + u32 otpprog; /* 0x318 */ + u32 otplayout; /* 0x31c */ + u32 otplayoutextension; /* 0x320 */ + u32 otpcontrol1; /* 0x324 */ + u32 otpprogdata; /* 0x328 */ + u32 PAD[52]; + u32 otpECCstatus; /* 0x3FC */ + u32 PAD[512]; + u32 lhl_core_capab_adr; /* 0xC00 */ + u32 lhl_main_ctl_adr; /* 0xC04 */ + u32 lhl_pmu_ctl_adr; /* 0xC08 */ + u32 lhl_extlpo_ctl_adr; /* 0xC0C */ + u32 lpo_ctl_adr; /* 0xC10 */ + u32 lhl_lpo2_ctl_adr; /* 0xC14 */ + u32 lhl_osc32k_ctl_adr; /* 0xC18 */ + u32 lhl_clk_status_adr; /* 0xC1C */ + u32 lhl_clk_det_ctl_adr; /* 0xC20 */ + u32 lhl_clk_sel_adr; /* 0xC24 */ + u32 hidoff_cnt_adr[2]; /* 0xC28-0xC2C */ + u32 lhl_autoclk_ctl_adr; /* 0xC30 */ + u32 PAD; + u32 lhl_hibtim_adr; /* 0xC38 */ + u32 lhl_wl_ilp_val_adr; /* 0xC3C */ + u32 lhl_wl_armtim0_intrp_adr; /* 0xC40 */ + u32 lhl_wl_armtim0_st_adr; /* 0xC44 */ + u32 lhl_wl_armtim0_adr; /* 0xC48 */ + u32 PAD[9]; + u32 lhl_wl_mactim0_intrp_adr; /* 0xC70 */ + u32 lhl_wl_mactim0_st_adr; /* 0xC74 */ + u32 lhl_wl_mactim_int0_adr; /* 0xC78 */ + u32 lhl_wl_mactim_frac0_adr; /* 0xC7C */ + u32 lhl_wl_mactim1_intrp_adr; /* 0xC80 */ + u32 lhl_wl_mactim1_st_adr; /* 0xC84 */ + u32 lhl_wl_mactim_int1_adr; /* 0xC88 */ + u32 lhl_wl_mactim_frac1_adr; /* 0xC8C */ + u32 PAD[8]; + u32 gpio_int_en_port_adr[4]; /* 0xCB0-0xCBC */ + u32 gpio_int_st_port_adr[4]; /* 0xCC0-0xCCC */ + u32 gpio_ctrl_iocfg_p_adr[64]; /* 0xCD0-0xDCC */ + u32 gpio_gctrl_iocfg_p0_p39_adr; /* 0xDD0 */ + u32 gpio_gdsctrl_iocfg_p0_p25_p30_p39_adr; /* 0xDD4 */ + u32 gpio_gdsctrl_iocfg_p26_p29_adr; /* 0xDD8 */ + u32 PAD[8]; + u32 lhl_gpio_din0_adr; /* 0xDFC */ + u32 lhl_gpio_din1_adr; /* 0xE00 */ + u32 lhl_wkup_status_adr; /* 0xE04 */ + u32 lhl_ctl_adr; /* 0xE08 */ + u32 lhl_adc_ctl_adr; /* 0xE0C */ + u32 lhl_qdxyz_in_dly_adr; /* 0xE10 */ + u32 lhl_optctl_adr; /* 0xE14 */ + u32 lhl_optct2_adr; /* 0xE18 */ + u32 lhl_scanp_cntr_init_val_adr; /* 0xE1C */ + u32 lhl_opt_togg_val_adr[6]; /* 0xE20-0xE34 */ + u32 lhl_optx_smp_val_adr; /* 0xE38 */ + u32 lhl_opty_smp_val_adr; /* 0xE3C */ + u32 lhl_optz_smp_val_adr; /* 0xE40 */ + u32 lhl_hidoff_keepstate_adr[3]; /* 0xE44-0xE4C */ + u32 lhl_bt_slmboot_ctl0_adr[4]; /* 0xE50-0xE5C */ + u32 lhl_wl_fw_ctl; /* 0xE60 */ + u32 lhl_wl_hw_ctl_adr[2]; /* 0xE64-0xE68 */ + u32 lhl_bt_hw_ctl_adr; /* 0xE6C */ + u32 lhl_top_pwrseq_en_adr; /* 0xE70 */ + u32 lhl_top_pwrdn_ctl_adr; /* 0xE74 */ + u32 lhl_top_pwrup_ctl_adr; /* 0xE78 */ + u32 lhl_top_pwrseq_ctl_adr; /* 0xE7C */ + u32 lhl_top_pwrdn2_ctl_adr; /* 0xE80 */ + u32 lhl_top_pwrup2_ctl_adr; /* 0xE84 */ + u32 wpt_regon_intrp_cfg_adr; /* 0xE88 */ + u32 bt_regon_intrp_cfg_adr; /* 0xE8C */ + u32 wl_regon_intrp_cfg_adr; /* 0xE90 */ + u32 regon_intrp_st_adr; /* 0xE94 */ + u32 regon_intrp_en_adr; /* 0xE98 */ + }; /* chipid */ From 93c98df688579fbb7e361a25c3dbf6bc1aab5325 Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Wed, 31 Jan 2018 15:35:45 +0530 Subject: [PATCH 076/213] brcmfmac: Set SDIO F1 MesBusyCtrl for CYW4373 Along with F2 watermark (existing) configuration, F1 MesBusyCtrl should be enabled & configured to avoid overflow errors. Signed-off-by: Madhan Mohan R --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 2 ++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h | 3 +++ 2 files changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 09a26862052e5..71f92b634458e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -53,6 +53,7 @@ #define DEFAULT_F2_WATERMARK 0x8 #define CY_4373_F2_WATERMARK 0x40 +#define CY_4373_F1_MESBUSYCTRL (CY_4373_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #define CY_43012_F2_WATERMARK 0x60 #ifdef DEBUG @@ -4323,6 +4324,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, &err); devctl |= SBSDIO_DEVCTL_F2WM_ENAB; brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, CY_4373_F1_MESBUSYCTRL, &err); break; case SDIO_DEVICE_ID_CYPRESS_43012: brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43012\n", diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index 8b9fd24820a9d..e5f4b2865e4f0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -107,6 +107,9 @@ #define SBSDIO_FUNC1_RFRAMEBCHI 0x1001C /* MesBusyCtl (rev 11) */ #define SBSDIO_FUNC1_MESBUSYCTRL 0x1001D +/* Enable busy capability for MES access */ +#define SBSDIO_MESBUSYCTRL_ENAB 0x80 + /* Sdio Core Rev 12 */ #define SBSDIO_FUNC1_WAKEUPCTRL 0x1001E #define SBSDIO_FUNC1_WCTRL_ALPWAIT_MASK 0x1 From 4350bf5206d5347ef1af67777cf6c9d69554e09f Mon Sep 17 00:00:00 2001 From: Winnie Chang Date: Fri, 24 Aug 2018 10:26:52 +0800 Subject: [PATCH 077/213] brcmfmac: add 4354 raw pcie device id Signed-off-by: Winnie Chang --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 1 + drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index e9fcbee61a7bc..9f97cbfb59419 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -1972,6 +1972,7 @@ static const struct dev_pm_ops brcmf_pciedrvr_pm = { static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4350_DEVICE_ID), + BRCMF_PCIE_DEVICE(BRCM_PCIE_4354_RAW_DEVICE_ID), BRCMF_PCIE_DEVICE_SUB(BRCM_PCIE_4355_DEVICE_ID,BRCM_PCIE_VENDOR_ID_BROADCOM,BRCM_PCIE_4355_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4356_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43567_DEVICE_ID), diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 31580360081c0..1bda1aa7af2b6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -74,6 +74,7 @@ /* PCIE Device IDs */ #define BRCM_PCIE_4350_DEVICE_ID 0x43a3 #define BRCM_PCIE_4354_DEVICE_ID 0x43df +#define BRCM_PCIE_4354_RAW_DEVICE_ID 0x4354 #define BRCM_PCIE_4355_DEVICE_ID 0x4355 #define BRCM_PCIE_4356_DEVICE_ID 0x43ec #define BRCM_PCIE_43567_DEVICE_ID 0x43d3 From a2044c70b4817ab06e86347f023bcb6880c3230a Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Wed, 29 Aug 2018 14:40:07 +0800 Subject: [PATCH 078/213] nl80211: add WPA3 definition for SAE authentication Add definition of WPA version 3 for SAE authentication. Signed-off-by: Chung-Hsien Hsu --- include/uapi/linux/nl80211.h | 1 + net/wireless/nl80211.c | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index fa43dd5a7b3dc..bb51b5c8f1696 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -4233,6 +4233,7 @@ enum nl80211_mfp { enum nl80211_wpa_versions { NL80211_WPA_VERSION_1 = 1 << 0, NL80211_WPA_VERSION_2 = 1 << 1, + NL80211_WPA_VERSION_3 = 1 << 2, }; /** diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index c149f3ffcd2bf..70f64e80d10e7 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -8406,7 +8406,8 @@ static int nl80211_dump_survey(struct sk_buff *skb, struct netlink_callback *cb) static bool nl80211_valid_wpa_versions(u32 wpa_versions) { return !(wpa_versions & ~(NL80211_WPA_VERSION_1 | - NL80211_WPA_VERSION_2)); + NL80211_WPA_VERSION_2 | + NL80211_WPA_VERSION_3)); } static int nl80211_authenticate(struct sk_buff *skb, struct genl_info *info) From 653c84145c4db6c99de6b1b081b5e93f358e3304 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Wed, 29 Aug 2018 14:57:12 +0800 Subject: [PATCH 079/213] cfg80211: add support for SAE authentication offload Let drivers advertise support for station-mode SAE authentication offload with a new NL80211_EXT_FEATURE_SAE_OFFLOAD flag. Signed-off-by: Chung-Hsien Hsu --- include/linux/ieee80211.h | 1 + include/net/cfg80211.h | 5 +++++ include/uapi/linux/nl80211.h | 18 ++++++++++++++++++ net/wireless/nl80211.c | 14 ++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index c83478271c2e0..6cf1971a46918 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2542,6 +2542,7 @@ enum ieee80211_key_len { #define FILS_ERP_MAX_RRK_LEN 64 #define PMK_MAX_LEN 64 +#define SAE_PASSWORD_MAX_LEN 128 /* Public action codes (IEEE Std 802.11-2016, 9.6.8.1, Table 9-307) */ enum ieee80211_pub_actioncode { diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 468deae5d603e..d5dc5788ec915 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -740,6 +740,9 @@ struct survey_info { * CFG80211_MAX_WEP_KEYS WEP keys * @wep_tx_key: key index (0..3) of the default TX static WEP key * @psk: PSK (for devices supporting 4-way-handshake offload) + * @sae_pwd: password for SAE authentication (for devices supporting SAE + * offload) + * @sae_pwd_len: length of SAE password (for devices supporting SAE offload) */ struct cfg80211_crypto_settings { u32 wpa_versions; @@ -755,6 +758,8 @@ struct cfg80211_crypto_settings { struct key_params *wep_keys; int wep_tx_key; const u8 *psk; + const u8 *sae_pwd; + u16 sae_pwd_len; }; /** diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index bb51b5c8f1696..5d4c8906c9dd9 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -234,6 +234,15 @@ * use in a FILS shared key connection with PMKSA caching. */ +/** + * DOC: SAE authentication offload + * + * By setting @NL80211_EXT_FEATURE_SAE_OFFLOAD flag drivers can indicate they + * support offloading SAE authentication for WPA3-Personal networks. In + * %NL80211_CMD_CONNECT the password for SAE should be specified using + * %NL80211_ATTR_SAE_PASSWORD. + */ + /** * enum nl80211_commands - supported nl80211 commands * @@ -2241,6 +2250,10 @@ enum nl80211_commands { * association request when used with NL80211_CMD_NEW_STATION). Can be set * only if %NL80211_STA_FLAG_WME is set. * + * @NL80211_ATTR_SAE_PASSWORD: attribute for passing SAE password material. It + * is used with %NL80211_CMD_CONNECT to provide password for offloading + * SAE authentication for WPA3-Personal networks. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -2682,6 +2695,8 @@ enum nl80211_attrs { NL80211_ATTR_HE_CAPABILITY, + NL80211_ATTR_SAE_PASSWORD, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -5224,6 +5239,8 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT: Driver/device can omit all data * except for supported rates from the probe request content if requested * by the %NL80211_SCAN_FLAG_MIN_PREQ_CONTENT flag. + * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in + * station mode (SAE password is passed as part of the connect command). * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. @@ -5260,6 +5277,7 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_TXQS, NL80211_EXT_FEATURE_SCAN_RANDOM_SN, NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT, + NL80211_EXT_FEATURE_SAE_OFFLOAD, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 70f64e80d10e7..3a1a839b38c44 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -456,6 +456,8 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { [NL80211_ATTR_FILS_CACHE_ID] = { .len = 2 }, [NL80211_ATTR_PMK] = { .type = NLA_BINARY, .len = PMK_MAX_LEN }, [NL80211_ATTR_SCHED_SCAN_MULTI] = { .type = NLA_FLAG }, + [NL80211_ATTR_SAE_PASSWORD] = { .type = NLA_BINARY, + .len = SAE_PASSWORD_MAX_LEN }, [NL80211_ATTR_EXTERNAL_AUTH_SUPPORT] = { .type = NLA_FLAG }, [NL80211_ATTR_TXQ_LIMIT] = { .type = NLA_U32 }, @@ -4184,6 +4186,8 @@ static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, return true; case NL80211_CMD_CONNECT: if (!(rdev->wiphy.features & NL80211_FEATURE_SAE) && + !wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD) && auth_type == NL80211_AUTHTYPE_SAE) return false; @@ -8646,6 +8650,16 @@ static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev, settings->psk = nla_data(info->attrs[NL80211_ATTR_PMK]); } + if (info->attrs[NL80211_ATTR_SAE_PASSWORD]) { + if (!wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD)) + return -EINVAL; + settings->sae_pwd = + nla_data(info->attrs[NL80211_ATTR_SAE_PASSWORD]); + settings->sae_pwd_len = + nla_len(info->attrs[NL80211_ATTR_SAE_PASSWORD]); + } + return 0; } From 4d40247772e0972ba78e3cb317e48393d4d4fbbe Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Wed, 29 Aug 2018 15:01:51 +0800 Subject: [PATCH 080/213] brcmfmac: add support for SAE authentication offload The firmware may have SAE authentication code built-in. This is detected by the driver and indicated in the wiphy features flags. User-space can use this flag to determine whether or not to provide the password material for SAE authentication in the nl80211 CONNECT command. Signed-off-by: Chung-Hsien Hsu --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 67 +++++++++++++++++-- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 3 +- .../broadcom/brcm80211/brcmfmac/feature.c | 1 + .../broadcom/brcm80211/brcmfmac/feature.h | 4 +- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 13 ++++ .../broadcom/brcm80211/include/brcmu_wifi.h | 2 + 6 files changed, 82 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index fae50c7b91e3a..300638120eb61 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1423,6 +1423,30 @@ static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) return err; } +static int brcmf_set_sae_password(struct brcmf_if *ifp, const u8 *pwd_data, + u16 pwd_len) +{ + struct brcmf_wsec_sae_pwd_le sae_pwd; + int err; + + if (pwd_len > BRCMF_WSEC_MAX_SAE_PASSWORD_LEN) { + brcmf_err("sae_password must be less than %d\n", + BRCMF_WSEC_MAX_SAE_PASSWORD_LEN); + return -EINVAL; + } + + sae_pwd.key_len = cpu_to_le16(pwd_len); + memcpy(sae_pwd.key, pwd_data, pwd_len); + + err = brcmf_fil_iovar_data_set(ifp, "sae_password", &sae_pwd, + sizeof(sae_pwd)); + if (err < 0) + brcmf_err("failed to set SAE password in firmware (len=%u)\n", + pwd_len); + + return err; +} + static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(vif->wdev.wiphy); @@ -1637,6 +1661,8 @@ static s32 brcmf_set_wpa_version(struct net_device *ndev, val = WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED; else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_2) val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; + else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3) + val = WPA3_AUTH_SAE_PSK; else val = WPA_AUTH_DISABLED; brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); @@ -1667,6 +1693,10 @@ static s32 brcmf_set_auth_type(struct net_device *ndev, val = 1; brcmf_dbg(CONN, "shared key\n"); break; + case NL80211_AUTHTYPE_SAE: + val = 3; + brcmf_dbg(CONN, "SAE authentication\n"); + break; default: val = 2; brcmf_dbg(CONN, "automatic, auth type (%d)\n", sme->auth_type); @@ -1832,10 +1862,23 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) sme->crypto.cipher_group); return -EINVAL; } + } else if (val & WPA3_AUTH_SAE_PSK) { + switch (sme->crypto.akm_suites[0]) { + case WLAN_AKM_SUITE_SAE: + val = WPA3_AUTH_SAE_PSK; + profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; + break; + default: + brcmf_err("invalid cipher group (%d)\n", + sme->crypto.cipher_group); + return -EINVAL; + } } if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) brcmf_dbg(INFO, "using 1X offload\n"); + else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) + brcmf_dbg(INFO, "using SAE offload\n"); if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MFP)) goto skip_mfp_config; @@ -1901,7 +1944,8 @@ brcmf_set_sharedkey(struct net_device *ndev, brcmf_dbg(CONN, "wpa_versions 0x%x cipher_pairwise 0x%x\n", sec->wpa_versions, sec->cipher_pairwise); - if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2)) + if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2 | + NL80211_WPA_VERSION_3)) return 0; if (!(sec->cipher_pairwise & @@ -2106,7 +2150,8 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, goto done; } - if (sme->crypto.psk) { + if (sme->crypto.psk && + profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { err = -EINVAL; goto done; @@ -2124,12 +2169,18 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, } } - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) { + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) err = brcmf_set_pmk(ifp, sme->crypto.psk, BRCMF_WSEC_MAX_PSK_LEN); - if (err) - goto done; + else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { + err = brcmf_set_sae_password(ifp, sme->crypto.sae_pwd, + sme->crypto.sae_pwd_len); + if (!err && sme->crypto.psk) + err = brcmf_set_pmk(ifp, sme->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); } + if (err) + goto done; /* Join with specific BSSID and cached SSID * If SSID is zero join based on BSSID only @@ -5475,7 +5526,8 @@ static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, if (event == BRCMF_E_SET_SSID && status == BRCMF_E_STATUS_SUCCESS) { brcmf_dbg(CONN, "Processing set ssid\n"); memcpy(vif->profile.bssid, e->addr, ETH_ALEN); - if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK) + if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK && + vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_SAE) return true; set_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state); @@ -6957,6 +7009,9 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD); } wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 8d514a851ded2..db80f61dfab71 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -142,7 +142,8 @@ struct brcmf_cfg80211_security { enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_NONE, BRCMF_PROFILE_FWSUP_PSK, - BRCMF_PROFILE_FWSUP_1X + BRCMF_PROFILE_FWSUP_1X, + BRCMF_PROFILE_FWSUP_SAE }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index c7c3563f9a53c..3069c6efba154 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -48,6 +48,7 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_MBSS, "mbss" }, { BRCMF_FEAT_MCHAN, "mchan" }, { BRCMF_FEAT_P2P, "p2p" }, + { BRCMF_FEAT_SAE, "sae" }, }; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 1ab4f16171120..48e679d4018aa 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -33,6 +33,7 @@ * MFP: 802.11w Management Frame Protection. * GSCAN: enhanced scan offload feature. * FWSUP: Firmware supplicant. + * SAE: simultaneous authentication of equals */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -48,7 +49,8 @@ BRCMF_FEAT_DEF(WOWL_ARP_ND) \ BRCMF_FEAT_DEF(MFP) \ BRCMF_FEAT_DEF(GSCAN) \ - BRCMF_FEAT_DEF(FWSUP) + BRCMF_FEAT_DEF(FWSUP) \ + BRCMF_FEAT_DEF(SAE) /* * Quirks: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 9616c94afb766..53b3b483a4112 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -53,6 +53,8 @@ #define BRCMF_WSEC_MAX_PSK_LEN 32 #define BRCMF_WSEC_PASSPHRASE BIT(0) +#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128 + /* primary (ie tx) key */ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 @@ -519,6 +521,17 @@ struct brcmf_wsec_pmk_le { u8 key[2 * BRCMF_WSEC_MAX_PSK_LEN + 1]; }; +/** + * struct brcmf_wsec_sae_pwd_le - firmware SAE password material. + * + * @key_len: number of octets in key materials. + * @key: SAE password material. + */ +struct brcmf_wsec_sae_pwd_le { + __le16 key_len; + u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; +}; + /* Used to get specific STA parameters */ struct brcmf_scb_val_le { __le32 val; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 91fca79e61583..cfe592fc4b040 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -240,6 +240,8 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_FT 0x4000 /* Fast BSS Transition */ #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ +#define WPA3_AUTH_SAE_PSK 0x40000 /* SAE with 4-way handshake */ + #define DOT11_DEFAULT_RTS_LEN 2347 #define DOT11_DEFAULT_FRAG_LEN 2346 From 1bc0b08e3d7d2026fd5c69ee482573a1bd04eb1e Mon Sep 17 00:00:00 2001 From: Double Lo Date: Wed, 5 Sep 2018 01:18:25 -0500 Subject: [PATCH 081/213] brcmfmac: fix 4339 CRC error under SDIO 3.0 SDR104 mode This patch fixes 4339 CRC error while runing Tput test with suspend/resume test script. The continuous failure messages before system crash: brcmfmac: brcmf_sdiod_sglist_rw: CMD53 sg block read failed -84 brcmfmac: brcmf_sdio_rxglom: glom read of 25600 bytes failed: -5 brcmfmac: brcmf_sdio_rxfail: abort command, terminate frame brcmfmac: brcmf_sdiod_sglist_rw: CMD53 sg block read failed -84 brcmfmac: brcmf_sdio_rxglom: glom read of 24576 bytes failed: -5 brcmfmac: brcmf_sdio_rxfail: abort command, terminate frame Signed-off-by: Lo-Hsiang Lo --- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 71f92b634458e..442dd594aad86 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -55,7 +55,9 @@ #define CY_4373_F2_WATERMARK 0x40 #define CY_4373_F1_MESBUSYCTRL (CY_4373_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #define CY_43012_F2_WATERMARK 0x60 - +#define CY_4339_F2_WATERMARK 48 +#define CY_4339_MES_WATERMARK 80 +#define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG #define BRCMF_TRAP_INFO_SIZE 80 @@ -4340,7 +4342,18 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, /* Register for ULP events */ brcmf_fweh_register(bus_if->drvr, BRCMF_E_ULP, brcmf_ulp_event_notify); - + break; + case SDIO_DEVICE_ID_BROADCOM_4339: + brcmf_sdiod_regwb(sdiodev, + SBSDIO_WATERMARK, + CY_4339_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, + SBSDIO_DEVICE_CTL, &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, + SBSDIO_DEVICE_CTL, devctl, &err); + brcmf_sdiod_regwb(sdiodev, + SBSDIO_FUNC1_MESBUSYCTRL, CY_4339_MESBUSYCTRL, &err); break; default: brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); From 7c19e119ba240fdc4a3faf107f2c1d14c7933a0f Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Tue, 4 Sep 2018 00:10:49 -0500 Subject: [PATCH 082/213] brcmfmac: fix the incorrect return value in brcmf_inform_single_bss(). The function brcmf_inform_single_bss returns the value as success, even when the length exceeds the maximum value. The fix is to send appropriate code on this error. This issue is observed when SVT reported random fmac crashes when running their tests and the path was identified from the crash logs. With this fix the random failure issue in SVT was resolved. Signed-off-by: Raveendran Somu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 300638120eb61..124615669f28b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2973,7 +2973,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, if (le32_to_cpu(bi->length) > WL_BSS_INFO_MAX) { brcmf_err("Bss info is larger than buffer. Discarding\n"); - return 0; + return -EINVAL; } if (!bi->ctl_ch) { From 1dfb8881c5c330b86f1269e4d2552c6557a183a3 Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Fri, 7 Sep 2018 13:49:55 -0500 Subject: [PATCH 083/213] brcmfmac: Fix double freeing in the fmac usb data path When the brcmf_fws_process_skb() fails to get hanger slot for queuing the skb, it tries to free the skb. But the caller brcmf_netdev_start_xmit() of that funciton frees the packet on error return value. This causes the double freeing and which caused the kernel crash. Signed-off-by: Raveendran Somu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 9abdbe6d91f11..2f6f32723c85e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2155,8 +2155,7 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb) brcmf_fws_enq(fws, BRCMF_FWS_SKBSTATE_DELAYED, fifo, skb); brcmf_fws_schedule_deq(fws); } else { - brcmf_err("drop skb: no hanger slot\n"); - brcmf_txfinalize(ifp, skb, false); + brcmf_err("no hanger slot available\n"); rc = -ENOMEM; } brcmf_fws_unlock(fws); From d4ef010167ff078eaf2afd604133af62b2a44ea4 Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Fri, 7 Sep 2018 14:07:07 -0500 Subject: [PATCH 084/213] brcmfmac: Fix driver crash on USB control transfer timeout When the control transfer gets timed out, the error status was returned without killing that urb, this leads to using the same urb. This issue causes the kernel crash as the same urb is sumbitted multiple times. The fix is to kill the urb for timeout tranfer before returning error Signed-off-by: Raveendran Somu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index b27170c12482d..7b3687552bab6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -339,11 +339,12 @@ static int brcmf_usb_tx_ctlpkt(struct device *dev, u8 *buf, u32 len) return err; } timeout = brcmf_usb_ioctl_resp_wait(devinfo); - clear_bit(0, &devinfo->ctl_op); if (!timeout) { brcmf_err("Txctl wait timed out\n"); + usb_kill_urb(devinfo->ctl_urb); err = -EIO; } + clear_bit(0, &devinfo->ctl_op); return err; } @@ -369,11 +370,12 @@ static int brcmf_usb_rx_ctlpkt(struct device *dev, u8 *buf, u32 len) } timeout = brcmf_usb_ioctl_resp_wait(devinfo); err = devinfo->ctl_urb_status; - clear_bit(0, &devinfo->ctl_op); if (!timeout) { brcmf_err("rxctl wait timed out\n"); + usb_kill_urb(devinfo->ctl_urb); err = -EIO; } + clear_bit(0, &devinfo->ctl_op); if (!err) return devinfo->ctl_urb_actual_length; else From 0f9d99cee4c4a39fee8d0d7347be3b808d118923 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Thu, 13 Sep 2018 03:07:29 -0500 Subject: [PATCH 085/213] brcmfmac: avoid network disconnection during suspend with linux-3.8 and above kernel From linux-3.8 kernel, wowlan packet filter is mandated to avoid the disconnection of connected network before suspend in commit 812569699119 ("cfg80211/mac80211: disconnect on suspend"). So as a dummy wowlan filter is configured for kernels linux-3.8 and above. Later all private command implementation of packet filters can be moved to wowlan based filters. Signed-off-by: Lo-Hsiang Lo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 124615669f28b..ce1b0cba32ed1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -7023,8 +7023,7 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy->vendor_commands = brcmf_vendor_cmds; wiphy->n_vendor_commands = BRCMF_VNDR_CMDS_LAST - 1; - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL)) - brcmf_wiphy_wowl_params(wiphy, ifp); + brcmf_wiphy_wowl_params(wiphy, ifp); err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_BANDLIST, &bandlist, sizeof(bandlist)); if (err) { From 0b0a2b448b08eb6e7eef0282bcf19b91c97b3da8 Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Tue, 11 Sep 2018 21:28:25 -0500 Subject: [PATCH 086/213] brcmfmac: Allow credit borrowing for all access categories Current credit borrowing allows only the access category BE to borrow the credits. This change is to fix the credit borrowing logic, to make borrowing available for all access categories and also to borrow only from the lower categories. Signed-off-by: Raveendran Somu --- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 2f6f32723c85e..da0e1bde9f39d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -416,6 +416,7 @@ struct brcmf_fws_mac_descriptor { }; #define BRCMF_FWS_HANGER_MAXITEMS 3072 +#define BRCMF_BORROW_RATIO 3 /** * enum brcmf_fws_hanger_item_state - state of hanger item. @@ -512,7 +513,8 @@ struct brcmf_fws_info { u32 fifo_enqpkt[BRCMF_FWS_FIFO_COUNT]; int fifo_credit[BRCMF_FWS_FIFO_COUNT]; int init_fifo_credit[BRCMF_FWS_FIFO_COUNT]; - int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1]; + int credits_borrowed[BRCMF_FWS_FIFO_AC_VO + 1] + [BRCMF_FWS_FIFO_AC_VO + 1]; int deq_node_pos[BRCMF_FWS_FIFO_COUNT]; u32 fifo_credit_map; u32 fifo_delay_map; @@ -1220,13 +1222,11 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, if (!credits) return; - fws->fifo_credit_map |= 1 << fifo; - - if ((fifo == BRCMF_FWS_FIFO_AC_BE) && - (fws->credits_borrowed[0])) { + if (fifo > BRCMF_FWS_FIFO_AC_BK && + fifo <= BRCMF_FWS_FIFO_AC_VO) { for (lender_ac = BRCMF_FWS_FIFO_AC_VO; lender_ac >= 0; lender_ac--) { - borrowed = &fws->credits_borrowed[lender_ac]; + borrowed = &fws->credits_borrowed[fifo][lender_ac]; if (*borrowed) { fws->fifo_credit_map |= (1 << lender_ac); fifo_credit = &fws->fifo_credit[lender_ac]; @@ -1243,7 +1243,11 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, } } - fws->fifo_credit[fifo] += credits; + if (credits) { + fws->fifo_credit[fifo] += credits; + fws->fifo_credit_map |= 1 << fifo; + } + if (fws->fifo_credit[fifo] > fws->init_fifo_credit[fifo]) fws->fifo_credit[fifo] = fws->init_fifo_credit[fifo]; @@ -2035,27 +2039,31 @@ static void brcmf_fws_rollback_toq(struct brcmf_fws_info *fws, } } -static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws) +static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws, + int highest_lender_ac, int borrower_ac, + bool borrow_all) { - int lender_ac; + int lender_ac, borrow_limit = 0; - if (time_after(fws->borrow_defer_timestamp, jiffies)) { - fws->fifo_credit_map &= ~(1 << BRCMF_FWS_FIFO_AC_BE); - return -ENAVAIL; - } + for (lender_ac = 0; lender_ac <= highest_lender_ac; lender_ac++) { - for (lender_ac = 0; lender_ac <= BRCMF_FWS_FIFO_AC_VO; lender_ac++) { - if (fws->fifo_credit[lender_ac] > 0) { - fws->credits_borrowed[lender_ac]++; + if (!borrow_all) + borrow_limit = + fws->fifo_credit[lender_ac] / BRCMF_BORROW_RATIO; + else + borrow_limit = 0; + + if (fws->fifo_credit[lender_ac] > borrow_limit) { + fws->credits_borrowed[borrower_ac][lender_ac]++; fws->fifo_credit[lender_ac]--; if (fws->fifo_credit[lender_ac] == 0) fws->fifo_credit_map &= ~(1 << lender_ac); - fws->fifo_credit_map |= (1 << BRCMF_FWS_FIFO_AC_BE); + fws->fifo_credit_map |= (1 << borrower_ac); brcmf_dbg(DATA, "borrow credit from: %d\n", lender_ac); return 0; } } - fws->fifo_credit_map &= ~(1 << BRCMF_FWS_FIFO_AC_BE); + fws->fifo_credit_map &= ~(1 << borrower_ac); return -ENAVAIL; } @@ -2244,9 +2252,10 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) } continue; } - while ((fws->fifo_credit[fifo] > 0) || + + while ((fws->fifo_credit[fifo]) || ((!fws->bcmc_credit_check) && - (fifo == BRCMF_FWS_FIFO_BCMC))) { + (fifo == BRCMF_FWS_FIFO_BCMC))) { skb = brcmf_fws_deq(fws, fifo); if (!skb) break; @@ -2256,10 +2265,14 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) if (fws->bus_flow_blocked) break; } - if ((fifo == BRCMF_FWS_FIFO_AC_BE) && - (fws->fifo_credit[fifo] <= 0) && - (!fws->bus_flow_blocked)) { - while (brcmf_fws_borrow_credit(fws) == 0) { + + if (fifo >= BRCMF_FWS_FIFO_AC_BE && + fifo <= BRCMF_FWS_FIFO_AC_VO && + fws->fifo_credit[fifo] == 0 && + !fws->bus_flow_blocked) { + while (brcmf_fws_borrow_credit(fws, + fifo - 1, fifo, + false) == 0) { skb = brcmf_fws_deq(fws, fifo); if (!skb) { brcmf_fws_return_credits(fws, fifo, 1); From 183cb5c347b732167292c6463c987fff125482f5 Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Tue, 11 Sep 2018 19:06:06 -0500 Subject: [PATCH 087/213] non-upstream: Changes to improve USB Tx throughput. The inbound buffer been duplicated and returned to the upper layer to increase the througput. Below the improvement observed in different traffic UDP Rx UDP Tx TCP Rx TCP Tx 237 138 161 71 Without Tx improvement 238 155 162 137 With Tx improvement Signed-off-by: Raveendran Somu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 7b3687552bab6..e38c56d4b9d04 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -621,6 +621,7 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) goto fail; } + skb_orphan(skb); req->skb = skb; req->devinfo = devinfo; usb_fill_bulk_urb(req->urb, devinfo->usbdev, devinfo->tx_pipe, From 84d346ef30b55cce5e73c5c9eb6993a36bd769f2 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 13 Sep 2018 22:12:17 -0500 Subject: [PATCH 088/213] non-upstream: reset two D11 cores if chip has two D11 cores There are two D11 cores in RSDB chips like 4359. We have to reset two D11 cores simutaneously before firmware download, or the firmware may not be initialized correctly and cause "fw initialized failed" error. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/chip.c | 50 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/chip.h | 1 + .../broadcom/brcm80211/brcmfmac/pcie.c | 2 +- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index faa5bc1376044..693d90d1a16a0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -456,11 +456,25 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, { struct brcmf_chip_priv *ci; int count; + struct brcmf_core *d11core2 = NULL; + struct brcmf_core_priv *d11priv2 = NULL; ci = core->chip; + /* special handle two D11 cores reset */ + if (core->pub.id == BCMA_CORE_80211) { + d11core2 = brcmf_chip_get_d11core(&ci->pub, 1); + if (d11core2) { + brcmf_dbg(INFO, "found two d11 cores, reset both\n"); + d11priv2 = container_of(d11core2, struct brcmf_core_priv, + pub); + } + } + /* must disable first to work for arbitrary current core state */ brcmf_chip_ai_coredisable(core, prereset, reset); + if (d11priv2) + brcmf_chip_ai_coredisable(d11priv2, prereset, reset); count = 0; while (ci->ops->read32(ci->ctx, core->wrapbase + BCMA_RESET_CTL) & @@ -472,9 +486,30 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, usleep_range(40, 60); } + if (d11priv2) { + count = 0; + while (ci->ops->read32(ci->ctx, + d11priv2->wrapbase + BCMA_RESET_CTL) & + BCMA_RESET_CTL_RESET) { + ci->ops->write32(ci->ctx, + d11priv2->wrapbase + BCMA_RESET_CTL, + 0); + count++; + if (count > 50) + break; + usleep_range(40, 60); + } + } + ci->ops->write32(ci->ctx, core->wrapbase + BCMA_IOCTL, postreset | BCMA_IOCTL_CLK); ci->ops->read32(ci->ctx, core->wrapbase + BCMA_IOCTL); + + if (d11priv2) { + ci->ops->write32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL, + postreset | BCMA_IOCTL_CLK); + ci->ops->read32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL); + } } static char *brcmf_chip_name(uint chipid, char *buf, uint len) @@ -1133,6 +1168,21 @@ void brcmf_chip_detach(struct brcmf_chip *pub) kfree(chip); } +struct brcmf_core *brcmf_chip_get_d11core(struct brcmf_chip *pub, u8 unit) +{ + struct brcmf_chip_priv *chip; + struct brcmf_core_priv *core; + + chip = container_of(pub, struct brcmf_chip_priv, pub); + list_for_each_entry(core, &chip->cores, list) { + if (core->pub.id == BCMA_CORE_80211) { + if (unit-- == 0) + return &core->pub; + } + } + return NULL; +} + struct brcmf_core *brcmf_chip_get_core(struct brcmf_chip *pub, u16 coreid) { struct brcmf_chip_priv *chip; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h index c7783dd056ee7..a7098d9301554 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.h @@ -100,5 +100,6 @@ void brcmf_chip_reset_watchdog(struct brcmf_chip *pub); void brcmf_chip_ulp_reset_lhl_regs(struct brcmf_chip *pub); void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub); void brcmf_chip_set_default_min_res_mask(struct brcmf_chip *pub); +struct brcmf_core *brcmf_chip_get_d11core(struct brcmf_chip *pub, u8 unit); #endif /* BRCMF_AXIDMP_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 9f97cbfb59419..e1454e0a4e1b2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -78,7 +78,7 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), }; -#define BRCMF_PCIE_FW_UP_TIMEOUT 2000 /* msec */ +#define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ #define BRCMF_PCIE_REG_MAP_SIZE (32 * 1024) From 3ec3856416b01c9d37ea1eac2bed352976d81b73 Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Tue, 28 Aug 2018 17:14:20 +0530 Subject: [PATCH 089/213] brcmfmac: reset PMU, backplane & all cores in CYW4373 during rmmod To do a clean reset of the chip during rmmod brcmfmac, program the PmuWatchdogCounter register. When a watchdog reset occurs, the PMU, backplane and all of the cores in the chip are reset. Signed-off-by: Madhan Mohan R --- .../broadcom/brcm80211/brcmfmac/chip.c | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index 693d90d1a16a0..fe1ca87225786 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -233,9 +233,15 @@ struct sbsocramregs { /* PMU CONTROL EXT mask for 43012C0 */ #define CY_43012_PMU_CONTROL_EXT_MASK 0x11 -/* PMU CONTROL EXT mask for 43012C0 */ +/* PMU Watchdog Counter Tick value for 43012C0 */ #define CY_43012_PMU_WATCHDOG_TICK_VAL 0x04 +/* PMU Watchdog Counter Tick value for 4373 */ +#define CY_4373_PMU_WATCHDOG_TICK_VAL 0x04 + +/* Minimum PMU resource mask for 4373 */ +#define CY_4373_PMU_MIN_RES_MASK 0xFCAFF7F + struct brcmf_core_priv { struct brcmf_core pub; u32 wrapbase; @@ -1587,7 +1593,18 @@ void brcmf_chip_reset_watchdog(struct brcmf_chip *pub) chip->ops->write32(chip->ctx, addr, CY_43012_PMU_MIN_RES_MASK); break; - + case CY_CC_4373_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_MIN_RES_MASK); + addr = CORE_CC_REG(base, watchdog_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_MIN_RES_MASK); + addr = CORE_CC_REG(base, pmuwatchdog); + chip->ops->write32(chip->ctx, addr, + CY_4373_PMU_WATCHDOG_TICK_VAL); + mdelay(100); + break; default: break; } From 5d9036a2e0e84ecc7ee2579c8925a5e8080042fb Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Mon, 17 Sep 2018 04:51:08 -0500 Subject: [PATCH 090/213] brcmfmac: introduce module parameter to configure default PM mode Add module parameter max_pm to allow using PM_MAX as default power management mode. Default PM mode is set to PM_MAX when max_pm=1, and is set to PM_FAST when max_pm=0 or max_pm is not set. Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 4 ++-- drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c | 6 ++++++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index ce1b0cba32ed1..0f74a5e3a35c8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2935,7 +2935,7 @@ brcmf_cfg80211_set_power_mgmt(struct wiphy *wiphy, struct net_device *ndev, goto done; } - pm = enabled ? PM_FAST : PM_OFF; + pm = enabled ? ifp->drvr->settings->default_pm : PM_OFF; /* Do not enable the power save after assoc if it is a p2p interface */ if (ifp->vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT) { brcmf_dbg(INFO, "Do not enable power save for P2P clients\n"); @@ -7095,7 +7095,7 @@ static s32 brcmf_config_dongle(struct brcmf_cfg80211_info *cfg) brcmf_dongle_scantime(ifp); - power_mode = cfg->pwr_save ? PM_FAST : PM_OFF; + power_mode = cfg->pwr_save ? ifp->drvr->settings->default_pm : PM_OFF; err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_PM, power_mode); if (err) goto default_conf_out; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index f65f2a5df97de..c6eed9ffe8f80 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -32,6 +32,7 @@ #include "firmware.h" #include "fweh.h" #include +#include "defs.h" MODULE_AUTHOR("Broadcom Corporation"); MODULE_DESCRIPTION("Broadcom 802.11 wireless LAN fullmac driver."); @@ -89,6 +90,10 @@ static int brcmf_frameburst; module_param_named(frameburst, brcmf_frameburst, int, 0); MODULE_PARM_DESC(frameburst, "Enable firmware frameburst feature"); +static int brcmf_max_pm; +module_param_named(max_pm, brcmf_max_pm, int, 0); +MODULE_PARM_DESC(max_pm, "Use max power management mode by default"); + #ifdef DEBUG /* always succeed brcmf_bus_started() */ static int brcmf_ignore_probe_fail; @@ -502,6 +507,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, settings->eap_restrict = !!brcmf_eap_restrict; settings->sdio_wq_highpri = !!brcmf_sdio_wq_highpri; settings->frameburst = !!brcmf_frameburst; + settings->default_pm = !!brcmf_max_pm ? PM_MAX : PM_FAST; #ifdef DEBUG settings->ignore_probe_fail = !!brcmf_ignore_probe_fail; #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 544382623191c..c53d39493f7f5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -52,6 +52,7 @@ extern struct brcmf_mp_global_t brcmf_mp_global; * @eap_restrict: Not allow data tx/rx until 802.1X auth succeeds * @sdio_wq_highpri: Tasks submitted to SDIO workqueue will run immediately. * @frameburst: Firmware frame burst mode. + * @default_pm: default power management (PM) mode. * @ignore_probe_fail: Ignore probe failure. * @country_codes: If available, pointer to struct for translating country codes * @bus: Bus specific platform data. Only SDIO at the mmoment. @@ -65,6 +66,7 @@ struct brcmf_mp_device { int sdio_dpc_prio; bool sdio_wq_highpri; bool frameburst; + int default_pm; bool ignore_probe_fail; struct brcmfmac_pd_cc *country_codes; union { From b7da7ac9db1d5dd2b0aa3fa412f12e18d727c9ba Mon Sep 17 00:00:00 2001 From: Double Lo Date: Thu, 20 Sep 2018 01:44:01 -0500 Subject: [PATCH 091/213] brcmfmac: configure wowl parameters in suspend function only if firmware support wowl This patch removes the redundant wowl configuration for none wowl FW. Signed-off-by: Lo-Hsiang Lo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 0f74a5e3a35c8..32b2a50eee4c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3928,8 +3928,9 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, brcmf_set_mpc(ifp, 1); } else { - /* Configure WOWL parameters */ - brcmf_configure_wowl(cfg, ifp, wowl); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL)) + /* Configure WOWL parameters */ + brcmf_configure_wowl(cfg, ifp, wowl); } exit: From 36ab7216c8b678b874916ffe6b56be6bd187f146 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 20 Sep 2018 16:41:35 +0800 Subject: [PATCH 092/213] brcmfmac: discard user-space RSNE for SAE authentication offload When offloading SAE authentication to firmware, user space doesn't have PMK/PMKID of the current connection but firmware has them. Since PMKID is expected for some re-association cases, a firmware-generated RSNE should be used rather than the user-space one. Signed-off-by: Chung-Hsien Hsu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 32b2a50eee4c0..36c41f5fd4ab4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2173,6 +2173,11 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, err = brcmf_set_pmk(ifp, sme->crypto.psk, BRCMF_WSEC_MAX_PSK_LEN); else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { + /* clean up user-space RSNE */ + if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { + brcmf_err("failed to clean up user-space RSNE\n"); + goto done; + } err = brcmf_set_sae_password(ifp, sme->crypto.sae_pwd, sme->crypto.sae_pwd_len); if (!err && sme->crypto.psk) From dd22e33ec6ce621133e18a3305d22a9c5c746d67 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 14 Nov 2018 23:13:02 -0600 Subject: [PATCH 093/213] brcmfmac: keep SDIO watchdog running when console_interval is non-zero brcmfmac host driver makes SDIO bus sleep and stops SDIO watchdog if no pending event or data. As a result, host driver does not poll firmware console buffer before buffer overflow, which leads to missing firmware logs. We should not stop SDIO watchdog if console_interval is non-zero in debug build. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 442dd594aad86..46af74002e226 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3846,7 +3846,11 @@ static void brcmf_sdio_bus_watchdog(struct brcmf_sdio *bus) if (bus->idlecount > bus->idletime) { brcmf_dbg(SDIO, "idle\n"); sdio_claim_host(bus->sdiodev->func[1]); - brcmf_sdio_wd_timer(bus, false); +#ifdef DEBUG + if (!BRCMF_FWCON_ON() || + bus->console_interval == 0) +#endif + brcmf_sdio_wd_timer(bus, false); bus->idlecount = 0; brcmf_sdio_bus_sleep(bus, true, false); sdio_release_host(bus->sdiodev->func[1]); From a0f6808aa9c55743aa135a803238c8062d59a430 Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Mon, 12 Nov 2018 20:32:11 -0600 Subject: [PATCH 094/213] brcmfmac: To fix kernel crash on out of boundary access To trunkcate the addtional bytes, if extra bytes been received. Current code only have a warning and proceed without handling it. But in one of the crash reported by DVT, these causes the crash intermittently. So the processing is limit to the skb->len. Signed-off-by: Raveendran Somu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index da0e1bde9f39d..55cfb82c04e0c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -1878,6 +1878,9 @@ void brcmf_fws_hdrpull(struct brcmf_if *ifp, s16 siglen, struct sk_buff *skb) WARN_ON(siglen > skb->len); + if (siglen > skb->len) + siglen = skb->len; + if (!siglen) return; /* if flow control disabled, skip to packet data and leave */ From 4aba9bba4708de5f12cc902efd35fb8e71a2bd67 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 20 Nov 2018 21:20:41 -0600 Subject: [PATCH 095/213] brcmfmac: reduce maximum station interface from 2 to 1 in RSDB mode The firmware state machines are not fully suitable for concurrent station interface support, it may hit unexpected error if we have 2 different SSIDs and the roaming scenarios concurrently. To avoid the bad user-experience if this is not fully validated, we dis-allow user to create two concurrent station interfaces. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 36c41f5fd4ab4..ad6c133942c55 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6747,7 +6747,7 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #AP <= 4, matching BI, channels = 1, 4 total * * no p2p and rsdb: - * #STA <= 2, #AP <= 2, channels = 2, 4 total + * #STA <= 1, #AP <= 2, channels = 2, 3 total * * p2p, no mchan, and mbss: * @@ -6762,8 +6762,14 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { * #AP <= 4, matching BI, channels = 1, 4 total * * p2p, rsdb, and no mbss: - * #STA <= 2, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, + * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, * channels = 2, 4 total + * + * p2p, rsdb, mbss + * #STA <= 1, #P2P-DEV <= 1, #{P2P-CL, P2P-GO} <= 2, AP <= 2, + * channels = 2, 4 total + * #AP <= 4, matching BI, channels = 1, 4 total + * */ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) { @@ -6802,7 +6808,7 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy->interface_modes |= BIT(NL80211_IFTYPE_P2P_CLIENT) | BIT(NL80211_IFTYPE_P2P_GO) | BIT(NL80211_IFTYPE_P2P_DEVICE); - c0_limits[i].max = 2; + c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_P2P_DEVICE); @@ -6811,7 +6817,7 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) BIT(NL80211_IFTYPE_P2P_GO); c0_limits[i].max = 2; c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); - combo[c].max_interfaces = 5; + combo[c].max_interfaces = 4; } else if (p2p) { if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MCHAN)) combo[c].num_different_channels = 2; @@ -6830,7 +6836,7 @@ static int brcmf_setup_ifmodes(struct wiphy *wiphy, struct brcmf_if *ifp) combo[c].max_interfaces = i; } else if (rsdb) { combo[c].num_different_channels = 2; - c0_limits[i].max = 2; + c0_limits[i].max = 1; c0_limits[i++].types = BIT(NL80211_IFTYPE_STATION); c0_limits[i].max = 2; c0_limits[i++].types = BIT(NL80211_IFTYPE_AP); From 3724a70accfecfab99702a53fc60433d4f0dc350 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 20 Nov 2018 21:29:49 -0600 Subject: [PATCH 096/213] Revert "brcmfmac: add creating station interface support" This reverts commit 36e7876df566b4996d45d234e798d996b3b2b509. Currently we don't need to support two concurrent station interfaces and the commit only supports version 1 interface creation, so we revert this commit until we need to support station interface creation. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 80 +++---------------- .../broadcom/brcm80211/brcmfmac/core.h | 1 - 2 files changed, 12 insertions(+), 69 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index ad6c133942c55..f45eed17bd525 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -239,19 +239,6 @@ struct parsed_vndr_ies { struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT]; }; -#define WL_INTERFACE_MAC_DONT_USE 0x0 -#define WL_INTERFACE_MAC_USE 0x2 - -#define WL_INTERFACE_CREATE_STA 0x0 -#define WL_INTERFACE_CREATE_AP 0x1 - -struct wl_interface_create { - u16 ver; - u32 flags; - u8 mac_addr[ETH_ALEN]; - u8 pad[13]; -}; - static u8 nl80211_band_to_fwil(enum nl80211_band band) { switch (band) { @@ -560,42 +547,6 @@ static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr) return -ENOMEM; } -static void brcmf_set_sta_iface_macaddr(struct brcmf_if *ifp, - struct wl_interface_create *iface) -{ - u8 mac_idx = ifp->drvr->sta_mac_idx; - - /* set difference MAC address with locally administered bit */ - iface->flags |= WL_INTERFACE_MAC_USE; - memcpy(iface->mac_addr, ifp->mac_addr, ETH_ALEN); - iface->mac_addr[0] |= 0x02; - iface->mac_addr[3] ^= mac_idx ? 0xC0 : 0xA0; - mac_idx++; - mac_idx = mac_idx % 2; - ifp->drvr->sta_mac_idx = mac_idx; -} - -static int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) -{ - struct wl_interface_create iface; - int err; - - memset(&iface, 0, sizeof(iface)); - - iface.ver = 0; - iface.flags = WL_INTERFACE_CREATE_STA; - if (!is_zero_ether_addr(macaddr)) { - /* set MAC address in cfg80211 params */ - memcpy(iface.mac_addr, macaddr, ETH_ALEN); - } else { - brcmf_set_sta_iface_macaddr(ifp, &iface); - } - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", &iface, - sizeof(iface)); - return err; -} - static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) { struct brcmf_mbss_ssid_le mbss_ssid_le; @@ -620,40 +571,33 @@ static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) } /** - * brcmf_ap_add_vif() - create a new AP or STA virtual interface + * brcmf_ap_add_vif() - create a new AP virtual interface for multiple BSS * * @wiphy: wiphy device of new interface. * @name: name of the new interface. - * @params: contains mac address for AP or STA device. + * @params: contains mac address for AP device. */ static -struct wireless_dev *brcmf_apsta_add_vif(struct wiphy *wiphy, const char *name, - struct vif_params *params, - enum nl80211_iftype type) +struct wireless_dev *brcmf_ap_add_vif(struct wiphy *wiphy, const char *name, + struct vif_params *params) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); struct brcmf_cfg80211_vif *vif; int err; - if (type != NL80211_IFTYPE_STATION && type != NL80211_IFTYPE_AP) - return ERR_PTR(-EINVAL); - if (brcmf_cfg80211_vif_event_armed(cfg)) return ERR_PTR(-EBUSY); brcmf_dbg(INFO, "Adding vif \"%s\"\n", name); - vif = brcmf_alloc_vif(cfg, type); + vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_AP); if (IS_ERR(vif)) return (struct wireless_dev *)vif; brcmf_cfg80211_arm_vif_event(cfg, vif); - if (type == NL80211_IFTYPE_STATION) - err = brcmf_cfg80211_request_sta_if(ifp, params->macaddr); - else - err = brcmf_cfg80211_request_ap_if(ifp); + err = brcmf_cfg80211_request_ap_if(ifp); if (err) { brcmf_cfg80211_arm_vif_event(cfg, NULL); goto fail; @@ -722,14 +666,14 @@ static struct wireless_dev *brcmf_cfg80211_add_iface(struct wiphy *wiphy, } switch (type) { case NL80211_IFTYPE_ADHOC: + case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_WDS: case NL80211_IFTYPE_MONITOR: case NL80211_IFTYPE_MESH_POINT: return ERR_PTR(-EOPNOTSUPP); - case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP: - wdev = brcmf_apsta_add_vif(wiphy, name, params, type); + wdev = brcmf_ap_add_vif(wiphy, name, params); break; case NL80211_IFTYPE_P2P_CLIENT: case NL80211_IFTYPE_P2P_GO: @@ -847,8 +791,8 @@ s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, return err; } -static int brcmf_cfg80211_del_apsta_iface(struct wiphy *wiphy, - struct wireless_dev *wdev) +static int brcmf_cfg80211_del_ap_iface(struct wiphy *wiphy, + struct wireless_dev *wdev) { struct brcmf_cfg80211_info *cfg = wiphy_priv(wiphy); struct net_device *ndev = wdev->netdev; @@ -904,14 +848,14 @@ int brcmf_cfg80211_del_iface(struct wiphy *wiphy, struct wireless_dev *wdev) switch (wdev->iftype) { case NL80211_IFTYPE_ADHOC: + case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_WDS: case NL80211_IFTYPE_MONITOR: case NL80211_IFTYPE_MESH_POINT: return -EOPNOTSUPP; - case NL80211_IFTYPE_STATION: case NL80211_IFTYPE_AP: - return brcmf_cfg80211_del_apsta_iface(wiphy, wdev); + return brcmf_cfg80211_del_ap_iface(wiphy, wdev); case NL80211_IFTYPE_P2P_CLIENT: case NL80211_IFTYPE_P2P_GO: case NL80211_IFTYPE_P2P_DEVICE: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index c2ca3b3d2da7c..e38f78e4d92c8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -145,7 +145,6 @@ struct brcmf_pub { u8 clmver[BRCMF_DCMD_SMLEN]; struct brcmf_pkt_filter_enable_le pkt_filter[MAX_PKT_FILTER_COUNT]; - u8 sta_mac_idx; }; From 607be9fac8ea9ddad3c24ca1ea2340286262672f Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 21 Nov 2018 03:16:55 -0600 Subject: [PATCH 097/213] brcmfmac: validate ifp pointer in brcmf_txfinalize We got ifp null pointer kernel panic in brcmf_txfinalize after removing Wi-Fi USB dongle when data was transmitting, The root cause is that interface was removed before calling brcmf_txfinalize in brcmf_fws_dequeue_worker and finally caused kernel panic. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 92f9db3768f5b..13d81c4f420ac 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -370,6 +370,11 @@ void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success) struct ethhdr *eh; u16 type; + if (!ifp) { + brcmu_pkt_buf_free_skb(txp); + return; + } + eh = (struct ethhdr *)(txp->data); type = ntohs(eh->h_proto); From e5370a95661d9d696dc780cb00a40a44382acdd9 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 21 Nov 2018 20:42:39 -0600 Subject: [PATCH 098/213] brcmfmac: clean up iface mac descriptor before de-initializing it We saw following warning message after removing USB Wi-Fi dongle with data connection. The root cause is that some skb were queued in iface mac descriptor psq and occupied the hanger slots, and we didn't de-queue the skb before de-initializing iface mac descriptor. So it triggered WARN_ON when cleaning up hanger slot eventually. backports/drivers/net/wireless/broadcom/brcm80211/brcmutil/utils.c:49 brcmu_pkt_buf_free_skb+0x21/0x30 [brcmutil] [ 977.523200] Modules linked in: brcmfmac(OE) brcmutil(OE) cfg80211(OE) compat(OE) sdhci_pci(OE) sdhci(OE) mmc_core(OE) rfkill ip6table_filter ip6_tables ebtable_nat ebtables dns_resolver fscache e1000e mei_me mei tpm_tis tpm_tis_core tpm iTCO_wdt ppdev iTCO_vendor_support pcspkr lpc_ich i2c_i801 mfd_core ptp pps_core parport_pc parport wmi uinput tcp_bic i915 iosf_mbi i2c_algo_bit drm_kms_helper drm i2c_core video [last unloaded: brcmfmac] [ 977.523219] CPU: 1 PID: 1306 Comm: kworker/1:1 Tainted: G OE 4.12.0 #1 [ 977.523220] Hardware name: /DH77EB, BIOS EBH7710H.86A.0100.2013.0312.1351 03/12/2013 [ 977.523223] Workqueue: usb_hub_wq hub_event [ 977.523224] task: ffff880118703600 task.stack: ffffc90000be4000 [ 977.523226] RIP: 0010:brcmu_pkt_buf_free_skb+0x21/0x30 [brcmutil] [ 977.523227] RSP: 0018:ffffc90000be7a98 EFLAGS: 00010086 [ 977.523228] RAX: 0000000000000045 RBX: ffffffffa03fa850 RCX: 0000000000000006 [ 977.523228] RDX: 0000000000000000 RSI: 0000000000000092 RDI: ffff8801106a3ce8 [ 977.523229] RBP: ffffc90000be7a98 R08: 0000000000000000 R09: 000000000000045b [ 977.523229] R10: 0000000000000001 R11: 0000000000aaaaaa R12: 0000000000000988 [ 977.523230] R13: ffff880110569938 R14: 0000000000000002 R15: ffff8801106a3ce8 [ 977.523231] FS: 0000000000000000(0000) GS:ffff88011f280000(0000) knlGS:0000000000000000 [ 977.523232] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 977.523232] CR2: 00007f7873490000 CR3: 0000000001c09000 CR4: 00000000001406e0 [ 977.523233] Call Trace: [ 977.523244] brcmf_fws_cleanup+0x1f2/0x230 [brcmfmac] [ 977.523251] brcmf_fws_del_interface+0x70/0x90 [brcmfmac] [ 977.523257] brcmf_proto_bcdc_del_if+0xe/0x10 [brcmfmac] [ 977.523262] brcmf_remove_interface+0x69/0x190 [brcmfmac] [ 977.523267] brcmf_detach+0x90/0xe0 [brcmfmac] [ 977.523273] brcmf_usb_disconnect+0x76/0xb0 [brcmfmac] [ 977.523275] usb_unbind_interface+0x72/0x260 [ 977.523279] device_release_driver_internal+0x141/0x200 [ 977.523280] device_release_driver+0x12/0x20 [ 977.523282] bus_remove_device+0xfd/0x170 [ 977.523283] device_del+0x1d9/0x300 [ 977.523284] usb_disable_device+0x9e/0x270 [ 977.523286] usb_disconnect+0x94/0x270 [ 977.523287] usb_disconnect+0x1f1/0x270 [ 977.523288] hub_event+0x76d/0x13b0 [ 977.523291] process_one_work+0x144/0x360 [ 977.523293] worker_thread+0x4d/0x3c0 [ 977.523294] kthread+0x112/0x150 [ 977.523295] ? rescuer_thread+0x340/0x340 [ 977.523296] ? kthread_park+0x60/0x60 [ 977.523298] ? SyS_exit_group+0x14/0x20 [ 977.523300] ret_from_fork+0x25/0x30 [ 977.523301] Code: 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 48 85 ff 74 15 48 83 3f 00 55 48 89 e5 75 0d be 01 00 00 00 e8 82 5a 3e e1 5d f3 c3 <0f> ff eb ef 66 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 44 00 00 55 Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 55cfb82c04e0c..0a47746b4e00d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2212,6 +2212,7 @@ void brcmf_fws_del_interface(struct brcmf_if *ifp) brcmf_fws_lock(fws); ifp->fws_desc = NULL; + brcmf_fws_macdesc_cleanup(fws, entry, ifp->ifidx); brcmf_dbg(TRACE, "deleting %s\n", entry->name); brcmf_fws_macdesc_deinit(entry); brcmf_fws_cleanup(fws, ifp->ifidx); From ad2d42bd4d998d175787453b1f3631bdf8e8052e Mon Sep 17 00:00:00 2001 From: Raveendran Somu Date: Wed, 7 Nov 2018 16:59:03 -0600 Subject: [PATCH 099/213] brcmfmac: To support printing USB console messages This change is to add support for printing the firmware console messges of a USB interface chip to the host. To enable this feature, build option '-msgtrace' should be enabled in the firmware. And in the host, debug=0x100000 should be provided as a module parameter. Signed-off-by: Raveendran Somu --- .../broadcom/brcm80211/brcmfmac/bus.h | 1 + .../broadcom/brcm80211/brcmfmac/core.c | 8 ++ .../broadcom/brcm80211/brcmfmac/debug.c | 82 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/debug.h | 24 ++++++ .../broadcom/brcm80211/brcmfmac/usb.c | 6 ++ 5 files changed, 121 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 0b76a615708e1..527753dba28fc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -256,6 +256,7 @@ void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state); int brcmf_bus_started(struct device *dev); s32 brcmf_iovar_data_set(struct device *dev, char *name, void *data, u32 len); void brcmf_bus_add_txhdrlen(struct device *dev, uint len); +int brcmf_fwlog_attach(struct device *dev); #ifdef CONFIG_BRCMFMAC_SDIO void brcmf_sdio_exit(void); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 13d81c4f420ac..48c7275eb7e94 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -928,6 +928,14 @@ int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings) return ret; } +int brcmf_fwlog_attach(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pub *drvr = bus_if->drvr; + + return brcmf_debug_fwlog_init(drvr); +} + static int brcmf_revinfo_read(struct seq_file *s, void *data) { struct brcmf_bus *bus_if = dev_get_drvdata(s->private); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c index 2d3e5e263a326..8ee3fd06f0a26 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c @@ -27,6 +27,82 @@ static struct dentry *root_folder; +static int +brcmf_debug_msgtrace_seqchk(u32 *prev, u32 cur) +{ + if ((cur == 0 && *prev == 0xFFFFFFFF) || ((cur - *prev) == 1)) { + goto done; + } else if (cur == *prev) { + brcmf_dbg(FWCON, "duplicate trace\n"); + return -1; + } else if (cur > *prev) { + brcmf_dbg(FWCON, "lost %d packets\n", cur - *prev); + } else { + brcmf_dbg(FWCON, "seq out of order, host %d, dongle %d\n", + *prev, cur); + } +done: + *prev = cur; + return 0; +} + +static int +brcmf_debug_msg_parser(void *event_data) +{ + int err = 0; + struct msgtrace_hdr *hdr; + char *data, *s; + static u32 seqnum_prev; + + hdr = (struct msgtrace_hdr *)event_data; + data = (char *)event_data + MSGTRACE_HDRLEN; + + /* There are 2 bytes available at the end of data */ + data[ntohs(hdr->len)] = '\0'; + + if (ntohl(hdr->discarded_bytes) || ntohl(hdr->discarded_printf)) { + brcmf_dbg(FWCON, "Discarded_bytes %d discarded_printf %d\n", + ntohl(hdr->discarded_bytes), + ntohl(hdr->discarded_printf)); + } + + err = brcmf_debug_msgtrace_seqchk(&seqnum_prev, ntohl(hdr->seqnum)); + if (err) + return err; + + while (*data != '\0' && (s = strstr(data, "\n")) != NULL) { + *s = '\0'; + brcmf_dbg(FWCON, "[FWLOG] %s\n", data); + data = s + 1; + } + if (*data) + brcmf_dbg(FWCON, "[FWLOG] %s", data); + + return err; +} + +static int +brcmf_debug_trace_parser(struct brcmf_if *ifp, + const struct brcmf_event_msg *evtmsg, + void *event_data) +{ + int err = 0; + struct msgtrace_hdr *hdr; + + hdr = (struct msgtrace_hdr *)event_data; + if (hdr->version != MSGTRACE_VERSION) { + brcmf_dbg(FWCON, "trace version mismatch host %d dngl %d\n", + MSGTRACE_VERSION, hdr->version); + err = -EPROTO; + return err; + } + + if (hdr->trace_type == MSGTRACE_HDR_TYPE_MSG) + err = brcmf_debug_msg_parser(event_data); + + return err; +} + int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, size_t len) { @@ -81,6 +157,12 @@ int brcmf_debug_attach(struct brcmf_pub *drvr) return PTR_ERR_OR_ZERO(drvr->dbgfs_dir); } +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr) +{ + return brcmf_fweh_register(drvr, BRCMF_E_TRACE, + brcmf_debug_trace_parser); +} + void brcmf_debug_detach(struct brcmf_pub *drvr) { brcmf_fweh_unregister(drvr, BRCMF_E_PSM_WATCHDOG); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 23a290f13a889..2fc9e06f98869 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -102,6 +102,10 @@ do { \ #endif /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ +#define MSGTRACE_VERSION 1 +#define MSGTRACE_HDR_TYPE_MSG 0 +#define MSGTRACE_HDR_TYPE_LOG 1 + #define brcmf_dbg_hex_dump(test, data, len, fmt, ...) \ do { \ trace_brcmf_hexdump((void *)data, len); \ @@ -123,6 +127,7 @@ int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, int (*read_fn)(struct seq_file *seq, void *data)); int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, size_t len); +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr); #else static inline void brcmf_debugfs_init(void) { @@ -149,6 +154,25 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, { return 0; } + +static inline +int brcmf_debug_fwlog_init(struct brcmf_pub *drvr) +{ + return 0; +} #endif +/* Message trace header */ +struct msgtrace_hdr { + u8 version; + u8 trace_type; + u16 len; /* Len of the trace */ + u32 seqnum; /* Sequence number of message */ + /* Number of discarded bytes because of trace overflow */ + u32 discarded_bytes; + /* Number of discarded printf because of trace overflow */ + u32 discarded_printf; +}; + +#define MSGTRACE_HDRLEN sizeof(struct msgtrace_hdr) #endif /* BRCMFMAC_DEBUG_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index e38c56d4b9d04..402dad97ecddf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -1168,6 +1168,12 @@ static int brcmf_usb_bus_setup(struct brcmf_usbdev_info *devinfo) return ret; } + if (BRCMF_FWCON_ON()) { + ret = brcmf_fwlog_attach(devinfo->dev); + if (ret) + goto fail; + } + ret = brcmf_usb_up(devinfo->dev); if (ret) goto fail; From bf749c18e583b89c56fd2cde8dc137e3f8ed8d60 Mon Sep 17 00:00:00 2001 From: Prasanna Kerekoppa Date: Tue, 4 Dec 2018 00:30:31 -0600 Subject: [PATCH 100/213] brcmfmac: To fix Bss Info flag definition Bug Bss info flag definition need to be fixed from 0x2 to 0x4 This flag is for rssi info received on channel. All Firmware branches defined as 0x4 and this is bug in brcmfmac. Signed-off-by: Prasanna Kerekoppa --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 53b3b483a4112..d642270533681 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -30,7 +30,7 @@ #define BRCMF_ARP_OL_PEER_AUTO_REPLY 0x00000008 #define BRCMF_BSS_INFO_VERSION 109 /* curr ver of brcmf_bss_info_le struct */ -#define BRCMF_BSS_RSSI_ON_CHANNEL 0x0002 +#define BRCMF_BSS_RSSI_ON_CHANNEL 0x0004 #define BRCMF_STA_WME 0x00000002 /* WMM association */ #define BRCMF_STA_AUTHE 0x00000008 /* Authenticated */ From 8054c5dd9536796462c7e441b190a707b3ad016d Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Mon, 17 Dec 2018 00:54:07 -0600 Subject: [PATCH 101/213] brcmfmac: disable command decode in sdio_aos for 4356 AOS is a part of the SDIOD core that becomes active when the rest of SDIOD is sleeping to keep SDIO bus alive responding to reduced set of commands. Transaction between AOS and SDIOD is not protected, and if cmd 52 is received in AOS and in the middle of response state changed from AOS to SDIOD, response is corrupted and it causes to SDIO Host controller to hang. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 46af74002e226..031f276816969 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -3573,6 +3573,7 @@ static void brcmf_sdio_sr_init(struct brcmf_sdio *bus) bus->ci->chip == CY_CC_4373_CHIP_ID || bus->ci->chip == BRCM_CC_4339_CHIP_ID || bus->ci->chip == BRCM_CC_4354_CHIP_ID || + bus->ci->chip == BRCM_CC_4356_CHIP_ID || bus->ci->chip == BRCM_CC_4345_CHIP_ID) { cardcap = SDIO_CCCR_BRCM_CARDCAP_CMD_NODEC; } else { From 6d6cc22f66bff2810101075564985a1ff1b6a9c2 Mon Sep 17 00:00:00 2001 From: Ryohei Kondo Date: Thu, 8 Nov 2018 10:37:29 +0900 Subject: [PATCH 102/213] brcmfmac: increase default max WOWL patterns to 16 4373 has support of 16 WOWL patterns thus increasing the default value Signed-off-by: Ryohei Kondo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index d642270533681..73b11242812d4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -127,7 +127,7 @@ /* Link Down indication in WoWL mode: */ #define BRCMF_WOWL_LINKDOWN (1 << 31) -#define BRCMF_WOWL_MAXPATTERNS 8 +#define BRCMF_WOWL_MAXPATTERNS 16 #define BRCMF_WOWL_MAXPATTERNSIZE 128 enum { From 1b67bbd0720b302bab4442bf76a687ea8efbf647 Mon Sep 17 00:00:00 2001 From: Robert Trask Date: Thu, 20 Dec 2018 11:55:30 -0600 Subject: [PATCH 103/213] brcmfmac: Enable, Process, and forward PHY_TEMP event. New code enables PHY_TEMP event in firmware, receives it, processes into cfg80211 vendor specific event, and forwards to waiting host. -1- Enable rx PHY_TEMP event from underlying hardware -2- Process PHY_TEMP event into vendor specific event -3- Forward vendor specific event to host layer Signed-off-by: Robert Trask --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 5 ++ .../broadcom/brcm80211/brcmfmac/fweh.h | 1 + .../broadcom/brcm80211/brcmfmac/vendor.c | 57 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/vendor.h | 19 +++++++ 4 files changed, 82 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index f45eed17bd525..4abe305509e61 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6908,6 +6908,7 @@ static void brcmf_wiphy_wowl_params(struct wiphy *wiphy, struct brcmf_if *ifp) static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) { struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); const struct ieee80211_iface_combination *combo; struct ieee80211_supported_band *band; u16 max_interfaces = 0; @@ -6978,6 +6979,10 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) /* vendor commands/events support */ wiphy->vendor_commands = brcmf_vendor_cmds; wiphy->n_vendor_commands = BRCMF_VNDR_CMDS_LAST - 1; + wiphy->vendor_events = brcmf_vendor_events; + wiphy->n_vendor_events = BRCMF_VNDR_EVTS_LAST; + brcmf_fweh_register(cfg->pub, BRCMF_E_PHY_TEMP, + brcmf_wiphy_phy_temp_evt_handler); brcmf_wiphy_wowl_params(wiphy, ifp); err = brcmf_fil_cmd_data_get(ifp, BRCMF_C_GET_BANDLIST, &bandlist, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index 47bd34e901a79..2cd2252c2ba5b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -101,6 +101,7 @@ struct brcmf_cfg80211_info; BRCMF_ENUM_DEF(FIFO_CREDIT_MAP, 74) \ BRCMF_ENUM_DEF(ACTION_FRAME_RX, 75) \ BRCMF_ENUM_DEF(TDLS_PEER_EVENT, 92) \ + BRCMF_ENUM_DEF(PHY_TEMP, 111) \ BRCMF_ENUM_DEF(BCMC_CREDIT_SUPPORT, 127) \ BRCMF_ENUM_DEF(ULP, 146) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index 2b88ba132bc8f..d969806bbacce 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -118,6 +118,56 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, return ret; } +s32 +brcmf_wiphy_phy_temp_evt_handler(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, void *data) + +{ + struct brcmf_cfg80211_info *cfg = ifp->drvr->config; + struct wiphy *wiphy = cfg_to_wiphy(cfg); + struct sk_buff *skb; + struct nlattr *phy_temp_data; + u32 version, temp, tempdelta; + struct brcmf_phy_temp_evt *phy_temp_evt; + + phy_temp_evt = (struct brcmf_phy_temp_evt *)data; + + version = le32_to_cpu(phy_temp_evt->version); + temp = le32_to_cpu(phy_temp_evt->temp); + tempdelta = le32_to_cpu(phy_temp_evt->tempdelta); + + skb = cfg80211_vendor_event_alloc(wiphy, NULL, + sizeof(*phy_temp_evt), + BRCMF_VNDR_EVTS_PHY_TEMP, + GFP_KERNEL); + + if (!skb) { + brcmf_dbg(EVENT, "NO MEM: can't allocate skb for vendor PHY_TEMP_EVENT\n"); + return -ENOMEM; + } + + phy_temp_data = nla_nest_start(skb, NL80211_ATTR_VENDOR_EVENTS); + if (!phy_temp_data) { + nla_nest_cancel(skb, phy_temp_data); + kfree_skb(skb); + brcmf_dbg(EVENT, "skb could not nest vendor attributes\n"); + return -EMSGSIZE; + } + + if (nla_put_u32(skb, BRCMF_NLATTR_VERS, version) || + nla_put_u32(skb, BRCMF_NLATTR_PHY_TEMP, temp) || + nla_put_u32(skb, BRCMF_NLATTR_PHY_TEMPDELTA, tempdelta)) { + kfree_skb(skb); + brcmf_dbg(EVENT, "NO ROOM in skb for vendor PHY_TEMP_EVENT\n"); + return -EMSGSIZE; + } + + nla_nest_end(skb, phy_temp_data); + + cfg80211_vendor_event(skb, GFP_KERNEL); + return 0; +} + const struct wiphy_vendor_command brcmf_vendor_cmds[] = { { { @@ -129,3 +179,10 @@ const struct wiphy_vendor_command brcmf_vendor_cmds[] = { .doit = brcmf_cfg80211_vndr_cmds_dcmd_handler }, }; + +const struct nl80211_vendor_cmd_info brcmf_vendor_events[] = { + { + .vendor_id = BROADCOM_OUI, + .subcmd = BRCMF_VNDR_EVTS_PHY_TEMP, + }, +}; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h index 061b7bfa2e1cf..f4495e9f893ce 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h @@ -25,6 +25,11 @@ enum brcmf_vndr_cmds { BRCMF_VNDR_CMDS_LAST }; +enum brcmf_vndr_evts { + BRCMF_VNDR_EVTS_PHY_TEMP, + BRCMF_VNDR_EVTS_LAST +}; + /** * enum brcmf_nlattrs - nl80211 message attributes * @@ -36,11 +41,21 @@ enum brcmf_nlattrs { BRCMF_NLATTR_LEN, BRCMF_NLATTR_DATA, + BRCMF_NLATTR_VERS, + BRCMF_NLATTR_PHY_TEMP, + BRCMF_NLATTR_PHY_TEMPDELTA, __BRCMF_NLATTR_AFTER_LAST, BRCMF_NLATTR_MAX = __BRCMF_NLATTR_AFTER_LAST - 1 }; +/* structure of event sent up by firmware: is this the right place for it? */ +struct brcmf_phy_temp_evt { + __le32 version; + __le32 temp; + __le32 tempdelta; +} __packed; + /** * struct brcmf_vndr_dcmd_hdr - message header for cfg80211 vendor command dcmd * support @@ -60,5 +75,9 @@ struct brcmf_vndr_dcmd_hdr { }; extern const struct wiphy_vendor_command brcmf_vendor_cmds[]; +extern const struct nl80211_vendor_cmd_info brcmf_vendor_events[]; +s32 brcmf_wiphy_phy_temp_evt_handler(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, + void *data); #endif /* _vendor_h_ */ From 54faf06cd36e9b519546e79e18a3ae5c846662de Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Sun, 2 Sep 2018 21:57:24 -0500 Subject: [PATCH 104/213] brcmfmac: add USB autosuspend feature support This patch is for adding auto suspend feature in FMAC USB bus. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/usb.c | 112 ++++++++++++------ 1 file changed, 74 insertions(+), 38 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 402dad97ecddf..9f0e4ab8a4937 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -175,7 +175,6 @@ struct brcmf_usbdev_info { struct urb *bulk_urb; /* used for FW download */ - bool wowl_enabled; struct brcmf_mp_device *settings; }; @@ -323,28 +322,44 @@ static int brcmf_usb_tx_ctlpkt(struct device *dev, u8 *buf, u32 len) int err = 0; int timeout = 0; struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); + struct usb_interface *intf = to_usb_interface(dev); brcmf_dbg(USB, "Enter\n"); - if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) - return -EIO; - if (test_and_set_bit(0, &devinfo->ctl_op)) - return -EIO; + err = usb_autopm_get_interface(intf); + if (err) + goto out; + + if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { + err = -EIO; + goto fail; + } + + if (test_and_set_bit(0, &devinfo->ctl_op)) { + err = -EIO; + goto fail; + } devinfo->ctl_completed = false; err = brcmf_usb_send_ctl(devinfo, buf, len); if (err) { brcmf_err("fail %d bytes: %d\n", err, len); clear_bit(0, &devinfo->ctl_op); - return err; + goto fail; } timeout = brcmf_usb_ioctl_resp_wait(devinfo); if (!timeout) { brcmf_err("Txctl wait timed out\n"); usb_kill_urb(devinfo->ctl_urb); + clear_bit(0, &devinfo->ctl_op); err = -EIO; + goto fail; } clear_bit(0, &devinfo->ctl_op); + +fail: + usb_autopm_put_interface(intf); +out: return err; } @@ -353,33 +368,48 @@ static int brcmf_usb_rx_ctlpkt(struct device *dev, u8 *buf, u32 len) int err = 0; int timeout = 0; struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); + struct usb_interface *intf = to_usb_interface(dev); brcmf_dbg(USB, "Enter\n"); - if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) - return -EIO; - if (test_and_set_bit(0, &devinfo->ctl_op)) - return -EIO; + err = usb_autopm_get_interface(intf); + if (err) + goto out; + + if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { + err = -EIO; + goto fail; + } + + if (test_and_set_bit(0, &devinfo->ctl_op)) { + err = -EIO; + goto fail; + } devinfo->ctl_completed = false; err = brcmf_usb_recv_ctl(devinfo, buf, len); if (err) { brcmf_err("fail %d bytes: %d\n", err, len); clear_bit(0, &devinfo->ctl_op); - return err; + goto fail; } timeout = brcmf_usb_ioctl_resp_wait(devinfo); err = devinfo->ctl_urb_status; if (!timeout) { brcmf_err("rxctl wait timed out\n"); usb_kill_urb(devinfo->ctl_urb); + clear_bit(0, &devinfo->ctl_op); err = -EIO; + goto fail; } clear_bit(0, &devinfo->ctl_op); + +fail: + usb_autopm_put_interface(intf); if (!err) return devinfo->ctl_urb_actual_length; - else - return err; +out: + return err; } static struct brcmf_usbreq *brcmf_usb_deq(struct brcmf_usbdev_info *devinfo, @@ -517,10 +547,12 @@ static void brcmf_usb_rx_complete(struct urb *urb) return; } - if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP) { + if (devinfo->bus_pub.state == BRCMFMAC_USB_STATE_UP || + devinfo->bus_pub.state == BRCMFMAC_USB_STATE_SLEEP) { skb_put(skb, urb->actual_length); brcmf_rx_frame(devinfo->dev, skb, true); brcmf_usb_rx_refill(devinfo, req); + usb_mark_last_busy(urb->dev); } else { brcmu_pkt_buf_free_skb(skb); brcmf_usb_enq(devinfo, &devinfo->rx_freeq, req, NULL); @@ -606,6 +638,11 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) struct brcmf_usbreq *req; int ret; unsigned long flags; + struct usb_interface *intf = to_usb_interface(dev); + + ret = usb_autopm_get_interface(intf); + if (ret) + goto out; brcmf_dbg(USB, "Enter, skb=%p\n", skb); if (devinfo->bus_pub.state != BRCMFMAC_USB_STATE_UP) { @@ -645,9 +682,10 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) devinfo->tx_flowblock = true; } spin_unlock_irqrestore(&devinfo->tx_flowblock_lock, flags); - return 0; fail: + usb_autopm_put_interface(intf); +out: return ret; } @@ -1005,20 +1043,32 @@ static int brcmf_usb_fw_download(struct brcmf_usbdev_info *devinfo) { int err; + struct usb_interface *intf; brcmf_dbg(USB, "Enter\n"); - if (devinfo == NULL) - return -ENODEV; + if (!devinfo) { + err = -ENODEV; + goto out; + } if (!devinfo->image) { brcmf_err("No firmware!\n"); - return -ENOENT; + err = -ENOENT; + goto out; } + intf = to_usb_interface(devinfo->dev); + err = usb_autopm_get_interface(intf); + if (err) + goto out; + err = brcmf_usb_dlstart(devinfo, (u8 *)devinfo->image, devinfo->image_len); if (err == 0) err = brcmf_usb_dlrun(devinfo); + + usb_autopm_put_interface(intf); +out: return err; } @@ -1119,18 +1169,6 @@ struct brcmf_usbdev *brcmf_usb_attach(struct brcmf_usbdev_info *devinfo, return NULL; } -static void brcmf_usb_wowl_config(struct device *dev, bool enabled) -{ - struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(dev); - - brcmf_dbg(USB, "Configuring WOWL, enabled=%d\n", enabled); - devinfo->wowl_enabled = enabled; - if (enabled) - device_set_wakeup_enable(devinfo->dev, true); - else - device_set_wakeup_enable(devinfo->dev, false); -} - static int brcmf_usb_get_fwname(struct device *dev, u32 chip, u32 chiprev, u8 *fw_name) { @@ -1153,7 +1191,6 @@ static const struct brcmf_bus_ops brcmf_usb_bus_ops = { .stop = brcmf_usb_down, .txctl = brcmf_usb_tx_ctlpkt, .rxctl = brcmf_usb_rx_ctlpkt, - .wowl_config = brcmf_usb_wowl_config, .get_fwname = brcmf_usb_get_fwname, }; @@ -1338,6 +1375,8 @@ brcmf_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) usb_set_intfdata(intf, devinfo); + intf->needs_remote_wakeup = 1; + /* Check that the device supports only one configuration */ if (usb->descriptor.bNumConfigurations != 1) { brcmf_err("Number of configurations: %d not supported\n", @@ -1451,10 +1490,8 @@ static int brcmf_usb_suspend(struct usb_interface *intf, pm_message_t state) brcmf_dbg(USB, "Enter\n"); devinfo->bus_pub.state = BRCMFMAC_USB_STATE_SLEEP; - if (devinfo->wowl_enabled) - brcmf_cancel_all_urbs(devinfo); - else - brcmf_detach(&usb->dev); + brcmf_cancel_all_urbs(devinfo); + device_set_wakeup_enable(devinfo->dev, true); return 0; } @@ -1467,11 +1504,9 @@ static int brcmf_usb_resume(struct usb_interface *intf) struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev); brcmf_dbg(USB, "Enter\n"); - if (!devinfo->wowl_enabled) - return brcmf_usb_bus_setup(devinfo); - devinfo->bus_pub.state = BRCMFMAC_USB_STATE_UP; brcmf_usb_rx_fill_all(devinfo); + device_set_wakeup_enable(devinfo->dev, false); return 0; } @@ -1519,6 +1554,7 @@ static struct usb_driver brcmf_usbdrvr = { .suspend = brcmf_usb_suspend, .resume = brcmf_usb_resume, .reset_resume = brcmf_usb_reset_resume, + .supports_autosuspend = true, .disable_hub_initiated_lpm = 1, }; From 1386d3beda95e081d2a7f882ec8b867a377cc42e Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Fri, 4 Jan 2019 14:42:04 +0530 Subject: [PATCH 105/213] non-upstream: workaround for 4373 USB WMM 5.2.27 test failure With the addition of skb_orphan in the datapath, though the throughput increases in TX path, it introduces an issue by removing the flowcontrol from upper layer and allowing more data to flow for different access category. This workaround is to disable using skb_orphan when running multi-stream data. This change will not be required in linux 4.15 and later versions. Signed-off-by: Raveendran Somu --- .../broadcom/brcm80211/brcmfmac/bus.h | 1 + .../broadcom/brcm80211/brcmfmac/fwsignal.c | 37 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/usb.c | 4 +- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 527753dba28fc..fc1b1e2cb1b9a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -160,6 +160,7 @@ struct brcmf_bus { const struct brcmf_bus_ops *ops; struct brcmf_bus_msgbuf *msgbuf; + bool allow_skborphan; }; /* diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 0a47746b4e00d..cfa2e1bcaf0ac 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -523,6 +523,7 @@ struct brcmf_fws_info { bool creditmap_received; u8 mode; bool avoid_queueing; + int fifo_init_credit[BRCMF_FWS_FIFO_COUNT]; }; /* @@ -1645,6 +1646,7 @@ static int brcmf_fws_notify_credit_map(struct brcmf_if *ifp, if (fws->fifo_credit[i] < 0) brcmf_err("fifo_credit[%d] value is negative(%d)\n", i, fws->fifo_credit[i]); + fws->fifo_init_credit[i] = fws->fifo_credit[i]; } brcmf_fws_schedule_deq(fws); brcmf_fws_unlock(fws); @@ -2219,6 +2221,36 @@ void brcmf_fws_del_interface(struct brcmf_if *ifp) brcmf_fws_unlock(fws); } +static bool brcmf_fws_ismultistream(struct brcmf_fws_info *fws) +{ + bool ret = false; + u8 credit_usage = 0; + + /* Check only for BE, VI and VO traffic */ + u32 delay_map = fws->fifo_delay_map & + ((1 << BRCMF_FWS_FIFO_AC_BE) | + (1 << BRCMF_FWS_FIFO_AC_VI) | + (1 << BRCMF_FWS_FIFO_AC_VO)); + + if (hweight_long(delay_map) > 1) { + ret = true; + } else { + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_BE] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_BE]) + credit_usage++; + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VI] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VI]) + credit_usage++; + if (fws->fifo_credit[BRCMF_FWS_FIFO_AC_VO] < + fws->fifo_init_credit[BRCMF_FWS_FIFO_AC_VO]) + credit_usage++; + + if (credit_usage > 1) + ret = true; + } + return ret; +} + static void brcmf_fws_dequeue_worker(struct work_struct *worker) { struct brcmf_fws_info *fws; @@ -2232,6 +2264,11 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) fws = container_of(worker, struct brcmf_fws_info, fws_dequeue_work); drvr = fws->drvr; + if (brcmf_fws_ismultistream(fws)) + drvr->bus_if->allow_skborphan = false; + else + drvr->bus_if->allow_skborphan = true; + brcmf_fws_lock(fws); for (fifo = BRCMF_FWS_FIFO_BCMC; fifo >= 0 && !fws->bus_flow_blocked; fifo--) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 9f0e4ab8a4937..63d42356d6299 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -658,7 +658,8 @@ static int brcmf_usb_tx(struct device *dev, struct sk_buff *skb) goto fail; } - skb_orphan(skb); + if (devinfo->bus_pub.bus->allow_skborphan) + skb_orphan(skb); req->skb = skb; req->devinfo = devinfo; usb_fill_bulk_urb(req->urb, devinfo->usbdev, devinfo->tx_pipe, @@ -1289,6 +1290,7 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) bus->ops = &brcmf_usb_bus_ops; bus->proto_type = BRCMF_PROTO_BCDC; bus->always_use_fws_queue = true; + bus->allow_skborphan = true; #ifdef CONFIG_PM bus->wowl_supported = true; #endif From 2fb47afbade1924d4a9a558318b101d0f46ba6c3 Mon Sep 17 00:00:00 2001 From: Stefan Wahren Date: Wed, 12 Dec 2018 20:20:06 +0100 Subject: [PATCH 106/213] brcmfmac: Fix access point mode commit 861cb5eb467f ("brcmfmac: Fix access point mode") upstream. Since commit 1204aa17f3b4 ("brcmfmac: set WIPHY_FLAG_HAVE_AP_SME flag") the Raspberry Pi 3 A+ (BCM43455) isn't able to operate in AP mode with hostapd (device_ap_sme=1 use_monitor=0): brcmfmac: brcmf_cfg80211_stop_ap: setting AP mode failed -52 So add the missing mgmt_stypes for AP mode to fix this. Fixes: 1204aa17f3b4 ("brcmfmac: set WIPHY_FLAG_HAVE_AP_SME flag") Suggested-by: Arend van Spriel Signed-off-by: Stefan Wahren Acked-by: Arend van Spriel Signed-off-by: Kalle Valo --- .../wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 4abe305509e61..39fa2b3c3cf4a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -6669,6 +6669,16 @@ brcmf_txrx_stypes[NUM_NL80211_IFTYPES] = { .tx = 0xffff, .rx = BIT(IEEE80211_STYPE_ACTION >> 4) | BIT(IEEE80211_STYPE_PROBE_REQ >> 4) + }, + [NL80211_IFTYPE_AP] = { + .tx = 0xffff, + .rx = BIT(IEEE80211_STYPE_ASSOC_REQ >> 4) | + BIT(IEEE80211_STYPE_REASSOC_REQ >> 4) | + BIT(IEEE80211_STYPE_PROBE_REQ >> 4) | + BIT(IEEE80211_STYPE_DISASSOC >> 4) | + BIT(IEEE80211_STYPE_AUTH >> 4) | + BIT(IEEE80211_STYPE_DEAUTH >> 4) | + BIT(IEEE80211_STYPE_ACTION >> 4) } }; From 55310cce51ef0e25ca1e681320f06050356d16e9 Mon Sep 17 00:00:00 2001 From: Brian Henriquez Date: Mon, 17 Dec 2018 12:26:07 +0900 Subject: [PATCH 107/213] brcmfmac: make compatible with Fully Preemptile Kernel (RT) Changed mutex locking to completions to make code compatible with Fully Preemptile Kernel (RT) configuration Signed-off-by: Brian Henriquez --- .../broadcom/brcm80211/brcmfmac/usb.c | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 63d42356d6299..3025b6ea9daa1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -103,7 +103,6 @@ static struct brcmf_firmware_mapping brcmf_usb_fwnames[] = { */ #define DL_IMAGE_TOOBIG 7 /* firmware image too big */ - struct trx_header_le { __le32 magic; /* "HDR0" */ __le32 len; /* Length of file including header */ @@ -160,7 +159,7 @@ struct brcmf_usbdev_info { struct usb_device *usbdev; struct device *dev; - struct mutex dev_init_lock; + struct completion fw_downloaded; int ctl_in_pipe, ctl_out_pipe; struct urb *ctl_urb; /* URB for control endpoint */ @@ -1257,11 +1256,11 @@ static void brcmf_usb_probe_phase2(struct device *dev, int ret, if (ret) goto error; - mutex_unlock(&devinfo->dev_init_lock); + complete(&devinfo->fw_downloaded); return; error: brcmf_dbg(TRACE, "failed: dev=%s, err=%d\n", dev_name(dev), ret); - mutex_unlock(&devinfo->dev_init_lock); + complete(&devinfo->fw_downloaded); device_release_driver(dev); } @@ -1308,7 +1307,7 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo) if (ret) goto fail; /* we are done */ - mutex_unlock(&devinfo->dev_init_lock); + complete(&devinfo->fw_downloaded); return 0; } bus->chip = bus_pub->devid; @@ -1369,11 +1368,9 @@ brcmf_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) devinfo->usbdev = usb; devinfo->dev = &usb->dev; - /* Take an init lock, to protect for disconnect while still loading. - * Necessary because of the asynchronous firmware load construction - */ - mutex_init(&devinfo->dev_init_lock); - mutex_lock(&devinfo->dev_init_lock); + + /* Initialize fw downloaded completion and mark it as not complete */ + init_completion(&devinfo->fw_downloaded); usb_set_intfdata(intf, devinfo); @@ -1453,7 +1450,7 @@ brcmf_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) return 0; fail: - mutex_unlock(&devinfo->dev_init_lock); + complete(&devinfo->fw_downloaded); kfree(devinfo); usb_set_intfdata(intf, NULL); return ret; @@ -1468,7 +1465,7 @@ brcmf_usb_disconnect(struct usb_interface *intf) devinfo = (struct brcmf_usbdev_info *)usb_get_intfdata(intf); if (devinfo) { - mutex_lock(&devinfo->dev_init_lock); + wait_for_completion(&devinfo->fw_downloaded); /* Make sure that devinfo still exists. Firmware probe routines * may have released the device and cleared the intfdata. */ From a5df95690bb739b54e58d805de8c47c3713fbe72 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Fri, 18 Jan 2019 03:57:34 -0600 Subject: [PATCH 108/213] brcmfmac: remove the duplicate line of writing BRCMF_PCIE_REG_SBMBX The code is duplicate so remove the line from pcie.c Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index e1454e0a4e1b2..63ad529a36446 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -671,7 +671,6 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) brcmf_pcie_write_tcm32(devinfo, addr, htod_mb_data); pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_SBMBX, 1); - pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_SBMBX, 1); return 0; } From 7d69dab8e56cfe48d313f602d971d1c02dbb5472 Mon Sep 17 00:00:00 2001 From: David Weng Date: Mon, 28 Jan 2019 01:05:36 -0600 Subject: [PATCH 109/213] brcmfmac: 43012 reloading FAMC driver failure on BU machine after it exiting DS1. Device is left in non-responding state when unloading FMAC driver in non-DS1 state if it ever enters DS1 at least once. It leaves that state only after a hard reset or power cycle 43012. SWWLAN-137253 Signed-off-by: David Weng --- .../net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 12 +++++++----- .../net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 2 -- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index fe1ca87225786..f35ae7735d72b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -1491,11 +1491,6 @@ void brcmf_chip_reset_pmu_regs(struct brcmf_chip *pub) addr = CORE_CC_REG(base, pmucontrol_ext); chip->ops->write32(chip->ctx, addr, CY_43012_PMU_CONTROL_EXT_MASK); - - /* PMU watchdog */ - addr = CORE_CC_REG(base, pmuwatchdog); - chip->ops->write32(chip->ctx, addr, - CY_43012_PMU_WATCHDOG_TICK_VAL); break; default: @@ -1588,10 +1583,17 @@ void brcmf_chip_reset_watchdog(struct brcmf_chip *pub) switch (pub->chip) { case CY_CC_43012_CHIP_ID: + addr = CORE_CC_REG(base, min_res_mask); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_MIN_RES_MASK); /* Watchdog res mask */ addr = CORE_CC_REG(base, watchdog_res_mask); chip->ops->write32(chip->ctx, addr, CY_43012_PMU_MIN_RES_MASK); + /* PMU watchdog */ + addr = CORE_CC_REG(base, pmuwatchdog); + chip->ops->write32(chip->ctx, addr, + CY_43012_PMU_WATCHDOG_TICK_VAL); break; case CY_CC_4373_CHIP_ID: addr = CORE_CC_REG(base, min_res_mask); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 031f276816969..99b56820892eb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4599,8 +4599,6 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) FMAC_ULP_ENTRY_RECV) { brcmf_chip_ulp_reset_lhl_regs(bus->ci); brcmf_chip_reset_pmu_regs(bus->ci); - brcmf_chip_set_default_min_res_mask( - bus->ci); } else { brcmf_chip_set_passive(bus->ci); } From 3f267bab6b77f6daa9a59c7fe3c098a86560f715 Mon Sep 17 00:00:00 2001 From: Arend Van Spriel Date: Wed, 8 Nov 2017 14:36:31 +0100 Subject: [PATCH 110/213] brcmfmac: handle FWHALT mailbox indication commit 2fd3877b5bb7 ("brcmfmac: handle FWHALT mailbox indication") upstream. The firmware uses a mailbox to communicate to the host what is going on. In the driver we validate the bit received. Various people seen the following message: brcmfmac: brcmf_sdio_hostmail: Unknown mailbox data content: 0x40012 Bit 4 is cause of this message, but this actually indicates the firmware has halted. Handle this bit by giving a more meaningful error message. Reviewed-by: Hante Meuleman Reviewed-by: Pieter-Paul Giesberts Reviewed-by: Franky Lin Signed-off-by: Arend van Spriel Signed-off-by: Kalle Valo --- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 99b56820892eb..6c99427e1e4a8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -271,10 +271,11 @@ struct rte_console { #define I_HMB_HOST_INT I_HMB_SW3 /* Miscellaneous Interrupt */ /* tohostmailboxdata */ -#define HMB_DATA_NAKHANDLED 1 /* retransmit NAK'd frame */ -#define HMB_DATA_DEVREADY 2 /* talk to host after enable */ -#define HMB_DATA_FC 4 /* per prio flowcontrol update flag */ -#define HMB_DATA_FWREADY 8 /* fw ready for protocol activity */ +#define HMB_DATA_NAKHANDLED 0x0001 /* retransmit NAK'd frame */ +#define HMB_DATA_DEVREADY 0x0002 /* talk to host after enable */ +#define HMB_DATA_FC 0x0004 /* per prio flowcontrol update flag */ +#define HMB_DATA_FWREADY 0x0008 /* fw ready for protocol activity */ +#define HMB_DATA_FWHALT 0x0010 /* firmware halted */ #define HMB_DATA_FCDATA_MASK 0xff000000 #define HMB_DATA_FCDATA_SHIFT 24 @@ -1117,6 +1118,10 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus, u32 *hmbd) offsetof(struct sdpcmd_regs, tosbmailbox)); bus->sdcnt.f1regdata += 2; + /* dongle indicates the firmware has halted/crashed */ + if (hmb_data & HMB_DATA_FWHALT) + brcmf_err("mailbox indicates firmware halted\n"); + /* Dongle recomposed rx frames, accept them again */ if (hmb_data & HMB_DATA_NAKHANDLED) { brcmf_dbg(SDIO, "Dongle reports NAK handled, expect rtx of %d\n", @@ -1174,6 +1179,7 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus, u32 *hmbd) HMB_DATA_NAKHANDLED | HMB_DATA_FC | HMB_DATA_FWREADY | + HMB_DATA_FWHALT | HMB_DATA_FCDATA_MASK | HMB_DATA_VERSION_MASK)) brcmf_err("Unknown mailbox data content: 0x%02x\n", hmb_data); From db04a79a935efe21a25b8f2eb129f9d7d50a634e Mon Sep 17 00:00:00 2001 From: Franky Lin Date: Wed, 16 May 2018 14:12:01 +0200 Subject: [PATCH 111/213] brcmfmac: validate user provided data for memdump before copying commit d2af9b566554 ("brcmfmac: validate user provided data for memdump before copying") upstream. In patch "brcmfmac: add support for sysfs initiated coredump", a new scenario of brcmf_debug_create_memdump was added in which the user of the function might not necessarily provide prefix data. Hence the function should not assume the data is always valid and should perform a check before copying. Reviewed-by: Arend van Spriel Signed-off-by: Franky Lin Signed-off-by: Arend van Spriel Signed-off-by: Kalle Valo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c index 8ee3fd06f0a26..e6bbbdffc33a5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.c @@ -118,7 +118,8 @@ int brcmf_debug_create_memdump(struct brcmf_bus *bus, const void *data, if (!dump) return -ENOMEM; - memcpy(dump, data, len); + if (data && len > 0) + memcpy(dump, data, len); err = brcmf_bus_get_memdump(bus, dump + len, ramsize); if (err) { vfree(dump); From 2c86033b8be9344f642c8237a742bf89b3d87d8a Mon Sep 17 00:00:00 2001 From: Justin Li Date: Wed, 23 Jan 2019 03:32:16 -0600 Subject: [PATCH 112/213] brcmfmac: Use FW priority definition to initialize WMM AC priority array It was observed that TCP Tx/Rx and UDP Tx showed very low throughput when fcmode=2. This is caused by incorrect default FIFO priority. BK FIFO queue was used for default 802.1D BE priority traffic. Hostapd relies on the correct priority setting to deliver good throughput. Signed-off-by: Justin Li --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 17 +++++++------- .../broadcom/brcm80211/brcmfmac/fwsignal.c | 22 ------------------ .../broadcom/brcm80211/brcmfmac/fwsignal.h | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 39fa2b3c3cf4a..b6f7cd4721694 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -33,6 +33,7 @@ #include "p2p.h" #include "btcoex.h" #include "pno.h" +#include "fwsignal.h" #include "cfg80211.h" #include "feature.h" #include "fwil.h" @@ -6048,14 +6049,14 @@ static void brcmf_init_wmm_prio(u8 *priority) * 802.1d prio 4,5 maps to VI * 802.1d prio 6,7 maps to VO */ - priority[0] = AC_BE; - priority[3] = AC_BE; - priority[1] = AC_BK; - priority[2] = AC_BK; - priority[4] = AC_VI; - priority[5] = AC_VI; - priority[6] = AC_VO; - priority[7] = AC_VO; + priority[0] = BRCMF_FWS_FIFO_AC_BE; + priority[3] = BRCMF_FWS_FIFO_AC_BE; + priority[1] = BRCMF_FWS_FIFO_AC_BK; + priority[2] = BRCMF_FWS_FIFO_AC_BK; + priority[4] = BRCMF_FWS_FIFO_AC_VI; + priority[5] = BRCMF_FWS_FIFO_AC_VI; + priority[6] = BRCMF_FWS_FIFO_AC_VO; + priority[7] = BRCMF_FWS_FIFO_AC_VO; } static void brcmf_register_event_handlers(struct brcmf_cfg80211_info *cfg) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index cfa2e1bcaf0ac..a90b045e61ecf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -322,28 +322,6 @@ struct brcmf_skbuff_cb { /* How long to defer borrowing in jiffies */ #define BRCMF_FWS_BORROW_DEFER_PERIOD (HZ / 10) -/** - * enum brcmf_fws_fifo - fifo indices used by dongle firmware. - * - * @BRCMF_FWS_FIFO_FIRST: first fifo, ie. background. - * @BRCMF_FWS_FIFO_AC_BK: fifo for background traffic. - * @BRCMF_FWS_FIFO_AC_BE: fifo for best-effort traffic. - * @BRCMF_FWS_FIFO_AC_VI: fifo for video traffic. - * @BRCMF_FWS_FIFO_AC_VO: fifo for voice traffic. - * @BRCMF_FWS_FIFO_BCMC: fifo for broadcast/multicast (AP only). - * @BRCMF_FWS_FIFO_ATIM: fifo for ATIM (AP only). - * @BRCMF_FWS_FIFO_COUNT: number of fifos. - */ -enum brcmf_fws_fifo { - BRCMF_FWS_FIFO_FIRST, - BRCMF_FWS_FIFO_AC_BK = BRCMF_FWS_FIFO_FIRST, - BRCMF_FWS_FIFO_AC_BE, - BRCMF_FWS_FIFO_AC_VI, - BRCMF_FWS_FIFO_AC_VO, - BRCMF_FWS_FIFO_BCMC, - BRCMF_FWS_FIFO_ATIM, - BRCMF_FWS_FIFO_COUNT -}; /** * enum brcmf_fws_txstatus - txstatus flag values. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h index ba07bd9720020..02540e6fa79c5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h @@ -18,6 +18,29 @@ #ifndef FWSIGNAL_H_ #define FWSIGNAL_H_ +/** + * enum brcmf_fws_fifo - fifo indices used by dongle firmware. + * + * @BRCMF_FWS_FIFO_FIRST: first fifo, ie. background. + * @BRCMF_FWS_FIFO_AC_BK: fifo for background traffic. + * @BRCMF_FWS_FIFO_AC_BE: fifo for best-effort traffic. + * @BRCMF_FWS_FIFO_AC_VI: fifo for video traffic. + * @BRCMF_FWS_FIFO_AC_VO: fifo for voice traffic. + * @BRCMF_FWS_FIFO_BCMC: fifo for broadcast/multicast (AP only). + * @BRCMF_FWS_FIFO_ATIM: fifo for ATIM (AP only). + * @BRCMF_FWS_FIFO_COUNT: number of fifos. + */ +enum brcmf_fws_fifo { + BRCMF_FWS_FIFO_FIRST, + BRCMF_FWS_FIFO_AC_BK = BRCMF_FWS_FIFO_FIRST, + BRCMF_FWS_FIFO_AC_BE, + BRCMF_FWS_FIFO_AC_VI, + BRCMF_FWS_FIFO_AC_VO, + BRCMF_FWS_FIFO_BCMC, + BRCMF_FWS_FIFO_ATIM, + BRCMF_FWS_FIFO_COUNT +}; + struct brcmf_fws_info *brcmf_fws_attach(struct brcmf_pub *drvr); void brcmf_fws_detach(struct brcmf_fws_info *fws); bool brcmf_fws_queue_skbs(struct brcmf_fws_info *fws); From 95ac2a537d30dd24ae8db02055a137dd06cdfa67 Mon Sep 17 00:00:00 2001 From: Jia-Shyr Chuang Date: Wed, 30 Jan 2019 21:22:04 -0600 Subject: [PATCH 113/213] brcmfmac: Fix P2P Group Formation failure via Go-neg method P2P group formation fails since either peer is not able to send go-neg confirm or dut is not able to send go-neg response. To fix this, retry limit should be increased and dwell time check should be added. Signed-off-by: Saint Chuang --- .../broadcom/brcm80211/brcmfmac/p2p.c | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index c8ad5c297468c..e603d0af62db1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -71,7 +71,7 @@ #define P2P_AF_MIN_DWELL_TIME 100 #define P2P_AF_MED_DWELL_TIME 400 #define P2P_AF_LONG_DWELL_TIME 1000 -#define P2P_AF_TX_MAX_RETRY 1 +#define P2P_AF_TX_MAX_RETRY 5 #define P2P_AF_MAX_WAIT_TIME msecs_to_jiffies(2000) #define P2P_INVALID_CHANNEL -1 #define P2P_CHANNEL_SYNC_RETRY 5 @@ -104,6 +104,9 @@ #define P2PSD_ACTION_ID_GAS_CRESP 0x0d /* GAS Comback Response AF */ #define BRCMF_P2P_DISABLE_TIMEOUT msecs_to_jiffies(500) + +/* Mask for retry counter of custom dwell time */ +#define CUSTOM_RETRY_MASK 0xff000000 /** * struct brcmf_p2p_disc_st_le - set discovery state in firmware. * @@ -1672,6 +1675,17 @@ static s32 brcmf_p2p_pub_af_tx(struct brcmf_cfg80211_info *cfg, return err; } +static bool brcmf_p2p_check_dwell_overflow(s32 requested_dwell, + unsigned long dwell_jiffies) +{ + if ((requested_dwell & CUSTOM_RETRY_MASK) && + (jiffies_to_msecs(jiffies - dwell_jiffies) > + (requested_dwell & ~CUSTOM_RETRY_MASK))) { + brcmf_err("Action frame TX retry time over dwell time!\n"); + return true; + } + return false; +} /** * brcmf_p2p_send_action_frame() - send action frame . * @@ -1695,6 +1709,10 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, s32 tx_retry; s32 extra_listen_time; uint delta_ms; + unsigned long dwell_jiffies = 0; + bool dwell_overflow = false; + + s32 requested_dwell = af_params->dwell_time; action_frame = &af_params->action_frame; action_frame_len = le16_to_cpu(action_frame->len); @@ -1806,12 +1824,18 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, /* update channel */ af_params->channel = cpu_to_le32(afx_hdl->peer_chan); } + dwell_jiffies = jiffies; + dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, + dwell_jiffies); tx_retry = 0; while (!p2p->block_gon_req_tx && - (ack == false) && (tx_retry < P2P_AF_TX_MAX_RETRY)) { + (!ack) && (tx_retry < P2P_AF_TX_MAX_RETRY) && + !dwell_overflow) { ack = !brcmf_p2p_tx_action_frame(p2p, af_params); tx_retry++; + dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, + dwell_jiffies); } if (ack == false) { brcmf_err("Failed to send Action Frame(retry %d)\n", tx_retry); From 8fb553fdd9660ca434cbe41289ca182b85fb3118 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 14 Feb 2019 16:33:53 +0800 Subject: [PATCH 114/213] nl80211: add authorized flag back to ROAM event Commit 503c1fb98ba3 ("cfg80211/nl80211: add a port authorized event") added the NL80211_CMD_PORT_AUTHORIZED event to indicate that a connection is authorized. It replaced the PORT_AUTHORIZED attribute and the authorized flag added in commit f45cbe6e691f ("nl80211: add authorized flag to ROAM event"). However, for offload FT, using PORT_AUTHORIZED event mechanism induces wpa_supplicant to start a full EAP exchange after a successful roaming. This patch adds the flag mechanism back to the ROAM event for drivers to fix the offload FT roaming issue. Signed-off-by: Chung-Hsien Hsu --- include/net/cfg80211.h | 4 ++++ include/uapi/linux/nl80211.h | 5 ++++- net/wireless/nl80211.c | 2 ++ net/wireless/sme.c | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index d5dc5788ec915..1f995205f9d36 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -5735,6 +5735,9 @@ cfg80211_connect_timeout(struct net_device *dev, const u8 *bssid, * @req_ie_len: association request IEs length * @resp_ie: association response IEs (may be %NULL) * @resp_ie_len: assoc response IEs length + * @authorized: true if the 802.1X authentication was done by the driver or is + * not needed (e.g., when Fast Transition protocol was used), false + * otherwise. Ignored for networks that don't use 802.1X authentication. * @fils: FILS related roaming information. */ struct cfg80211_roam_info { @@ -5745,6 +5748,7 @@ struct cfg80211_roam_info { size_t req_ie_len; const u8 *resp_ie; size_t resp_ie_len; + bool authorized; struct cfg80211_fils_resp_params fils; }; diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 5d4c8906c9dd9..a9047482f65df 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -2222,7 +2222,10 @@ enum nl80211_commands { * in %NL80211_CMD_CONNECT to indicate that for 802.1X authentication it * wants to use the supported offload of the 4-way handshake. * @NL80211_ATTR_PMKR0_NAME: PMK-R0 Name for offloaded FT. - * @NL80211_ATTR_PORT_AUTHORIZED: (reserved) + * @NL80211_ATTR_PORT_AUTHORIZED: flag attribute used in %NL80211_CMD_ROAMED + * notification indicating that that 802.1X authentication was done by + * the driver or is not needed (because roaming used the Fast Transition + * protocol). * * @NL80211_ATTR_EXTERNAL_AUTH_ACTION: Identify the requested external * authentication operation (u32 attribute with an diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 3a1a839b38c44..9612b20f7dd09 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -14537,6 +14537,8 @@ void nl80211_send_roamed(struct cfg80211_registered_device *rdev, (info->resp_ie && nla_put(msg, NL80211_ATTR_RESP_IE, info->resp_ie_len, info->resp_ie)) || + (info->authorized && + nla_put_flag(msg, NL80211_ATTR_PORT_AUTHORIZED)) || (info->fils.update_erp_next_seq_num && nla_put_u16(msg, NL80211_ATTR_FILS_ERP_NEXT_SEQ_NUM, info->fils.erp_next_seq_num)) || diff --git a/net/wireless/sme.c b/net/wireless/sme.c index 07c2196e9d573..9bc610c88cf73 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -996,6 +996,7 @@ void cfg80211_roamed(struct net_device *dev, struct cfg80211_roam_info *info, if (info->fils.update_erp_next_seq_num) ev->rm.fils.erp_next_seq_num = info->fils.erp_next_seq_num; ev->rm.bss = info->bss; + ev->rm.authorized = info->authorized; spin_lock_irqsave(&wdev->event_lock, flags); list_add_tail(&ev->list, &wdev->event_list); From 8b794589486131cc85049b9a08fb4d14a535dece Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 14 Feb 2019 17:28:22 +0800 Subject: [PATCH 115/213] brcmfmac: set authorized flag in ROAM event for offload FT roaming When using PORT_AUTHORIZED event mechanism for offload FT, wpa_supplicant started a full EAP exchange after a successful roaming. It was caused by setting portEnabled to FALSE to get EAP state machine out of the SUCCESS state and eapSuccess cleared when handling ROAM event in wap_supplicant. With this patch, the authorized flag in the ROAM event is used to indicate the connection is authorized. Wpa_supplicant sets portEnabled according to the flag and no full EAP exchange is performed after the roaming. Signed-off-by: Chung-Hsien Hsu --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index b6f7cd4721694..0531526bce559 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5788,14 +5788,12 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, roam_info.resp_ie = conn_info->resp_ie; roam_info.resp_ie_len = conn_info->resp_ie_len; + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) + roam_info.authorized = true; + cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); brcmf_dbg(CONN, "Report roaming result\n"); - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) { - cfg80211_port_authorized(ndev, profile->bssid, GFP_KERNEL); - brcmf_dbg(CONN, "Report port authorized\n"); - } - set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); brcmf_dbg(TRACE, "Exit\n"); return err; From 5431f6b06e0ed384b801b88e0e2fae50d603aa9b Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 27 Feb 2019 01:29:27 -0600 Subject: [PATCH 116/213] brcmfmac: allocate msgbuf pktid from 1 to size of pktids array Some PCIE firmwares drop txstatus with pktid=0, that makes packet stays in host side and not release. If the packet is 8021x packet, the pend_8021x_cnt will be always larger than 0 and see "Timed out waiting for no pending 802.1x packets" error when sending key to dongle. Signed-off-by: Wright Feng --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index d2c834c3b2fc9..1a7b203e21827 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -330,7 +330,7 @@ brcmf_msgbuf_alloc_pktid(struct device *dev, do { (*idx)++; if (*idx == pktids->array_size) - *idx = 0; + *idx = 1; if (array[*idx].allocated.counter == 0) if (atomic_cmpxchg(&array[*idx].allocated, 0, 1) == 0) break; From ab64f252c8a16219b1e494fd466b4a9e17763382 Mon Sep 17 00:00:00 2001 From: Justin Li Date: Wed, 27 Feb 2019 02:47:49 -0600 Subject: [PATCH 117/213] brcmfmac: Add P2P Action Frame retry delay to fix GAS Comeback Response failure issue It was observed that P2P Cert. 5.1.19: DEVUT responds to Service Discovery request failed due to DUT did not send GAS Comeback Response after receiving request from test bed P2P peer. To fix this issue, we need to add P2P Action Frame retry delay to enhance P2P connection under VSDB and noisy environment, since the peer can be in other channels under VSDB. Signed-off-by: Justin Li --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index e603d0af62db1..fd449817bd723 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -77,6 +77,7 @@ #define P2P_CHANNEL_SYNC_RETRY 5 #define P2P_AF_FRM_SCAN_MAX_WAIT msecs_to_jiffies(450) #define P2P_DEFAULT_SLEEP_TIME_VSDB 200 +#define P2P_AF_RETRY_DELAY_TIME 40 /* WiFi P2P Public Action Frame OUI Subtypes */ #define P2P_PAF_GON_REQ 0 /* Group Owner Negotiation Req */ @@ -1832,6 +1833,9 @@ bool brcmf_p2p_send_action_frame(struct brcmf_cfg80211_info *cfg, while (!p2p->block_gon_req_tx && (!ack) && (tx_retry < P2P_AF_TX_MAX_RETRY) && !dwell_overflow) { + if (af_params->channel) + msleep(P2P_AF_RETRY_DELAY_TIME); + ack = !brcmf_p2p_tx_action_frame(p2p, af_params); tx_retry++; dwell_overflow = brcmf_p2p_check_dwell_overflow(requested_dwell, From 4da9915fb743b42e3282b8c13e514820903623bb Mon Sep 17 00:00:00 2001 From: Madhan Mohan R Date: Wed, 27 Feb 2019 03:36:42 -0600 Subject: [PATCH 118/213] brcmfmac: Use default FW priority when EDCA params same for all ACs In brcmfmac driver, all ACs were assgined a FW priority based on the EDCA parameters from AP. In a specific scenario where EDCA parameters are configured to be same for all ACs, we propose to use the default FW priority definition to avoid queuing packets of all ACs to the same priority queue. Also in case of fcmode=2, throughput of any AC would have depended on available credits of a single AC, without this fix. Fixes 11AC CERT 5.2.33 TXOP Limit test Signed-off-by: Madhan Mohan R --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 70 +++++++++++-------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 0531526bce559..62853524453bc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5581,6 +5581,25 @@ u8 brcmf_map_prio_to_aci(void *config, u8 prio) return prio; } +static void brcmf_init_wmm_prio(u8 *priority) +{ + /* Initialize AC priority array to default + * 802.1d priority as per following table: + * 802.1d prio 0,3 maps to BE + * 802.1d prio 1,2 maps to BK + * 802.1d prio 4,5 maps to VI + * 802.1d prio 6,7 maps to VO + */ + priority[0] = BRCMF_FWS_FIFO_AC_BE; + priority[3] = BRCMF_FWS_FIFO_AC_BE; + priority[1] = BRCMF_FWS_FIFO_AC_BK; + priority[2] = BRCMF_FWS_FIFO_AC_BK; + priority[4] = BRCMF_FWS_FIFO_AC_VI; + priority[5] = BRCMF_FWS_FIFO_AC_VI; + priority[6] = BRCMF_FWS_FIFO_AC_VO; + priority[7] = BRCMF_FWS_FIFO_AC_VO; +} + static void brcmf_wifi_prioritize_acparams(const struct brcmf_cfg80211_edcf_acparam *acp, u8 *priority) { @@ -5633,22 +5652,30 @@ static void brcmf_wifi_prioritize_acparams(const * Use ACI prio to get the new priority value for * each 802.1d traffic type, in this range. */ + if (!(aci_prio[AC_BE] == aci_prio[AC_BK] && + aci_prio[AC_BK] == aci_prio[AC_VI] && + aci_prio[AC_VI] == aci_prio[AC_VO])) { - /* 802.1d 0,3 maps to BE */ - priority[0] = aci_prio[AC_BE]; - priority[3] = aci_prio[AC_BE]; + /* 802.1d 0,3 maps to BE */ + priority[0] = aci_prio[AC_BE]; + priority[3] = aci_prio[AC_BE]; - /* 802.1d 1,2 maps to BK */ - priority[1] = aci_prio[AC_BK]; - priority[2] = aci_prio[AC_BK]; + /* 802.1d 1,2 maps to BK */ + priority[1] = aci_prio[AC_BK]; + priority[2] = aci_prio[AC_BK]; - /* 802.1d 4,5 maps to VO */ - priority[4] = aci_prio[AC_VI]; - priority[5] = aci_prio[AC_VI]; + /* 802.1d 4,5 maps to VO */ + priority[4] = aci_prio[AC_VI]; + priority[5] = aci_prio[AC_VI]; - /* 802.1d 6,7 maps to VO */ - priority[6] = aci_prio[AC_VO]; - priority[7] = aci_prio[AC_VO]; + /* 802.1d 6,7 maps to VO */ + priority[6] = aci_prio[AC_VO]; + priority[7] = aci_prio[AC_VO]; + + } else { + /* Initialize to default priority */ + brcmf_init_wmm_prio(priority); + } brcmf_dbg(CONN, "Adj prio BE 0->%d, BK 1->%d, BK 2->%d, BE 3->%d\n", priority[0], priority[1], priority[2], priority[3]); @@ -6038,25 +6065,6 @@ static void brcmf_init_conf(struct brcmf_cfg80211_conf *conf) conf->retry_long = (u32)-1; } -static void brcmf_init_wmm_prio(u8 *priority) -{ - /* Initialize AC priority array to default - * 802.1d priority as per following table: - * 802.1d prio 0,3 maps to BE - * 802.1d prio 1,2 maps to BK - * 802.1d prio 4,5 maps to VI - * 802.1d prio 6,7 maps to VO - */ - priority[0] = BRCMF_FWS_FIFO_AC_BE; - priority[3] = BRCMF_FWS_FIFO_AC_BE; - priority[1] = BRCMF_FWS_FIFO_AC_BK; - priority[2] = BRCMF_FWS_FIFO_AC_BK; - priority[4] = BRCMF_FWS_FIFO_AC_VI; - priority[5] = BRCMF_FWS_FIFO_AC_VI; - priority[6] = BRCMF_FWS_FIFO_AC_VO; - priority[7] = BRCMF_FWS_FIFO_AC_VO; -} - static void brcmf_register_event_handlers(struct brcmf_cfg80211_info *cfg) { brcmf_fweh_register(cfg->pub, BRCMF_E_LINK, From 887c2b2546f0cff9026f9e8e8c521fbc3fc74315 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Sun, 3 Mar 2019 20:30:32 -0600 Subject: [PATCH 119/213] brcmfmac: set authorized flag in ROAM event for PMK caching With 4-way handshake offload for 802.1X authentication, the authorized flag in ROAM event should be set for a successful roaming with PMK caching. The roaming is identified by checking the existence of PMKID within the (Re)Association Request frame with this patch. Signed-off-by: Chung-Hsien Hsu --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 62853524453bc..127df6f48d4f1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5755,6 +5755,47 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, return err; } +static bool +brcmf_has_pmkid(const u8 *parse, u32 len) +{ + const struct brcmf_tlv *rsn_ie; + const u8 *ie; + u32 ie_len; + u32 offset; + u16 count; + + rsn_ie = brcmf_parse_tlvs(parse, len, WLAN_EID_RSN); + if (!rsn_ie) + goto done; + ie = (const u8 *)rsn_ie; + ie_len = rsn_ie->len + TLV_HDR_LEN; + /* Skip group data cipher suite */ + offset = TLV_HDR_LEN + WPA_IE_VERSION_LEN + WPA_IE_MIN_OUI_LEN; + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip pairwise cipher suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip auth key management suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + RSN_CAP_LEN >= ie_len) + goto done; + /* Skip rsn capabilities */ + offset += RSN_CAP_LEN; + if (offset + RSN_PMKID_COUNT_LEN > ie_len) + goto done; + /* Extract PMKID count */ + count = ie[offset] + (ie[offset + 1] << 8); + if (count) + return true; + +done: + return false; +} + static s32 brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, struct net_device *ndev, @@ -5815,7 +5856,9 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, roam_info.resp_ie = conn_info->resp_ie; roam_info.resp_ie_len = conn_info->resp_ie_len; - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && + (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) || + profile->is_ft)) roam_info.authorized = true; cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); From 74799aed67a9d2ea8592ae05bebfa9aab627195a Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Wed, 27 Feb 2019 02:06:18 -0600 Subject: [PATCH 120/213] brcmfmac: fix continuous 802.1x tx pending timeout error The race condition in brcmf_msgbuf_txflow and brcmf_msgbuf_delete_flowring makes tx_msghdr writing after brcmf_msgbuf_remove_flowring. Host driver should delete flowring after txflow complete and all txstatus back, or pend_8021x_cnt will never be zero and cause every connection 950 milliseconds(MAX_WAIT_FOR_8021X_TX) delay. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/core.c | 4 +++- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 23 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 48c7275eb7e94..4f8dc6c90be51 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1145,8 +1145,10 @@ int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp) !brcmf_get_pend_8021x_cnt(ifp), MAX_WAIT_FOR_8021X_TX); - if (!err) + if (!err) { brcmf_err("Timed out waiting for no pending 802.1x packets\n"); + atomic_set(&ifp->pend_8021x_cnt, 0); + } return !err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 1a7b203e21827..cff2b9ad3a36a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -78,6 +78,7 @@ #define BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS 32 #define BRCMF_MSGBUF_UPDATE_RX_PTR_THRS 48 +#define BRCMF_MAX_TXSTATUS_WAIT_RETRIES 10 struct msgbuf_common_hdr { u8 msgtype; @@ -793,8 +794,12 @@ static int brcmf_msgbuf_tx_queue_data(struct brcmf_pub *drvr, int ifidx, flowid = brcmf_flowring_lookup(flow, eh->h_dest, skb->priority, ifidx); if (flowid == BRCMF_FLOWRING_INVALID_ID) { flowid = brcmf_msgbuf_flowring_create(msgbuf, ifidx, skb); - if (flowid == BRCMF_FLOWRING_INVALID_ID) + if (flowid == BRCMF_FLOWRING_INVALID_ID) { return -ENOMEM; + } else { + brcmf_flowring_enqueue(flow, flowid, skb); + return 0; + } } queue_count = brcmf_flowring_enqueue(flow, flowid, skb); force = ((queue_count % BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS) == 0); @@ -1324,9 +1329,25 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; struct msgbuf_tx_flowring_delete_req *delete; struct brcmf_commonring *commonring; + struct brcmf_commonring *commonring_del; + void *ret_ptr; u8 ifidx; int err; + int retry = BRCMF_MAX_TXSTATUS_WAIT_RETRIES; + + /* wait for commonring txflow finished */ + commonring_del = msgbuf->flowrings[flowid]; + brcmf_commonring_lock(commonring_del); + while (retry && atomic_read(&commonring_del->outstanding_tx)) { + usleep_range(5000, 10000); + retry--; + } + brcmf_commonring_unlock(commonring_del); + if (!retry && atomic_read(&commonring_del->outstanding_tx)) { + brcmf_err("timed out waiting for txstatus\n"); + atomic_set(&commonring_del->outstanding_tx, 0); + } commonring = msgbuf->commonrings[BRCMF_H2D_MSGRING_CONTROL_SUBMIT]; brcmf_commonring_lock(commonring); From ca9862e62c2f33115d7573db3e3ea91d5931a46c Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 7 Mar 2019 01:37:27 -0600 Subject: [PATCH 121/213] brcmfmac: add sleep in bus suspend and cfg80211 resume functions With asynchronous suspend/resume feature, suspend and resume callbacks to be executed in parallel with each other. It makes bus changes the state to BRCMF_BUS_DOWN before all brcmf_cfg80211_suspend IOVAR executions. The same situation also happens in resume procedure and causes PM mode keeps in PM_MAX after resume. In order to fix the race condition, We add one second sleep in bus suspend and cfg80211 resume function. Signed-off-by: Wright Feng --- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 19 ++++++++++++-- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 25 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 10 ++++++++ .../broadcom/brcm80211/brcmfmac/pcie.c | 12 +++++++++ .../broadcom/brcm80211/brcmfmac/usb.c | 15 +++++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index ee918f8f65118..f0363f62c14f7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -47,6 +47,7 @@ #include "sdio.h" #include "core.h" #include "common.h" +#include "cfg80211.h" #define SDIOH_API_ACCESS_RETRY_LIMIT 2 @@ -1184,6 +1185,7 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, dev_set_drvdata(&func->dev, bus_if); dev_set_drvdata(&sdiodev->func[1]->dev, bus_if); sdiodev->dev = &sdiodev->func[1]->dev; + dev_set_drvdata(&sdiodev->func[2]->dev, bus_if); brcmf_sdiod_change_state(sdiodev, BRCMF_SDIOD_DOWN); @@ -1200,6 +1202,7 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, fail: dev_set_drvdata(&func->dev, NULL); dev_set_drvdata(&sdiodev->func[1]->dev, NULL); + dev_set_drvdata(&sdiodev->func[2]->dev, NULL); kfree(sdiodev->func[0]); kfree(sdiodev); kfree(bus_if); @@ -1256,14 +1259,26 @@ static int brcmf_ops_sdio_suspend(struct device *dev) struct brcmf_bus *bus_if; struct brcmf_sdio_dev *sdiodev; mmc_pm_flag_t sdio_flags; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; func = container_of(dev, struct sdio_func, dev); + bus_if = dev_get_drvdata(dev); + config = bus_if->drvr->config; + brcmf_dbg(SDIO, "Enter: F%d\n", func->num); + + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err("timed out wait for cfg80211 suspended\n"); + if (func->num != SDIO_FUNC_1) return 0; - - bus_if = dev_get_drvdata(dev); sdiodev = bus_if->bus_priv.sdio; brcmf_sdiod_freezer_on(sdiodev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 127df6f48d4f1..f899b7c3877f1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3747,10 +3747,24 @@ static s32 brcmf_cfg80211_resume(struct wiphy *wiphy) struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct net_device *ndev = cfg_to_ndev(cfg); struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_bus *bus_if = drvr->bus_if; + struct brcmf_cfg80211_info *config = drvr->config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(TRACE, "Enter\n"); + config->pm_state = BRCMF_CFG80211_PM_STATE_RESUMING; + if (cfg->wowl.active) { + /* wait for bus resumed */ + while (retry && bus_if->state != BRCMF_BUS_UP) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && bus_if->state != BRCMF_BUS_UP) + brcmf_err("timed out wait for bus resume\n"); + brcmf_report_wowl_wakeind(wiphy, ifp); brcmf_fil_iovar_int_set(ifp, "wowl_clear", 0); brcmf_config_wowl_pattern(ifp, "clr", NULL, 0, NULL, 0); @@ -3771,6 +3785,7 @@ static s32 brcmf_cfg80211_resume(struct wiphy *wiphy) brcmf_pktfilter_enable(ifp->ndev, false); } + config->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED; return 0; } @@ -3840,9 +3855,12 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, struct net_device *ndev = cfg_to_ndev(cfg); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_vif *vif; + struct brcmf_cfg80211_info *config = ifp->drvr->config; brcmf_dbg(TRACE, "Enter\n"); + config->pm_state = BRCMF_CFG80211_PM_STATE_SUSPENDING; + /* if the primary net_device is not READY there is nothing * we can do but pray resume goes smoothly. */ @@ -3884,9 +3902,13 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, } exit: - brcmf_dbg(TRACE, "Exit\n"); + /* set cfg80211 pm state to cfg80211 suspended state */ + config->pm_state = BRCMF_CFG80211_PM_STATE_SUSPENDED; + /* clear any scanning activity */ cfg->scan_status = 0; + + brcmf_dbg(TRACE, "Exit\n"); return 0; } @@ -7429,6 +7451,7 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->wiphy = wiphy; cfg->ops = ops; cfg->pub = drvr; + cfg->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index db80f61dfab71..11883c814782c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -103,6 +103,8 @@ #define BRCMF_VIF_EVENT_TIMEOUT msecs_to_jiffies(1500) +#define BRCMF_PM_WAIT_MAXRETRY 100 + /* cfg80211 wowlan definitions */ #define WL_WOWLAN_MAX_PATTERNS 8 #define WL_WOWLAN_MIN_PATTERN_LEN 1 @@ -182,6 +184,13 @@ enum brcmf_vif_status { BRCMF_VIF_STATUS_ASSOC_SUCCESS, }; +enum brcmf_cfg80211_pm_state { + BRCMF_CFG80211_PM_STATE_RESUMED, + BRCMF_CFG80211_PM_STATE_RESUMING, + BRCMF_CFG80211_PM_STATE_SUSPENDED, + BRCMF_CFG80211_PM_STATE_SUSPENDING, +}; + /** * struct vif_saved_ie - holds saved IEs for a virtual interface. * @@ -372,6 +381,7 @@ struct brcmf_cfg80211_info { struct brcmf_cfg80211_wowl wowl; struct brcmf_pno_info *pno; u8 ac_priority[MAX_8021D_PRIO]; + u8 pm_state; }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 63ad529a36446..dc2fc226b372a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -39,6 +39,7 @@ #include "chip.h" #include "core.h" #include "common.h" +#include "cfg80211.h" enum brcmf_pcie_state { @@ -1887,11 +1888,22 @@ static int brcmf_pcie_pm_enter_D3(struct device *dev) { struct brcmf_pciedev_info *devinfo; struct brcmf_bus *bus; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(PCIE, "Enter\n"); bus = dev_get_drvdata(dev); devinfo = bus->bus_priv.pcie->devinfo; + config = bus->drvr->config; + + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err("timed out wait for cfg80211 suspended\n"); brcmf_bus_change_state(bus, BRCMF_BUS_DOWN); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 3025b6ea9daa1..6e893d613b552 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -30,6 +30,7 @@ #include "core.h" #include "common.h" #include "bcdc.h" +#include "cfg80211.h" #define IOCTL_RESP_TIMEOUT msecs_to_jiffies(2000) @@ -1486,8 +1487,22 @@ static int brcmf_usb_suspend(struct usb_interface *intf, pm_message_t state) { struct usb_device *usb = interface_to_usbdev(intf); struct brcmf_usbdev_info *devinfo = brcmf_usb_get_businfo(&usb->dev); + struct brcmf_bus *bus; + struct brcmf_cfg80211_info *config; + int retry = BRCMF_PM_WAIT_MAXRETRY; brcmf_dbg(USB, "Enter\n"); + + bus = devinfo->bus_pub.bus; + config = bus->drvr->config; + while (retry && + config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) { + usleep_range(10000, 20000); + retry--; + } + if (!retry && config->pm_state == BRCMF_CFG80211_PM_STATE_SUSPENDING) + brcmf_err("timed out wait for cfg80211 suspended\n"); + devinfo->bus_pub.state = BRCMFMAC_USB_STATE_SLEEP; brcmf_cancel_all_urbs(devinfo); device_set_wakeup_enable(devinfo->dev, true); From 05fbce5f0e3864e66201419e47a7e3214171635e Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Thu, 7 Mar 2019 21:13:02 -0600 Subject: [PATCH 122/213] brcmfmac: fix 43455 CRC error under SDIO 3.0 SDR104 mode This patch fixes 43455 CRC error while running throughput test with suspend/resume stress test. The continuous failure messages before system crash: brcmfmac: brcmf_sdiod_sglist_rw: CMD53 sg block read failed -84 brcmfmac: brcmf_sdio_rxglom: glom read of 25600 bytes failed: -5 brcmfmac: brcmf_sdio_rxfail: abort command, terminate frame brcmfmac: brcmf_sdiod_sglist_rw: CMD53 sg block read failed -84 brcmfmac: brcmf_sdio_rxglom: glom read of 24576 bytes failed: -5 brcmfmac: brcmf_sdio_rxfail: abort command, terminate frame Signed-off-by: Wright Feng --- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 6c99427e1e4a8..8a72e91c178aa 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -55,6 +55,10 @@ #define CY_4373_F2_WATERMARK 0x40 #define CY_4373_F1_MESBUSYCTRL (CY_4373_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #define CY_43012_F2_WATERMARK 0x60 +#define CY_43455_F2_WATERMARK 0x60 +#define CY_43455_MES_WATERMARK 0x50 +#define CY_43455_MESBUSYCTRL (CY_43455_MES_WATERMARK | \ + SBSDIO_MESBUSYCTRL_ENAB) #define CY_4339_F2_WATERMARK 48 #define CY_4339_MES_WATERMARK 80 #define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) @@ -4354,6 +4358,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_fweh_register(bus_if->drvr, BRCMF_E_ULP, brcmf_ulp_event_notify); break; + case SDIO_DEVICE_ID_BROADCOM_43455: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43455\n", + CY_43455_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, + CY_43455_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, + CY_43455_MESBUSYCTRL, &err); + break; case SDIO_DEVICE_ID_BROADCOM_4339: brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, From a38f5ba9c4ca08dd024bd349c30b5141af357783 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Wed, 3 Apr 2019 04:39:43 -0500 Subject: [PATCH 123/213] brcmfmac: set F2 blocksize and watermark for 4359 Set F2 blocksize to 256 bytes and watermark to 0x40 for 4359. Also enable and configure F1 MesBusyCtrl. It fixes DMA error while having UDP bi-directional traffic. Signed-off-by: Chung-Hsien Hsu --- .../wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 10 +++++++++- .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index f0363f62c14f7..4bcb15d1d0d41 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -56,6 +56,7 @@ #define SDIO_FUNC1_BLOCKSIZE 64 #define SDIO_FUNC2_BLOCKSIZE 512 #define SDIO_4373_FUNC2_BLOCKSIZE 256 +#define SDIO_4359_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -1053,8 +1054,15 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) goto out; } - if (sdiodev->func[0]->device == SDIO_DEVICE_ID_CYPRESS_4373) { + switch (sdiodev->func[0]->device) { + case SDIO_DEVICE_ID_CYPRESS_4373: f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE; + break; + case SDIO_DEVICE_ID_BROADCOM_4359: + f2_blksz = SDIO_4359_FUNC2_BLOCKSIZE; + break; + default: + break; } ret = sdio_set_block_size(sdiodev->func[2], f2_blksz); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 8a72e91c178aa..14f999cd172ee 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -62,6 +62,8 @@ #define CY_4339_F2_WATERMARK 48 #define CY_4339_MES_WATERMARK 80 #define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) +#define CY_4359_F2_WATERMARK 0x40 +#define CY_4359_F1_MESBUSYCTRL (CY_4359_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG #define BRCMF_TRAP_INFO_SIZE 80 @@ -4383,6 +4385,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, CY_4339_MESBUSYCTRL, &err); break; + case SDIO_DEVICE_ID_BROADCOM_4359: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", + CY_4359_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, + CY_4359_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, + CY_4359_F1_MESBUSYCTRL, &err); + break; default: brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); break; From 657f0218cb30d3cf3b41188b87516845a639c36e Mon Sep 17 00:00:00 2001 From: Arend van Spriel Date: Thu, 14 Feb 2019 13:43:48 +0100 Subject: [PATCH 124/213] brcmfmac: add subtype check for event handling in data path commit a4176ec356c7 ("brcmfmac: add subtype check for event handling in data path") upstream. For USB there is no separate channel being used to pass events from firmware to the host driver and as such are passed over the data path. In order to detect mock event messages an additional check is needed on event subtype. This check is added conditionally using unlikely() keyword. Reviewed-by: Hante Meuleman Reviewed-by: Pieter-Paul Giesberts Reviewed-by: Franky Lin Signed-off-by: Arend van Spriel Signed-off-by: Kalle Valo --- .../wireless/broadcom/brcm80211/brcmfmac/core.c | 5 +++-- .../wireless/broadcom/brcm80211/brcmfmac/fweh.h | 16 ++++++++++++---- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 4f8dc6c90be51..4a4904bce1dad 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -344,7 +344,8 @@ void brcmf_rx_frame(struct device *dev, struct sk_buff *skb, bool handle_event) } else { /* Process special event packets */ if (handle_event) - brcmf_fweh_process_skb(ifp->drvr, skb); + brcmf_fweh_process_skb(ifp->drvr, skb, + BCMILCP_SUBTYPE_VENDOR_LONG); brcmf_netif_rx(ifp, skb); } @@ -361,7 +362,7 @@ void brcmf_rx_event(struct device *dev, struct sk_buff *skb) if (brcmf_rx_hdrpull(drvr, skb, &ifp)) return; - brcmf_fweh_process_skb(ifp->drvr, skb); + brcmf_fweh_process_skb(ifp->drvr, skb, 0); brcmu_pkt_buf_free_skb(skb); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index 2cd2252c2ba5b..9ed8e637a3839 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -213,7 +213,7 @@ enum brcmf_fweh_event_code { */ #define BRCM_OUI "\x00\x10\x18" #define BCMILCP_BCM_SUBTYPE_EVENT 1 - +#define BCMILCP_SUBTYPE_VENDOR_LONG 32769 /** * struct brcm_ethhdr - broadcom specific ether header. @@ -358,10 +358,10 @@ void brcmf_fweh_process_event(struct brcmf_pub *drvr, void brcmf_fweh_p2pdev_setup(struct brcmf_if *ifp, bool ongoing); static inline void brcmf_fweh_process_skb(struct brcmf_pub *drvr, - struct sk_buff *skb) + struct sk_buff *skb, u16 stype) { struct brcmf_event *event_packet; - u16 usr_stype; + u16 subtype, usr_stype; /* only process events when protocol matches */ if (skb->protocol != cpu_to_be16(ETH_P_LINK_CTL)) @@ -370,8 +370,16 @@ static inline void brcmf_fweh_process_skb(struct brcmf_pub *drvr, if ((skb->len + ETH_HLEN) < sizeof(*event_packet)) return; - /* check for BRCM oui match */ event_packet = (struct brcmf_event *)skb_mac_header(skb); + + /* check subtype if needed */ + if (unlikely(stype)) { + subtype = get_unaligned_be16(&event_packet->hdr.subtype); + if (subtype != stype) + return; + } + + /* check for BRCM oui match */ if (memcmp(BRCM_OUI, &event_packet->hdr.oui[0], sizeof(event_packet->hdr.oui))) return; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index cff2b9ad3a36a..32ee82761873c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -1117,7 +1117,7 @@ static void brcmf_msgbuf_process_event(struct brcmf_msgbuf *msgbuf, void *buf) skb->protocol = eth_type_trans(skb, ifp->ndev); - brcmf_fweh_process_skb(ifp->drvr, skb); + brcmf_fweh_process_skb(ifp->drvr, skb, 0); exit: brcmu_pkt_buf_free_skb(skb); From 3584e0bf5ce6a0eda5177a08b1f66f58b70653a1 Mon Sep 17 00:00:00 2001 From: Arend van Spriel Date: Thu, 14 Feb 2019 13:43:47 +0100 Subject: [PATCH 125/213] brcmfmac: assure SSID length from firmware is limited commit 1b5e2423164b ("brcmfmac: assure SSID length from firmware is limited") upstream. The SSID length as received from firmware should not exceed IEEE80211_MAX_SSID_LEN as that would result in heap overflow. Reviewed-by: Hante Meuleman Reviewed-by: Pieter-Paul Giesberts Reviewed-by: Franky Lin Signed-off-by: Arend van Spriel Signed-off-by: Kalle Valo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index f899b7c3877f1..b093dc176552d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3647,6 +3647,8 @@ brcmf_wowl_nd_results(struct brcmf_if *ifp, const struct brcmf_event_msg *e, } netinfo = brcmf_get_netinfo_array(pfn_result); + if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) + netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; memcpy(cfg->wowl.nd->ssid.ssid, netinfo->SSID, netinfo->SSID_len); cfg->wowl.nd->ssid.ssid_len = netinfo->SSID_len; cfg->wowl.nd->n_channels = 1; From 0511f2ab87c6e047235308c6fd45d65ec0f12109 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 30 Apr 2019 17:52:41 -0500 Subject: [PATCH 126/213] nl80211: add authorized flag to CONNECT event Add authorized flag to CONNECT event. It is used for 802.1X 4-way handshake offload with PMK caching. Signed-off-by: Chung-Hsien Hsu --- include/net/cfg80211.h | 3 +++ net/wireless/nl80211.c | 2 ++ net/wireless/sme.c | 1 + 3 files changed, 6 insertions(+) diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 1f995205f9d36..39656e379bcbc 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -5580,6 +5580,8 @@ struct cfg80211_fils_resp_params { * not known. This value is used only if @status < 0 to indicate that the * failure is due to a timeout and not due to explicit rejection by the AP. * This value is ignored in other cases (@status >= 0). + * @authorized: Indicates whether the connection is ready to transport + * data packets. */ struct cfg80211_connect_resp_params { int status; @@ -5591,6 +5593,7 @@ struct cfg80211_connect_resp_params { size_t resp_ie_len; struct cfg80211_fils_resp_params fils; enum nl80211_timeout_reason timeout_reason; + bool authorized; }; /** diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 9612b20f7dd09..b184cd9500b28 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -14480,6 +14480,8 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev, (nla_put_flag(msg, NL80211_ATTR_TIMED_OUT) || nla_put_u32(msg, NL80211_ATTR_TIMEOUT_REASON, cr->timeout_reason))) || + (cr->authorized && + nla_put_flag(msg, NL80211_ATTR_PORT_AUTHORIZED)) || (cr->req_ie && nla_put(msg, NL80211_ATTR_REQ_IE, cr->req_ie_len, cr->req_ie)) || (cr->resp_ie && diff --git a/net/wireless/sme.c b/net/wireless/sme.c index 9bc610c88cf73..3a7b4baf01adb 100644 --- a/net/wireless/sme.c +++ b/net/wireless/sme.c @@ -863,6 +863,7 @@ void cfg80211_connect_done(struct net_device *dev, ev->cr.bss = params->bss; ev->cr.status = params->status; ev->cr.timeout_reason = params->timeout_reason; + ev->cr.authorized = params->authorized; spin_lock_irqsave(&wdev->event_lock, flags); list_add_tail(&ev->list, &wdev->event_list); From e3ff15482ee710cc03091028ae24ff253772b639 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 30 Apr 2019 17:55:02 -0500 Subject: [PATCH 127/213] brcmfmac: set authorized flag in CONNECT event for PMK caching With 4-way handshake offload for 802.1X authentication, the authorized flag in CONNECT event should be set for a successful connection with PMK caching. The connection is identified by checking the existence of PMKID within the Association Request frame with this patch. Signed-off-by: Chung-Hsien Hsu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index b093dc176552d..fa9bf626b86bd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -5922,6 +5922,11 @@ brcmf_bss_connect_done(struct brcmf_cfg80211_info *cfg, conn_params.req_ie_len = conn_info->req_ie_len; conn_params.resp_ie = conn_info->resp_ie; conn_params.resp_ie_len = conn_info->resp_ie_len; + + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && + brcmf_has_pmkid(conn_params.req_ie, conn_params.req_ie_len)) + conn_params.authorized = true; + cfg80211_connect_done(ndev, &conn_params, GFP_KERNEL); brcmf_dbg(CONN, "Report connect result - connection %s\n", completed ? "succeeded" : "failed"); From 307ed3b420ee392e6e520477a725339d2c9a7de4 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 28 May 2019 04:20:20 -0500 Subject: [PATCH 128/213] brcmfmac: add support for Opportunistic Key Caching The firmware may have OKC management. This is detected by the driver and supported via providing the PMK. The authorized flag in ROAM event should be set for a successful roaming with OKC. Signed-off-by: Darren Li Signed-off-by: Chung-Hsien Hsu --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 25 ++++++++++++++++--- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 1 + 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index fa9bf626b86bd..6bb12af73dea9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1740,6 +1740,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; s32 val; s32 err; + s32 okc_enable; const struct brcmf_tlv *rsn_ie; const u8 *ie; u32 ie_len; @@ -1750,6 +1751,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; profile->is_ft = false; + profile->is_okc = false; if (!sme->crypto.n_akm_suites) return 0; @@ -1820,10 +1822,19 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) } } - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) { brcmf_dbg(INFO, "using 1X offload\n"); - else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + brcmf_err("get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { brcmf_dbg(INFO, "using SAE offload\n"); + } if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MFP)) goto skip_mfp_config; @@ -5340,6 +5351,7 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, const struct cfg80211_pmk_conf *conf) { struct brcmf_if *ifp; + int ret; brcmf_dbg(TRACE, "enter\n"); @@ -5348,6 +5360,13 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X)) return -EINVAL; + if (ifp->vif->profile.is_okc) { + ret = brcmf_fil_iovar_data_set(ifp, "okc_info_pmk", conf->pmk, + conf->pmk_len); + if (ret < 0) + brcmf_err("okc_info_pmk iovar failed: ret=%d\n", ret); + } + return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len); } @@ -5882,7 +5901,7 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) || - profile->is_ft)) + profile->is_ft || profile->is_okc)) roam_info.authorized = true; cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 11883c814782c..53b3247b97a10 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -161,6 +161,7 @@ struct brcmf_cfg80211_profile { struct brcmf_wsec_key key[BRCMF_MAX_DEFAULT_KEYS]; enum brcmf_profile_fwsup use_fwsup; bool is_ft; + bool is_okc; }; /** From bcd85a6b79ebf4113ffba1bbbf075e025863b8ec Mon Sep 17 00:00:00 2001 From: Amar Shankar Date: Mon, 15 Oct 2018 05:32:26 -0500 Subject: [PATCH 129/213] brcmfmac: reserve 2 credits for host tx control path It is observed that sometimes when sdiod is low in tx credits in low rssi scenarios, the data path consumes all sdiod rx all credits and there is no sdiod rx credit available for control path causing host and card to go out of sync resulting in link loss between host and card. So in order to prevent it some credits are reserved for control path. Note that TXCTL_CREDITS can't be larger than the firmware default credit update threshold 2; otherwise there will be a deadlock for both side waiting for each other. Signed-off-by: Amar Shankar Signed-off-by: Jia-Shyr Chuang Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/sdio.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 14f999cd172ee..9233d7a5f85a5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -662,6 +662,8 @@ static struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012) }; +#define TXCTL_CREDITS 2 + static void pkt_align(struct sk_buff *p, int len, int align) { uint datalign; @@ -672,8 +674,17 @@ static void pkt_align(struct sk_buff *p, int len, int align) __skb_trim(p, len); } -/* To check if there's window offered */ +/* To check if there's window offered + * Reserve 3 credits for txctl + */ static bool data_ok(struct brcmf_sdio *bus) +{ + return (u8)(bus->tx_max - bus->tx_seq) > TXCTL_CREDITS && + ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0; +} + +/* To check if there's window offered */ +static bool txctl_ok(struct brcmf_sdio *bus) { return (u8)(bus->tx_max - bus->tx_seq) != 0 && ((u8)(bus->tx_max - bus->tx_seq) & 0x80) == 0; @@ -2859,7 +2870,7 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) brcmf_sdio_clrintr(bus); if (bus->ctrl_frame_stat && (bus->clkstate == CLK_AVAIL) && - data_ok(bus) && brcmf_sdio_f2_ready(bus)) { + txctl_ok(bus) && brcmf_sdio_f2_ready(bus)) { sdio_claim_host(bus->sdiodev->func[1]); if (bus->ctrl_frame_stat) { err = brcmf_sdio_tx_ctrlframe(bus, @@ -2868,6 +2879,9 @@ static void brcmf_sdio_dpc(struct brcmf_sdio *bus) bus->ctrl_frame_err = err; wmb(); /* drain writebuffer */ bus->ctrl_frame_stat = false; + if (err) + brcmf_err("sdio ctrlframe tx failed err=%d\n", + err); } sdio_release_host(bus->sdiodev->func[1]); brcmf_sdio_wait_event_wakeup(bus); From 82bc859add0b1af3a4143d680c69d9a145b5779e Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 20 Jun 2019 01:31:16 -0500 Subject: [PATCH 130/213] brcmfmac: update tx status flags to sync with firmware There is a mismatch of tx status flag values between host and firmware. It makes the host mistake the flags and have incorrect behavior of credit returns. So update the flags to sync with the firmware ones. Signed-off-by: Chung-Hsien Hsu --- .../wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index a90b045e61ecf..4a41f126bb713 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -334,6 +334,10 @@ struct brcmf_skbuff_cb { * firmware suppress the packet as device is already in PS mode. * @BRCMF_FWS_TXSTATUS_FW_TOSSED: * firmware tossed the packet. + * @BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK: + * firmware tossed the packet after retries. + * @BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED: + * firmware wrongly reported suppressed previously, now fixing to acked. * @BRCMF_FWS_TXSTATUS_HOST_TOSSED: * host tossed the packet. */ @@ -342,6 +346,8 @@ enum brcmf_fws_txstatus { BRCMF_FWS_TXSTATUS_CORE_SUPPRESS, BRCMF_FWS_TXSTATUS_FW_PS_SUPPRESS, BRCMF_FWS_TXSTATUS_FW_TOSSED, + BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK, + BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED, BRCMF_FWS_TXSTATUS_HOST_TOSSED }; @@ -1467,6 +1473,10 @@ brcmf_fws_txs_process(struct brcmf_fws_info *fws, u8 flags, u32 hslot, remove_from_hanger = false; } else if (flags == BRCMF_FWS_TXSTATUS_FW_TOSSED) fws->stats.txs_tossed += compcnt; + else if (flags == BRCMF_FWS_TXSTATUS_FW_DISCARD_NOACK) + fws->stats.txs_discard += compcnt; + else if (flags == BRCMF_FWS_TXSTATUS_FW_SUPPRESS_ACKED) + fws->stats.txs_discard += compcnt; else if (flags == BRCMF_FWS_TXSTATUS_HOST_TOSSED) fws->stats.txs_host_tossed += compcnt; else From bceef274f6af2cada89f219ad178af40ce809ee5 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 20 Jun 2019 02:59:26 -0500 Subject: [PATCH 131/213] brcmfmac: fix credit reserve for each access category Commit 1c99c7d6551f ("brcmfmac: Allow credit borrowing for all access categories") added a ratio to reserve the credits for each access category. However, calculating the number of reserved credits with the runtime one let an access category lend all its credits. Fix this by using the number of initial credits to determine the reserved one. Signed-off-by: Chung-Hsien Hsu --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 4a41f126bb713..91d6d633e496a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2042,7 +2042,7 @@ static int brcmf_fws_borrow_credit(struct brcmf_fws_info *fws, if (!borrow_all) borrow_limit = - fws->fifo_credit[lender_ac] / BRCMF_BORROW_RATIO; + fws->init_fifo_credit[lender_ac] / BRCMF_BORROW_RATIO; else borrow_limit = 0; From 28b4e7268764ed02f1ba3df87896e54a7719fd09 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Thu, 27 Jun 2019 04:00:02 -0500 Subject: [PATCH 132/213] brcmfmac: fix throughput zero stalls on PM 1 mode due to credit map This patch move the credit map setting to right place to avoid brcmf_fws_return_credits() return without setting the credit map. It fix the thoughput zero stalls issue in softAP mode when STA using PM 1 mode. Signed-off-by: Double Lo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index 91d6d633e496a..e91038dc2f96a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -1207,6 +1207,8 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, if (!credits) return; + fws->fifo_credit_map |= 1 << fifo; + if (fifo > BRCMF_FWS_FIFO_AC_BK && fifo <= BRCMF_FWS_FIFO_AC_VO) { for (lender_ac = BRCMF_FWS_FIFO_AC_VO; lender_ac >= 0; @@ -1230,7 +1232,6 @@ static void brcmf_fws_return_credits(struct brcmf_fws_info *fws, if (credits) { fws->fifo_credit[fifo] += credits; - fws->fifo_credit_map |= 1 << fifo; } if (fws->fifo_credit[fifo] > fws->init_fifo_credit[fifo]) From 0105f881c6ad08c2bef3d4a8f2d9b451080de9c0 Mon Sep 17 00:00:00 2001 From: Double Lo Date: Sun, 18 Aug 2019 22:31:19 -0500 Subject: [PATCH 133/213] brcmfmac: 43012 Update MES Watermark Set MES watermark size to 0x50 for 43012. It fixes SDIO bus hang issue when running at high throughput. Signed-off-by: Double Lo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 9233d7a5f85a5..b112883f38ccc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -55,6 +55,9 @@ #define CY_4373_F2_WATERMARK 0x40 #define CY_4373_F1_MESBUSYCTRL (CY_4373_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #define CY_43012_F2_WATERMARK 0x60 +#define CY_43012_MES_WATERMARK 0x50 +#define CY_43012_MESBUSYCTRL (CY_43012_MES_WATERMARK | \ + SBSDIO_MESBUSYCTRL_ENAB) #define CY_43455_F2_WATERMARK 0x60 #define CY_43455_MES_WATERMARK 0x50 #define CY_43455_MESBUSYCTRL (CY_43455_MES_WATERMARK | \ @@ -4373,6 +4376,8 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, /* Register for ULP events */ brcmf_fweh_register(bus_if->drvr, BRCMF_E_ULP, brcmf_ulp_event_notify); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, + CY_43012_MESBUSYCTRL, &err); break; case SDIO_DEVICE_ID_BROADCOM_43455: brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes for 43455\n", From 76a56bb55f3c84add7428c690647bc66adf9fe2a Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Tue, 8 Oct 2019 01:28:40 -0500 Subject: [PATCH 134/213] brcmfmac: add support for CYW89359 SDIO chipset Add support for CYW89359 SDIO chipset. CYW89359 is a 2x2 dual-band 11ac chipset with 20/40/80Mhz channel support. Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 2 ++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 1 + include/linux/mmc/sdio_ids.h | 1 + 3 files changed, 4 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 4bcb15d1d0d41..53ede75935ee4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -1059,6 +1059,7 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE; break; case SDIO_DEVICE_ID_BROADCOM_4359: + case SDIO_DEVICE_ID_CYPRESS_89359: f2_blksz = SDIO_4359_FUNC2_BLOCKSIZE; break; default: @@ -1127,6 +1128,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = { BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4359), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_4373), BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_43012), + BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_CYPRESS_89359), { /* end: all zeroes */ } }; MODULE_DEVICE_TABLE(sdio, brcmf_sdmmc_ids); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index b112883f38ccc..3638b50eec73e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -4405,6 +4405,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, SBSDIO_FUNC1_MESBUSYCTRL, CY_4339_MESBUSYCTRL, &err); break; case SDIO_DEVICE_ID_BROADCOM_4359: + case SDIO_DEVICE_ID_CYPRESS_89359: brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", CY_4359_F2_WATERMARK); brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index ae4ba5d6244e9..73fd3f4318cc8 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -44,6 +44,7 @@ #define SDIO_DEVICE_ID_BROADCOM_4359 0x4355 #define SDIO_DEVICE_ID_CYPRESS_4373 0x4373 #define SDIO_DEVICE_ID_CYPRESS_43012 43012 +#define SDIO_DEVICE_ID_CYPRESS_89359 0x4359 #define SDIO_VENDOR_ID_INTEL 0x0089 #define SDIO_DEVICE_ID_INTEL_IWMC3200WIMAX 0x1402 From e32f4d2f758c75e6cc7b6565f77b7f9d711c895c Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Mon, 4 Nov 2019 12:13:09 -0600 Subject: [PATCH 135/213] brcmfmac: add CYW43570 PCIE device CYW43570 is a 3-antenna, 2x2 MIMO,802.11a/b/g/n/ac, PCIe 3.0 for WLAN. It is BT/WIFI combo. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 1 + drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index dc2fc226b372a..1011caf496812 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -1988,6 +1988,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4356_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43567_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_DEVICE_ID), + BRCMF_PCIE_DEVICE(BRCM_PCIE_43570_RAW_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4358_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4359_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_43602_DEVICE_ID), diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index 1bda1aa7af2b6..dcd4d52d7a6c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -79,6 +79,7 @@ #define BRCM_PCIE_4356_DEVICE_ID 0x43ec #define BRCM_PCIE_43567_DEVICE_ID 0x43d3 #define BRCM_PCIE_43570_DEVICE_ID 0x43d9 +#define BRCM_PCIE_43570_RAW_DEVICE_ID 0xaa31 #define BRCM_PCIE_4358_DEVICE_ID 0x43e9 #define BRCM_PCIE_4359_DEVICE_ID 0x43ef #define BRCM_PCIE_43602_DEVICE_ID 0x43ba From df53e59966bb6b9a6a9c649b9c12d2cde2a5c0c5 Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Mon, 4 Nov 2019 13:05:35 -0600 Subject: [PATCH 136/213] brcmfmac: Use seq/seq_len and set iv_initialize when plumbing of rxiv in (GTK) keys When plumbing rxiv for (GTK) keys, current code does not use seq/seq_len when present nor set iv_initialized for iovar wsec_key. This could result in missing broadcast traffic after GTK rekey. The fix is setting iv_initialized and using seq/seq_len for iovar wsec_key. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 6bb12af73dea9..60f92bab96489 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2459,6 +2459,17 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, if (!ext_key) key->flags = BRCMF_PRIMARY_KEY; + if (params->seq && params->seq_len == 6) { + /* rx iv */ + u8 *ivptr; + + ivptr = (u8 *)params->seq; + key->rxiv.hi = (ivptr[5] << 24) | (ivptr[4] << 16) | + (ivptr[3] << 8) | ivptr[2]; + key->rxiv.lo = (ivptr[1] << 8) | ivptr[0]; + key->iv_initialized = true; + } + switch (params->cipher) { case WLAN_CIPHER_SUITE_WEP40: key->algo = CRYPTO_ALGO_WEP1; From 79a825855aa28a3ee0fc772263d4449b4ed97ca0 Mon Sep 17 00:00:00 2001 From: Ryohei Kondo Date: Tue, 5 Nov 2019 12:39:56 +0900 Subject: [PATCH 137/213] brcmfmac: use actframe_abort to cancel ongoing action frame The driver sends an action frame down and waits for dwell time to be completed or aborted before sending out the next action frame. Driver issues "scan abort" to cancel the current time slot, but this doesn't have any effect because, we are not using scan engine for sending action frame. Fix is to use "actframe_abort" to cancels the current action frame. Signed-off-by: Ryohei Kondo Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/p2p.c | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index fd449817bd723..40f940df23b12 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1276,6 +1276,30 @@ bool brcmf_p2p_scan_finding_common_channel(struct brcmf_cfg80211_info *cfg, return true; } +/** + * brcmf_p2p_abort_action_frame() - abort action frame. + * + * @cfg: common configuration struct. + * + */ +static s32 brcmf_p2p_abort_action_frame(struct brcmf_cfg80211_info *cfg) +{ + struct brcmf_p2p_info *p2p = &cfg->p2p; + struct brcmf_cfg80211_vif *vif; + s32 err; + s32 int_val = 1; + + brcmf_dbg(TRACE, "Enter\n"); + + vif = p2p->bss_idx[P2PAPI_BSSCFG_DEVICE].vif; + err = brcmf_fil_bsscfg_data_set(vif->ifp, "actframe_abort", &int_val, + sizeof(s32)); + if (err) + brcmf_err(" aborting action frame has failed (%d)\n", err); + + return err; +} + /** * brcmf_p2p_stop_wait_next_action_frame() - finish scan if af tx complete. * @@ -1287,6 +1311,7 @@ brcmf_p2p_stop_wait_next_action_frame(struct brcmf_cfg80211_info *cfg) { struct brcmf_p2p_info *p2p = &cfg->p2p; struct brcmf_if *ifp = p2p->bss_idx[P2PAPI_BSSCFG_PRIMARY].vif->ifp; + s32 err; if (test_bit(BRCMF_P2P_STATUS_SENDING_ACT_FRAME, &p2p->status) && (test_bit(BRCMF_P2P_STATUS_ACTION_TX_COMPLETED, &p2p->status) || @@ -1295,8 +1320,13 @@ brcmf_p2p_stop_wait_next_action_frame(struct brcmf_cfg80211_info *cfg) /* if channel is not zero, "actfame" uses off channel scan. * So abort scan for off channel completion. */ - if (p2p->af_sent_channel) - brcmf_notify_escan_complete(cfg, ifp, true, true); + if (p2p->af_sent_channel) { + /* abort actframe using actframe_abort or abort scan */ + err = brcmf_p2p_abort_action_frame(cfg); + if (err) + brcmf_notify_escan_complete( + cfg, ifp, true, true); + } } else if (test_bit(BRCMF_P2P_STATUS_WAITING_NEXT_AF_LISTEN, &p2p->status)) { brcmf_dbg(TRACE, "*** Wake UP ** abort listen for next af frame\n"); From 6f3d3dd8143409ecfa06283063991fe43d000268 Mon Sep 17 00:00:00 2001 From: Wright Feng Date: Tue, 5 Nov 2019 19:54:22 -0600 Subject: [PATCH 138/213] brcmfmac: fix scheduling while atomic issue when deleting flowring We should not sleep while holding the spin lock. It makes 'scheduling while atomic' in brcmf_msgbuf_delete_flowring. And to avoid race condition between deleting flowring and txflow, we only hold spin lock when seting flowring status to RING_CLOSING. Signed-off-by: Wright Feng Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/flowring.c | 5 +---- .../wireless/broadcom/brcm80211/brcmfmac/msgbuf.c | 14 ++++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c index d0b738da24582..dd328faa5ffbc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c @@ -427,7 +427,6 @@ void brcmf_flowring_configure_addr_mode(struct brcmf_flowring *flow, int ifidx, flowid = flow->hash[i].flowid; if (flow->rings[flowid]->status != RING_OPEN) continue; - flow->rings[flowid]->status = RING_CLOSING; brcmf_msgbuf_delete_flowring(drvr, flowid); } } @@ -466,10 +465,8 @@ void brcmf_flowring_delete_peer(struct brcmf_flowring *flow, int ifidx, if ((sta || (memcmp(hash[i].mac, peer, ETH_ALEN) == 0)) && (hash[i].ifidx == ifidx)) { flowid = flow->hash[i].flowid; - if (flow->rings[flowid]->status == RING_OPEN) { - flow->rings[flowid]->status = RING_CLOSING; + if (flow->rings[flowid]->status == RING_OPEN) brcmf_msgbuf_delete_flowring(drvr, flowid); - } } } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 32ee82761873c..6b8842da6f843 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -1329,22 +1329,24 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; struct msgbuf_tx_flowring_delete_req *delete; struct brcmf_commonring *commonring; - struct brcmf_commonring *commonring_del; - + struct brcmf_commonring *commonring_del = msgbuf->flowrings[flowid]; + struct brcmf_flowring *flow = msgbuf->flow; void *ret_ptr; u8 ifidx; int err; int retry = BRCMF_MAX_TXSTATUS_WAIT_RETRIES; - /* wait for commonring txflow finished */ - commonring_del = msgbuf->flowrings[flowid]; + /* make sure it is not in txflow */ brcmf_commonring_lock(commonring_del); + flow->rings[flowid]->status = RING_CLOSING; + brcmf_commonring_unlock(commonring_del); + + /* wait for commonring txflow finished */ while (retry && atomic_read(&commonring_del->outstanding_tx)) { usleep_range(5000, 10000); retry--; } - brcmf_commonring_unlock(commonring_del); - if (!retry && atomic_read(&commonring_del->outstanding_tx)) { + if (!retry) { brcmf_err("timed out waiting for txstatus\n"); atomic_set(&commonring_del->outstanding_tx, 0); } From 551a6b1e4eab4b0429ba41a169ee040db5f3baf5 Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Wed, 6 Nov 2019 16:31:12 -0600 Subject: [PATCH 139/213] brcmfmac: Increase message buffer packet size In wifi firmware, max length of IOCTL/IOVAR buffer size is 8192. Increase the message buffer max size same as wifi firmware. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 6b8842da6f843..dc232cde52901 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -63,7 +63,7 @@ #define BRCMF_IOCTL_REQ_PKTID 0xFFFE -#define BRCMF_MSGBUF_MAX_PKT_SIZE 2048 +#define BRCMF_MSGBUF_MAX_PKT_SIZE 8192 #define BRCMF_MSGBUF_RXBUFPOST_THRESHOLD 32 #define BRCMF_MSGBUF_MAX_IOCTLRESPBUF_POST 8 #define BRCMF_MSGBUF_MAX_EVENTBUF_POST 8 From 8d0be33d6c946574f358e697aad87da199d4eb31 Mon Sep 17 00:00:00 2001 From: Franky Lin Date: Thu, 26 Apr 2018 12:16:51 +0200 Subject: [PATCH 140/213] brcmfmac: coarse support for PCIe shared structure rev7 commit f56324baf329bc9362a52ad77a4a1a0f3356d1bc upstream Revision 7 of PCIe dongle interface increases the item size of tx and rx complete rings to accommodate extra payload for new feature. This patch simply bump up the size of these two rings without adding the support for utilizing the new space. This makes brcmfmac compatible with rev7 firmware. Reviewed-by: Arend van Spriel Signed-off-by: Franky Lin Signed-off-by: Arend van Spriel Signed-off-by: Kalle Valo --- .../broadcom/brcm80211/brcmfmac/msgbuf.h | 6 ++-- .../broadcom/brcm80211/brcmfmac/pcie.c | 28 +++++++++++++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h index f93ba6be1ef87..692235d252779 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h @@ -27,8 +27,10 @@ #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE 40 #define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE 32 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_ITEMSIZE 24 -#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE 16 -#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE 32 +#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE_PRE_V7 16 +#define BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE 24 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE_PRE_V7 32 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE 40 #define BRCMF_H2D_TXFLOWRING_ITEMSIZE 48 struct msgbuf_buf_addr { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 1011caf496812..c33723b7116dc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -134,9 +134,11 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_PCIE_MB_INT_D2H2_DB1 | \ BRCMF_PCIE_MB_INT_D2H3_DB0 | \ BRCMF_PCIE_MB_INT_D2H3_DB1) - -#define BRCMF_PCIE_MIN_SHARED_VERSION 5 -#define BRCMF_PCIE_MAX_SHARED_VERSION 6 +#define BRCMF_PCIE_SHARED_VERSION_5 5 +#define BRCMF_PCIE_SHARED_VERSION_6 6 +#define BRCMF_PCIE_SHARED_VERSION_7 7 +#define BRCMF_PCIE_MIN_SHARED_VERSION BRCMF_PCIE_SHARED_VERSION_5 +#define BRCMF_PCIE_MAX_SHARED_VERSION BRCMF_PCIE_SHARED_VERSION_7 #define BRCMF_PCIE_SHARED_VERSION_MASK 0x00FF #define BRCMF_PCIE_SHARED_DMA_INDEX 0x10000 #define BRCMF_PCIE_SHARED_DMA_2B_IDX 0x100000 @@ -316,6 +318,14 @@ static const u32 brcmf_ring_max_item[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM }; +static const u32 brcmf_ring_itemsize_pre_v7[BRCMF_NROF_COMMON_MSGRINGS] = { + BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE, + BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE, + BRCMF_D2H_MSGRING_CONTROL_COMPLETE_ITEMSIZE, + BRCMF_D2H_MSGRING_TX_COMPLETE_ITEMSIZE_PRE_V7, + BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE_PRE_V7 +}; + static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE, BRCMF_H2D_MSGRING_RXPOST_SUBMIT_ITEMSIZE, @@ -998,8 +1008,14 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, struct brcmf_pcie_ringbuf *ring; u32 size; u32 addr; + const u32 *ring_itemsize_array; + + if (devinfo->shared.version < BRCMF_PCIE_SHARED_VERSION_7) + ring_itemsize_array = brcmf_ring_itemsize_pre_v7; + else + ring_itemsize_array = brcmf_ring_itemsize; - size = brcmf_ring_max_item[ring_id] * brcmf_ring_itemsize[ring_id]; + size = brcmf_ring_max_item[ring_id] * ring_itemsize_array[ring_id]; dma_buf = brcmf_pcie_init_dmabuffer_for_device(devinfo, size, tcm_ring_phys_addr + BRCMF_RING_MEM_BASE_ADDR_OFFSET, &dma_handle); @@ -1009,7 +1025,7 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, addr = tcm_ring_phys_addr + BRCMF_RING_MAX_ITEM_OFFSET; brcmf_pcie_write_tcm16(devinfo, addr, brcmf_ring_max_item[ring_id]); addr = tcm_ring_phys_addr + BRCMF_RING_LEN_ITEMS_OFFSET; - brcmf_pcie_write_tcm16(devinfo, addr, brcmf_ring_itemsize[ring_id]); + brcmf_pcie_write_tcm16(devinfo, addr, ring_itemsize_array[ring_id]); ring = kzalloc(sizeof(*ring), GFP_KERNEL); if (!ring) { @@ -1018,7 +1034,7 @@ brcmf_pcie_alloc_dma_and_ring(struct brcmf_pciedev_info *devinfo, u32 ring_id, return NULL; } brcmf_commonring_config(&ring->commonring, brcmf_ring_max_item[ring_id], - brcmf_ring_itemsize[ring_id], dma_buf); + ring_itemsize_array[ring_id], dma_buf); ring->dma_handle = dma_handle; ring->devinfo = devinfo; brcmf_commonring_register_cb(&ring->commonring, From 9d77d58b65d9e434ff954cfc10bc22b07e947901 Mon Sep 17 00:00:00 2001 From: "alep@cypress.com" Date: Fri, 26 Jul 2019 04:03:06 -0500 Subject: [PATCH 141/213] brcmfmac: Support 89459 pcie Adds support of 89459 chip pcie device and save restore support. Signed-off-by: Joseph chuang Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c | 8 ++++++++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 4 ++++ .../net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h | 4 +++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index f35ae7735d72b..0f83946ab8fe5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -752,6 +752,8 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) return 0x200000; case CY_CC_4373_CHIP_ID: return 0x160000; + case CY_CC_89459_CHIP_ID: + return ((ci->pub.chiprev < 9) ? 0x180000 : 0x160000); default: brcmf_err("unknown chip: %s\n", ci->pub.name); break; @@ -1452,6 +1454,12 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub) reg = chip->ops->read32(chip->ctx, addr); return (reg & (PMU_RCTL_MACPHY_DISABLE_MASK | PMU_RCTL_LOGIC_DISABLE_MASK)) == 0; + case CY_CC_89459_CHIP_ID: + /* explicitly check SR engine enable bit */ + sr_eng_en = BIT(0); + addr = CORE_CC_REG(base, sr_control0); + reg = chip->ops->read32(chip->ctx, addr); + return (reg & sr_eng_en) != 0; default: addr = CORE_CC_REG(pmu->base, pmucapabilities_ext); reg = chip->ops->read32(chip->ctx, addr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index c33723b7116dc..341a4c659d099 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -59,6 +59,7 @@ BRCMF_FW_NVRAM_DEF(4365C, "brcmfmac4365c-pcie.bin", "brcmfmac4365c-pcie.txt"); BRCMF_FW_NVRAM_DEF(4366B, "brcmfmac4366b-pcie.bin", "brcmfmac4366b-pcie.txt"); BRCMF_FW_NVRAM_DEF(4366C, "brcmfmac4366c-pcie.bin", "brcmfmac4366c-pcie.txt"); BRCMF_FW_NVRAM_DEF(4371, "brcmfmac4371-pcie.bin", "brcmfmac4371-pcie.txt"); +BRCMF_FW_NVRAM_DEF(4355, "brcmfmac89459-pcie.bin", "brcmfmac89459-pcie.txt"); static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602), @@ -77,6 +78,7 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0x0000000F, 4366B), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4366_CHIP_ID, 0xFFFFFFF0, 4366C), BRCMF_FW_NVRAM_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371), + BRCMF_FW_NVRAM_ENTRY(CY_CC_89459_CHIP_ID, 0xFFFFFFFF, 4355), }; #define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ @@ -2019,6 +2021,8 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_2G_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_5G_DEVICE_ID), BRCMF_PCIE_DEVICE(BRCM_PCIE_4371_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_89459_DEVICE_ID), + BRCMF_PCIE_DEVICE(CY_PCIE_89459_RAW_DEVICE_ID), { /* end: all zeroes */ } }; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index dcd4d52d7a6c0..459b251868069 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -60,6 +60,7 @@ #define BRCM_CC_4371_CHIP_ID 0x4371 #define CY_CC_4373_CHIP_ID 0x4373 #define CY_CC_43012_CHIP_ID 43012 +#define CY_CC_89459_CHIP_ID 0x4355 /* USB Device IDs */ #define BRCM_USB_43143_DEVICE_ID 0xbd1e @@ -93,7 +94,8 @@ #define BRCM_PCIE_4366_2G_DEVICE_ID 0x43c4 #define BRCM_PCIE_4366_5G_DEVICE_ID 0x43c5 #define BRCM_PCIE_4371_DEVICE_ID 0x440d - +#define CY_PCIE_89459_DEVICE_ID 0x4415 +#define CY_PCIE_89459_RAW_DEVICE_ID 0x4355 /* brcmsmac IDs */ #define BCM4313_D11N2G_ID 0x4727 /* 4313 802.11n 2.4G device */ From ea2d94721580ca2d3fa2f0bbc2daf1ab7efacef1 Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Thu, 7 Nov 2019 12:49:40 -0600 Subject: [PATCH 142/213] brcmfmac: Fix for unable to return to visible SSID Unable to change back to visiable SSID because there is no disable hidden ssid routine. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 60f92bab96489..ef0da7e6e373f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4817,12 +4817,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, goto exit; } - if (settings->hidden_ssid) { - err = brcmf_fil_iovar_int_set(ifp, "closednet", 1); - if (err) { - brcmf_err("closednet error (%d)\n", err); - goto exit; - } + err = brcmf_fil_iovar_int_set(ifp, "closednet", + settings->hidden_ssid); + if (err) { + brcmf_err("%s closednet error (%d)\n", + settings->hidden_ssid ? + "enabled" : "disabled", + err); + goto exit; } brcmf_dbg(TRACE, "AP mode configuration complete\n"); From 0accc307b384c910bc8e998fcdc02cd664362c1e Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Wed, 6 Nov 2019 18:57:12 -0600 Subject: [PATCH 143/213] brcmfmac: Fix for wrong disconnection event source information. Add event source argument on link down handler. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index ef0da7e6e373f..9f1d27cc756e5 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1392,7 +1392,8 @@ static int brcmf_set_sae_password(struct brcmf_if *ifp, const u8 *pwd_data, return err; } -static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason) +static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, + bool locally_generated) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(vif->wdev.wiphy); s32 err = 0; @@ -1409,7 +1410,7 @@ static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason) if ((vif->wdev.iftype == NL80211_IFTYPE_STATION) || (vif->wdev.iftype == NL80211_IFTYPE_P2P_CLIENT)) cfg80211_disconnected(vif->wdev.netdev, reason, NULL, 0, - true, GFP_KERNEL); + locally_generated, GFP_KERNEL); } clear_bit(BRCMF_VIF_STATUS_CONNECTING, &vif->sme_state); clear_bit(BRCMF_SCAN_STATUS_SUPPRESS, &cfg->scan_status); @@ -1586,7 +1587,7 @@ brcmf_cfg80211_leave_ibss(struct wiphy *wiphy, struct net_device *ndev) return 0; } - brcmf_link_down(ifp->vif, WLAN_REASON_DEAUTH_LEAVING); + brcmf_link_down(ifp->vif, WLAN_REASON_DEAUTH_LEAVING, true); brcmf_net_setcarrier(ifp, false); brcmf_dbg(TRACE, "Exit\n"); @@ -3909,7 +3910,7 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, * disassociate from AP to save power while system is * in suspended state */ - brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED); + brcmf_link_down(vif, WLAN_REASON_UNSPECIFIED, true); /* Make sure WPA_Supplicant receives all the event * generated due to DISASSOC call to the fw to keep * the state fw and WPA_Supplicant state consistent @@ -6051,7 +6052,11 @@ brcmf_notify_connect_status(struct brcmf_if *ifp, if (!brcmf_is_ibssmode(ifp->vif)) { brcmf_bss_connect_done(cfg, ndev, e, false); brcmf_link_down(ifp->vif, - brcmf_map_fw_linkdown_reason(e)); + brcmf_map_fw_linkdown_reason(e), + e->event_code & + (BRCMF_E_DEAUTH_IND | + BRCMF_E_DISASSOC_IND) + ? false : true); brcmf_init_prof(ndev_to_prof(ndev)); if (ndev != cfg_to_ndev(cfg)) complete(&cfg->vif_disabled); @@ -7232,7 +7237,7 @@ static s32 __brcmf_cfg80211_down(struct brcmf_if *ifp) * from AP to save power */ if (check_vif_up(ifp->vif)) { - brcmf_link_down(ifp->vif, WLAN_REASON_UNSPECIFIED); + brcmf_link_down(ifp->vif, WLAN_REASON_UNSPECIFIED, true); /* Make sure WPA_Supplicant receives all the event generated due to DISASSOC call to the fw to keep From da916456e7857d8dae4f200d7cc14c5c5b5b9460 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Fri, 1 Nov 2019 02:04:07 -0500 Subject: [PATCH 144/213] Revert "brcmfmac: discard user-space RSNE for SAE authentication offload" This reverts commit e82453a83c953dda6c8b9ddee0ffb98393a57af4. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 9f1d27cc756e5..33ce619888729 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2130,11 +2130,6 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, err = brcmf_set_pmk(ifp, sme->crypto.psk, BRCMF_WSEC_MAX_PSK_LEN); else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { - /* clean up user-space RSNE */ - if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { - brcmf_err("failed to clean up user-space RSNE\n"); - goto done; - } err = brcmf_set_sae_password(ifp, sme->crypto.sae_pwd, sme->crypto.sae_pwd_len); if (!err && sme->crypto.psk) From a13c3ead63e9987a4946b11b0b26980a9467cd2a Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Fri, 1 Nov 2019 02:11:26 -0500 Subject: [PATCH 145/213] Revert "brcmfmac: add support for SAE authentication offload" This reverts commit 364b349e598dbf3c7fcc23edb2d65ae0a238b0c9. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin Conflicts: drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 67 ++----------------- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 3 +- .../broadcom/brcm80211/brcmfmac/feature.c | 1 - .../broadcom/brcm80211/brcmfmac/feature.h | 4 +- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 13 ---- .../broadcom/brcm80211/include/brcmu_wifi.h | 2 - 6 files changed, 8 insertions(+), 82 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 33ce619888729..1485fa6284309 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1368,30 +1368,6 @@ static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) return err; } -static int brcmf_set_sae_password(struct brcmf_if *ifp, const u8 *pwd_data, - u16 pwd_len) -{ - struct brcmf_wsec_sae_pwd_le sae_pwd; - int err; - - if (pwd_len > BRCMF_WSEC_MAX_SAE_PASSWORD_LEN) { - brcmf_err("sae_password must be less than %d\n", - BRCMF_WSEC_MAX_SAE_PASSWORD_LEN); - return -EINVAL; - } - - sae_pwd.key_len = cpu_to_le16(pwd_len); - memcpy(sae_pwd.key, pwd_data, pwd_len); - - err = brcmf_fil_iovar_data_set(ifp, "sae_password", &sae_pwd, - sizeof(sae_pwd)); - if (err < 0) - brcmf_err("failed to set SAE password in firmware (len=%u)\n", - pwd_len); - - return err; -} - static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, bool locally_generated) { @@ -1607,8 +1583,6 @@ static s32 brcmf_set_wpa_version(struct net_device *ndev, val = WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED; else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_2) val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; - else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3) - val = WPA3_AUTH_SAE_PSK; else val = WPA_AUTH_DISABLED; brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); @@ -1639,10 +1613,6 @@ static s32 brcmf_set_auth_type(struct net_device *ndev, val = 1; brcmf_dbg(CONN, "shared key\n"); break; - case NL80211_AUTHTYPE_SAE: - val = 3; - brcmf_dbg(CONN, "SAE authentication\n"); - break; default: val = 2; brcmf_dbg(CONN, "automatic, auth type (%d)\n", sme->auth_type); @@ -1810,17 +1780,6 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) sme->crypto.cipher_group); return -EINVAL; } - } else if (val & WPA3_AUTH_SAE_PSK) { - switch (sme->crypto.akm_suites[0]) { - case WLAN_AKM_SUITE_SAE: - val = WPA3_AUTH_SAE_PSK; - profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; - break; - default: - brcmf_err("invalid cipher group (%d)\n", - sme->crypto.cipher_group); - return -EINVAL; - } } if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) { @@ -1833,8 +1792,6 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); profile->is_okc = okc_enable; } - } else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { - brcmf_dbg(INFO, "using SAE offload\n"); } if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MFP)) @@ -1901,8 +1858,7 @@ brcmf_set_sharedkey(struct net_device *ndev, brcmf_dbg(CONN, "wpa_versions 0x%x cipher_pairwise 0x%x\n", sec->wpa_versions, sec->cipher_pairwise); - if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2 | - NL80211_WPA_VERSION_3)) + if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2)) return 0; if (!(sec->cipher_pairwise & @@ -2107,8 +2063,7 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, goto done; } - if (sme->crypto.psk && - profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { + if (sme->crypto.psk) { if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { err = -EINVAL; goto done; @@ -2126,18 +2081,12 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, } } - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) { err = brcmf_set_pmk(ifp, sme->crypto.psk, BRCMF_WSEC_MAX_PSK_LEN); - else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { - err = brcmf_set_sae_password(ifp, sme->crypto.sae_pwd, - sme->crypto.sae_pwd_len); - if (!err && sme->crypto.psk) - err = brcmf_set_pmk(ifp, sme->crypto.psk, - BRCMF_WSEC_MAX_PSK_LEN); + if (err) + goto done; } - if (err) - goto done; /* Join with specific BSSID and cached SSID * If SSID is zero join based on BSSID only @@ -5529,8 +5478,7 @@ static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, if (event == BRCMF_E_SET_SSID && status == BRCMF_E_STATUS_SUCCESS) { brcmf_dbg(CONN, "Processing set ssid\n"); memcpy(vif->profile.bssid, e->addr, ETH_ALEN); - if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK && - vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_SAE) + if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK) return true; set_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state); @@ -7087,9 +7035,6 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X); - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE)) - wiphy_ext_feature_set(wiphy, - NL80211_EXT_FEATURE_SAE_OFFLOAD); } wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 53b3247b97a10..3b6d01182b2b1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -144,8 +144,7 @@ struct brcmf_cfg80211_security { enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_NONE, BRCMF_PROFILE_FWSUP_PSK, - BRCMF_PROFILE_FWSUP_1X, - BRCMF_PROFILE_FWSUP_SAE + BRCMF_PROFILE_FWSUP_1X }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 3069c6efba154..c7c3563f9a53c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -48,7 +48,6 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_MBSS, "mbss" }, { BRCMF_FEAT_MCHAN, "mchan" }, { BRCMF_FEAT_P2P, "p2p" }, - { BRCMF_FEAT_SAE, "sae" }, }; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 48e679d4018aa..1ab4f16171120 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -33,7 +33,6 @@ * MFP: 802.11w Management Frame Protection. * GSCAN: enhanced scan offload feature. * FWSUP: Firmware supplicant. - * SAE: simultaneous authentication of equals */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -49,8 +48,7 @@ BRCMF_FEAT_DEF(WOWL_ARP_ND) \ BRCMF_FEAT_DEF(MFP) \ BRCMF_FEAT_DEF(GSCAN) \ - BRCMF_FEAT_DEF(FWSUP) \ - BRCMF_FEAT_DEF(SAE) + BRCMF_FEAT_DEF(FWSUP) /* * Quirks: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 73b11242812d4..2ab973a4ff206 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -53,8 +53,6 @@ #define BRCMF_WSEC_MAX_PSK_LEN 32 #define BRCMF_WSEC_PASSPHRASE BIT(0) -#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128 - /* primary (ie tx) key */ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 @@ -521,17 +519,6 @@ struct brcmf_wsec_pmk_le { u8 key[2 * BRCMF_WSEC_MAX_PSK_LEN + 1]; }; -/** - * struct brcmf_wsec_sae_pwd_le - firmware SAE password material. - * - * @key_len: number of octets in key materials. - * @key: SAE password material. - */ -struct brcmf_wsec_sae_pwd_le { - __le16 key_len; - u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; -}; - /* Used to get specific STA parameters */ struct brcmf_scb_val_le { __le32 val; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index cfe592fc4b040..91fca79e61583 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -240,8 +240,6 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_FT 0x4000 /* Fast BSS Transition */ #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ -#define WPA3_AUTH_SAE_PSK 0x40000 /* SAE with 4-way handshake */ - #define DOT11_DEFAULT_RTS_LEN 2347 #define DOT11_DEFAULT_FRAG_LEN 2346 From 38e4d061a4b8f4010e04c8aa60680a05f965fd91 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Fri, 1 Nov 2019 02:14:26 -0500 Subject: [PATCH 146/213] Revert "cfg80211: add support for SAE authentication offload" This reverts commit 4d115f96cdc68ea7a44807c44a6e8180f629e411. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- include/linux/ieee80211.h | 1 - include/net/cfg80211.h | 5 ----- include/uapi/linux/nl80211.h | 18 ------------------ net/wireless/nl80211.c | 14 -------------- 4 files changed, 38 deletions(-) diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 6cf1971a46918..c83478271c2e0 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2542,7 +2542,6 @@ enum ieee80211_key_len { #define FILS_ERP_MAX_RRK_LEN 64 #define PMK_MAX_LEN 64 -#define SAE_PASSWORD_MAX_LEN 128 /* Public action codes (IEEE Std 802.11-2016, 9.6.8.1, Table 9-307) */ enum ieee80211_pub_actioncode { diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 39656e379bcbc..3ac846ec00a49 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -740,9 +740,6 @@ struct survey_info { * CFG80211_MAX_WEP_KEYS WEP keys * @wep_tx_key: key index (0..3) of the default TX static WEP key * @psk: PSK (for devices supporting 4-way-handshake offload) - * @sae_pwd: password for SAE authentication (for devices supporting SAE - * offload) - * @sae_pwd_len: length of SAE password (for devices supporting SAE offload) */ struct cfg80211_crypto_settings { u32 wpa_versions; @@ -758,8 +755,6 @@ struct cfg80211_crypto_settings { struct key_params *wep_keys; int wep_tx_key; const u8 *psk; - const u8 *sae_pwd; - u16 sae_pwd_len; }; /** diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index a9047482f65df..8d9040edf8b47 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -234,15 +234,6 @@ * use in a FILS shared key connection with PMKSA caching. */ -/** - * DOC: SAE authentication offload - * - * By setting @NL80211_EXT_FEATURE_SAE_OFFLOAD flag drivers can indicate they - * support offloading SAE authentication for WPA3-Personal networks. In - * %NL80211_CMD_CONNECT the password for SAE should be specified using - * %NL80211_ATTR_SAE_PASSWORD. - */ - /** * enum nl80211_commands - supported nl80211 commands * @@ -2253,10 +2244,6 @@ enum nl80211_commands { * association request when used with NL80211_CMD_NEW_STATION). Can be set * only if %NL80211_STA_FLAG_WME is set. * - * @NL80211_ATTR_SAE_PASSWORD: attribute for passing SAE password material. It - * is used with %NL80211_CMD_CONNECT to provide password for offloading - * SAE authentication for WPA3-Personal networks. - * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -2698,8 +2685,6 @@ enum nl80211_attrs { NL80211_ATTR_HE_CAPABILITY, - NL80211_ATTR_SAE_PASSWORD, - /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -5242,8 +5227,6 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT: Driver/device can omit all data * except for supported rates from the probe request content if requested * by the %NL80211_SCAN_FLAG_MIN_PREQ_CONTENT flag. - * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in - * station mode (SAE password is passed as part of the connect command). * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. @@ -5280,7 +5263,6 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_TXQS, NL80211_EXT_FEATURE_SCAN_RANDOM_SN, NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT, - NL80211_EXT_FEATURE_SAE_OFFLOAD, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index b184cd9500b28..b51979afd4b6e 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -456,8 +456,6 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { [NL80211_ATTR_FILS_CACHE_ID] = { .len = 2 }, [NL80211_ATTR_PMK] = { .type = NLA_BINARY, .len = PMK_MAX_LEN }, [NL80211_ATTR_SCHED_SCAN_MULTI] = { .type = NLA_FLAG }, - [NL80211_ATTR_SAE_PASSWORD] = { .type = NLA_BINARY, - .len = SAE_PASSWORD_MAX_LEN }, [NL80211_ATTR_EXTERNAL_AUTH_SUPPORT] = { .type = NLA_FLAG }, [NL80211_ATTR_TXQ_LIMIT] = { .type = NLA_U32 }, @@ -4186,8 +4184,6 @@ static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, return true; case NL80211_CMD_CONNECT: if (!(rdev->wiphy.features & NL80211_FEATURE_SAE) && - !wiphy_ext_feature_isset(&rdev->wiphy, - NL80211_EXT_FEATURE_SAE_OFFLOAD) && auth_type == NL80211_AUTHTYPE_SAE) return false; @@ -8650,16 +8646,6 @@ static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev, settings->psk = nla_data(info->attrs[NL80211_ATTR_PMK]); } - if (info->attrs[NL80211_ATTR_SAE_PASSWORD]) { - if (!wiphy_ext_feature_isset(&rdev->wiphy, - NL80211_EXT_FEATURE_SAE_OFFLOAD)) - return -EINVAL; - settings->sae_pwd = - nla_data(info->attrs[NL80211_ATTR_SAE_PASSWORD]); - settings->sae_pwd_len = - nla_len(info->attrs[NL80211_ATTR_SAE_PASSWORD]); - } - return 0; } From 426d588edc6c74935c5addad5090320c1d0b1db7 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Fri, 1 Nov 2019 02:17:26 -0500 Subject: [PATCH 147/213] non-upstream: update enum nl80211_attrs and nl80211_ext_feature_index This patch updates enum nl80211_attrs and nl80211_ext_feature_index to sync with the ones in wireless-drivers-next.git include/uapi/linux/nl80211.h as of 2019-06-13. It is required for the following SAE offload patches. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- include/uapi/linux/nl80211.h | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 8d9040edf8b47..c8787af93d9ae 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -2244,6 +2244,36 @@ enum nl80211_commands { * association request when used with NL80211_CMD_NEW_STATION). Can be set * only if %NL80211_STA_FLAG_WME is set. * + * @NL80211_ATTR_FTM_RESPONDER: nested attribute which user-space can include + * in %NL80211_CMD_START_AP or %NL80211_CMD_SET_BEACON for fine timing + * measurement (FTM) responder functionality and containing parameters as + * possible, see &enum nl80211_ftm_responder_attr + * + * @NL80211_ATTR_FTM_RESPONDER_STATS: Nested attribute with FTM responder + * statistics, see &enum nl80211_ftm_responder_stats. + * + * @NL80211_ATTR_TIMEOUT: Timeout for the given operation in milliseconds (u32), + * if the attribute is not given no timeout is requested. Note that 0 is an + * invalid value. + * + * @NL80211_ATTR_PEER_MEASUREMENTS: peer measurements request (and result) + * data, uses nested attributes specified in + * &enum nl80211_peer_measurement_attrs. + * This is also used for capability advertisement in the wiphy information, + * with the appropriate sub-attributes. + * + * @NL80211_ATTR_AIRTIME_WEIGHT: Station's weight when scheduled by the airtime + * scheduler. + * + * @NL80211_ATTR_STA_TX_POWER_SETTING: Transmit power setting type (u8) for + * station associated with the AP. See &enum nl80211_tx_power_setting for + * possible values. + * @NL80211_ATTR_STA_TX_POWER: Transmit power level (s16) in dBm units. This + * allows to set Tx power for a station. If this attribute is not included, + * the default per-interface tx power setting will be overriding. Driver + * should be picking up the lowest tx power, either tx power per-interface + * or per-station. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -2685,6 +2715,18 @@ enum nl80211_attrs { NL80211_ATTR_HE_CAPABILITY, + NL80211_ATTR_FTM_RESPONDER, + + NL80211_ATTR_FTM_RESPONDER_STATS, + + NL80211_ATTR_TIMEOUT, + + NL80211_ATTR_PEER_MEASUREMENTS, + + NL80211_ATTR_AIRTIME_WEIGHT, + NL80211_ATTR_STA_TX_POWER_SETTING, + NL80211_ATTR_STA_TX_POWER, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -5227,6 +5269,28 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT: Driver/device can omit all data * except for supported rates from the probe request content if requested * by the %NL80211_SCAN_FLAG_MIN_PREQ_CONTENT flag. + * @NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER: Driver supports enabling fine + * timing measurement responder role. + * + * @NL80211_EXT_FEATURE_CAN_REPLACE_PTK0: Driver/device confirm that they are + * able to rekey an in-use key correctly. Userspace must not rekey PTK keys + * if this flag is not set. Ignoring this can leak clear text packets and/or + * freeze the connection. + * @NL80211_EXT_FEATURE_EXT_KEY_ID: Driver supports "Extended Key ID for + * Individually Addressed Frames" from IEEE802.11-2016. + * + * @NL80211_EXT_FEATURE_AIRTIME_FAIRNESS: Driver supports getting airtime + * fairness for transmitted packets and has enabled airtime fairness + * scheduling. + * + * @NL80211_EXT_FEATURE_AP_PMKSA_CACHING: Driver/device supports PMKSA caching + * (set/del PMKSA operations) in AP mode. + * + * @NL80211_EXT_FEATURE_SCHED_SCAN_BAND_SPECIFIC_RSSI_THOLD: Driver supports + * filtering of sched scan results using band specific RSSI thresholds. + * + * @NL80211_EXT_FEATURE_STA_TX_PWR: This driver supports controlling tx power + * to a station. * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. @@ -5263,6 +5327,13 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_TXQS, NL80211_EXT_FEATURE_SCAN_RANDOM_SN, NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT, + NL80211_EXT_FEATURE_CAN_REPLACE_PTK0, + NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER, + NL80211_EXT_FEATURE_AIRTIME_FAIRNESS, + NL80211_EXT_FEATURE_AP_PMKSA_CACHING, + NL80211_EXT_FEATURE_SCHED_SCAN_BAND_SPECIFIC_RSSI_THOLD, + NL80211_EXT_FEATURE_EXT_KEY_ID, + NL80211_EXT_FEATURE_STA_TX_PWR, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, From e94c1e20a33718685826c9d95995d61d931b7b52 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Thu, 9 May 2019 09:49:06 +0000 Subject: [PATCH 148/213] nl80211: add support for SAE authentication offload commit 26f7044e95042daabcf1c71796a0e804a83c979f upstream. Let drivers advertise support for station-mode SAE authentication offload with a new NL80211_EXT_FEATURE_SAE_OFFLOAD flag. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin Signed-off-by: Johannes Berg Conflicts: include/linux/ieee80211.h net/wireless/nl80211.c --- include/linux/ieee80211.h | 1 + include/net/cfg80211.h | 5 +++++ include/uapi/linux/nl80211.h | 19 +++++++++++++++++++ net/wireless/nl80211.c | 14 ++++++++++++++ 4 files changed, 39 insertions(+) diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index c83478271c2e0..6cf1971a46918 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -2542,6 +2542,7 @@ enum ieee80211_key_len { #define FILS_ERP_MAX_RRK_LEN 64 #define PMK_MAX_LEN 64 +#define SAE_PASSWORD_MAX_LEN 128 /* Public action codes (IEEE Std 802.11-2016, 9.6.8.1, Table 9-307) */ enum ieee80211_pub_actioncode { diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 3ac846ec00a49..8de9ad84c31b9 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -740,6 +740,9 @@ struct survey_info { * CFG80211_MAX_WEP_KEYS WEP keys * @wep_tx_key: key index (0..3) of the default TX static WEP key * @psk: PSK (for devices supporting 4-way-handshake offload) + * @sae_pwd: password for SAE authentication (for devices supporting SAE + * offload) + * @sae_pwd_len: length of SAE password (for devices supporting SAE offload) */ struct cfg80211_crypto_settings { u32 wpa_versions; @@ -755,6 +758,8 @@ struct cfg80211_crypto_settings { struct key_params *wep_keys; int wep_tx_key; const u8 *psk; + const u8 *sae_pwd; + u8 sae_pwd_len; }; /** diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index c8787af93d9ae..9a33598928166 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -234,6 +234,15 @@ * use in a FILS shared key connection with PMKSA caching. */ +/** + * DOC: SAE authentication offload + * + * By setting @NL80211_EXT_FEATURE_SAE_OFFLOAD flag drivers can indicate they + * support offloading SAE authentication for WPA3-Personal networks. In + * %NL80211_CMD_CONNECT the password for SAE should be specified using + * %NL80211_ATTR_SAE_PASSWORD. + */ + /** * enum nl80211_commands - supported nl80211 commands * @@ -2274,6 +2283,10 @@ enum nl80211_commands { * should be picking up the lowest tx power, either tx power per-interface * or per-station. * + * @NL80211_ATTR_SAE_PASSWORD: attribute for passing SAE password material. It + * is used with %NL80211_CMD_CONNECT to provide password for offloading + * SAE authentication for WPA3-Personal networks. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -2727,6 +2740,8 @@ enum nl80211_attrs { NL80211_ATTR_STA_TX_POWER_SETTING, NL80211_ATTR_STA_TX_POWER, + NL80211_ATTR_SAE_PASSWORD, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -5292,6 +5307,9 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_STA_TX_PWR: This driver supports controlling tx power * to a station. * + * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in + * station mode (SAE password is passed as part of the connect command). + * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. */ @@ -5334,6 +5352,7 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_SCHED_SCAN_BAND_SPECIFIC_RSSI_THOLD, NL80211_EXT_FEATURE_EXT_KEY_ID, NL80211_EXT_FEATURE_STA_TX_PWR, + NL80211_EXT_FEATURE_SAE_OFFLOAD, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index b51979afd4b6e..b184cd9500b28 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -456,6 +456,8 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { [NL80211_ATTR_FILS_CACHE_ID] = { .len = 2 }, [NL80211_ATTR_PMK] = { .type = NLA_BINARY, .len = PMK_MAX_LEN }, [NL80211_ATTR_SCHED_SCAN_MULTI] = { .type = NLA_FLAG }, + [NL80211_ATTR_SAE_PASSWORD] = { .type = NLA_BINARY, + .len = SAE_PASSWORD_MAX_LEN }, [NL80211_ATTR_EXTERNAL_AUTH_SUPPORT] = { .type = NLA_FLAG }, [NL80211_ATTR_TXQ_LIMIT] = { .type = NLA_U32 }, @@ -4184,6 +4186,8 @@ static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, return true; case NL80211_CMD_CONNECT: if (!(rdev->wiphy.features & NL80211_FEATURE_SAE) && + !wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD) && auth_type == NL80211_AUTHTYPE_SAE) return false; @@ -8646,6 +8650,16 @@ static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev, settings->psk = nla_data(info->attrs[NL80211_ATTR_PMK]); } + if (info->attrs[NL80211_ATTR_SAE_PASSWORD]) { + if (!wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD)) + return -EINVAL; + settings->sae_pwd = + nla_data(info->attrs[NL80211_ATTR_SAE_PASSWORD]); + settings->sae_pwd_len = + nla_len(info->attrs[NL80211_ATTR_SAE_PASSWORD]); + } + return 0; } From 9d373211884795a401d3104d6114e4dd9e6af736 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Wed, 2 Oct 2019 09:31:12 +0000 Subject: [PATCH 149/213] brcmfmac: add support for SAE authentication offload commit 3b1e0a7bdfeedbde49b8d424aeb88ac3c0cf8182 upstream. The firmware may have SAE authentication code built-in. This is detected by the driver and indicated in the wiphy features flags. User-space can use this flag to determine whether or not to provide the password material for SAE authentication in the nl80211 CONNECT command. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin Signed-off-by: Kalle Valo --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 72 +++++++++++++++++-- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 3 +- .../broadcom/brcm80211/brcmfmac/feature.c | 1 + .../broadcom/brcm80211/brcmfmac/feature.h | 4 +- .../broadcom/brcm80211/brcmfmac/fwil_types.h | 13 ++++ .../broadcom/brcm80211/include/brcmu_wifi.h | 2 + 6 files changed, 88 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 1485fa6284309..2ba6495497e7f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -1368,6 +1368,30 @@ static int brcmf_set_pmk(struct brcmf_if *ifp, const u8 *pmk_data, u16 pmk_len) return err; } +static int brcmf_set_sae_password(struct brcmf_if *ifp, const u8 *pwd_data, + u16 pwd_len) +{ + struct brcmf_wsec_sae_pwd_le sae_pwd; + int err; + + if (pwd_len > BRCMF_WSEC_MAX_SAE_PASSWORD_LEN) { + brcmf_err("sae_password must be less than %d\n", + BRCMF_WSEC_MAX_SAE_PASSWORD_LEN); + return -EINVAL; + } + + sae_pwd.key_len = cpu_to_le16(pwd_len); + memcpy(sae_pwd.key, pwd_data, pwd_len); + + err = brcmf_fil_iovar_data_set(ifp, "sae_password", &sae_pwd, + sizeof(sae_pwd)); + if (err < 0) + brcmf_err("failed to set SAE password in firmware (len=%u)\n", + pwd_len); + + return err; +} + static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, bool locally_generated) { @@ -1583,6 +1607,8 @@ static s32 brcmf_set_wpa_version(struct net_device *ndev, val = WPA_AUTH_PSK | WPA_AUTH_UNSPECIFIED; else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_2) val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; + else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3) + val = WPA3_AUTH_SAE_PSK; else val = WPA_AUTH_DISABLED; brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); @@ -1613,6 +1639,10 @@ static s32 brcmf_set_auth_type(struct net_device *ndev, val = 1; brcmf_dbg(CONN, "shared key\n"); break; + case NL80211_AUTHTYPE_SAE: + val = 3; + brcmf_dbg(CONN, "SAE authentication\n"); + break; default: val = 2; brcmf_dbg(CONN, "automatic, auth type (%d)\n", sme->auth_type); @@ -1780,6 +1810,16 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) sme->crypto.cipher_group); return -EINVAL; } + } else if (val & WPA3_AUTH_SAE_PSK) { + switch (sme->crypto.akm_suites[0]) { + case WLAN_AKM_SUITE_SAE: + val = WPA3_AUTH_SAE_PSK; + break; + default: + brcmf_err("invalid cipher group (%d)\n", + sme->crypto.cipher_group); + return -EINVAL; + } } if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) { @@ -1858,7 +1898,8 @@ brcmf_set_sharedkey(struct net_device *ndev, brcmf_dbg(CONN, "wpa_versions 0x%x cipher_pairwise 0x%x\n", sec->wpa_versions, sec->cipher_pairwise); - if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2)) + if (sec->wpa_versions & (NL80211_WPA_VERSION_1 | NL80211_WPA_VERSION_2 | + NL80211_WPA_VERSION_3)) return 0; if (!(sec->cipher_pairwise & @@ -2063,7 +2104,13 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, goto done; } - if (sme->crypto.psk) { + if (sme->crypto.sae_pwd) { + brcmf_dbg(INFO, "using SAE offload\n"); + profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; + } + + if (sme->crypto.psk && + profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { err = -EINVAL; goto done; @@ -2081,12 +2128,23 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, } } - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) { + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) err = brcmf_set_pmk(ifp, sme->crypto.psk, BRCMF_WSEC_MAX_PSK_LEN); - if (err) + else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { + /* clean up user-space RSNE */ + if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { + brcmf_err("failed to clean up user-space RSNE\n"); goto done; + } + err = brcmf_set_sae_password(ifp, sme->crypto.sae_pwd, + sme->crypto.sae_pwd_len); + if (!err && sme->crypto.psk) + err = brcmf_set_pmk(ifp, sme->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); } + if (err) + goto done; /* Join with specific BSSID and cached SSID * If SSID is zero join based on BSSID only @@ -5478,7 +5536,8 @@ static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, if (event == BRCMF_E_SET_SSID && status == BRCMF_E_STATUS_SUCCESS) { brcmf_dbg(CONN, "Processing set ssid\n"); memcpy(vif->profile.bssid, e->addr, ETH_ALEN); - if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK) + if (vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_PSK && + vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_SAE) return true; set_bit(BRCMF_VIF_STATUS_ASSOC_SUCCESS, &vif->sme_state); @@ -7035,6 +7094,9 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD); } wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 3b6d01182b2b1..53b3247b97a10 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -144,7 +144,8 @@ struct brcmf_cfg80211_security { enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_NONE, BRCMF_PROFILE_FWSUP_PSK, - BRCMF_PROFILE_FWSUP_1X + BRCMF_PROFILE_FWSUP_1X, + BRCMF_PROFILE_FWSUP_SAE }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index c7c3563f9a53c..3069c6efba154 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -48,6 +48,7 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_MBSS, "mbss" }, { BRCMF_FEAT_MCHAN, "mchan" }, { BRCMF_FEAT_P2P, "p2p" }, + { BRCMF_FEAT_SAE, "sae" }, }; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 1ab4f16171120..48e679d4018aa 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -33,6 +33,7 @@ * MFP: 802.11w Management Frame Protection. * GSCAN: enhanced scan offload feature. * FWSUP: Firmware supplicant. + * SAE: simultaneous authentication of equals */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -48,7 +49,8 @@ BRCMF_FEAT_DEF(WOWL_ARP_ND) \ BRCMF_FEAT_DEF(MFP) \ BRCMF_FEAT_DEF(GSCAN) \ - BRCMF_FEAT_DEF(FWSUP) + BRCMF_FEAT_DEF(FWSUP) \ + BRCMF_FEAT_DEF(SAE) /* * Quirks: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index 2ab973a4ff206..73b11242812d4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -53,6 +53,8 @@ #define BRCMF_WSEC_MAX_PSK_LEN 32 #define BRCMF_WSEC_PASSPHRASE BIT(0) +#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128 + /* primary (ie tx) key */ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 @@ -519,6 +521,17 @@ struct brcmf_wsec_pmk_le { u8 key[2 * BRCMF_WSEC_MAX_PSK_LEN + 1]; }; +/** + * struct brcmf_wsec_sae_pwd_le - firmware SAE password material. + * + * @key_len: number of octets in key materials. + * @key: SAE password material. + */ +struct brcmf_wsec_sae_pwd_le { + __le16 key_len; + u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; +}; + /* Used to get specific STA parameters */ struct brcmf_scb_val_le { __le32 val; diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 91fca79e61583..cfe592fc4b040 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -240,6 +240,8 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_FT 0x4000 /* Fast BSS Transition */ #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ +#define WPA3_AUTH_SAE_PSK 0x40000 /* SAE with 4-way handshake */ + #define DOT11_DEFAULT_RTS_LEN 2347 #define DOT11_DEFAULT_FRAG_LEN 2346 From 42943ab08589922941e210f88e92ab833f5af0c0 Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Wed, 6 Nov 2019 17:36:25 -0600 Subject: [PATCH 150/213] brcmfmac: Support multiple AP interfaces and fix STA disconnection issue Support multiple AP interfaces for STA + AP + AP usecase. And fix STA disconnection when deactivating AP interface. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 49 +++++++++++++++---- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 1 + .../broadcom/brcm80211/brcmfmac/common.c | 5 ++ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 2ba6495497e7f..75e37d69fb0de 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4664,6 +4664,7 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, settings->inactivity_timeout); dev_role = ifp->vif->wdev.iftype; mbss = ifp->vif->mbss; + brcmf_dbg(TRACE, "mbss %s\n", mbss ? "enabled" : "disabled"); /* store current 11d setting */ if (brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY, @@ -4866,6 +4867,9 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, if ((err) && (!mbss)) { brcmf_set_mpc(ifp, 1); brcmf_configure_arp_nd_offload(ifp, true); + } else { + cfg->num_softap++; + brcmf_dbg(TRACE, "Num of SoftAP %u\n", cfg->num_softap); } return err; } @@ -4873,9 +4877,11 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) { struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); s32 err; struct brcmf_fil_bss_enable_le bss_enable; struct brcmf_join_params join_params; + s32 apsta = 0; brcmf_dbg(TRACE, "Enter\n"); @@ -4884,6 +4890,27 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) /* first to make sure they get processed by fw. */ msleep(400); + cfg->num_softap--; + + /* Clear bss configuration and SSID */ + bss_enable.bsscfgidx = cpu_to_le32(ifp->bsscfgidx); + bss_enable.enable = cpu_to_le32(0); + err = brcmf_fil_iovar_data_set(ifp, "bss", &bss_enable, + sizeof(bss_enable)); + if (err < 0) + brcmf_err("bss_enable config failed %d\n", err); + + memset(&join_params, 0, sizeof(join_params)); + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, + &join_params, sizeof(join_params)); + if (err < 0) + brcmf_err("SET SSID error (%d)\n", err); + + if (cfg->num_softap) { + brcmf_dbg(TRACE, "Num of SoftAP %u\n", cfg->num_softap); + return 0; + } + if (ifp->vif->mbss) { err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); return err; @@ -4893,17 +4920,18 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) if (ifp->bsscfgidx == 0) brcmf_fil_iovar_int_set(ifp, "closednet", 0); - memset(&join_params, 0, sizeof(join_params)); - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, sizeof(join_params)); - if (err < 0) - brcmf_err("SET SSID error (%d)\n", err); - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); + err = brcmf_fil_iovar_int_get(ifp, "apsta", &apsta); if (err < 0) - brcmf_err("BRCMF_C_DOWN error %d\n", err); - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 0); - if (err < 0) - brcmf_err("setting AP mode failed %d\n", err); + brcmf_err("wl apsta failed (%d)\n", err); + + if (!apsta) { + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_DOWN, 1); + if (err < 0) + brcmf_err("BRCMF_C_DOWN error %d\n", err); + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 0); + if (err < 0) + brcmf_err("setting AP mode failed %d\n", err); + } if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_MBSS)) brcmf_fil_iovar_int_set(ifp, "mbss", 0); brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_REGULATORY, @@ -7498,6 +7526,7 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->ops = ops; cfg->pub = drvr; cfg->pm_state = BRCMF_CFG80211_PM_STATE_RESUMED; + cfg->num_softap = 0; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 53b3247b97a10..d61a545630137 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -383,6 +383,7 @@ struct brcmf_cfg80211_info { struct brcmf_pno_info *pno; u8 ac_priority[MAX_8021D_PRIO]; u8 pm_state; + u8 num_softap; }; /** diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index c6eed9ffe8f80..000a505d9eec2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -343,6 +343,11 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) brcmf_dbg(INFO, "CLM version = %s\n", clmver); } + /* set apsta */ + err = brcmf_fil_iovar_int_set(ifp, "apsta", 1); + if (err) + brcmf_info("failed setting apsta, %d\n", err); + /* set mpc */ err = brcmf_fil_iovar_int_set(ifp, "mpc", 1); if (err) { From 01c136fa6048a064f7f3ecb0087d44ddd73105a8 Mon Sep 17 00:00:00 2001 From: Soontak Lee Date: Mon, 4 Nov 2019 18:33:29 -0600 Subject: [PATCH 151/213] brcmfmac: Support custom PCIE BAR window size. Certain host processors cannot support 4MB PCIE BAR window size. For example, AMLogic A113D can support 2MB size only. This patch is for host processor which support lower than 4MB PCIE BAR window size. Signed-off-by: Soontak Lee Signed-off-by: Chi-Hsien Lin --- .../net/wireless/broadcom/brcm80211/Kconfig | 11 + .../broadcom/brcm80211/brcmfmac/pcie.c | 241 +++++++++++++++++- 2 files changed, 246 insertions(+), 6 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/Kconfig b/drivers/net/wireless/broadcom/brcm80211/Kconfig index 9d99eb42d9176..d8c6fd64d3eea 100644 --- a/drivers/net/wireless/broadcom/brcm80211/Kconfig +++ b/drivers/net/wireless/broadcom/brcm80211/Kconfig @@ -85,3 +85,14 @@ config BRCMDBG select WANT_DEV_COREDUMP ---help--- Selecting this enables additional code for debug purposes. + +config BRCMFMAC_PCIE_BARWIN_SZ + bool "Custom PCIE BAR window size support for FullMAC driver" + depends on BRCMFMAC + depends on PCI + default n + ---help--- + If you say Y here, the FMAC driver will use custom PCIE BAR + window size. Say Y to allow developers to use custom PCIE + BAR window size when HOST PCIE IP can support less then 4MB + BAR window. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 341a4c659d099..026e6c79ca2e9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -270,6 +270,9 @@ struct brcmf_pciedev_info { void (*write_ptr)(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u16 value); struct brcmf_mp_device *settings; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + ulong bar1_size; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ }; struct brcmf_pcie_ringbuf { @@ -336,6 +339,9 @@ static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE }; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ +DEFINE_RAW_SPINLOCK(pcie_lock); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ static u32 brcmf_pcie_read_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset) @@ -360,8 +366,24 @@ static u8 brcmf_pcie_read_tcm8(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + u8 value; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread8(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread8(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -369,8 +391,24 @@ static u16 brcmf_pcie_read_tcm16(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u16 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread16(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread16(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -379,8 +417,22 @@ brcmf_pcie_write_tcm16(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u16 value) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } iowrite16(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else + iowrite16(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -407,8 +459,24 @@ static u32 brcmf_pcie_read_tcm32(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { void __iomem *address = devinfo->tcm + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u32 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread32(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); + return value; +#else return (ioread32(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -417,17 +485,47 @@ brcmf_pcie_write_tcm32(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u32 value) { void __iomem *address = devinfo->tcm + mem_offset; - +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + iowrite32(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else iowrite32(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } static u32 brcmf_pcie_read_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset) { - void __iomem *addr = devinfo->tcm + devinfo->ci->rambase + mem_offset; + void __iomem *address = devinfo->tcm + devinfo->ci->rambase + + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + u32 value; + unsigned long flags; + + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + value = ioread32(address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); - return (ioread32(addr)); + return value; +#else + return (ioread32(address)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -435,9 +533,23 @@ static void brcmf_pcie_write_ram32(struct brcmf_pciedev_info *devinfo, u32 mem_offset, u32 value) { - void __iomem *addr = devinfo->tcm + devinfo->ci->rambase + mem_offset; + void __iomem *address = devinfo->tcm + devinfo->ci->rambase + + mem_offset; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; - iowrite32(value, addr); + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= devinfo->bar1_size) { + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } + iowrite32(value, address); + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#else + iowrite32(value, address); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -449,12 +561,30 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, __le32 *src32; __le16 *src16; u8 *src8; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (((ulong)address & 4) || ((ulong)srcaddr & 4) || (len & 4)) { if (((ulong)address & 2) || ((ulong)srcaddr & 2) || (len & 2)) { src8 = (u8 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite8(*src8, address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address++; src8++; len--; @@ -463,7 +593,22 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 2; src16 = (__le16 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite16(le16_to_cpu(*src16), address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 2; src16++; len--; @@ -473,12 +618,29 @@ brcmf_pcie_copy_mem_todev(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 4; src32 = (__le32 *)srcaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ iowrite32(le32_to_cpu(*src32), address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 4; src32++; len--; } } +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -490,12 +652,30 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, __le32 *dst32; __le16 *dst16; u8 *dst8; +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + unsigned long flags; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (((ulong)address & 4) || ((ulong)dstaddr & 4) || (len & 4)) { if (((ulong)address & 2) || ((ulong)dstaddr & 2) || (len & 2)) { dst8 = (u8 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst8 = ioread8(address); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address++; dst8++; len--; @@ -504,7 +684,22 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 2; dst16 = (__le16 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - + devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst16 = cpu_to_le16(ioread16(address)); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 2; dst16++; len--; @@ -514,12 +709,29 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset, len = len / 4; dst32 = (__le32 *)dstaddr; while (len) { +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_lock_irqsave(&pcie_lock, flags); + if ((address - devinfo->tcm) >= + devinfo->bar1_size) { + pci_write_config_dword + (devinfo->pdev, + BCMA_PCI_BAR1_WIN, + devinfo->bar1_size); + address = address - devinfo->bar1_size; + } +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ *dst32 = cpu_to_le32(ioread32(address)); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + raw_spin_unlock_irqrestore(&pcie_lock, flags); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ address += 4; dst32++; len--; } } +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + pci_write_config_dword(devinfo->pdev, BCMA_PCI_BAR1_WIN, 0x0); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ } @@ -562,6 +774,9 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) BRCMF_PCIE_CFGREG_MSI_ADDR_L, BRCMF_PCIE_CFGREG_MSI_ADDR_H, BRCMF_PCIE_CFGREG_MSI_DATA, +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + BCMA_PCI_BAR1_WIN, +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ BRCMF_PCIE_CFGREG_LINK_STATUS_CTRL2, BRCMF_PCIE_CFGREG_RBAR_CTRL, BRCMF_PCIE_CFGREG_PML1_SUB_CTRL1, @@ -1106,9 +1321,14 @@ static int brcmf_pcie_init_ringbuffers(struct brcmf_pciedev_info *devinfo) u16 max_flowrings; u16 max_submissionrings; u16 max_completionrings; - +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + brcmf_pcie_copy_dev_tomem(devinfo, devinfo->shared.ring_info_addr, + &ringinfo, sizeof(ringinfo)); +#else memcpy_fromio(&ringinfo, devinfo->tcm + devinfo->shared.ring_info_addr, sizeof(ringinfo)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ + if (devinfo->shared.version >= 6) { max_submissionrings = le16_to_cpu(ringinfo.max_submissionrings); max_flowrings = le16_to_cpu(ringinfo.max_flowrings); @@ -1177,8 +1397,14 @@ static int brcmf_pcie_init_ringbuffers(struct brcmf_pciedev_info *devinfo) ringinfo.d2h_r_idx_hostaddr.high_addr = cpu_to_le32(address >> 32); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + brcmf_pcie_copy_mem_todev(devinfo, + devinfo->shared.ring_info_addr, + &ringinfo, sizeof(ringinfo)); +#else memcpy_toio(devinfo->tcm + devinfo->shared.ring_info_addr, &ringinfo, sizeof(ringinfo)); +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ brcmf_dbg(PCIE, "Using host memory indices\n"); } @@ -1572,6 +1798,9 @@ static int brcmf_pcie_get_resource(struct brcmf_pciedev_info *devinfo) devinfo->regs = ioremap_nocache(bar0_addr, BRCMF_PCIE_REG_MAP_SIZE); devinfo->tcm = ioremap_nocache(bar1_addr, bar1_size); +#ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ + devinfo->bar1_size = bar1_size; +#endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ if (!devinfo->regs || !devinfo->tcm) { brcmf_err("ioremap() failed (%p,%p)\n", devinfo->regs, From 33769e46a2dedc6b01a7595a056c974336466b0a Mon Sep 17 00:00:00 2001 From: Frank Kao Date: Mon, 25 Nov 2019 15:41:13 +0800 Subject: [PATCH 152/213] brcmfmac: set F2 blocksize and watermark for 4354 Set F2 blocksize to 256 bytes and watermark to 0x40 for 4354. Also enable and configure F1 MesBusyCtrl. It would resolve random driver crash issue. Signed-off-by: Frank Kao Signed-off-by: Chi-Hsien Lin --- .../wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c | 4 ++++ .../wireless/broadcom/brcm80211/brcmfmac/sdio.c | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 53ede75935ee4..de92306c89d1f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -57,6 +57,7 @@ #define SDIO_FUNC2_BLOCKSIZE 512 #define SDIO_4373_FUNC2_BLOCKSIZE 256 #define SDIO_4359_FUNC2_BLOCKSIZE 256 +#define SDIO_4354_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -1062,6 +1063,9 @@ static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) case SDIO_DEVICE_ID_CYPRESS_89359: f2_blksz = SDIO_4359_FUNC2_BLOCKSIZE; break; + case SDIO_DEVICE_ID_BROADCOM_4354: + f2_blksz = SDIO_4354_FUNC2_BLOCKSIZE; + break; default: break; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 3638b50eec73e..4128a17132a8e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -67,6 +67,8 @@ #define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #define CY_4359_F2_WATERMARK 0x40 #define CY_4359_F1_MESBUSYCTRL (CY_4359_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) +#define CY_4354_F2_WATERMARK 0x40 +#define CY_4354_F1_MESBUSYCTRL (CY_4354_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG #define BRCMF_TRAP_INFO_SIZE 80 @@ -4418,6 +4420,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, CY_4359_F1_MESBUSYCTRL, &err); break; + case SDIO_DEVICE_ID_BROADCOM_4354: + brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", + CY_4354_F2_WATERMARK); + brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, + CY_4354_F2_WATERMARK, &err); + devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, + &err); + devctl |= SBSDIO_DEVCTL_F2WM_ENAB; + brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, + &err); + brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, + CY_4354_F1_MESBUSYCTRL, &err); + break; default: brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); break; From 88f79d84ccb2afd05bc7096dac01366ed4de182c Mon Sep 17 00:00:00 2001 From: "Lo(Double)Hsiang Lo" Date: Mon, 9 Dec 2019 20:49:17 -0600 Subject: [PATCH 153/213] brcmfmac: support for virtual interface creation from firmware Allow interface creation via IF_ADD event from firmware. Signed-off-by: Double Lo Signed-off-by: Chi-Hsien Lin SWLINUX-1291 --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 75e37d69fb0de..39ee3e37047bb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -240,6 +240,9 @@ struct parsed_vndr_ies { struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT]; }; +#define WLC_E_IF_ROLE_STA 0 /* Infra STA */ +#define WLC_E_IF_ROLE_AP 1 /* Access Point */ + static u8 nl80211_band_to_fwil(enum nl80211_band band) { switch (band) { @@ -5544,8 +5547,10 @@ void brcmf_cfg80211_free_netdev(struct net_device *ndev) ifp = netdev_priv(ndev); vif = ifp->vif; - if (vif) + if (vif) { brcmf_free_vif(vif); + ifp->vif = NULL; + } } static bool brcmf_is_linkup(struct brcmf_cfg80211_vif *vif, @@ -6149,6 +6154,9 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, struct brcmf_if_event *ifevent = (struct brcmf_if_event *)data; struct brcmf_cfg80211_vif_event *event = &cfg->vif_event; struct brcmf_cfg80211_vif *vif; + enum nl80211_iftype iftype = NL80211_IFTYPE_UNSPECIFIED; + bool vif_pend = false; + int err; brcmf_dbg(TRACE, "Enter: action %u flags %u ifidx %u bsscfgidx %u\n", ifevent->action, ifevent->flags, ifevent->ifidx, @@ -6161,9 +6169,28 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, switch (ifevent->action) { case BRCMF_E_IF_ADD: /* waiting process may have timed out */ - if (!cfg->vif_event.vif) { + if (!vif) { + /* handle IF_ADD event from firmware */ spin_unlock(&event->vif_event_lock); - return -EBADF; + vif_pend = true; + if (ifevent->role == WLC_E_IF_ROLE_STA) + iftype = NL80211_IFTYPE_STATION; + else if (ifevent->role == WLC_E_IF_ROLE_AP) + iftype = NL80211_IFTYPE_AP; + else + vif_pend = false; + + if (vif_pend) { + vif = brcmf_alloc_vif(cfg, iftype); + if (IS_ERR(vif)) { + brcmf_err("Role:%d failed to alloc vif\n", + ifevent->role); + return PTR_ERR(vif); + } + } else { + brcmf_err("Invalid Role:%d\n", ifevent->role); + return -EBADF; + } } ifp->vif = vif; @@ -6173,6 +6200,18 @@ static s32 brcmf_notify_vif_event(struct brcmf_if *ifp, ifp->ndev->ieee80211_ptr = &vif->wdev; SET_NETDEV_DEV(ifp->ndev, wiphy_dev(cfg->wiphy)); } + + if (vif_pend) { + err = brcmf_net_attach(ifp, false); + if (err) { + brcmf_err("netdevice register failed with err:%d\n", + err); + brcmf_free_vif(vif); + free_netdev(ifp->ndev); + } + return err; + } + spin_unlock(&event->vif_event_lock); wake_up(&event->vif_wq); return 0; From b7740d1f1233a3e96a5a53fa8356f34339e7eab2 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 10 Dec 2019 14:21:26 -0600 Subject: [PATCH 154/213] nl80211: support 4-way handshake offloading for WPA/WPA2-PSK in AP mode Let drivers advertise support for AP-mode WPA/WPA2-PSK 4-way handshake offloading with a new NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK flag. Extend use of NL80211_ATTR_PMK attribute indicating it might be passed as part of NL80211_CMD_START_AP command, and contain the PSK (which is the PMK, hence the name). The driver is assumed to handle the 4-way handshake by itself in this case, instead of relying on user space. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- include/uapi/linux/nl80211.h | 41 ++++++++++++++++++++++++------------ net/wireless/nl80211.c | 4 +++- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 9a33598928166..e20f5962cda1f 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -178,18 +178,27 @@ * * By setting @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK flag drivers * can indicate they support offloading EAPOL handshakes for WPA/WPA2 - * preshared key authentication. In %NL80211_CMD_CONNECT the preshared - * key should be specified using %NL80211_ATTR_PMK. Drivers supporting - * this offload may reject the %NL80211_CMD_CONNECT when no preshared - * key material is provided, for example when that driver does not - * support setting the temporal keys through %CMD_NEW_KEY. + * preshared key authentication in station mode. In %NL80211_CMD_CONNECT + * the preshared key should be specified using %NL80211_ATTR_PMK. Drivers + * supporting this offload may reject the %NL80211_CMD_CONNECT when no + * preshared key material is provided, for example when that driver does + * not support setting the temporal keys through %NL80211_CMD_NEW_KEY. * * Similarly @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_1X flag can be * set by drivers indicating offload support of the PTK/GTK EAPOL - * handshakes during 802.1X authentication. In order to use the offload - * the %NL80211_CMD_CONNECT should have %NL80211_ATTR_WANT_1X_4WAY_HS - * attribute flag. Drivers supporting this offload may reject the - * %NL80211_CMD_CONNECT when the attribute flag is not present. + * handshakes during 802.1X authentication in station mode. In order to + * use the offload the %NL80211_CMD_CONNECT should have + * %NL80211_ATTR_WANT_1X_4WAY_HS attribute flag. Drivers supporting this + * offload may reject the %NL80211_CMD_CONNECT when the attribute flag is + * not present. + * + * By setting @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK flag drivers + * can indicate they support offloading EAPOL handshakes for WPA/WPA2 + * preshared key authentication in AP mode. In %NL80211_CMD_START_AP + * the preshared key should be specified using %NL80211_ATTR_PMK. Drivers + * supporting this offload may reject the %NL80211_CMD_START_AP when no + * preshared key material is provided, for example when that driver does + * not support setting the temporal keys through %NL80211_CMD_NEW_KEY. * * For 802.1X the PMK or PMK-R0 are set by providing %NL80211_ATTR_PMK * using %NL80211_CMD_SET_PMK. For offloaded FT support also @@ -2208,10 +2217,11 @@ enum nl80211_commands { * * @NL80211_ATTR_PMK: attribute for passing PMK key material. Used with * %NL80211_CMD_SET_PMKSA for the PMKSA identified by %NL80211_ATTR_PMKID. - * For %NL80211_CMD_CONNECT it is used to provide PSK for offloading 4-way - * handshake for WPA/WPA2-PSK networks. For 802.1X authentication it is - * used with %NL80211_CMD_SET_PMK. For offloaded FT support this attribute - * specifies the PMK-R0 if NL80211_ATTR_PMKR0_NAME is included as well. + * For %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP it is used to provide + * PSK for offloading 4-way handshake for WPA/WPA2-PSK networks. For 802.1X + * authentication it is used with %NL80211_CMD_SET_PMK. For offloaded FT + * support this attribute specifies the PMK-R0 if NL80211_ATTR_PMKR0_NAME + * is included as well. * * @NL80211_ATTR_SCHED_SCAN_MULTI: flag attribute which user-space shall use to * indicate that it supports multiple active scheduled scan requests. @@ -5310,6 +5320,10 @@ enum nl80211_feature_flags { * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in * station mode (SAE password is passed as part of the connect command). * + * @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK: Device wants to do 4-way + * handshake with PSK in AP mode (PSK is passed as part of the start AP + * command). + * * @NUM_NL80211_EXT_FEATURES: number of extended features. * @MAX_NL80211_EXT_FEATURES: highest extended feature index. */ @@ -5353,6 +5367,7 @@ enum nl80211_ext_feature_index { NL80211_EXT_FEATURE_EXT_KEY_ID, NL80211_EXT_FEATURE_STA_TX_PWR, NL80211_EXT_FEATURE_SAE_OFFLOAD, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK, /* add new features before the definition below */ NUM_NL80211_EXT_FEATURES, diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index b184cd9500b28..a32c43b11412a 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -8645,7 +8645,9 @@ static int nl80211_crypto_settings(struct cfg80211_registered_device *rdev, if (nla_len(info->attrs[NL80211_ATTR_PMK]) != WLAN_PMK_LEN) return -EINVAL; if (!wiphy_ext_feature_isset(&rdev->wiphy, - NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK)) + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK) && + !wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK)) return -EINVAL; settings->psk = nla_data(info->attrs[NL80211_ATTR_PMK]); } From 739a220efcf0001419e5f07f579648659f60b70a Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 10 Dec 2019 14:22:56 -0600 Subject: [PATCH 155/213] brcmfmac: support 4-way handshake offloading for WPA/WPA2-PSK in AP mode Firmware may have authenticator code built-in. This is detected by the driver and indicated in the wiphy features flags. User space can use this flag to determine whether or not to provide the pre-shared key material in the nl80211 start AP command to offload the 4-way handshake in AP mode. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 24 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 12 ++++++++++ .../broadcom/brcm80211/brcmfmac/feature.c | 1 + .../broadcom/brcm80211/brcmfmac/feature.h | 4 +++- 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 39ee3e37047bb..33c007d97c380 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4644,6 +4644,8 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, s32 ie_offset; struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; + struct cfg80211_crypto_settings *crypto = &settings->crypto; const struct brcmf_tlv *ssid_ie; const struct brcmf_tlv *country_ie; struct brcmf_ssid_le ssid_le; @@ -4808,6 +4810,18 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, brcmf_err("BRCMF_C_UP error (%d)\n", err); goto exit; } + + if (crypto->psk) { + brcmf_dbg(INFO, "using PSK offload\n"); + profile->use_fwauth |= BIT(BRCMF_PROFILE_FWAUTH_PSK); + err = brcmf_set_pmk(ifp, crypto->psk, + BRCMF_WSEC_MAX_PSK_LEN); + if (err < 0) + goto exit; + } + if (profile->use_fwauth == 0) + profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); + /* On DOWN the firmware removes the WEP keys, reconfigure * them if they were set. */ @@ -4881,6 +4895,7 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) { struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; s32 err; struct brcmf_fil_bss_enable_le bss_enable; struct brcmf_join_params join_params; @@ -4895,6 +4910,12 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) cfg->num_softap--; + if (profile->use_fwauth != BIT(BRCMF_PROFILE_FWAUTH_NONE)) { + if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_PSK)) + brcmf_set_pmk(ifp, NULL, 0); + profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); + } + /* Clear bss configuration and SSID */ bss_enable.bsscfgidx = cpu_to_le32(ifp->bsscfgidx); bss_enable.enable = cpu_to_le32(0); @@ -7165,6 +7186,9 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SAE_OFFLOAD); } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWAUTH)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK); wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PNO)) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index d61a545630137..7ab3f86811f00 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -148,6 +148,17 @@ enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_SAE }; +/** + * enum brcmf_profile_fwauth - firmware authenticator profile + * + * @BRCMF_PROFILE_FWAUTH_NONE: no firmware authenticator + * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for WPA/WPA2-PSK + */ +enum brcmf_profile_fwauth { + BRCMF_PROFILE_FWAUTH_NONE, + BRCMF_PROFILE_FWAUTH_PSK +}; + /** * struct brcmf_cfg80211_profile - profile information. * @@ -160,6 +171,7 @@ struct brcmf_cfg80211_profile { struct brcmf_cfg80211_security sec; struct brcmf_wsec_key key[BRCMF_MAX_DEFAULT_KEYS]; enum brcmf_profile_fwsup use_fwsup; + u16 use_fwauth; bool is_ft; bool is_okc; }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 3069c6efba154..a0f27611d7a29 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -49,6 +49,7 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_MCHAN, "mchan" }, { BRCMF_FEAT_P2P, "p2p" }, { BRCMF_FEAT_SAE, "sae" }, + { BRCMF_FEAT_FWAUTH, "idauth" }, }; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 48e679d4018aa..67ce8cb87b76c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -34,6 +34,7 @@ * GSCAN: enhanced scan offload feature. * FWSUP: Firmware supplicant. * SAE: simultaneous authentication of equals + * FWAUTH: Firmware authenticator */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -50,7 +51,8 @@ BRCMF_FEAT_DEF(MFP) \ BRCMF_FEAT_DEF(GSCAN) \ BRCMF_FEAT_DEF(FWSUP) \ - BRCMF_FEAT_DEF(SAE) + BRCMF_FEAT_DEF(SAE) \ + BRCMF_FEAT_DEF(FWAUTH) /* * Quirks: From 1631f3edefedc337d62535d879355d84134b8294 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 10 Dec 2019 14:24:10 -0600 Subject: [PATCH 156/213] nl80211: support SAE authentication offload in AP mode Add support for SAE authentication offload in AP mode. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- include/uapi/linux/nl80211.h | 10 ++++++---- net/wireless/nl80211.c | 5 +++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index e20f5962cda1f..bc11e2ac3ed3b 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -247,9 +247,10 @@ * DOC: SAE authentication offload * * By setting @NL80211_EXT_FEATURE_SAE_OFFLOAD flag drivers can indicate they - * support offloading SAE authentication for WPA3-Personal networks. In - * %NL80211_CMD_CONNECT the password for SAE should be specified using - * %NL80211_ATTR_SAE_PASSWORD. + * support offloading SAE authentication for WPA3-Personal networks. The + * password for SAE should be specified using %NL80211_ATTR_SAE_PASSWORD in + * %NL80211_CMD_CONNECT and %NL80211_CMD_START_AP for station and AP mode + * respectively. */ /** @@ -5318,7 +5319,8 @@ enum nl80211_feature_flags { * to a station. * * @NL80211_EXT_FEATURE_SAE_OFFLOAD: Device wants to do SAE authentication in - * station mode (SAE password is passed as part of the connect command). + * station mode (SAE password is passed as part of the connect command) + * or AP mode (SAE password is passed as part of the start AP command). * * @NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK: Device wants to do 4-way * handshake with PSK in AP mode (PSK is passed as part of the start AP diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index a32c43b11412a..774c182b06494 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -4202,8 +4202,9 @@ static bool nl80211_valid_auth_type(struct cfg80211_registered_device *rdev, return false; return true; case NL80211_CMD_START_AP: - /* SAE not supported yet */ - if (auth_type == NL80211_AUTHTYPE_SAE) + if (!wiphy_ext_feature_isset(&rdev->wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD) && + auth_type == NL80211_AUTHTYPE_SAE) return false; /* FILS not supported yet */ if (auth_type == NL80211_AUTHTYPE_FILS_SK || From 59794a6c8c285f3530bb0b3613679eb49a9dfd05 Mon Sep 17 00:00:00 2001 From: Chung-Hsien Hsu Date: Tue, 10 Dec 2019 14:25:14 -0600 Subject: [PATCH 157/213] brcmfmac: support SAE authentication offload in AP mode Firmware may have SAE authenticator code built-in. This is detected by the driver and indicated in the wiphy features flags. User space can use this flag to determine whether or not to provide the password material in the nl80211 start AP command to offload the SAE authentication in AP mode. Signed-off-by: Chung-Hsien Hsu Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 28 ++++++++++++++++--- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 4 ++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 33c007d97c380..79e5b25a9afd2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -66,6 +66,7 @@ #define RSN_AKM_PSK 2 /* Pre-shared Key */ #define RSN_AKM_SHA256_1X 5 /* SHA256, 802.1X */ #define RSN_AKM_SHA256_PSK 6 /* SHA256, Pre-shared Key */ +#define RSN_AKM_SAE 8 /* SAE */ #define RSN_CAP_LEN 2 /* Length of RSN capabilities */ #define RSN_CAP_PTK_REPLAY_CNTR_MASK (BIT(2) | BIT(3)) #define RSN_CAP_MFPR_MASK BIT(6) @@ -4245,6 +4246,10 @@ brcmf_configure_wpaie(struct brcmf_if *ifp, brcmf_dbg(TRACE, "RSN_AKM_MFP_1X\n"); wpa_auth |= WPA2_AUTH_1X_SHA256; break; + case RSN_AKM_SAE: + brcmf_dbg(TRACE, "RSN_AKM_SAE\n"); + wpa_auth |= WPA3_AUTH_SAE_PSK; + break; default: brcmf_err("Invalid key mgmt info\n"); } @@ -4262,11 +4267,12 @@ brcmf_configure_wpaie(struct brcmf_if *ifp, brcmf_dbg(TRACE, "MFP Required\n"); mfp = BRCMF_MFP_REQUIRED; /* Firmware only supports mfp required in - * combination with WPA2_AUTH_PSK_SHA256 or - * WPA2_AUTH_1X_SHA256. + * combination with WPA2_AUTH_PSK_SHA256, + * WPA2_AUTH_1X_SHA256, or WPA3_AUTH_SAE_PSK. */ if (!(wpa_auth & (WPA2_AUTH_PSK_SHA256 | - WPA2_AUTH_1X_SHA256))) { + WPA2_AUTH_1X_SHA256 | + WPA3_AUTH_SAE_PSK))) { err = -EINVAL; goto exit; } @@ -4819,6 +4825,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, if (err < 0) goto exit; } + if (crypto->sae_pwd) { + brcmf_dbg(INFO, "using SAE offload\n"); + profile->use_fwauth |= BIT(BRCMF_PROFILE_FWAUTH_SAE); + err = brcmf_set_sae_password(ifp, crypto->sae_pwd, + crypto->sae_pwd_len); + if (err < 0) + goto exit; + } if (profile->use_fwauth == 0) profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); @@ -4913,6 +4927,8 @@ static int brcmf_cfg80211_stop_ap(struct wiphy *wiphy, struct net_device *ndev) if (profile->use_fwauth != BIT(BRCMF_PROFILE_FWAUTH_NONE)) { if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_PSK)) brcmf_set_pmk(ifp, NULL, 0); + if (profile->use_fwauth & BIT(BRCMF_PROFILE_FWAUTH_SAE)) + brcmf_set_sae_password(ifp, NULL, 0); profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); } @@ -7186,9 +7202,13 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_SAE_OFFLOAD); } - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWAUTH)) + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWAUTH)) { wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE)) + wiphy_ext_feature_set(wiphy, + NL80211_EXT_FEATURE_SAE_OFFLOAD); + } wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PNO)) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 7ab3f86811f00..20cad8303abe9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -153,10 +153,12 @@ enum brcmf_profile_fwsup { * * @BRCMF_PROFILE_FWAUTH_NONE: no firmware authenticator * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for WPA/WPA2-PSK + * @BRCMF_PROFILE_FWAUTH_PSK: authenticator for SAE */ enum brcmf_profile_fwauth { BRCMF_PROFILE_FWAUTH_NONE, - BRCMF_PROFILE_FWAUTH_PSK + BRCMF_PROFILE_FWAUTH_PSK, + BRCMF_PROFILE_FWAUTH_SAE }; /** From a6fb452858ba1f15cf94be31d44d1f6d1ce2ec19 Mon Sep 17 00:00:00 2001 From: Franky Lin Date: Wed, 16 May 2018 14:12:02 +0200 Subject: [PATCH 158/213] brcmfmac: trigger memory dump upon firmware halt signal commit 8a3ab2f38f1669e3be6433a1f6b82a077b38c4c7 upstream. PCIe dongle firmware signals a halt/trap through mailbox interrupt. Trigger a memory dump upon receiving such signal could help to provide useful information for issue debug. Reviewed-by: Arend van Spriel Signed-off-by: Franky Lin Signed-off-by: Arend van Spriel Signed-off-by: Kalle Valo --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 026e6c79ca2e9..cca5bd1686a50 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -183,6 +183,7 @@ static struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_D2H_DEV_D3_ACK 0x00000001 #define BRCMF_D2H_DEV_DS_ENTER_REQ 0x00000002 #define BRCMF_D2H_DEV_DS_EXIT_NOTE 0x00000004 +#define BRCMF_D2H_DEV_FWHALT 0x10000000 #define BRCMF_H2D_HOST_D3_INFORM 0x00000001 #define BRCMF_H2D_HOST_DS_ACK 0x00000002 @@ -932,6 +933,10 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) devinfo->mbdata_completed = true; wake_up(&devinfo->mbdata_resp_wait); } + if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) { + brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); + brcmf_dev_coredump(&devinfo->pdev->dev); + } } From ecec9902a38276a87bce51a08f237d7ac22e8deb Mon Sep 17 00:00:00 2001 From: Arend Van Spriel Date: Wed, 16 May 2018 14:11:59 +0200 Subject: [PATCH 159/213] brcmfmac: add support for sysfs initiated coredump commit 8e072168f75ebce85b96cbcefea2b10ddbd5913f upstream. The driver already supports device coredump initiated by firmware event. Since commit 3c47d19ff4dc ("drivers: base: add coredump driver ops") it is also possible to initiate it from user-space through sysfs. This patch adds support for SDIO and PCIe devices. Reviewed-by: Hante Meuleman Reviewed-by: Pieter-Paul Giesberts Reviewed-by: Franky Lin Signed-off-by: Arend van Spriel Signed-off-by: Kalle Valo [chi-hsien.lin@cypress.com: removed .coredump in bcmsdh.c and pcie.c] Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h | 2 ++ drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index fc1b1e2cb1b9a..b6c996c08e218 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -250,6 +250,8 @@ int brcmf_attach(struct device *dev, struct brcmf_mp_device *settings); void brcmf_detach(struct device *dev); /* Indication from bus module that dongle should be reset */ void brcmf_dev_reset(struct device *dev); +/* Request from bus module to initiate a coredump */ +void brcmf_dev_coredump(struct device *dev); /* Configure the "global" bus state used by upper layers */ void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 4a4904bce1dad..f610b4c08df81 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1084,6 +1084,14 @@ void brcmf_dev_reset(struct device *dev) brcmf_fil_cmd_int_set(drvr->iflist[0], BRCMF_C_TERMINATED, 1); } +void brcmf_dev_coredump(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + + if (brcmf_debug_create_memdump(bus_if, NULL, 0) < 0) + brcmf_dbg(TRACE, "failed to create coredump\n"); +} + void brcmf_detach(struct device *dev) { s32 i; From 46a66b0dc7e4aaa61f3099eda286732f69e15876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Tue, 26 Feb 2019 14:11:16 +0100 Subject: [PATCH 160/213] brcmfmac: support repeated brcmf_fw_alloc_request() calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit c9692820710f57c826b2e43a6fb1e4cd307508b0 upstream. During a normal brcmfmac lifetime brcmf_fw_alloc_request() is called once only during the probe. It's safe to assume provided array is clear. Further brcmfmac improvements may require calling it multiple times though. This patch allows it by fixing invalid firmware paths like: brcm/brcmfmac4366c-pcie.binbrcm/brcmfmac4366c-pcie.bin Signed-off-by: RafaÅ‚ MiÅ‚ecki Reviewed-by: Arend van Spriel Signed-off-by: Kalle Valo [chi-hsien.lin@cypress.com: use fw_names instead of fwnames] Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c index 091b52979e030..e5ffb55dc1b85 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c @@ -581,6 +581,7 @@ int brcmf_fw_map_chip_to_name(u32 chip, u32 chiprev, return -ENODEV; } + fw_name[0] = '\0'; /* check if firmware path is provided by module parameter */ if (brcmf_mp_global.firmware_path[0] != '\0') { strlcpy(fw_name, brcmf_mp_global.firmware_path, From 1c140737918d80927c63295e4eb9d83a8187d4b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Tue, 26 Feb 2019 14:11:18 +0100 Subject: [PATCH 161/213] brcmfmac: add a function designated for handling firmware fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit a2ec87ddbf1637f854ffcfff9d12d392fa30758b upstream. This improves handling PCIe firmware halts by printing a clear error message and replaces a similar code in the SDIO bus support. It will also allow further improvements like trying to recover from a firmware crash. Signed-off-by: RafaÅ‚ MiÅ‚ecki Reviewed-by: Arend van Spriel Signed-off-by: Kalle Valo [chi-hsien.lin@cypress.com: replace bphy_err() with brcmf_err()] Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h | 2 ++ .../net/wireless/broadcom/brcm80211/brcmfmac/core.c | 10 ++++++++++ .../net/wireless/broadcom/brcm80211/brcmfmac/pcie.c | 2 +- .../net/wireless/broadcom/brcm80211/brcmfmac/sdio.c | 6 ++++-- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index b6c996c08e218..98e79b7a849a2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -252,6 +252,8 @@ void brcmf_detach(struct device *dev); void brcmf_dev_reset(struct device *dev); /* Request from bus module to initiate a coredump */ void brcmf_dev_coredump(struct device *dev); +/* Indication that firmware has halted or crashed */ +void brcmf_fw_crashed(struct device *dev); /* Configure the "global" bus state used by upper layers */ void brcmf_bus_change_state(struct brcmf_bus *bus, enum brcmf_bus_state state); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index f610b4c08df81..eb4635a5471a0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1092,6 +1092,16 @@ void brcmf_dev_coredump(struct device *dev) brcmf_dbg(TRACE, "failed to create coredump\n"); } +void brcmf_fw_crashed(struct device *dev) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pub *drvr = bus_if->drvr; + + brcmf_err("Firmware has halted or crashed\n"); + + brcmf_dev_coredump(dev); +} + void brcmf_detach(struct device *dev) { s32 i; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index cca5bd1686a50..fb93341924fe7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -935,7 +935,7 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) } if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) { brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); - brcmf_dev_coredump(&devinfo->pdev->dev); + brcmf_fw_crashed(&devinfo->pdev->dev); } } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 4128a17132a8e..851eadb5a46ed 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -1141,8 +1141,10 @@ static u32 brcmf_sdio_hostmail(struct brcmf_sdio *bus, u32 *hmbd) bus->sdcnt.f1regdata += 2; /* dongle indicates the firmware has halted/crashed */ - if (hmb_data & HMB_DATA_FWHALT) - brcmf_err("mailbox indicates firmware halted\n"); + if (hmb_data & HMB_DATA_FWHALT) { + brcmf_dbg(SDIO, "mailbox indicates firmware halted\n"); + brcmf_fw_crashed(bus->sdiodev->dev); + } /* Dongle recomposed rx frames, accept them again */ if (hmb_data & HMB_DATA_NAKHANDLED) { From 4318178635dc9040514793d02db7c53859b2044a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Tue, 26 Feb 2019 14:11:19 +0100 Subject: [PATCH 162/213] brcmfmac: reset PCIe bus on a firmware crash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 4684997d9eea29380000e062755aa6d368d789a3 upstream. This includes bus reset & reloading a firmware. It should be sufficient for a user space to (setup and) use a wireless device again. Support for reset on USB & SDIO can be added later. Signed-off-by: RafaÅ‚ MiÅ‚ecki Reviewed-by: Arend van Spriel Signed-off-by: Kalle Valo [chi-hsien.lin@cypress.com: merge to 4.14 based CY branch] [chi-hsien.lin@cypress.com: remove brcmf_fw_request usage for 4.14] Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/bus.h | 10 +++++ .../broadcom/brcm80211/brcmfmac/core.c | 12 +++++ .../broadcom/brcm80211/brcmfmac/core.h | 2 + .../broadcom/brcm80211/brcmfmac/pcie.c | 45 +++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 98e79b7a849a2..95efdb6076687 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -90,6 +90,7 @@ struct brcmf_bus_ops { int (*get_memdump)(struct device *dev, void *data, size_t len); int (*get_fwname)(struct device *dev, uint chip, uint chiprev, unsigned char *fw_name); + int (*reset)(struct device *dev); }; @@ -235,6 +236,15 @@ int brcmf_bus_get_fwname(struct brcmf_bus *bus, uint chip, uint chiprev, return bus->ops->get_fwname(bus->dev, chip, chiprev, fw_name); } +static inline +int brcmf_bus_reset(struct brcmf_bus *bus) +{ + if (!bus->ops->reset) + return -EOPNOTSUPP; + + return bus->ops->reset(bus->dev); +} + /* * interface functions from common layer */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index eb4635a5471a0..5fa852d22a28d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -967,6 +967,14 @@ static int brcmf_revinfo_read(struct seq_file *s, void *data) return 0; } +static void brcmf_core_bus_reset(struct work_struct *work) +{ + struct brcmf_pub *drvr = container_of(work, struct brcmf_pub, + bus_reset); + + brcmf_bus_reset(drvr->bus_if); +} + int brcmf_bus_started(struct device *dev) { int ret = -1; @@ -1043,6 +1051,8 @@ int brcmf_bus_started(struct device *dev) #endif #endif /* CONFIG_INET */ + INIT_WORK(&drvr->bus_reset, brcmf_core_bus_reset); + return 0; fail: @@ -1100,6 +1110,8 @@ void brcmf_fw_crashed(struct device *dev) brcmf_err("Firmware has halted or crashed\n"); brcmf_dev_coredump(dev); + + schedule_work(&drvr->bus_reset); } void brcmf_detach(struct device *dev) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index e38f78e4d92c8..9418d7169a950 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -143,6 +143,8 @@ struct brcmf_pub { struct notifier_block inet6addr_notifier; struct brcmf_mp_device *settings; + struct work_struct bus_reset; + u8 clmver[BRCMF_DCMD_SMLEN]; struct brcmf_pkt_filter_enable_le pkt_filter[MAX_PKT_FILTER_COUNT]; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index fb93341924fe7..d6c82743a94de 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -340,6 +340,10 @@ static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = { BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE }; +static void brcmf_pcie_setup(struct device *dev, int ret, + const struct firmware *fw, + void *nvram, u32 nvram_len); + #ifdef CONFIG_BRCMFMAC_PCIE_BARWIN_SZ DEFINE_RAW_SPINLOCK(pcie_lock); #endif /* CONFIG_BRCMFMAC_PCIE_BARWIN_SZ */ @@ -1618,6 +1622,46 @@ static int brcmf_pcie_get_fwname(struct device *dev, u32 chip, u32 chiprev, return ret; } +static int brcmf_pcie_reset(struct device *dev) +{ + struct brcmf_bus *bus; + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + struct brcmf_pciedev_info *devinfo = buspub->devinfo; + int err; + u16 domain_nr; + u16 bus_nr; + + domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1; + bus_nr = devinfo->pdev->bus->number; + brcmf_dbg(PCIE, "Enter %x:%x (%d/%d)\n", devinfo->pdev->vendor, + devinfo->pdev->device, domain_nr, bus_nr); + bus = dev_get_drvdata(dev); + + brcmf_detach(dev); + + brcmf_pcie_release_irq(devinfo); + brcmf_pcie_release_scratchbuffers(devinfo); + brcmf_pcie_release_ringbuffers(devinfo); + brcmf_pcie_reset_device(devinfo); + + err = brcmf_fw_map_chip_to_name(devinfo->ci->chip, + devinfo->ci->chiprev, + brcmf_pcie_fwnames, + ARRAY_SIZE(brcmf_pcie_fwnames), + devinfo->fw_name, devinfo->nvram_name); + if (err) + return err; + + err = brcmf_fw_get_firmwares_pcie(bus->dev, BRCMF_FW_REQUEST_NVRAM | + BRCMF_FW_REQ_NV_OPTIONAL, + devinfo->fw_name, devinfo->nvram_name, + brcmf_pcie_setup, domain_nr, bus_nr); + if (err == 0) + return 0; + return err; +} + static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .txdata = brcmf_pcie_tx, .stop = brcmf_pcie_down, @@ -1627,6 +1671,7 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .get_ramsize = brcmf_pcie_get_ramsize, .get_memdump = brcmf_pcie_get_memdump, .get_fwname = brcmf_pcie_get_fwname, + .reset = brcmf_pcie_reset, }; From b3b4c76241e55b364f2012d29c5cd48980c47262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Sun, 1 Sep 2019 13:34:35 +0200 Subject: [PATCH 163/213] brcmfmac: add stub version of brcmf_debugfs_get_devdir() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit cb34212b1c25f7656a315f956d72696777e88340 upstream. In case of compiling driver without DEBUG expose a stub function to make writing debug code much simpler (no extra conditions). This will allow e.g. using debugfs_create_file() without any magic if or #ifdef. Signed-off-by: RafaÅ‚ MiÅ‚ecki Signed-off-by: Kalle Valo [chi-hsien.lin@cypress.com: merge to Cypress 4.14 branch] Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 2fc9e06f98869..4ad813643bce1 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -142,6 +142,11 @@ static inline int brcmf_debug_attach(struct brcmf_pub *drvr) static inline void brcmf_debug_detach(struct brcmf_pub *drvr) { } + +static inline struct dentry *brcmf_debugfs_get_devdir(struct brcmf_pub *drvr) +{ + return ERR_PTR(-ENOENT); +} static inline int brcmf_debugfs_add_entry(struct brcmf_pub *drvr, const char *fn, int (*read_fn)(struct seq_file *seq, void *data)) From 473924f1b1d9c39c011f83794c70d97011fc2600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Sun, 1 Sep 2019 13:34:36 +0200 Subject: [PATCH 164/213] brcmfmac: add "reset" debugfs entry for testing reset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 2f8c8e62cd50d72ac68de884a09c6f5a969a269c upstream. This is a trivial debugfs entry for triggering reset just like in case of firmware crash. It works by writing 1 to it: echo 1 > reset Signed-off-by: RafaÅ‚ MiÅ‚ecki Signed-off-by: Kalle Valo [chi-hsien.lin@cypress.com: merge to 4.14 Cypress codebase] Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/core.c | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 5fa852d22a28d..c671996bd71bb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -975,6 +975,29 @@ static void brcmf_core_bus_reset(struct work_struct *work) brcmf_bus_reset(drvr->bus_if); } +static ssize_t bus_reset_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct brcmf_pub *drvr = file->private_data; + u8 value; + + if (kstrtou8_from_user(user_buf, count, 0, &value)) + return -EINVAL; + + if (value != 1) + return -EINVAL; + + schedule_work(&drvr->bus_reset); + + return count; +} + +static const struct file_operations bus_reset_fops = { + .open = simple_open, + .llseek = no_llseek, + .write = bus_reset_write, +}; + int brcmf_bus_started(struct device *dev) { int ret = -1; @@ -1053,6 +1076,10 @@ int brcmf_bus_started(struct device *dev) INIT_WORK(&drvr->bus_reset, brcmf_core_bus_reset); + /* populate debugfs */ + debugfs_create_file("reset", 0600, brcmf_debugfs_get_devdir(drvr), drvr, + &bus_reset_fops); + return 0; fail: From 5667b1d5e0145c66852e8eef3b7743ad1d5d2195 Mon Sep 17 00:00:00 2001 From: Chi-Hsien Lin Date: Mon, 4 Nov 2019 02:03:38 -0600 Subject: [PATCH 165/213] brcmfmac: reset SDIO bus on a firmware crash commit 4684997d9eea ("brcmfmac: reset PCIe bus on a firmware crash") adds a reset function to recover firmware trap for PCIe bus. This commit adds an implementation for SDIO bus. Upon SDIO firmware trap, do below: - Remove the device - Reset hardware - Probe the device again Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 4 +-- .../broadcom/brcm80211/brcmfmac/firmware.c | 2 ++ .../broadcom/brcm80211/brcmfmac/sdio.c | 32 +++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/sdio.h | 3 ++ 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index de92306c89d1f..41ff7c82a9ea2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -1004,7 +1004,7 @@ static void brcmf_sdiod_freezer_detach(struct brcmf_sdio_dev *sdiodev) } #endif /* CONFIG_PM_SLEEP */ -static int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev) { sdiodev->state = BRCMF_SDIOD_DOWN; if (sdiodev->bus) { @@ -1039,7 +1039,7 @@ static void brcmf_sdiod_host_fixup(struct mmc_host *host) host->caps |= MMC_CAP_NONREMOVABLE; } -static int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) +int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) { int ret = 0; unsigned int f2_blksz = SDIO_FUNC2_BLOCKSIZE; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c index e5ffb55dc1b85..e5947beff98f0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c @@ -582,6 +582,8 @@ int brcmf_fw_map_chip_to_name(u32 chip, u32 chiprev, } fw_name[0] = '\0'; + if (nvram_name) + nvram_name[0] = '\0'; /* check if firmware path is provided by module parameter */ if (brcmf_mp_global.firmware_path[0] != '\0') { strlcpy(fw_name, brcmf_mp_global.firmware_path, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 851eadb5a46ed..dbfac6a174923 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -4273,6 +4274,36 @@ static int brcmf_sdio_get_fwname(struct device *dev, u32 chip, u32 chiprev, return ret; } +static int brcmf_sdio_bus_reset(struct device *dev) +{ + int ret = 0; + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio; + + brcmf_dbg(SDIO, "Enter\n"); + + /* start by unregistering irqs */ + brcmf_sdiod_intr_unregister(sdiodev); + + brcmf_sdiod_remove(sdiodev); + + /* reset the adapter */ + sdio_claim_host(sdiodev->func[1]); + mmc_hw_reset(sdiodev->func[1]->card->host); + sdio_release_host(sdiodev->func[1]); + + brcmf_bus_change_state(sdiodev->bus_if, BRCMF_BUS_DOWN); + + ret = brcmf_sdiod_probe(sdiodev); + if (ret) { + brcmf_err("Failed to probe after sdio device reset: ret %d\n", + ret); + brcmf_sdiod_remove(sdiodev); + } + + return ret; +} + static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .stop = brcmf_sdio_bus_stop, .preinit = brcmf_sdio_bus_preinit, @@ -4284,6 +4315,7 @@ static const struct brcmf_bus_ops brcmf_sdio_bus_ops = { .get_ramsize = brcmf_sdio_bus_get_ramsize, .get_memdump = brcmf_sdio_bus_get_memdump, .get_fwname = brcmf_sdio_get_fwname, + .reset = brcmf_sdio_bus_reset, }; static void brcmf_sdio_firmware_callback(struct device *dev, int err, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index e5f4b2865e4f0..f73a0058187d6 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -401,6 +401,9 @@ static inline void brcmf_sdiod_freezer_uncount(struct brcmf_sdio_dev *sdiodev) } #endif /* CONFIG_PM_SLEEP */ +int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev); +int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev); + struct brcmf_sdio *brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); void brcmf_sdio_remove(struct brcmf_sdio *bus); void brcmf_sdio_isr(struct brcmf_sdio *bus); From 65e625bb651b26f27b4c1c8aa3aa38fc8a8d5fde Mon Sep 17 00:00:00 2001 From: Jia-Shyr Chuang Date: Thu, 12 Dec 2019 00:02:45 -0600 Subject: [PATCH 166/213] brcmfmac: set security after reiniting interface FMAC parses and sets security params into FW passed by supplicant. This has to be done after reiniting interface in the firmware. Signed-off-by: Joseph Chuang Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 89 ++++++++++++------- 1 file changed, 57 insertions(+), 32 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 79e5b25a9afd2..8f8c401808a8a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -4643,6 +4643,48 @@ brcmf_config_ap_mgmt_ie(struct brcmf_cfg80211_vif *vif, return err; } +static s32 +brcmf_parse_configure_security(struct brcmf_if *ifp, + struct cfg80211_ap_settings *settings, + enum nl80211_iftype dev_role) +{ + const struct brcmf_tlv *rsn_ie; + const struct brcmf_vs_tlv *wpa_ie; + s32 err = 0; + + /* find the RSN_IE */ + rsn_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail, + settings->beacon.tail_len, WLAN_EID_RSN); + + /* find the WPA_IE */ + wpa_ie = brcmf_find_wpaie((u8 *)settings->beacon.tail, + settings->beacon.tail_len); + + if (wpa_ie || rsn_ie) { + brcmf_dbg(TRACE, "WPA(2) IE is found\n"); + if (wpa_ie) { + /* WPA IE */ + err = brcmf_configure_wpaie(ifp, wpa_ie, false); + if (err < 0) + return err; + } else { + struct brcmf_vs_tlv *tmp_ie; + + tmp_ie = (struct brcmf_vs_tlv *)rsn_ie; + + /* RSN IE */ + err = brcmf_configure_wpaie(ifp, tmp_ie, true); + if (err < 0) + return err; + } + } else { + brcmf_dbg(TRACE, "No WPA(2) IEs found\n"); + brcmf_configure_opensecurity(ifp); + } + + return err; +} + static s32 brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, struct cfg80211_ap_settings *settings) @@ -4656,8 +4698,6 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, const struct brcmf_tlv *country_ie; struct brcmf_ssid_le ssid_le; s32 err = -EPERM; - const struct brcmf_tlv *rsn_ie; - const struct brcmf_vs_tlv *wpa_ie; struct brcmf_join_params join_params; enum nl80211_iftype dev_role; struct brcmf_fil_bss_enable_le bss_enable; @@ -4712,36 +4752,6 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, brcmf_configure_arp_nd_offload(ifp, false); } - /* find the RSN_IE */ - rsn_ie = brcmf_parse_tlvs((u8 *)settings->beacon.tail, - settings->beacon.tail_len, WLAN_EID_RSN); - - /* find the WPA_IE */ - wpa_ie = brcmf_find_wpaie((u8 *)settings->beacon.tail, - settings->beacon.tail_len); - - if ((wpa_ie != NULL || rsn_ie != NULL)) { - brcmf_dbg(TRACE, "WPA(2) IE is found\n"); - if (wpa_ie != NULL) { - /* WPA IE */ - err = brcmf_configure_wpaie(ifp, wpa_ie, false); - if (err < 0) - goto exit; - } else { - struct brcmf_vs_tlv *tmp_ie; - - tmp_ie = (struct brcmf_vs_tlv *)rsn_ie; - - /* RSN IE */ - err = brcmf_configure_wpaie(ifp, tmp_ie, true); - if (err < 0) - goto exit; - } - } else { - brcmf_dbg(TRACE, "No WPA(2) IEs found\n"); - brcmf_configure_opensecurity(ifp); - } - /* Parameters shared by all radio interfaces */ if (!mbss) { if ((supports_11d) && (is_11d != ifp->vif->is_11d)) { @@ -4836,6 +4846,13 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, if (profile->use_fwauth == 0) profile->use_fwauth = BIT(BRCMF_PROFILE_FWAUTH_NONE); + err = brcmf_parse_configure_security(ifp, settings, + NL80211_IFTYPE_AP); + if (err < 0) { + brcmf_err("brcmf_parse_configure_security error\n"); + goto exit; + } + /* On DOWN the firmware removes the WEP keys, reconfigure * them if they were set. */ @@ -4870,6 +4887,14 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, chanspec, err); goto exit; } + + err = brcmf_parse_configure_security(ifp, settings, + NL80211_IFTYPE_P2P_GO); + if (err < 0) { + brcmf_err("brcmf_parse_configure_security error\n"); + goto exit; + } + err = brcmf_fil_bsscfg_data_set(ifp, "ssid", &ssid_le, sizeof(ssid_le)); if (err < 0) { From 792a5c11d7f97c61fdcf8bccaf47e66223cb8b36 Mon Sep 17 00:00:00 2001 From: "Lo(Double)Hsiang Lo" Date: Mon, 16 Dec 2019 03:41:09 -0600 Subject: [PATCH 167/213] brcmfmac: increase dcmd maximum buffer size Increase dcmd maximum buffer size to match firmware configuration for new chips. Signed-off-by: Double Lo Signed-off-by: Chi-Hsien Lin SWLINUX-1273 --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c index 9f2d0b0cf6e5c..dece0d3ba182e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c @@ -98,6 +98,8 @@ struct brcmf_proto_bcdc_header { * plus any space that might be needed * for bus alignment padding. */ +#define ROUND_UP_MARGIN 2048 + struct brcmf_bcdc { u16 reqid; u8 bus_header[BUS_HEADER_LEN]; @@ -471,7 +473,7 @@ int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr) drvr->hdrlen += BCDC_HEADER_LEN + BRCMF_PROT_FW_SIGNAL_MAX_TXBYTES; drvr->bus_if->maxctl = BRCMF_DCMD_MAXLEN + - sizeof(struct brcmf_proto_bcdc_dcmd); + sizeof(struct brcmf_proto_bcdc_dcmd) + ROUND_UP_MARGIN; return 0; fail: From 6e137869d3687f9989f056203c41969ae1db3e55 Mon Sep 17 00:00:00 2001 From: Frank Kao Date: Wed, 18 Dec 2019 16:11:14 +0800 Subject: [PATCH 168/213] brcmfmac: set F2 blocksize and watermark for 4356 SDIO Set F2 blocksize to 256 bytes and watermark to 0x40 for 4356 SDIO. Also enable and configure F1 MesBusyCtrl. It would resolve random driver crash issue. Signed-off-by: Frank Kao Signed-off-by: Chi-Hsien Lin --- .../broadcom/brcm80211/brcmfmac/bcmsdh.c | 8 +++--- .../broadcom/brcm80211/brcmfmac/sdio.c | 25 +++++-------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index 41ff7c82a9ea2..ea439eed18a67 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -56,8 +56,7 @@ #define SDIO_FUNC1_BLOCKSIZE 64 #define SDIO_FUNC2_BLOCKSIZE 512 #define SDIO_4373_FUNC2_BLOCKSIZE 256 -#define SDIO_4359_FUNC2_BLOCKSIZE 256 -#define SDIO_4354_FUNC2_BLOCKSIZE 256 +#define SDIO_435X_FUNC2_BLOCKSIZE 256 /* Maximum milliseconds to wait for F2 to come up */ #define SDIO_WAIT_F2RDY 3000 @@ -1061,10 +1060,9 @@ int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) break; case SDIO_DEVICE_ID_BROADCOM_4359: case SDIO_DEVICE_ID_CYPRESS_89359: - f2_blksz = SDIO_4359_FUNC2_BLOCKSIZE; - break; case SDIO_DEVICE_ID_BROADCOM_4354: - f2_blksz = SDIO_4354_FUNC2_BLOCKSIZE; + case SDIO_DEVICE_ID_BROADCOM_4356: + f2_blksz = SDIO_435X_FUNC2_BLOCKSIZE; break; default: break; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index dbfac6a174923..ba695922f1a60 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -66,10 +66,8 @@ #define CY_4339_F2_WATERMARK 48 #define CY_4339_MES_WATERMARK 80 #define CY_4339_MESBUSYCTRL (CY_4339_MES_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) -#define CY_4359_F2_WATERMARK 0x40 -#define CY_4359_F1_MESBUSYCTRL (CY_4359_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) -#define CY_4354_F2_WATERMARK 0x40 -#define CY_4354_F1_MESBUSYCTRL (CY_4354_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) +#define CY_435X_F2_WATERMARK 0x40 +#define CY_435X_F1_MESBUSYCTRL (CY_435X_F2_WATERMARK | SBSDIO_MESBUSYCTRL_ENAB) #ifdef DEBUG #define BRCMF_TRAP_INFO_SIZE 80 @@ -4442,30 +4440,19 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err, break; case SDIO_DEVICE_ID_BROADCOM_4359: case SDIO_DEVICE_ID_CYPRESS_89359: - brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", - CY_4359_F2_WATERMARK); - brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, - CY_4359_F2_WATERMARK, &err); - devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, - &err); - devctl |= SBSDIO_DEVCTL_F2WM_ENAB; - brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, - &err); - brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, - CY_4359_F1_MESBUSYCTRL, &err); - break; case SDIO_DEVICE_ID_BROADCOM_4354: + case SDIO_DEVICE_ID_BROADCOM_4356: brcmf_dbg(INFO, "set F2 watermark to 0x%x*4 bytes\n", - CY_4354_F2_WATERMARK); + CY_435X_F2_WATERMARK); brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, - CY_4354_F2_WATERMARK, &err); + CY_435X_F2_WATERMARK, &err); devctl = brcmf_sdiod_regrb(sdiodev, SBSDIO_DEVICE_CTL, &err); devctl |= SBSDIO_DEVCTL_F2WM_ENAB; brcmf_sdiod_regwb(sdiodev, SBSDIO_DEVICE_CTL, devctl, &err); brcmf_sdiod_regwb(sdiodev, SBSDIO_FUNC1_MESBUSYCTRL, - CY_4354_F1_MESBUSYCTRL, &err); + CY_435X_F1_MESBUSYCTRL, &err); break; default: brcmf_sdiod_regwb(sdiodev, SBSDIO_WATERMARK, DEFAULT_F2_WATERMARK, &err); From 7d21d2148b84e9c1722b9d01b2bcde76097150fe Mon Sep 17 00:00:00 2001 From: Jia-Shyr Chuang Date: Mon, 23 Dec 2019 02:19:50 -0600 Subject: [PATCH 169/213] brcmfmac: enable credit borrow all for WFA 11n certs 5.2.27 5.2.27 test fails due to lacking credit for VI. Allow VI to borrow more credits from BE and gain higher throughput Signed-off-by: Joseph Chuang Signed-off-by: Chi-Hsien Lin --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c index e91038dc2f96a..cda807c46c18b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c @@ -2302,7 +2302,7 @@ static void brcmf_fws_dequeue_worker(struct work_struct *worker) !fws->bus_flow_blocked) { while (brcmf_fws_borrow_credit(fws, fifo - 1, fifo, - false) == 0) { + true) == 0) { skb = brcmf_fws_deq(fws, fifo); if (!skb) { brcmf_fws_return_credits(fws, fifo, 1); From 7c42b41985c97cfdb88038f57938c363e75e2448 Mon Sep 17 00:00:00 2001 From: Kurt Lee/TAIPEI Date: Tue, 31 Dec 2019 03:47:08 -0600 Subject: [PATCH 170/213] brcmfmac: set net carrier on via test tool for AP mode Host parses ioctl cmd via test tool, then set itself iftype to ap mode and report netif_carrier_on to upper layer Signed-off-by: Kurt Lee Signed-off-by: Chi-Hsien Lin SWLINUX-1322 --- .../net/wireless/broadcom/brcm80211/brcmfmac/vendor.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d969806bbacce..10003e808b4eb 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -74,6 +74,15 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } + if (cmdhdr->cmd == BRCMF_C_SET_AP) { + if (*(int *)(dcmd_buf) == 1) { + ifp->vif->wdev.iftype = NL80211_IFTYPE_AP; + brcmf_net_setcarrier(ifp, true); + } else { + ifp->vif->wdev.iftype = NL80211_IFTYPE_STATION; + } + } + if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); From db7fad32d35a8d5368614a4e626e3a17232c071c Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 29 Jan 2018 13:20:10 -0600 Subject: [PATCH 171/213] ar1021_i2c: add invert/swap options Signed-off-by: Robert Nelson --- drivers/input/touchscreen/ar1021_i2c.c | 45 +++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/drivers/input/touchscreen/ar1021_i2c.c b/drivers/input/touchscreen/ar1021_i2c.c index b35b640fdadfa..5c6fe04403997 100644 --- a/drivers/input/touchscreen/ar1021_i2c.c +++ b/drivers/input/touchscreen/ar1021_i2c.c @@ -18,6 +18,7 @@ #define AR1021_MAX_X 4095 #define AR1021_MAX_Y 4095 +#define AR1021_MAX_PRESSURE 255 #define AR1021_CMD 0x55 @@ -27,6 +28,9 @@ struct ar1021_i2c { struct i2c_client *client; struct input_dev *input; u8 data[AR1021_TOCUH_PKG_SIZE]; + bool invert_x; + bool invert_y; + bool swap_xy; }; static irqreturn_t ar1021_i2c_irq(int irq, void *dev_id) @@ -50,9 +54,22 @@ static irqreturn_t ar1021_i2c_irq(int irq, void *dev_id) x = ((data[2] & 0x1f) << 7) | (data[1] & 0x7f); y = ((data[4] & 0x1f) << 7) | (data[3] & 0x7f); - input_report_abs(input, ABS_X, x); - input_report_abs(input, ABS_Y, y); + if (ar1021->invert_x) + x = AR1021_MAX_X - x; + + if (ar1021->invert_y) + y = AR1021_MAX_Y - y; + + if (ar1021->swap_xy) { + input_report_abs(input, ABS_X, y); + input_report_abs(input, ABS_Y, x); + } else { + input_report_abs(input, ABS_X, x); + input_report_abs(input, ABS_Y, y); + } + input_report_key(input, BTN_TOUCH, button); + input_report_abs(input, ABS_PRESSURE, AR1021_MAX_PRESSURE); input_sync(input); out: @@ -117,10 +134,28 @@ static int ar1021_i2c_probe(struct i2c_client *client, input->open = ar1021_i2c_open; input->close = ar1021_i2c_close; + ar1021->invert_x = device_property_read_bool(&client->dev, "touchscreen-inverted-x"); + ar1021->invert_y = device_property_read_bool(&client->dev, "touchscreen-inverted-y"); + ar1021->swap_xy = device_property_read_bool(&client->dev, "touchscreen-swapped-x-y"); + __set_bit(INPUT_PROP_DIRECT, input->propbit); - input_set_capability(input, EV_KEY, BTN_TOUCH); - input_set_abs_params(input, ABS_X, 0, AR1021_MAX_X, 0, 0); - input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_Y, 0, 0); + //input_set_capability(input, EV_KEY, BTN_TOUCH); + + input->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS); + input->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH); + + if(ar1021->swap_xy) + { + input_set_abs_params(input, ABS_X, 0, AR1021_MAX_Y, 0, 0); + input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_X, 0, 0); + } + else + { + input_set_abs_params(input, ABS_X, 0, AR1021_MAX_X, 0, 0); + input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_Y, 0, 0); + } + + input_set_abs_params(input, ABS_PRESSURE, 0, AR1021_MAX_PRESSURE, 0, 0); input_set_drvdata(input, ar1021); From c0fdf57f25307801c8eef9eca83cbfa9d5e3df80 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 29 Jan 2018 13:23:40 -0600 Subject: [PATCH 172/213] ar1021_i2c.c: introduce offsets to manually re-calbrate screen Signed-off-by: Robert Nelson --- drivers/input/touchscreen/ar1021_i2c.c | 44 ++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/drivers/input/touchscreen/ar1021_i2c.c b/drivers/input/touchscreen/ar1021_i2c.c index 5c6fe04403997..a1d4019c5a743 100644 --- a/drivers/input/touchscreen/ar1021_i2c.c +++ b/drivers/input/touchscreen/ar1021_i2c.c @@ -24,6 +24,10 @@ #define AR1021_CMD_ENABLE_TOUCH 0x12 +#define AR1021_CMD 0x55 + +#define AR1021_CMD_ENABLE_TOUCH 0x12 + struct ar1021_i2c { struct i2c_client *client; struct input_dev *input; @@ -33,6 +37,24 @@ struct ar1021_i2c { bool swap_xy; }; +static bool ar1021_get_prop_u32(struct device *dev, + const char *property, + unsigned int default_value, + unsigned int *value) +{ + u32 val; + int error; + + error = device_property_read_u32(dev, property, &val); + if (error) { + *value = default_value; + return false; + } + + *value = val; + return true; +} + static irqreturn_t ar1021_i2c_irq(int irq, void *dev_id) { struct ar1021_i2c *ar1021 = dev_id; @@ -111,6 +133,8 @@ static int ar1021_i2c_probe(struct i2c_client *client, struct ar1021_i2c *ar1021; struct input_dev *input; int error; + unsigned int offset_x, offset_y; + bool data_present; if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { dev_err(&client->dev, "i2c_check_functionality error\n"); @@ -138,6 +162,22 @@ static int ar1021_i2c_probe(struct i2c_client *client, ar1021->invert_y = device_property_read_bool(&client->dev, "touchscreen-inverted-y"); ar1021->swap_xy = device_property_read_bool(&client->dev, "touchscreen-swapped-x-y"); + data_present = ar1021_get_prop_u32(&client->dev, + "touchscreen-offset-x", + 0, + &offset_x); + + if (data_present) + dev_info(&client->dev, "touchscreen-offset-x: %d\n", offset_x); + + data_present = ar1021_get_prop_u32(&client->dev, + "touchscreen-offset-y", + 0, + &offset_y); + + if (data_present) + dev_info(&client->dev, "touchscreen-offset-y: %d\n", offset_y); + __set_bit(INPUT_PROP_DIRECT, input->propbit); //input_set_capability(input, EV_KEY, BTN_TOUCH); @@ -151,8 +191,8 @@ static int ar1021_i2c_probe(struct i2c_client *client, } else { - input_set_abs_params(input, ABS_X, 0, AR1021_MAX_X, 0, 0); - input_set_abs_params(input, ABS_Y, 0, AR1021_MAX_Y, 0, 0); + input_set_abs_params(input, ABS_X, offset_x, AR1021_MAX_X-offset_x, 0, 0); + input_set_abs_params(input, ABS_Y, offset_y, AR1021_MAX_Y-offset_y, 0, 0); } input_set_abs_params(input, ABS_PRESSURE, 0, AR1021_MAX_PRESSURE, 0, 0); From 72fdbe2bbbf688e0d66f752b80b98c48b21fa58a Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 7 Jan 2019 13:26:25 -0600 Subject: [PATCH 173/213] btrfs: backport CONFIG_RAID6_PQ_BENCHMARK Signed-off-by: Robert Nelson --- lib/Kconfig | 8 +++++ lib/raid6/algos.c | 81 +++++++++++++++++++++++++---------------------- 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/lib/Kconfig b/lib/Kconfig index a3928d4438b50..fcb00bc06f1ad 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -10,6 +10,14 @@ menu "Library routines" config RAID6_PQ tristate +config RAID6_PQ_BENCHMARK + bool "Automatically choose fastest RAID6 PQ functions" + depends on RAID6_PQ + default y + help + Benchmark all available RAID6 PQ functions on init and choose the + fastest one. + config BITREVERSE tristate diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 5065b1e7e3275..7e4f7a8ffa8e2 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -34,64 +34,64 @@ struct raid6_calls raid6_call; EXPORT_SYMBOL_GPL(raid6_call); const struct raid6_calls * const raid6_algos[] = { -#if defined(__ia64__) - &raid6_intx16, - &raid6_intx32, -#endif #if defined(__i386__) && !defined(__arch_um__) - &raid6_mmxx1, - &raid6_mmxx2, - &raid6_sse1x1, - &raid6_sse1x2, - &raid6_sse2x1, - &raid6_sse2x2, -#ifdef CONFIG_AS_AVX2 - &raid6_avx2x1, - &raid6_avx2x2, -#endif #ifdef CONFIG_AS_AVX512 - &raid6_avx512x1, &raid6_avx512x2, + &raid6_avx512x1, #endif -#endif -#if defined(__x86_64__) && !defined(__arch_um__) - &raid6_sse2x1, - &raid6_sse2x2, - &raid6_sse2x4, #ifdef CONFIG_AS_AVX2 - &raid6_avx2x1, &raid6_avx2x2, - &raid6_avx2x4, + &raid6_avx2x1, +#endif + &raid6_sse2x2, + &raid6_sse2x1, + &raid6_sse1x2, + &raid6_sse1x1, + &raid6_mmxx2, + &raid6_mmxx1, #endif +#if defined(__x86_64__) && !defined(__arch_um__) #ifdef CONFIG_AS_AVX512 - &raid6_avx512x1, - &raid6_avx512x2, &raid6_avx512x4, + &raid6_avx512x2, + &raid6_avx512x1, #endif +#ifdef CONFIG_AS_AVX2 + &raid6_avx2x4, + &raid6_avx2x2, + &raid6_avx2x1, +#endif + &raid6_sse2x4, + &raid6_sse2x2, + &raid6_sse2x1, #endif #ifdef CONFIG_ALTIVEC - &raid6_altivec1, - &raid6_altivec2, - &raid6_altivec4, - &raid6_altivec8, - &raid6_vpermxor1, - &raid6_vpermxor2, - &raid6_vpermxor4, &raid6_vpermxor8, + &raid6_vpermxor4, + &raid6_vpermxor2, + &raid6_vpermxor1, + &raid6_altivec8, + &raid6_altivec4, + &raid6_altivec2, + &raid6_altivec1, #endif #if defined(CONFIG_S390) &raid6_s390vx8, #endif - &raid6_intx1, - &raid6_intx2, - &raid6_intx4, - &raid6_intx8, #ifdef CONFIG_KERNEL_MODE_NEON - &raid6_neonx1, - &raid6_neonx2, - &raid6_neonx4, &raid6_neonx8, + &raid6_neonx4, + &raid6_neonx2, + &raid6_neonx1, #endif +#if defined(__ia64__) + &raid6_intx32, + &raid6_intx16, +#endif + &raid6_intx8, + &raid6_intx4, + &raid6_intx2, + &raid6_intx1, NULL }; @@ -163,6 +163,11 @@ static inline const struct raid6_calls *raid6_choose_gen( if ((*algo)->valid && !(*algo)->valid()) continue; + if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { + best = *algo; + break; + } + perf = 0; preempt_disable(); From 752097406f95b7de8e9f178c93db951d9df6ffcc Mon Sep 17 00:00:00 2001 From: David Hsu Date: Sun, 30 Apr 2017 22:10:56 -0500 Subject: [PATCH 174/213] pwm: Create device class for pwm channels #https://lkml.org/lkml/2016/6/14/967 Pwm channels don't send uevents when exported, this change adds the channels to a pwm class and set their device type to pwm_channel so uevents are sent. To do this properly, the device names need to change to uniquely identify a channel. This change is from pwmN to pwm-(chip->base):N Signed-off-by: David Hsu Signed-off-by: Greg Kroah-Hartman Signed-off-by: Robert Nelson --- Documentation/pwm.txt | 6 ++++-- drivers/pwm/sysfs.c | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Documentation/pwm.txt b/Documentation/pwm.txt index 8fbf0aa3ba2d5..72db8dd15f53b 100644 --- a/Documentation/pwm.txt +++ b/Documentation/pwm.txt @@ -85,9 +85,11 @@ will find: The PWM channels are numbered using a per-chip index from 0 to npwm-1. -When a PWM channel is exported a pwmX directory will be created in the +When a PWM channel is exported a pwm-N:X directory will be created in the pwmchipN directory it is associated with, where X is the number of the -channel that was exported. The following properties will then be available: +channel that was exported. It will also be exposed at /sys/class/pwm/ and +can be identified by the pwm_channel device type. +The following properties will then be available: period The total period of the PWM signal (read/write). diff --git a/drivers/pwm/sysfs.c b/drivers/pwm/sysfs.c index 89cf5af3baac6..9035664aee370 100644 --- a/drivers/pwm/sysfs.c +++ b/drivers/pwm/sysfs.c @@ -23,6 +23,8 @@ #include #include +static struct class pwm_class; + struct pwm_export { struct device child; struct pwm_device *pwm; @@ -239,6 +241,10 @@ static struct attribute *pwm_attrs[] = { }; ATTRIBUTE_GROUPS(pwm); +static const struct device_type pwm_channel_type = { + .name = "pwm_channel", +}; + static void pwm_export_release(struct device *child) { struct pwm_export *export = child_to_pwm_export(child); @@ -248,6 +254,7 @@ static void pwm_export_release(struct device *child) static int pwm_export_child(struct device *parent, struct pwm_device *pwm) { + struct pwm_chip *chip = dev_get_drvdata(parent); struct pwm_export *export; int ret; @@ -265,9 +272,11 @@ static int pwm_export_child(struct device *parent, struct pwm_device *pwm) export->child.release = pwm_export_release; export->child.parent = parent; + export->child.type = &pwm_channel_type; export->child.devt = MKDEV(0, 0); + export->child.class = &pwm_class; export->child.groups = pwm_groups; - dev_set_name(&export->child, "pwm%u", pwm->hwpwm); + dev_set_name(&export->child, "pwm-%d:%u", chip->base, pwm->hwpwm); ret = device_register(&export->child); if (ret) { @@ -373,7 +382,6 @@ ATTRIBUTE_GROUPS(pwm_chip); static struct class pwm_class = { .name = "pwm", .owner = THIS_MODULE, - .dev_groups = pwm_chip_groups, }; static int pwmchip_sysfs_match(struct device *parent, const void *data) @@ -389,7 +397,8 @@ void pwmchip_sysfs_export(struct pwm_chip *chip) * If device_create() fails the pwm_chip is still usable by * the kernel its just not exported. */ - parent = device_create(&pwm_class, chip->dev, MKDEV(0, 0), chip, + parent = device_create_with_groups(&pwm_class, chip->dev, MKDEV(0, 0), + chip, pwm_chip_groups, "pwmchip%d", chip->base); if (IS_ERR(parent)) { dev_warn(chip->dev, From 1ea44a5d64827879599398fb520a155a35011883 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 11 Jul 2019 22:08:14 -0500 Subject: [PATCH 175/213] sound: give us SND_SOC_PCM5102A Signed-off-by: Robert Nelson --- sound/soc/ti/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/soc/ti/Kconfig b/sound/soc/ti/Kconfig index 598e7f98a4575..eaa96755ad5b4 100644 --- a/sound/soc/ti/Kconfig +++ b/sound/soc/ti/Kconfig @@ -28,6 +28,7 @@ config SND_SOC_DAVINCI_MCASP select SND_SOC_TI_EDMA_PCM if TI_EDMA select SND_SOC_TI_SDMA_PCM if DMA_OMAP select SND_SOC_TI_UDMA_PCM if TI_K3_UDMA + select SND_SOC_PCM5102A help Say Y or M here if you want to have support for McASP IP found in various Texas Instruments SoCs like: From 6d0d3b0497edaa41f3054d7b23ebddf6fd50de28 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 4 Aug 2016 11:26:37 -0500 Subject: [PATCH 176/213] NFM: spi: spidev: allow use of spidev in DT This reverts commit 956b200a846e324322f6211034c734c65a38e550. Signed-off-by: Robert Nelson --- drivers/spi/spidev.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/spi/spidev.c b/drivers/spi/spidev.c index 028725573e632..3cb76605eda54 100644 --- a/drivers/spi/spidev.c +++ b/drivers/spi/spidev.c @@ -727,9 +727,9 @@ static int spidev_probe(struct spi_device *spi) * compatible string, it is a Linux implementation thing * rather than a description of the hardware. */ - WARN(spi->dev.of_node && - of_device_is_compatible(spi->dev.of_node, "spidev"), - "%pOF: buggy DT: spidev listed directly in DT\n", spi->dev.of_node); +// WARN(spi->dev.of_node && +// of_device_is_compatible(spi->dev.of_node, "spidev"), +// "%pOF: buggy DT: spidev listed directly in DT\n", spi->dev.of_node); spidev_probe_acpi(spi); From 26b9516b44fdca02a5e9d75bf9a9584c811f947f Mon Sep 17 00:00:00 2001 From: Jason Kridner Date: Mon, 22 Jan 2018 09:12:34 +0000 Subject: [PATCH 177/213] fb_ssd1306: add support for MikroElectronica OLED B Click https://www.mikroe.com/oled-b-click --- drivers/staging/fbtft/fb_ssd1306.c | 67 +++++++++++++++++++++++----- drivers/staging/fbtft/fbtft_device.c | 19 ++++++++ 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/drivers/staging/fbtft/fb_ssd1306.c b/drivers/staging/fbtft/fb_ssd1306.c index 50172ddd94aea..f7be4f9606aea 100644 --- a/drivers/staging/fbtft/fb_ssd1306.c +++ b/drivers/staging/fbtft/fb_ssd1306.c @@ -55,6 +55,8 @@ static int init_display(struct fbtft_par *par) write_reg(par, 0x3F); else if (par->info->var.yres == 48) write_reg(par, 0x2F); + else if (par->info->var.yres == 39) + write_reg(par, 0x27); /* https://libstock.mikroe.com/projects/download/1111/2577/1411057038_oled_b_click___e_mikroc_arm.zip */ else write_reg(par, 0x1F); @@ -72,16 +74,24 @@ static int init_display(struct fbtft_par *par) /* Set Memory Addressing Mode */ write_reg(par, 0x20); - /* Vertical addressing mode */ + /* Vertical Addressing mode */ write_reg(par, 0x01); /* Set Segment Re-map */ - /* column address 127 is mapped to SEG0 */ - write_reg(par, 0xA0 | 0x1); + if (par->info->var.yres == 39) + /* no segment re-map */ + write_reg(par, 0xA0 | 0x0); + else + /* column address 127 is mapped to SEG0 */ + write_reg(par, 0xA0 | 0x1); /* Set COM Output Scan Direction */ - /* remapped mode. Scan from COM[N-1] to COM0 */ - write_reg(par, 0xC8); + if (par->info->var.yres == 39) + /* no columnt re-map mode. Scan from COM0 to COM[N-1] */ + write_reg(par, 0xC0 | 0x0); + else + /* remapped mode. Scan from COM[N-1] to COM0 */ + write_reg(par, 0xC0 | 0x8); /* Set COM Pins Hardware Configuration */ write_reg(par, 0xDA); @@ -91,18 +101,27 @@ static int init_display(struct fbtft_par *par) else if (par->info->var.yres == 48) /* A[4]=1b, Alternative COM pin configuration */ write_reg(par, 0x12); + else if (par->info->var.yres == 39) + /* A[4]=1b, Alternative COM pin configuration */ + write_reg(par, 0x12); else /* A[4]=0b, Sequential COM pin configuration */ write_reg(par, 0x02); /* Set Pre-charge Period */ write_reg(par, 0xD9); - write_reg(par, 0xF1); + if (par->info->var.yres == 39) + write_reg(par, 0x25); + else + write_reg(par, 0xF1); /* Set VCOMH Deselect Level */ write_reg(par, 0xDB); - /* according to the datasheet, this value is out of bounds */ - write_reg(par, 0x40); + if (par->info->var.yres == 39) + write_reg(par, 0x20); + else + /* according to the datasheet, this value is out of bounds */ + write_reg(par, 0x40); /* Entire Display ON */ /* Resume to RAM content display. Output follows RAM content */ @@ -133,17 +152,35 @@ static void set_addr_win_64x48(struct fbtft_par *par) write_reg(par, 0x5); } +static void set_addr_win_96x39(struct fbtft_par *par) +{ + /* Set Page Address */ + write_reg(par, 0xB0); + + /* Set Column Address */ + write_reg(par, 0x21); + write_reg(par, 0x00); + write_reg(par, 0x5F); + + /* Set Page Address Range */ + write_reg(par, 0x22); + write_reg(par, 0x0); + write_reg(par, 0x4); +} + static void set_addr_win(struct fbtft_par *par, int xs, int ys, int xe, int ye) { - /* Set Lower Column Start Address for Page Addressing Mode */ + /* Set Lower Column Start Address for Vertical Addressing Mode */ write_reg(par, 0x00 | 0x0); - /* Set Higher Column Start Address for Page Addressing Mode */ + /* Set Higher Column Start Address for Vertical Addressing Mode */ write_reg(par, 0x10 | 0x0); /* Set Display Start Line */ write_reg(par, 0x40 | 0x0); if (par->info->var.xres == 64 && par->info->var.yres == 48) set_addr_win_64x48(par); + else if (par->info->var.xres == 96 && par->info->var.yres == 39) + set_addr_win_96x39(par); } static int blank(struct fbtft_par *par, bool on) @@ -187,11 +224,19 @@ static int write_vmem(struct fbtft_par *par, size_t offset, size_t len) *buf |= (vmem16[(y * 8 + i) * xres + x] ? 1 : 0) << i; buf++; } + if (yres % 8) { + *buf = 0x00; + for (i = 0; i < (yres - (y * 8)); i++) + *buf |= (vmem16[(y * 8 + i) * xres + x] ? 1 : 0) << i; + buf++; + y++; + } } /* Write data */ gpio_set_value(par->gpio.dc, 1); - ret = par->fbtftops.write(par, par->txbuf.buf, xres * yres / 8); + ret = par->fbtftops.write(par, par->txbuf.buf, xres * (yres / 8 + (yres % 8 != 0))); + //ret = par->fbtftops.write(par, par->txbuf.buf, xres * 128 / 8); if (ret < 0) dev_err(par->info->device, "write failed and returned: %d\n", ret); diff --git a/drivers/staging/fbtft/fbtft_device.c b/drivers/staging/fbtft/fbtft_device.c index 50e97da993e79..7e1e0bb83cbcc 100644 --- a/drivers/staging/fbtft/fbtft_device.c +++ b/drivers/staging/fbtft/fbtft_device.c @@ -387,6 +387,25 @@ static struct fbtft_device_display displays[] = { }, } } + }, { + .name = "mikroe_oledb", + .spi = &(struct spi_board_info) { + .modalias = "fb_ssd1306", + .max_speed_hz = 16000000, + .mode = SPI_MODE_0, + .platform_data = &(struct fbtft_platform_data) { + .display = { + .width = 96, + .height = 39, + .buswidth = 8, + }, + .gpios = (const struct fbtft_gpio []) { + { "reset", 45 }, + { "dc", 110 }, + {}, + }, + } + } }, { .name = "admatec_c-berry28", .spi = &(struct spi_board_info) { From b83a4105f1261d6297938950fe1f746c6a19eb9b Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Fri, 28 Oct 2016 14:14:27 -0500 Subject: [PATCH 178/213] HACK: tps65217_pwr_but Signed-off-by: Robert Nelson --- drivers/input/misc/tps65218-pwrbutton.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/input/misc/tps65218-pwrbutton.c b/drivers/input/misc/tps65218-pwrbutton.c index a4455bb12ae0c..b6d4321d421fc 100644 --- a/drivers/input/misc/tps65218-pwrbutton.c +++ b/drivers/input/misc/tps65218-pwrbutton.c @@ -36,7 +36,7 @@ struct tps6521x_data { static const struct tps6521x_data tps65217_data = { .reg_status = TPS65217_REG_STATUS, .pb_mask = TPS65217_STATUS_PB, - .name = "tps65217_pwrbutton", + .name = "tps65217_pwr_but", }; static const struct tps6521x_data tps65218_data = { From 4ea006ac0e80caa2da1794563a73e180de51bcb2 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Tue, 14 Nov 2017 10:15:31 -0600 Subject: [PATCH 179/213] cpufreq: opp: dont fail _opp_add_static_v2, temp till overlays repo moves to v4.14.x base Signed-off-by: Robert Nelson --- drivers/opp/of.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 20988c4266501..dc14671e27fca 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -374,7 +374,7 @@ static int _opp_add_static_v2(struct opp_table *opp_table, struct device *dev, free_opp: _opp_free(new_opp); - return ret; + return 0; } /* Initializes OPP tables based on new bindings */ From 0c61cf6427427bbfd37ed90eacff1034afd14295 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 5 Oct 2017 15:30:04 -0500 Subject: [PATCH 180/213] wiznet: w5100-spi: hack dt support for w5500 Signed-off-by: Robert Nelson --- drivers/net/ethernet/wiznet/w5100-spi.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/net/ethernet/wiznet/w5100-spi.c b/drivers/net/ethernet/wiznet/w5100-spi.c index 93a2d3c07303e..e981d2ab2336a 100644 --- a/drivers/net/ethernet/wiznet/w5100-spi.c +++ b/drivers/net/ethernet/wiznet/w5100-spi.c @@ -442,6 +442,21 @@ static int w5100_spi_remove(struct spi_device *spi) return w5100_remove(&spi->dev); } +#ifdef CONFIG_OF +static const struct of_device_id w5100_spi_of_match[] = { + { .compatible = "wiznet,w5500" }, + { } +}; +MODULE_DEVICE_TABLE(of, w5100_spi_of_match); +#endif + +#ifdef CONFIG_OF +static const struct spi_device_id w5100_spi_ids[] = { + { "w5500", W5500 }, + {} +}; +MODULE_DEVICE_TABLE(spi, w5100_spi_ids); +#else static const struct spi_device_id w5100_spi_ids[] = { { "w5100", W5100 }, { "w5200", W5200 }, @@ -449,11 +464,13 @@ static const struct spi_device_id w5100_spi_ids[] = { {} }; MODULE_DEVICE_TABLE(spi, w5100_spi_ids); +#endif static struct spi_driver w5100_spi_driver = { .driver = { .name = "w5100", .pm = &w5100_pm_ops, + .of_match_table = of_match_ptr(w5100_spi_of_match), }, .probe = w5100_spi_probe, .remove = w5100_spi_remove, From 697868fb5ef31a2a1df062fb0ca9364a40d426a6 Mon Sep 17 00:00:00 2001 From: Pantelis Antoniou Date: Sat, 6 Dec 2014 20:10:49 +0200 Subject: [PATCH 181/213] ARM: DT: Enable symbols when CONFIG_OF_OVERLAY is used --- arch/arm/boot/Makefile | 4 ++++ arch/arm/boot/dts/Makefile | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile index a3af4dc08c3e9..5e9c1d6bd88eb 100644 --- a/arch/arm/boot/Makefile +++ b/arch/arm/boot/Makefile @@ -29,6 +29,10 @@ export ZRELADDR INITRD_PHYS PARAMS_PHYS targets := Image zImage xipImage bootpImage uImage +ifeq ($(CONFIG_OF_OVERLAY),y) +DTC_FLAGS += -@ +endif + ifeq ($(CONFIG_XIP_KERNEL),y) cmd_deflate_xip_data = $(CONFIG_SHELL) -c \ diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 0416e9503d546..89354571bc448 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1,4 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 + +ifeq ($(CONFIG_OF_OVERLAY),y) +DTC_FLAGS += -@ +endif + dtb-$(CONFIG_ARCH_ALPINE) += \ alpine-db.dtb dtb-$(CONFIG_MACH_ARTPEC6) += \ From 9539edfab06906046e3d97d62b13a24fd90c408c Mon Sep 17 00:00:00 2001 From: Jay at Control Module Industries Date: Tue, 19 May 2015 13:35:14 -0500 Subject: [PATCH 182/213] cpsw: search for phy I have encountered the same issue(s) on A6A boards. I couldn't find a patch, so I wrote this patch to update the device tree in the davinci_mdio driver in the 3.15.1 tree, it seems to correct it. I would welcome any input on a different approach. https://groups.google.com/d/msg/beagleboard/9mctrG26Mc8/SRlnumt0LoMJ v4.1-rcX: added hack around CONFIG_OF_OVERLAY v4.2-rc3+: added if (of_machine_is_compatible("ti,am335x-bone")) so we do not break dual ethernet am335x devices Signed-off-by: Robert Nelson --- drivers/net/ethernet/ti/davinci_mdio.c | 97 ++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c index 876d5241391d8..e19e51a3b8d80 100644 --- a/drivers/net/ethernet/ti/davinci_mdio.c +++ b/drivers/net/ethernet/ti/davinci_mdio.c @@ -108,6 +108,10 @@ struct davinci_mdio_data { u32 clk_div; }; +#if IS_ENABLED(CONFIG_OF) +static void davinci_mdio_update_dt_from_phymask(u32 phy_mask); +#endif + static void davinci_mdio_init_clk(struct davinci_mdio_data *data) { u32 mdio_in, div, mdio_out_khz, access_time; @@ -175,6 +179,12 @@ static int davinci_mdio_reset(struct mii_bus *bus) /* restrict mdio bus to live phys only */ dev_info(data->dev, "detected phy mask %x\n", ~phy_mask); phy_mask = ~phy_mask; + + #if IS_ENABLED(CONFIG_OF) + if (of_machine_is_compatible("ti,am335x-bone")) + davinci_mdio_update_dt_from_phymask(phy_mask); + #endif + } else { /* desperately scan all phys */ dev_warn(data->dev, "no live phy, scanning all\n"); @@ -488,6 +498,93 @@ static int davinci_mdio_runtime_resume(struct device *dev) davinci_mdio_enable(data); return 0; } +static void davinci_mdio_update_dt_from_phymask(u32 phy_mask) +{ + int i, len, skip; + u32 addr; + __be32 *old_phy_p, *phy_id_p; + struct property *phy_id_property = NULL; + struct device_node *node_p, *slave_p; + + addr = 0; + + for (i = 0; i < PHY_MAX_ADDR; i++) { + if ((phy_mask & (1 << i)) == 0) { + addr = (u32) i; + break; + } + } + + for_each_compatible_node(node_p, NULL, "ti,cpsw") { + for_each_node_by_name(slave_p, "slave") { + +#if IS_ENABLED(CONFIG_OF_OVERLAY) + skip = 1; + // Hack, the overlay fixup "slave" doesn't have phy-mode... + old_phy_p = (__be32 *) of_get_property(slave_p, "phy-mode", &len); + + if (len != (sizeof(__be32 *) * 1)) + { + skip = 0; + } + + if (skip) { +#endif + + old_phy_p = (__be32 *) of_get_property(slave_p, "phy_id", &len); + + if (len != (sizeof(__be32 *) * 2)) + goto err_out; + + if (old_phy_p) { + + phy_id_property = kzalloc(sizeof(*phy_id_property), GFP_KERNEL); + + if (! phy_id_property) + goto err_out; + + phy_id_property->length = len; + phy_id_property->name = kstrdup("phy_id", GFP_KERNEL); + phy_id_property->value = kzalloc(len, GFP_KERNEL); + + if (! phy_id_property->name) + goto err_out; + + if (! phy_id_property->value) + goto err_out; + + memcpy(phy_id_property->value, old_phy_p, len); + + phy_id_p = (__be32 *) phy_id_property->value + 1; + + *phy_id_p = cpu_to_be32(addr); + + of_update_property(slave_p, phy_id_property); + pr_info("davinci_mdio: dt: updated phy_id[%d] from phy_mask[%x]\n", addr, phy_mask); + + ++addr; + } +#if IS_ENABLED(CONFIG_OF_OVERLAY) + } +#endif + } + } + + return; + +err_out: + + if (phy_id_property) { + if (phy_id_property->name) + kfree(phy_id_property->name); + + if (phy_id_property->value) + kfree(phy_id_property->value); + + if (phy_id_property) + kfree(phy_id_property); + } +} #endif #ifdef CONFIG_PM_SLEEP From 78b98521bf3cc211eb2a643cda13c87e3922ecb9 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 4 Mar 2019 14:54:40 -0600 Subject: [PATCH 183/213] ti: dra7: etnaviv: 2d acceleration Signed-off-by: Robert Nelson --- arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi | 4 ++++ arch/arm/boot/dts/dra7.dtsi | 10 ++++++++++ drivers/gpu/drm/etnaviv/Kconfig | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi index a3a9844937ec8..b7b81a8749c49 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi +++ b/arch/arm/boot/dts/am57xx-beagle-x15-common.dtsi @@ -223,6 +223,10 @@ }; }; +//&bb2d { +// status = "okay"; +//}; + &i2c1 { status = "okay"; clock-frequency = <400000>; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index 83c1706fe83b0..4310e45746623 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -1095,6 +1095,16 @@ status = "disabled"; }; +// bb2d: bb2d@59000000 { +// compatible = "ti,dra7-bb2d","vivante,gc"; +// reg = <0x59000000 0x0700>; +// interrupts = ; +// ti,hwmods = "bb2d"; +// clocks = <&dpll_core_h24x2_ck>; +// clock-names = "fclk"; +// status = "disabled"; +// }; +//s i2c1: i2c@48070000 { compatible = "ti,omap4-i2c"; reg = <0x48070000 0x100>; diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig index 041a77e400d4e..9219a6c3b1ab7 100644 --- a/drivers/gpu/drm/etnaviv/Kconfig +++ b/drivers/gpu/drm/etnaviv/Kconfig @@ -2,7 +2,7 @@ config DRM_ETNAVIV tristate "ETNAVIV (DRM support for Vivante GPU IP cores)" depends on DRM - depends on ARCH_MXC || ARCH_DOVE || (ARM && COMPILE_TEST) + depends on ARCH_MXC || ARCH_DOVE || ARCH_OMAP2PLUS || (ARM && COMPILE_TEST) depends on MMU select SHMEM select SYNC_FILE From dc08e475bb2ef256a4ab1a61fe11ac5d0eab4f06 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 29 Jan 2018 13:42:52 -0600 Subject: [PATCH 184/213] tieqep: forward port of Nathaniel Lewis eQEP driver Signed-off-by: Robert Nelson --- drivers/misc/Kconfig | 11 + drivers/misc/Makefile | 1 + drivers/misc/tieqep.c | 738 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 750 insertions(+) create mode 100644 drivers/misc/tieqep.c diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 6d3635552160e..e1f1b77c2f807 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -518,6 +518,17 @@ config DMA_BUF_PHYS help Exports CPU physical address of DMA-BUF to user-space. +config TIEQEP + tristate "EQEP Hardware quadrature encoder controller" + depends on SOC_AM33XX + select PWM_TIPWMSS + help + Driver support for the EQEP quadrature encoder controller AM33XX + TI SOC + + To compile this driver as a module, choose M here: the module + will be called tieqep. + source "drivers/misc/c2port/Kconfig" source "drivers/misc/eeprom/Kconfig" source "drivers/misc/cb710/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index efe8447c115b1..59258a3d58d51 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -43,6 +43,7 @@ obj-y += ti-st/ obj-y += lis3lv02d/ obj-$(CONFIG_USB_SWITCH_FSA9480) += fsa9480.o obj-$(CONFIG_ALTERA_STAPL) +=altera-stapl/ +obj-$(CONFIG_TIEQEP) += tieqep.o obj-$(CONFIG_INTEL_MEI) += mei/ obj-$(CONFIG_VMWARE_VMCI) += vmw_vmci/ obj-$(CONFIG_LATTICE_ECP3_CONFIG) += lattice-ecp3-config.o diff --git a/drivers/misc/tieqep.c b/drivers/misc/tieqep.c new file mode 100644 index 0000000000000..d2628847bb8e3 --- /dev/null +++ b/drivers/misc/tieqep.c @@ -0,0 +1,738 @@ +/* + * TI eQEP driver for AM33xx devices + * + * Copyright (C) 2013 Nathaniel R. Lewis - http://teknoman117.wordpress.com/ + * Copyright (C) 2015 SoftPLC Corporation, Dick Hollenbeck + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * + * sysfs entries + * - position = absolute - current position; relative - last latched value + * - mode => 0 - absolute; 1 - relative + * - period => sampling period for the hardware + * - enable => 0 - eQEP disabled, 1 - eQEP enabled + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* eQEP register offsets from its base IO address */ +#define QPOSCNT 0x0000 +#define QPOSINIT 0x0004 +#define QPOSMAX 0x0008 +#define QPOSCMP 0x000C +#define QPOSILAT 0x0010 +#define QPOSSLAT 0x0014 +#define QPOSLAT 0x0018 +#define QUTMR 0x001C +#define QUPRD 0x0020 +#define QWDTMR 0x0024 +#define QWDPRD 0x0026 +#define QDECCTL 0x0028 +#define QEPCTL 0x002A +#define QCAPCTL 0x002C +#define QPOSCTL 0x002E +#define QEINT 0x0030 +#define QFLG 0x0032 +#define QCLR 0x0034 +#define QFRC 0x0036 +#define QEPSTS 0x0038 +#define QCTMR 0x003A +#define QCPRD 0x003C +#define QCTMRLAT 0x003E +#define QCPRDLAT 0x0040 +#define QREVID 0x005C + +#if 0 /* if you wanted another way to modify IP registers... */ +typedef volatile u32 REG32; +typedef volatile u16 REG16; +struct EQEP_REGS { + REG32 q_poscnt; /* 0x00 position counter */ + REG32 q_posinit; /* 0x04 position counter initialization */ + REG32 q_posmax; /* 0x08 maximum position count */ + REG32 q_poscmp; /* 0x0C position compare */ + REG32 q_posilat; /* 0x10 index position latch */ + REG32 q_posslat; /* 0x14 strobe position latch */ + REG32 q_poslat; /* 0x18 position counter latch */ + REG32 q_utmr; /* 0x1C unit timer */ + REG32 q_uprd; /* 0x20 unit period */ + REG16 q_wdtmr; /* 0x24 watchdog timer */ + REG16 q_wdprd; /* 0x26 watchdog period */ + REG16 q_decctl; /* 0x28 decoder control */ + REG16 q_epctl; /* 0x2A control register */ + REG16 q_capctl; /* 0x2C capture control */ + REG16 q_posctl; /* 0x2E position compare control */ + REG16 q_eint; /* 0x30 interrupt enable */ + REG16 q_flg; /* 0x32 interrupt flag */ + REG16 q_clr; /* 0x34 interrupt clear */ + REG16 q_frc; /* 0x36 interrupt force */ + REG16 q_epsts; /* 0x38 status */ + REG16 q_ctmr; /* 0x3A capture timer */ + REG16 q_cprd; /* 0x3C capture period */ + REG16 q_ctmrlat; /* 0x3E capture timer latch */ + REG16 q_prdlat; /* 0x40 capture period latch */ + char q_fill1[0x5c-0x40]; + REG32 q_revid; /* 0x5C revision id */ +}; +#endif + + +/* Bits for the QDECTL register */ +#define QSRC1 (1 << 15) +#define QSRC0 (1 << 14) +#define SOEN (1 << 13) +#define SPSEL (1 << 12) +#define XCR (1 << 11) +#define SWAP (1 << 10) +#define IGATE (1 << 9) +#define QAP (1 << 8) +#define QBP (1 << 7) +#define QIP (1 << 6) +#define QSP (1 << 5) + +/* Bits for the QEPCTL register */ +#define FREESOFT1 (1 << 15) +#define FREESOFT0 (1 << 14) +#define PCRM1 (1 << 13) +#define PCRM0 (1 << 12) +#define SEI1 (1 << 11) +#define SEI0 (1 << 10) +#define IEI1 (1 << 9) +#define IEI0 (1 << 8) +#define SWI (1 << 7) +#define SEL (1 << 6) +#define IEL1 (1 << 5) +#define IEL0 (1 << 4) +#define PHEN (1 << 3) +#define QCLM (1 << 2) +#define UTE (1 << 1) +#define WDE (1 << 0) + +/* Bits for the QCAPCTL register */ +#define CEN (1 << 15) +#define CCPS2 (1 << 6) +#define CCPS0 (1 << 5) +#define CCPS1 (1 << 4) +#define UPPS3 (1 << 3) +#define UPPS2 (1 << 2) +#define UPPS1 (1 << 1) +#define UPPS0 (1 << 0) + +/* Bits for the QPOSCTL register */ +#define PCSHDW (1 << 15) +#define PCLOAD (1 << 14) +#define PCPOL (1 << 13) +#define PCE (1 << 12) +#define PCSPW11 (1 << 11) +#define PCSPW10 (1 << 10) +#define PCSPW9 (1 << 9) +#define PCSPW8 (1 << 8) +#define PCSPW7 (1 << 7) +#define PCSPW6 (1 << 6) +#define PCSPW5 (1 << 5) +#define PCSPW4 (1 << 4) +#define PCSPW3 (1 << 3) +#define PCSPW2 (1 << 2) +#define PCSPW1 (1 << 1) +#define PCSPW0 (1 << 0) + +/* Bits for the interrupt registers */ +#define EQEP_INTERRUPT_MASK 0x0FFF +#define UTOF (1 << 11) + +/* Bits to control the clock in the PWMSS subsystem */ +#define PWMSS_EQEPCLK_EN BIT(4) +#define PWMSS_EQEPCLK_STOP_REQ BIT(5) +#define PWMSS_EQEPCLK_EN_ACK BIT(4) + +/* + * Modes for the eQEP unit + * Absolute - the position entry represents the current position of the encoder. + * Poll this value and it will be notified every period nanoseconds + * Relative - the position entry represents the last latched position of the encoder + * This value is latched every period nanoseconds and the internal counter + * is subsequenty reset + */ +#define TIEQEP_MODE_ABSOLUTE 0 +#define TIEQEP_MODE_RELATIVE 1 + +/* Structure defining the characteristics of the eQEP unit */ +struct eqep_chip +{ + /* Platform device for this eQEP unit */ + struct platform_device *pdev; + + /* Pointer to the base of the memory of the eQEP unit */ + void __iomem *mmio_base; + + /* SYSCLKOUT to the eQEP unit */ + u32 clk_rate; + + /* IRQ for the eQEP unit */ + u16 irq; + + /* Mode of the eQEP unit */ + u8 op_mode; + + /* work stuct for the notify userspace work */ + struct work_struct notify_work; + + /* Backup for driver suspension */ + u16 prior_qepctl; + u16 prior_qeint; +}; + +/* Notify userspace work */ +static void notify_handler(struct work_struct *work) +{ + /* Get a reference to the eQEP driver */ + struct eqep_chip *eqep = container_of(work, struct eqep_chip, notify_work); + + /* Notify the userspace */ + sysfs_notify(&eqep->pdev->dev.kobj, NULL, "position"); +} + +/* eQEP Interrupt handler */ +static irqreturn_t eqep_irq_handler(int irq, void *dev_id) +{ + /* Get the instance information */ + struct platform_device *pdev = dev_id; + struct eqep_chip *eqep = platform_get_drvdata(pdev); + + /* Get the interrupt flags */ + u16 iflags = readw(eqep->mmio_base + QFLG) & EQEP_INTERRUPT_MASK; + + /* Check the interrupt source(s) */ + if (iflags & UTOF) { + /* Handle the unit timer overflow interrupt by notifying any potential pollers */ + schedule_work(&eqep->notify_work); + } + + /* Clear interrupt flags (write back triggered flags to the clear register) */ + writew(iflags, eqep->mmio_base + QCLR); + + /* Return that the IRQ was handled successfully */ + return IRQ_HANDLED; +} + +/* Function to read whether the eQEP unit is enabled or disabled */ +static ssize_t eqep_get_enabled(struct device *dev, struct device_attribute *attr, char *buf) +{ + /* Get the instance structure */ + struct eqep_chip *eqep = dev_get_drvdata(dev); + + /* Read the qep control register and mask all but the enabled bit */ + u16 enabled = readw(eqep->mmio_base + QEPCTL) & PHEN; + + /* Return the target in string format */ + return sprintf(buf, "%u\n", (enabled) ? 1 : 0); +} + +/* Function to set if the eQEP is enabled */ +static ssize_t eqep_set_enabled(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + /* Get the instance structure */ + int rc; + u16 val; + u8 enabled; + struct eqep_chip *eqep = dev_get_drvdata(dev); + + /* Convert the input string to an 8 bit uint */ + if ((rc = kstrtou8(buf, 0, &enabled))) + return rc; + + /* Get the existing state of QEPCTL */ + val = readw(eqep->mmio_base + QEPCTL); + + /* If we passed a number that is not 0, enable the eQEP */ + if (enabled) + /* Enable the eQEP (Set PHEN in QEPCTL) */ + val |= PHEN; + else + /* Disable the eQEP (Clear PHEN in QEPCTL) */ + val &= ~PHEN; + + /* Write flags back to control register */ + writew(val, eqep->mmio_base + QEPCTL); + + /* Return buffer length consumed (all) */ + return count; +} + +/* Function to read the current position of the eQEP */ +static ssize_t eqep_get_position(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct eqep_chip *eqep = dev_get_drvdata(dev); + + s32 position = 0; + + if (eqep->op_mode == TIEQEP_MODE_ABSOLUTE) { + position = readl(eqep->mmio_base + QPOSCNT); + } else if (eqep->op_mode == TIEQEP_MODE_RELATIVE) { + /* in relative mode, use the last latched value of the eQEP hardware */ + position = readl(eqep->mmio_base + QPOSLAT); + dev_dbg(dev, "get_position:0x%08x\n", position); + } + + return sprintf(buf, "%d\n", position); +} + +/* Function to set the position of the eQEP hardware */ +static ssize_t eqep_set_position(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + int rc; + s32 position; + struct eqep_chip *eqep = dev_get_drvdata(dev); + + if ((rc = kstrtos32(buf, 0, &position))) + return rc; + + /* + * If we are in absolute mode, set the position of the encoder, + * discard relative mode because thats pointless + */ + if (eqep->op_mode == TIEQEP_MODE_ABSOLUTE) { + /* If absolute mode, set the current value of the eQEP hardware */ + writel(position, eqep->mmio_base + QPOSCNT); + } + + /* Return buffer length consumed (all) */ + return count; +} + +/* Function to read the period of the unit time event timer */ +static ssize_t eqep_get_timer_period(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct eqep_chip *eqep = dev_get_drvdata(dev); + u64 period; + + /* Convert from counts per interrupt back into period_ns */ + period = readl(eqep->mmio_base + QUPRD); + period = period * NSEC_PER_SEC; + do_div(period, eqep->clk_rate); + + /* Otherwise write out the data */ + return sprintf(buf, "%llu\n", period); +} + +/* Function to set the unit timer period. 0 = off, greater than zero sets the period */ +static ssize_t eqep_set_timer_period(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + int rc; + u16 tmp; + u64 period; + + struct eqep_chip *eqep = dev_get_drvdata(dev); + + if ((rc = kstrtou64(buf, 0, &period))) + return rc; + + /* Disable the unit timer before modifying its period register */ + tmp = readw(eqep->mmio_base + QEPCTL); + tmp &= ~(UTE | QCLM); + writew(tmp, eqep->mmio_base + QEPCTL); + + /* Zero the unit timer counter register */ + writel(0, eqep->mmio_base + QUTMR); + + /* If the timer is enabled (a non-zero period has been passed) */ + if (period) { + /* update the period */ + period = period * eqep->clk_rate; + do_div(period, NSEC_PER_SEC); + + dev_dbg(dev, "eqep_set_timer_period:%llu\n", period); + + writel(period, eqep->mmio_base + QUPRD); + + /* Enable unit timer, and latch QPOSLAT to QPOSCNT on timer expiration */ + tmp |= UTE | QCLM; + writew(tmp, eqep->mmio_base + QEPCTL); + } + + return count; +} + +/* Function to read the mode of the eQEP hardware */ +static ssize_t eqep_get_mode(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct eqep_chip *eqep = dev_get_drvdata(dev); + + return sprintf(buf, "%u\n", eqep->op_mode); +} + +/* Function to set the mode of the eQEP hardware */ +static ssize_t eqep_set_mode(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) +{ + int rc; + u16 val; + u8 tmp_mode; + struct eqep_chip *eqep = dev_get_drvdata(dev); + + if ((rc = kstrtou8(buf, 0, &tmp_mode))) + return rc; + + dev_dbg(dev, "eqep_set_mode:%d\n", tmp_mode); + + val = readw(eqep->mmio_base + QEPCTL); + + if (tmp_mode == TIEQEP_MODE_ABSOLUTE) { + /* + * In absolute mode, don't reset the hardware based on time, + * so disable the unit timer position reset (Set PCRM[1:0] = 0) + */ + val &= ~(PCRM1 | PCRM0); + + eqep->op_mode = TIEQEP_MODE_ABSOLUTE; + } else if (tmp_mode == TIEQEP_MODE_RELATIVE) { + /* + * In relative mode, latch the value of the eQEP hardware on the + * overflow of the unit timer. So enable the unit timer position reset + * (Set PCRM[1:0] = 3) + */ + val |= PCRM1 | PCRM0; + + eqep->op_mode = TIEQEP_MODE_RELATIVE; + } + + writew(val, eqep->mmio_base + QEPCTL); + + return count; +} + +/* Bind read/write functions to sysfs entries */ +static DEVICE_ATTR(enabled, 0644, eqep_get_enabled, eqep_set_enabled); +static DEVICE_ATTR(position, 0644, eqep_get_position, eqep_set_position); +static DEVICE_ATTR(period, 0644, eqep_get_timer_period, eqep_set_timer_period); +static DEVICE_ATTR(mode, 0644, eqep_get_mode, eqep_set_mode); + +/* Array holding all of the sysfs entries */ +static const struct attribute *eqep_attrs[] = { + &dev_attr_enabled.attr, + &dev_attr_position.attr, + &dev_attr_period.attr, + &dev_attr_mode.attr, + NULL, +}; + +/* Driver function group */ +static const struct attribute_group eqep_device_attr_group = { + .attrs = (struct attribute **) eqep_attrs, +}; + +/* Driver compatibility list */ +static struct of_device_id eqep_of_match[] = +{ + { .compatible = "ti,am33xx-eqep" }, + { } +}; + +/* Register our compatibilities for device trees */ +MODULE_DEVICE_TABLE(of, eqep_of_match); + +/* Create an instance of the eQEP driver */ +static int eqep_probe(struct platform_device *pdev) +{ + struct resource *r; + struct clk *clk; + struct eqep_chip *eqep; + struct pinctrl *pinctrl; + + u64 period; + u16 status; + u32 value; + + dev_info(&pdev->dev, "ver. 1.0\n"); + + /* Select pins provided through the device tree */ + pinctrl = devm_pinctrl_get_select_default(&pdev->dev); + if (IS_ERR(pinctrl)) + { + dev_warn(&pdev->dev, "unable to select pin group\n"); + } + + /* Allocate a eqep_driver object */ + eqep = devm_kzalloc(&pdev->dev, sizeof(struct eqep_chip), GFP_KERNEL); + if (!eqep) { + dev_err(&pdev->dev, "failed to allocate memory\n"); + return -ENOMEM; + } + + /* Get a handle to the system clock object */ + clk = devm_clk_get(pdev->dev.parent, "fck"); + if (IS_ERR(clk)) { + dev_err(&pdev->dev, "failed to get clock\n"); + return PTR_ERR(clk); + } + + /* Get the frequency of the system clock */ + eqep->clk_rate = clk_get_rate(clk); + if (!eqep->clk_rate) { + dev_err(&pdev->dev, "failed to get clock rate\n"); + return -EINVAL; + } + + /* Get a resource containing the IRQ for this eQEP controller */ + r = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (unlikely(!r)) { + dev_err(&pdev->dev, "Invalid IRQ resource\n"); + return -ENODEV; + } + + /* Store the irq */ + eqep->irq = r->start; + + /* Get a resource containing the requested (from DT) memory address and range of eQEP controller */ + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!r) { + dev_err(&pdev->dev, "no memory resource defined\n"); + return -ENODEV; + } + + /* Remap the eQEP controller memory into our own memory space */ + eqep->mmio_base = devm_ioremap_resource(&pdev->dev, r); + if (IS_ERR(eqep->mmio_base)) + return PTR_ERR(eqep->mmio_base); + + /* Store the platform device in our eQEP data structure for later usage */ + eqep->pdev = pdev; + + /* Subscribe to the eQEP interrupt */ + if (request_irq(eqep->irq, eqep_irq_handler, IRQF_IRQPOLL, "eqep_interrupt", pdev)) + { + dev_err(&pdev->dev, "unable to request irq for eQEP\n"); + return -ENODEV; + } + + /* Register controls to sysfs */ + if (sysfs_create_group(&pdev->dev.kobj, &eqep_device_attr_group)) + { + dev_err(&pdev->dev, "sysfs creation failed\n"); + return -EINVAL; + } + + /* set QDECCTL */ + status = 0; /* default to Quadrature count mode, QSRC1 & QSRC0 = 0 */ + + /* set QSRC1 & QSRC0 bits, one of 4 count_modes. */ + if (!of_property_read_u32(pdev->dev.of_node, "count_mode", &value) && value <= 3) { + status |= value << 14; + + /* + * in count up or count down mode, count on rising edge only + * not on both edges. + */ + if (value >= 2) + status |= XCR; + } + dev_info(&pdev->dev, "count_mode:%d\n", value); + + /* Should we invert the qa input */ + if (!of_property_read_u32(pdev->dev.of_node, "invert_qa", &value)) + status = value ? status | QAP : status & ~QAP; + dev_info(&pdev->dev, "invert_qa:%d\n", value); + + /* Should we invert the qb input */ + if (!of_property_read_u32(pdev->dev.of_node, "invert_qb", &value)) + status = value ? status | QBP : status & ~QBP; + dev_info(&pdev->dev, "invert_qb:%d\n", value); + + /* Should we invert the index input */ + if (!of_property_read_u32(pdev->dev.of_node, "invert_qi", &value)) + status = value ? status | QIP : status & ~QIP; + dev_info(&pdev->dev, "invert_qi:%d\n", value); + + /* Should we invert the strobe input */ + if (!of_property_read_u32(pdev->dev.of_node, "invert_qs", &value)) + status = value ? status | QSP : status & ~QSP; + dev_info(&pdev->dev, "invert_qs:%d\n", value); + + /* Should we swap the cha and chb inputs */ + if (!of_property_read_u32(pdev->dev.of_node, "swap_inputs", &value)) + status = value ? status | SWAP : status & ~SWAP; + dev_info(&pdev->dev, "swap_inputs:%d\n", value); + + dev_info(&pdev->dev, "QDECCTL:0x%04x\n", status); + + /* Write the decoder control settings back to the control register */ + writew(status, eqep->mmio_base + QDECCTL); + + writel( 0, eqep->mmio_base + QPOSINIT); + writel(~0, eqep->mmio_base + QPOSMAX); + writel( 0, eqep->mmio_base + QPOSCNT); + + dev_info(&pdev->dev, "QPOSINIT:0x%08x\n", readl(eqep->mmio_base + QPOSINIT)); + dev_info(&pdev->dev, "QPOSMAX:0x%08x\n", readl(eqep->mmio_base + QPOSMAX)); + dev_info(&pdev->dev, "QPOSCNT:0x%08x\n", readl(eqep->mmio_base + QPOSCNT)); + + status = UTOF; /* Enable Unit Time Period interrupt. */ + if (!of_property_read_u32(pdev->dev.of_node, "omit_interrupt", &value) && value) { + status = 0; /* no interrupt */ + } + writew(status, eqep->mmio_base + QEINT); + dev_info(&pdev->dev, "omit_interrupt:%d\n", value); + dev_info(&pdev->dev, "QEINT:0x%04x\n", status); + + /* Calculate the timer ticks per second */ + period = 1000000000; + period = period * eqep->clk_rate; + do_div(period, NSEC_PER_SEC); + + /* Set this period into the unit timer period register */ + writel(period, eqep->mmio_base + QUPRD); + dev_info(&pdev->dev, "QUPRD:0x%08x\n", (u32) period); + + /* + * Enable the eQEP with basic position counting turned on + * PHEN - Quadrature position counter enable bit + * UTE - unit timer enable + * QCLM - latch QPOSLAT to QPOSCNT upon unit timer overflow + * IEL0 - Latch QPOSILAT on index signal. Rising or falling, IEL[1:0] = 0 is reserved + * SWI - Software initialization of position count register, i.e. set QPOSCNT <= QPOSINIT, + * but this bit was not being reset by hardware as advertised in TRM, + * (so omit & clear QPOSCNT manually elsewhere?) + */ + status = PHEN | UTE | QCLM | IEL0 | SWI; + writew(status, eqep->mmio_base + QEPCTL); + dev_info(&pdev->dev, "QEPCTL:0x%04x write\n", status); + dev_info(&pdev->dev, "QEPCTL:0x%04x read\n", readw(eqep->mmio_base + QEPCTL)); + + /* We default to absolute mode */ + eqep->op_mode = TIEQEP_MODE_ABSOLUTE; + + /* Enable the power management runtime */ + pm_runtime_enable(&pdev->dev); + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(&pdev->dev); + + /* Initialize the notify work struture */ + INIT_WORK(&eqep->notify_work, notify_handler); + + /* Decrement the device usage count (twice) and run pm_runtime_idle() if zero */ + pm_runtime_put_sync(&pdev->dev); + + /* Set the platform driver data to the data object we've been creating for the eQEP unit */ + platform_set_drvdata(pdev, eqep); + + /* Success! */ + dev_info(&pdev->dev, "irq:%d, clk_rate:%u\n", eqep->irq, eqep->clk_rate); + return 0; +} + +/* Remove an instance of the eQEP driver */ +static int eqep_remove(struct platform_device *pdev) +{ + /* Get the eQEP driver data from the platform device structure */ + struct eqep_chip *eqep = platform_get_drvdata(pdev); + + /* Cancel work */ + cancel_work_sync(&eqep->notify_work); + + /* Unmap from sysfs */ + sysfs_remove_group(&pdev->dev.kobj, &eqep_device_attr_group); + + /* Release important assets */ + free_irq(eqep->irq, pdev); + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(&pdev->dev); + + /* Decrement the device usage count (twice) and run pm_runtime_idle() if zero */ + pm_runtime_put_sync(&pdev->dev); + pm_runtime_put_sync(&pdev->dev); + + /* Disable the runtime power management of this device */ + pm_runtime_disable(&pdev->dev); + + /* Return success */ + return 0; +} + +/* Power management suspend device */ +static int eqep_suspend(struct device *dev) +{ + /* Get the eqep driver information */ + struct eqep_chip *eqep = dev_get_drvdata(dev); + u16 tmp; + + /* Shut down interrupts */ + eqep->prior_qeint = readw(eqep->mmio_base + QEINT); + tmp = eqep->prior_qeint & ~UTOF; + writew(tmp, eqep->mmio_base + QEINT); + + /* Get the existing state of QEPCTL */ + eqep->prior_qepctl = readw(eqep->mmio_base + QEPCTL); + + /* Disable eQEP controller */ + writew(eqep->prior_qepctl & ~PHEN, eqep->mmio_base + QEPCTL); + + /* Decrement the device usage count and run pm_runtime_idle() if zero */ + pm_runtime_put_sync(dev); + + /* Return success */ + return 0; +} + +/* Power management wake device back up */ +static int eqep_resume(struct device *dev) +{ + /* Get the eqep driver information */ + struct eqep_chip *eqep = dev_get_drvdata(dev); + + /* Restore interrupt enabled register */ + writew(eqep->prior_qeint, eqep->mmio_base + QEINT); + + /* Restore prior qep control register */ + writew(eqep->prior_qepctl, eqep->mmio_base + QEPCTL); + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); + + /* Success */ + return 0; +} + +/* create pm functions object */ +static SIMPLE_DEV_PM_OPS(eqep_pm_ops, eqep_suspend, eqep_resume); + +/* Platform driver information */ +static struct platform_driver eqep_driver = { + .driver = { + .name = "eqep", + .owner = THIS_MODULE, + .pm = &eqep_pm_ops, + .of_match_table = eqep_of_match, + }, + .probe = eqep_probe, + .remove = eqep_remove, +}; + +/* Register this platform driver */ +module_platform_driver(eqep_driver); + +/* Module information */ +MODULE_DESCRIPTION("TI eQEP driver"); +MODULE_AUTHOR("Nathaniel R. Lewis"); +MODULE_LICENSE("GPL"); From 3f754da24d0d67484337510a8580801be9aa1fec Mon Sep 17 00:00:00 2001 From: Drew Fustini Date: Thu, 2 Feb 2017 05:19:11 -0600 Subject: [PATCH 185/213] tieqep: fix unhandled fault on eQEP register access Call pm_runtime_get_sync() at the beginning of any functions that will read or write to the memory mapped eQEP registers. This is to ensure that the eQEP peripheral is running and its clock is enabled. Before this patch, an attempt to read the position file via sysfs would results in a segmentation fault. The kernel log would be contain this error: [ 2591.653471] Unhandled fault: external abort on non-linefetch (0x1028) at 0xfa304180 [ 2591.915165] [] (eqep_get_position [tieqep]) from [] (dev_attr_show+0x2c/0x58) More details: https://gist.github.com/pdp7/fe07082d23f2bfbc362c733a7b0aea72 BeagleBoard mailing list thread: https://groups.google.com/d/msg/beagleboard/_TdTH7oPEXE/MNvU-mY6DgAJ --- drivers/misc/tieqep.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/misc/tieqep.c b/drivers/misc/tieqep.c index d2628847bb8e3..bb69ad4468e26 100644 --- a/drivers/misc/tieqep.c +++ b/drivers/misc/tieqep.c @@ -241,9 +241,13 @@ static ssize_t eqep_get_enabled(struct device *dev, struct device_attribute *att { /* Get the instance structure */ struct eqep_chip *eqep = dev_get_drvdata(dev); + u16 enabled = 0; + + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); /* Read the qep control register and mask all but the enabled bit */ - u16 enabled = readw(eqep->mmio_base + QEPCTL) & PHEN; + enabled = readw(eqep->mmio_base + QEPCTL) & PHEN; /* Return the target in string format */ return sprintf(buf, "%u\n", (enabled) ? 1 : 0); @@ -262,6 +266,8 @@ static ssize_t eqep_set_enabled(struct device *dev, struct device_attribute *att if ((rc = kstrtou8(buf, 0, &enabled))) return rc; + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); /* Get the existing state of QEPCTL */ val = readw(eqep->mmio_base + QEPCTL); @@ -286,6 +292,8 @@ static ssize_t eqep_get_position(struct device *dev, struct device_attribute *at struct eqep_chip *eqep = dev_get_drvdata(dev); s32 position = 0; + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); if (eqep->op_mode == TIEQEP_MODE_ABSOLUTE) { position = readl(eqep->mmio_base + QPOSCNT); @@ -308,6 +316,8 @@ static ssize_t eqep_set_position(struct device *dev, struct device_attribute *at if ((rc = kstrtos32(buf, 0, &position))) return rc; + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); /* * If we are in absolute mode, set the position of the encoder, * discard relative mode because thats pointless @@ -327,6 +337,8 @@ static ssize_t eqep_get_timer_period(struct device *dev, struct device_attribute struct eqep_chip *eqep = dev_get_drvdata(dev); u64 period; + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); /* Convert from counts per interrupt back into period_ns */ period = readl(eqep->mmio_base + QUPRD); period = period * NSEC_PER_SEC; @@ -348,6 +360,8 @@ static ssize_t eqep_set_timer_period(struct device *dev, struct device_attribute if ((rc = kstrtou64(buf, 0, &period))) return rc; + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); /* Disable the unit timer before modifying its period register */ tmp = readw(eqep->mmio_base + QEPCTL); tmp &= ~(UTE | QCLM); @@ -395,6 +409,8 @@ static ssize_t eqep_set_mode(struct device *dev, struct device_attribute *attr, dev_dbg(dev, "eqep_set_mode:%d\n", tmp_mode); + /* Increment the device usage count and run pm_runtime_resume() */ + pm_runtime_get_sync(dev); val = readw(eqep->mmio_base + QEPCTL); if (tmp_mode == TIEQEP_MODE_ABSOLUTE) { From 190a05afcef95cc47ac197d7c5a6e3cff8022254 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Fri, 30 Oct 2015 11:14:50 -0500 Subject: [PATCH 186/213] ARM: samples seccomp no -m32 Signed-off-by: Robert Nelson --- samples/seccomp/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/seccomp/Makefile b/samples/seccomp/Makefile index cf34ff6b40657..4c2038bbc7180 100644 --- a/samples/seccomp/Makefile +++ b/samples/seccomp/Makefile @@ -18,6 +18,7 @@ bpf-direct-objs := bpf-direct.o # Try to match the kernel target. ifndef CONFIG_64BIT +ifndef CONFIG_ARM # s390 has -m31 flag to build 31 bit binaries ifndef CONFIG_S390 @@ -36,3 +37,4 @@ HOSTLDLIBS_dropper += $(MFLAG) endif always := $(hostprogs-m) endif +endif From af709abf2d2695f2e738b785ff2551d314167eb1 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 16 Jul 2015 17:24:57 -0500 Subject: [PATCH 187/213] quiet: 8250_omap.c use pr_info over pr_err Signed-off-by: Robert Nelson --- drivers/tty/serial/8250/8250_omap.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 979e4c861a6b7..0c6e5cfcadd06 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -1616,10 +1616,10 @@ static int __init omap8250_console_fixup(void) } add_preferred_console("ttyS", idx, options); - pr_err("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n", + pr_info("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n", idx, idx); - pr_err("This ensures that you still see kernel messages. Please\n"); - pr_err("update your kernel commandline.\n"); + pr_info("This ensures that you still see kernel messages. Please\n"); + pr_info("update your kernel commandline.\n"); return 0; } console_initcall(omap8250_console_fixup); From 4c9af45aaffa70c78c4c62378e89454431b19b4c Mon Sep 17 00:00:00 2001 From: "bigguiness@gmail.com" Date: Wed, 3 Jan 2018 15:34:07 -0800 Subject: [PATCH 188/213] ti_am335x_tsc.c driver ------=_Part_422_1349561576.1515022447432 Content-Type: text/plain; charset="UTF-8" Hello all, The TI touch screen driver does not work _right_ with the libts-bin package in the jessie image. $ cat /etc/dogtag BeagleBoard.org Debian Image 2018-01-01 $ lsb_release -a No LSB modules are available. Distributor ID: Debian Description: Debian GNU/Linux 8.10 (jessie) Release: 8.10 Codename: jessie $ dpkg -l | grep libts-bin ii libts-bin 1.14-1rcnee0~jessie+20171122 armhf touch screen library utilities $ sudo ts_calibrate ts_setup: No such file or directory It is possible to make it work by setting the TSLIB_TSDEVICE environment variable: $ sudo su # export TSLIB_TSDEVICE=/dev/input/event2 # ts_calibrate But, that's a bit of a pain since the environment variable always needs to be set in order to use the touchscreen. It appears that this version of the utilities uses the INPUT_PROP_DIRECT propbit to automatically detect which /dev/input/event device is the touchscreen. It looks like the following is the only change needed to make it work. Unfortunately, I don't have currently have a way to build a custom kernel for the BeagleBone in order to test it. If there is anyone that could I would appreciate it. Regards, Hartley --- drivers/input/touchscreen/ti_am335x_tsc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c index 9e8684ab48f4f..87807b99d7845 100644 --- a/drivers/input/touchscreen/ti_am335x_tsc.c +++ b/drivers/input/touchscreen/ti_am335x_tsc.c @@ -459,6 +459,7 @@ static int titsc_probe(struct platform_device *pdev) input_dev->name = "ti-tsc"; input_dev->dev.parent = &pdev->dev; + __set_bit(INPUT_PROP_DIRECT, input_dev->propbit); input_dev->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS); input_dev->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH); From addd29613ed7922b668ca3fda81b29248720fb83 Mon Sep 17 00:00:00 2001 From: frost Date: Fri, 12 May 2017 00:22:29 +0800 Subject: [PATCH 189/213] ti_am335x_tsc: correct formula code to calculate pressure; fix touchscreen jitter problem --- drivers/input/touchscreen/ti_am335x_tsc.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/input/touchscreen/ti_am335x_tsc.c b/drivers/input/touchscreen/ti_am335x_tsc.c index 87807b99d7845..d921c35b9afba 100644 --- a/drivers/input/touchscreen/ti_am335x_tsc.c +++ b/drivers/input/touchscreen/ti_am335x_tsc.c @@ -34,6 +34,7 @@ #define ADCFSM_STEPID 0x10 #define SEQ_SETTLE 275 #define MAX_12BIT ((1 << 12) - 1) +#define PRESSURE_MAX 1000 #define TSC_IRQENB_MASK (IRQENB_FIFO0THRES | IRQENB_EOS | IRQENB_HW_PEN) @@ -234,6 +235,7 @@ static void titsc_read_coordinates(struct titsc *ts_dev, for (i = 0; i < creads; i++) { xvals[i] = titsc_readl(ts_dev, REG_FIFO0); xvals[i] &= 0xfff; + pr_debug("i %d xval %d yval %d z1 %d z2 %d\n", i, xvals[i], yvals[i], *z1, *z2); } /* @@ -312,13 +314,13 @@ static irqreturn_t titsc_irq(int irq, void *dev) * Resistance(touch) = x plate resistance * * x postion/4096 * ((z2 / z1) - 1) */ - z = z1 - z2; + z = z2 - z1; z *= x; z *= ts_dev->x_plate_resistance; - z /= z2; + z /= z1; z = (z + 2047) >> 12; - - if (z <= MAX_12BIT) { + pr_debug("x %d y %d z1 %d z2 %d z %d\n", x, y, z1, z2, z); + if (z <= PRESSURE_MAX) { input_report_abs(input_dev, ABS_X, x); input_report_abs(input_dev, ABS_Y, y); input_report_abs(input_dev, ABS_PRESSURE, z); From 9186691d0408b3aa623f1ed0f42ca70d2b842eb1 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 29 Jan 2018 13:43:56 -0600 Subject: [PATCH 190/213] BeagleBone pinmux helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authors: Pantelis Antoniou Charles Steinkuehler Jason Kridner Robert Nelson Tobias Müller Matthijs van Duin This patch was derived from 19 commits: https://github.com/RobertCNelson/linux-dev/tree/35e301ae8436e9f56f65bf1a7440021eda42f948/patches/drivers/ti/gpio Signed-off-by: Robert Nelson --- drivers/gpio/Kconfig | 14 + drivers/gpio/Makefile | 1 + drivers/gpio/gpio-of-helper.c | 435 ++++++++++++++++++ drivers/gpio/gpiolib-sysfs.c | 35 +- drivers/misc/Kconfig | 1 + drivers/misc/Makefile | 1 + drivers/misc/cape/Kconfig | 5 + drivers/misc/cape/Makefile | 5 + drivers/misc/cape/beaglebone/Kconfig | 10 + drivers/misc/cape/beaglebone/Makefile | 5 + .../misc/cape/beaglebone/bone-pinmux-helper.c | 242 ++++++++++ 11 files changed, 751 insertions(+), 3 deletions(-) create mode 100644 drivers/gpio/gpio-of-helper.c create mode 100644 drivers/misc/cape/Kconfig create mode 100644 drivers/misc/cape/Makefile create mode 100644 drivers/misc/cape/beaglebone/Kconfig create mode 100644 drivers/misc/cape/beaglebone/Makefile create mode 100644 drivers/misc/cape/beaglebone/bone-pinmux-helper.c diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 3bed235a38f39..ce5ad09a9e6c4 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -73,6 +73,20 @@ config GPIO_SYSFS Kernel drivers may also request that a particular GPIO be exported to userspace; this can be useful when debugging. +config GPIO_OF_HELPER + bool "GPIO OF helper device (EXPERIMENTAL)" + depends on OF_GPIO + help + Say Y here to add an GPIO OF helper driver + + Allows you specify a GPIO helper based on OF + which allows simple export of GPIO functionality + in user-space. + + Features include, value set/get, direction control, + interrupt/value change poll support, event counting + and others. + config GPIO_GENERIC depends on HAS_IOMEM # Only for IOMEM drivers tristate diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index c256aff66a656..2c11fc095c673 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_GPIOLIB) += gpiolib-devprop.o obj-$(CONFIG_OF_GPIO) += gpiolib-of.o obj-$(CONFIG_GPIO_SYSFS) += gpiolib-sysfs.o obj-$(CONFIG_GPIO_ACPI) += gpiolib-acpi.o +obj-$(CONFIG_GPIO_OF_HELPER) += gpio-of-helper.o # Device drivers. Generally keep list sorted alphabetically obj-$(CONFIG_GPIO_GENERIC) += gpio-generic.o diff --git a/drivers/gpio/gpio-of-helper.c b/drivers/gpio/gpio-of-helper.c new file mode 100644 index 0000000000000..83f362f7da5ff --- /dev/null +++ b/drivers/gpio/gpio-of-helper.c @@ -0,0 +1,435 @@ +/* + * GPIO OF based helper + * + * A simple DT based driver to provide access to GPIO functionality + * to user-space via sysfs. + * + * Copyright (C) 2013 Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* fwd decl. */ +struct gpio_of_helper_info; + +enum gpio_type { + GPIO_TYPE_INPUT = 0, + GPIO_TYPE_OUTPUT = 1, +}; + +struct gpio_of_entry { + int id; + struct gpio_of_helper_info *info; + struct device_node *node; + enum gpio_type type; + int gpio; + int irq; + const char *name; + atomic64_t counter; + unsigned int count_flags; +#define COUNT_RISING_EDGE (1 << 0) +#define COUNT_FALLING_EDGE (1 << 1) +}; + +struct gpio_of_helper_info { + struct platform_device *pdev; + struct idr idr; +}; + +static const struct of_device_id gpio_of_helper_of_match[] = { + { + .compatible = "gpio-of-helper", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, gpio_of_helper_of_match); + +static ssize_t gpio_of_helper_show_status(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct platform_device *pdev = to_platform_device(dev); + struct gpio_of_helper_info *info = platform_get_drvdata(pdev); + struct gpio_of_entry *entry; + char *p, *e; + int id, n; + + p = buf; + e = p + PAGE_SIZE; + n = 0; + idr_for_each_entry(&info->idr, entry, id) { + switch (entry->type) { + case GPIO_TYPE_INPUT: + n = snprintf(p, e - p, "%2d %-24s %3d %-3s %llu\n", + entry->id, entry->name, entry->gpio, "IN", + (unsigned long long) + atomic64_read(&entry->counter)); + break; + case GPIO_TYPE_OUTPUT: + n = snprintf(p, e - p, "%2d %-24s %3d %-3s\n", + entry->id, entry->name, entry->gpio, "OUT"); + break; + } + p += n; + } + + return p - buf; +} + +static DEVICE_ATTR(status, S_IRUGO, + gpio_of_helper_show_status, NULL); + +static irqreturn_t gpio_of_helper_handler(int irq, void *ptr) +{ + struct gpio_of_entry *entry = ptr; + + /* caution - low speed interfaces only! */ + atomic64_inc(&entry->counter); + + return IRQ_HANDLED; +} + +static struct gpio_of_entry * +gpio_of_entry_create(struct gpio_of_helper_info *info, + struct device_node *node) +{ + struct platform_device *pdev = info->pdev; + struct device *dev = &pdev->dev; + struct gpio_of_entry *entry; + int err, gpio, irq; + unsigned int req_flags, count_flags, irq_flags; + enum gpio_type type; + enum of_gpio_flags gpio_flags; + const char *name; + + /* get the type of the node first */ + if (of_property_read_bool(node, "input")) + type = GPIO_TYPE_INPUT; + else if (of_property_read_bool(node, "output") + || of_property_read_bool(node, "init-low") + || of_property_read_bool(node, "init-high")) + type = GPIO_TYPE_OUTPUT; + else { + dev_err(dev, "Not valid gpio node type\n"); + err = -EINVAL; + goto err_bad_node; + } + + /* get the name */ + if (of_property_read_string(node, "line-name", &name)) + if (of_property_read_string(node, "gpio-name", &name)) + name = node->name; + + err = of_get_named_gpio_flags(node, "gpio", 0, &gpio_flags); + if (IS_ERR_VALUE(err)) { + dev_err(dev, "Failed to get gpio property of '%s'\n", name); + goto err_bad_node; + } + gpio = err; + + req_flags = 0; + count_flags = 0; + + /* set the request flags */ + switch (type) { + case GPIO_TYPE_INPUT: + req_flags = GPIOF_DIR_IN | GPIOF_EXPORT; + if (of_property_read_bool(node, "count-falling-edge")) + count_flags |= COUNT_FALLING_EDGE; + if (of_property_read_bool(node, "count-rising-edge")) + count_flags |= COUNT_RISING_EDGE; + break; + case GPIO_TYPE_OUTPUT: + req_flags = GPIOF_DIR_OUT | GPIOF_EXPORT; + if (of_property_read_bool(node, "init-high")) + req_flags |= GPIOF_OUT_INIT_HIGH; + else if (of_property_read_bool(node, "init-low")) + req_flags |= GPIOF_OUT_INIT_LOW; + break; + } + if (of_property_read_bool(node, "dir-changeable")) + req_flags |= GPIOF_EXPORT_CHANGEABLE; + if (gpio_flags & OF_GPIO_ACTIVE_LOW) + req_flags |= GPIOF_ACTIVE_LOW; + if (gpio_flags & OF_GPIO_SINGLE_ENDED) { + if (gpio_flags & OF_GPIO_ACTIVE_LOW) + req_flags |= GPIOF_OPEN_DRAIN; + else + req_flags |= GPIOF_OPEN_SOURCE; + } + + /* request the gpio */ + err = devm_gpio_request_one(dev, gpio, req_flags, name); + if (err != 0) { + dev_err(dev, "Failed to request gpio '%s'\n", name); + goto err_bad_node; + } + + irq = -1; + irq_flags = 0; + + /* counter mode requested - need an interrupt */ + if (count_flags != 0) { + irq = gpio_to_irq(gpio); + if (IS_ERR_VALUE(irq)) { + dev_err(dev, "Failed to request gpio '%s'\n", name); + goto err_bad_node; + } + + if (count_flags & COUNT_RISING_EDGE) + irq_flags |= IRQF_TRIGGER_RISING; + if (count_flags & COUNT_FALLING_EDGE) + irq_flags |= IRQF_TRIGGER_FALLING; + } + +// if (!idr_pre_get(&info->idr, GFP_KERNEL)) { +// dev_err(dev, "Failed on idr_pre_get of '%s'\n", name); +// err = -ENOMEM; +// goto err_no_mem; +// } + + idr_preload(GFP_KERNEL); + + entry = devm_kzalloc(dev, sizeof(*entry), GFP_KERNEL); + if (entry == NULL) { + dev_err(dev, "Failed to allocate gpio entry of '%s'\n", name); + err = -ENOMEM; + goto err_no_mem; + } + + entry->id = -1; + entry->info = info; + entry->node = of_node_get(node); /* get node reference */ + entry->type = type; + entry->gpio = gpio; + entry->irq = irq; + entry->name = name; + + /* interrupt enable is last thing done */ + if (irq >= 0) { + atomic64_set(&entry->counter, 0); + entry->count_flags = count_flags; + err = devm_request_irq(dev, irq, gpio_of_helper_handler, + irq_flags, name, entry); + if (err != 0) { + dev_err(dev, "Failed to request irq of '%s'\n", name); + goto err_no_irq; + } + } + + /* all done; insert */ +// err = idr_get_new(&info->idr, entry, &entry->id); +// if (IS_ERR_VALUE(err)) { +// dev_err(dev, "Failed to idr_get_new of '%s'\n", name); +// goto err_fail_idr; +// } + + err = idr_alloc(&info->idr, entry, 0, 0, GFP_NOWAIT); + if (err >= 0) + entry->id = err; + + idr_preload_end(); + + if (err < 0) { + dev_err(dev, "Failed to idr_get_new of '%s'\n", name); + goto err_fail_idr; + } + + dev_dbg(dev, "Allocated GPIO id=%d name='%s'\n", entry->id, name); + + return entry; + +err_fail_idr: + /* nothing to do */ +err_no_irq: + /* release node ref */ + of_node_put(node); + /* nothing else needs to be done, devres handles it */ +err_no_mem: +err_bad_node: + return ERR_PTR(err); +} + +static int gpio_of_entry_destroy(struct gpio_of_entry *entry) +{ + struct gpio_of_helper_info *info = entry->info; + struct platform_device *pdev = info->pdev; + struct device *dev = &pdev->dev; + + dev_dbg(dev, "Destroying GPIO id=%d\n", entry->id); + + /* remove from the IDR */ + idr_remove(&info->idr, entry->id); + + /* remove node ref */ + of_node_put(entry->node); + + /* free gpio */ + devm_gpio_free(dev, entry->gpio); + + /* gree irq */ + if (entry->irq >= 0) + devm_free_irq(dev, entry->irq, entry); + + /* and free */ + devm_kfree(dev, entry); + + return 0; +} + +static int gpio_of_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct gpio_of_helper_info *info; + struct gpio_of_entry *entry; + struct device_node *pnode = pdev->dev.of_node; + struct device_node *cnode; + struct pinctrl *pinctrl; + int err; + + /* we only support OF */ + if (pnode == NULL) { + dev_err(&pdev->dev, "No platform of_node!\n"); + return -ENODEV; + } + + pinctrl = devm_pinctrl_get_select_default(&pdev->dev); + if (IS_ERR(pinctrl)) { + /* special handling for probe defer */ + if (PTR_ERR(pinctrl) == -EPROBE_DEFER) + return -EPROBE_DEFER; + + dev_warn(&pdev->dev, + "pins are not configured from the driver\n"); + } + + info = devm_kzalloc(&pdev->dev, sizeof(*info), GFP_KERNEL); + if (info == NULL) { + dev_err(&pdev->dev, "Failed to allocate info\n"); + err = -ENOMEM; + goto err_no_mem; + } + platform_set_drvdata(pdev, info); + info->pdev = pdev; + + idr_init(&info->idr); + + err = device_create_file(dev, &dev_attr_status); + if (err != 0) { + dev_err(dev, "Failed to create status sysfs attribute\n"); + goto err_no_sysfs; + } + + for_each_child_of_node(pnode, cnode) { + + entry = gpio_of_entry_create(info, cnode); + if (IS_ERR_OR_NULL(entry)) { + dev_err(dev, "Failed to create gpio entry\n"); + err = PTR_ERR(entry); + goto err_fail_entry; + } + } + + dev_info(&pdev->dev, "ready\n"); + + return 0; +err_fail_entry: + device_remove_file(&pdev->dev, &dev_attr_status); +err_no_sysfs: +err_no_mem: + return err; +} + +static int gpio_of_helper_remove(struct platform_device *pdev) +{ + struct gpio_of_helper_info *info = platform_get_drvdata(pdev); + struct gpio_of_entry *entry; + int id; + + dev_info(&pdev->dev, "removing\n"); + + device_remove_file(&pdev->dev, &dev_attr_status); + + id = 0; + idr_for_each_entry(&info->idr, entry, id) { + /* destroy each and every one */ + gpio_of_entry_destroy(entry); + } + + return 0; +} + +#ifdef CONFIG_PM +//#ifdef CONFIG_PM_RUNTIME +static int gpio_of_helper_runtime_suspend(struct device *dev) +{ + /* place holder */ + return 0; +} + +static int gpio_of_helper_runtime_resume(struct device *dev) +{ + /* place holder */ + return 0; +} +//#endif /* CONFIG_PM_RUNTIME */ + +static struct dev_pm_ops gpio_of_helper_pm_ops = { + SET_RUNTIME_PM_OPS(gpio_of_helper_runtime_suspend, + gpio_of_helper_runtime_resume, NULL) +}; +#define GPIO_OF_HELPER_PM_OPS (&gpio_of_helper_pm_ops) +#else +#define GPIO_OF_HELPER_PM_OPS NULL +#endif /* CONFIG_PM */ + +struct platform_driver gpio_of_helper_driver = { + .probe = gpio_of_helper_probe, + .remove = gpio_of_helper_remove, + .driver = { + .name = "gpio-of-helper", + .owner = THIS_MODULE, + .pm = GPIO_OF_HELPER_PM_OPS, + .of_match_table = gpio_of_helper_of_match, + }, +}; + +module_platform_driver(gpio_of_helper_driver); + +MODULE_AUTHOR("Pantelis Antoniou "); +MODULE_DESCRIPTION("GPIO OF Helper driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:gpio-of-helper"); diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index 3dbaf489a8a52..7015730d84f99 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -37,10 +37,10 @@ static DEFINE_MUTEX(sysfs_lock); /* * /sys/class/gpio/gpioN... only for GPIOs that are exported * /direction - * * MAY BE OMITTED if kernel won't allow direction changes * * is read/write as "in" or "out" * * may also be written as "high" or "low", initializing * output value as specified ("out" implies "low") + * * read-only if kernel won't allow direction changes * /value * * always readable, subject to hardware behavior * * may be writable, as zero/nonzero @@ -53,6 +53,8 @@ static DEFINE_MUTEX(sysfs_lock); * * is read/write as zero/nonzero * * also affects existing and subsequent "falling" and "rising" * /edge configuration + * /label + * * descriptor label */ static ssize_t direction_show(struct device *dev, @@ -83,7 +85,9 @@ static ssize_t direction_store(struct device *dev, mutex_lock(&data->mutex); - if (sysfs_streq(buf, "high")) + if (!data->direction_can_change) + status = -EPERM; + else if (sysfs_streq(buf, "high")) status = gpiod_direction_output_raw(desc, 1); else if (sysfs_streq(buf, "out") || sysfs_streq(buf, "low")) status = gpiod_direction_output_raw(desc, 0); @@ -362,6 +366,23 @@ static ssize_t active_low_store(struct device *dev, } static DEVICE_ATTR_RW(active_low); +static ssize_t label_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct gpiod_data *data = dev_get_drvdata(dev); + struct gpio_desc *desc = data->desc; + ssize_t status; + + mutex_lock(&data->mutex); + + status = sprintf(buf, "%s\n", desc->label); + + mutex_unlock(&data->mutex); + + return status; +} +static DEVICE_ATTR_RO(label); + static umode_t gpio_is_visible(struct kobject *kobj, struct attribute *attr, int n) { @@ -373,12 +394,15 @@ static umode_t gpio_is_visible(struct kobject *kobj, struct attribute *attr, if (attr == &dev_attr_direction.attr) { if (!show_direction) - mode = 0; + mode &= 0444; } else if (attr == &dev_attr_edge.attr) { if (gpiod_to_irq(desc) < 0) mode = 0; if (!show_direction && test_bit(FLAG_IS_OUT, &desc->flags)) mode = 0; + } else if (attr == &dev_attr_value.attr) { + if (!show_direction && !test_bit(FLAG_IS_OUT, &desc->flags)) + mode &= 0444; } return mode; @@ -389,6 +413,7 @@ static struct attribute *gpio_attrs[] = { &dev_attr_edge.attr, &dev_attr_value.attr, &dev_attr_active_low.attr, + &dev_attr_label.attr, NULL, }; @@ -402,6 +427,10 @@ static const struct attribute_group *gpio_groups[] = { NULL }; +/* bwlegh, a second device in the same file... get out of my namespace! */ +#define dev_attr_label dev_attr_chip_label +#define label_show chip_label_show + /* * /sys/class/gpio/gpiochipN/ * /base ... matching gpio_chip.base (N) diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index e1f1b77c2f807..7ed24b23df1ab 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -539,6 +539,7 @@ source "drivers/misc/mei/Kconfig" source "drivers/misc/vmw_vmci/Kconfig" source "drivers/misc/mic/Kconfig" source "drivers/misc/genwqe/Kconfig" +source "drivers/misc/cape/Kconfig" source "drivers/misc/echo/Kconfig" source "drivers/misc/cxl/Kconfig" source "drivers/misc/ocxl/Kconfig" diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile index 59258a3d58d51..64cf8d8ebbc27 100644 --- a/drivers/misc/Makefile +++ b/drivers/misc/Makefile @@ -51,6 +51,7 @@ obj-$(CONFIG_SRAM) += sram.o obj-$(CONFIG_SRAM_EXEC) += sram-exec.o obj-y += mic/ obj-$(CONFIG_GENWQE) += genwqe/ +obj-y += cape/ obj-$(CONFIG_ECHO) += echo/ obj-$(CONFIG_VEXPRESS_SYSCFG) += vexpress-syscfg.o obj-$(CONFIG_CXL_BASE) += cxl/ diff --git a/drivers/misc/cape/Kconfig b/drivers/misc/cape/Kconfig new file mode 100644 index 0000000000000..a2ef85e0415c5 --- /dev/null +++ b/drivers/misc/cape/Kconfig @@ -0,0 +1,5 @@ +# +# Capes +# + +source "drivers/misc/cape/beaglebone/Kconfig" diff --git a/drivers/misc/cape/Makefile b/drivers/misc/cape/Makefile new file mode 100644 index 0000000000000..7c4eb96ab29e4 --- /dev/null +++ b/drivers/misc/cape/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for cape like devices +# + +obj-y += beaglebone/ diff --git a/drivers/misc/cape/beaglebone/Kconfig b/drivers/misc/cape/beaglebone/Kconfig new file mode 100644 index 0000000000000..eeb67828fd829 --- /dev/null +++ b/drivers/misc/cape/beaglebone/Kconfig @@ -0,0 +1,10 @@ +# +# Beaglebone capes +# + +config BEAGLEBONE_PINMUX_HELPER + tristate "Beaglebone Pinmux Helper" + depends on ARCH_OMAP2PLUS && OF + default n + help + Say Y here to include support for the pinmux helper diff --git a/drivers/misc/cape/beaglebone/Makefile b/drivers/misc/cape/beaglebone/Makefile new file mode 100644 index 0000000000000..7f4617a01e262 --- /dev/null +++ b/drivers/misc/cape/beaglebone/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for beaglebone capes +# + +obj-$(CONFIG_BEAGLEBONE_PINMUX_HELPER) += bone-pinmux-helper.o diff --git a/drivers/misc/cape/beaglebone/bone-pinmux-helper.c b/drivers/misc/cape/beaglebone/bone-pinmux-helper.c new file mode 100644 index 0000000000000..d81363a77b103 --- /dev/null +++ b/drivers/misc/cape/beaglebone/bone-pinmux-helper.c @@ -0,0 +1,242 @@ +/* + * Pinmux helper driver + * + * Copyright (C) 2013 Pantelis Antoniou + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const struct of_device_id bone_pinmux_helper_of_match[] = { + { + .compatible = "bone-pinmux-helper", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, bone_pinmux_helper_of_match); + +struct pinmux_helper_data { + struct pinctrl *pinctrl; + char *selected_state_name; +}; + +static ssize_t pinmux_helper_show_state(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct platform_device *pdev = to_platform_device(dev); + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + const char *name; + + name = data->selected_state_name; + if (name == NULL || strlen(name) == 0) + name = "none"; + return sprintf(buf, "%s\n", name); +} + +static ssize_t pinmux_helper_store_state(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct platform_device *pdev = to_platform_device(dev); + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + struct pinctrl_state *state; + char *state_name; + char *s; + int err; + + /* duplicate (as a null terminated string) */ + state_name = kmalloc(count + 1, GFP_KERNEL); + if (state_name == NULL) + return -ENOMEM; + memcpy(state_name, buf, count); + state_name[count] = '\0'; + + /* and chop off newline */ + s = strchr(state_name, '\n'); + if (s != NULL) + *s = '\0'; + + /* try to select default state at first (if it exists) */ + state = pinctrl_lookup_state(data->pinctrl, state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) + dev_err(dev, "Failed to select state %s\n", + state_name); + } else { + dev_err(dev, "Failed to find state %s\n", state_name); + err = PTR_RET(state); + } + + if (err == 0) { + kfree(data->selected_state_name); + data->selected_state_name = state_name; + } + + return err ? err : count; +} + +static DEVICE_ATTR(state, S_IWUSR | S_IRUGO, + pinmux_helper_show_state, pinmux_helper_store_state); + +static struct attribute *pinmux_helper_attributes[] = { + &dev_attr_state.attr, + NULL +}; + +static const struct attribute_group pinmux_helper_attr_group = { + .attrs = pinmux_helper_attributes, +}; + +static int bone_pinmux_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pinmux_helper_data *data; + struct pinctrl_state *state; + char *state_name; + const char *mode_name; + int mode_len; + int err; + + data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); + if (data == NULL) { + dev_err(dev, "Failed to allocate data\n"); + err = -ENOMEM; + goto err_no_mem; + } + + state_name = kmalloc(strlen(PINCTRL_STATE_DEFAULT) + 1, + GFP_KERNEL); + if (state_name == NULL) { + dev_err(dev, "Failed to allocate state name\n"); + err = -ENOMEM; + goto err_no_state_mem; + } + data->selected_state_name = state_name; + strcpy(data->selected_state_name, PINCTRL_STATE_DEFAULT); + + platform_set_drvdata(pdev, data); + + data->pinctrl = devm_pinctrl_get(dev); + if (IS_ERR(data->pinctrl)) { + dev_err(dev, "Failed to get pinctrl\n"); + err = PTR_RET(data->pinctrl); + goto err_no_pinctrl; + } + + /* See if an initial mode is specified in the device tree */ + mode_name = of_get_property(dev->of_node, "mode", &mode_len); + + err = -1; + if (mode_name != NULL ) { + state_name = kmalloc(mode_len + 1, GFP_KERNEL); + if (state_name == NULL) { + dev_err(dev, "Failed to allocate state name\n"); + err = -ENOMEM; + goto err_no_mode_mem; + } + strncpy(state_name, mode_name, mode_len); + + /* try to select requested mode */ + state = pinctrl_lookup_state(data->pinctrl, state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) { + dev_warn(dev, "Unable to select requested mode %s\n", state_name); + kfree(state_name); + } else { + kfree(data->selected_state_name); + data->selected_state_name = state_name; + dev_notice(dev, "Set initial pinmux mode to %s\n", state_name); + } + } + } + + /* try to select default state if mode_name failed */ + if ( err != 0) { + state = pinctrl_lookup_state(data->pinctrl, + data->selected_state_name); + if (!IS_ERR(state)) { + err = pinctrl_select_state(data->pinctrl, state); + if (err != 0) { + dev_err(dev, "Failed to select default state\n"); + goto err_no_state; + } + } else { + data->selected_state_name = '\0'; + } + } + + /* Register sysfs hooks */ + err = sysfs_create_group(&dev->kobj, &pinmux_helper_attr_group); + if (err) { + dev_err(dev, "Failed to create sysfs group\n"); + goto err_no_sysfs; + } + + return 0; + +err_no_sysfs: +err_no_state: +err_no_mode_mem: + devm_pinctrl_put(data->pinctrl); +err_no_pinctrl: + devm_kfree(dev, data->selected_state_name); +err_no_state_mem: + devm_kfree(dev, data); +err_no_mem: + return err; +} + +static int bone_pinmux_helper_remove(struct platform_device *pdev) +{ + struct pinmux_helper_data *data = platform_get_drvdata(pdev); + struct device *dev = &pdev->dev; + + sysfs_remove_group(&dev->kobj, &pinmux_helper_attr_group); + kfree(data->selected_state_name); + devm_pinctrl_put(data->pinctrl); + devm_kfree(dev, data); + + return 0; +} + +struct platform_driver bone_pinmux_helper_driver = { + .probe = bone_pinmux_helper_probe, + .remove = bone_pinmux_helper_remove, + .driver = { + .name = "bone-pinmux-helper", + .owner = THIS_MODULE, + .of_match_table = bone_pinmux_helper_of_match, + }, +}; + +module_platform_driver(bone_pinmux_helper_driver); + +MODULE_AUTHOR("Pantelis Antoniou"); +MODULE_DESCRIPTION("Beaglebone pinmux helper driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:bone-pinmux-helper"); From f48419f2171aef37019826601d6e14f322f860c1 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Fri, 15 Feb 2019 14:33:29 -0600 Subject: [PATCH 191/213] hack: gpiolib: yes we have drivers stomping on each other... Signed-off-by: Robert Nelson --- drivers/gpio/gpiolib.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index a24f13df42f47..42e05783af2ef 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2312,10 +2312,10 @@ static int gpiod_request_commit(struct gpio_desc *desc, const char *label) if (test_and_set_bit(FLAG_REQUESTED, &desc->flags) == 0) { desc_set_label(desc, label ? : "?"); status = 0; - } else { - kfree_const(label); - status = -EBUSY; - goto done; +// } else { +// kfree_const(label); +// status = -EBUSY; +// goto done; } if (chip->request) { From d82b6b6394240967626cb6ae40f7a255cf40cf86 Mon Sep 17 00:00:00 2001 From: Jason Kridner Date: Tue, 30 Jul 2019 16:34:46 -0500 Subject: [PATCH 192/213] uio: add uio_pruss_shmem driver Signed-off-by: Jason Kridner --- drivers/uio/Kconfig | 8 ++ drivers/uio/Makefile | 1 + drivers/uio/uio_pruss_shmem.c | 163 ++++++++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 drivers/uio/uio_pruss_shmem.c diff --git a/drivers/uio/Kconfig b/drivers/uio/Kconfig index 7e8dc78a97961..8c829dc8769a8 100644 --- a/drivers/uio/Kconfig +++ b/drivers/uio/Kconfig @@ -142,6 +142,14 @@ config UIO_PRUSS To compile this driver as a module, choose M here: the module will be called uio_pruss. +config UIO_PRUSS_SHMEM + tristate "Texas Instruments PRUSS_SHMEM driver" + help + PRUSS SHMEM driver for AM3X and AM5X devices + + To compile this driver as a module, choose M here: the module + will be called uio_pruss_shmem. + config UIO_MF624 tristate "Humusoft MF624 DAQ PCI card driver" depends on PCI diff --git a/drivers/uio/Makefile b/drivers/uio/Makefile index c285dd2a4539b..a4bdec8960350 100644 --- a/drivers/uio/Makefile +++ b/drivers/uio/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_UIO_SERCOS3) += uio_sercos3.o obj-$(CONFIG_UIO_PCI_GENERIC) += uio_pci_generic.o obj-$(CONFIG_UIO_NETX) += uio_netx.o obj-$(CONFIG_UIO_PRUSS) += uio_pruss.o +obj-$(CONFIG_UIO_PRUSS_SHMEM) += uio_pruss_shmem.o obj-$(CONFIG_UIO_MF624) += uio_mf624.o obj-$(CONFIG_UIO_FSL_ELBC_GPCM) += uio_fsl_elbc_gpcm.o obj-$(CONFIG_UIO_HV_GENERIC) += uio_hv_generic.o diff --git a/drivers/uio/uio_pruss_shmem.c b/drivers/uio/uio_pruss_shmem.c new file mode 100644 index 0000000000000..2259ba70e4f04 --- /dev/null +++ b/drivers/uio/uio_pruss_shmem.c @@ -0,0 +1,163 @@ +/* + * Programmable Real-Time Unit Sub System (PRUSS) UIO driver (uio_pruss) + * + * This driver exports PRUSS host event out interrupts and PRUSS, L3 RAM, + * and DDR RAM to user space for applications interacting with PRUSS firmware + * + * Copyright (C) 2010-11 Texas Instruments Incorporated - http://www.ti.com/ + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation version 2. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_NAME "pruss_uio_shmem" +#define DRV_VERSION "1.0" + +struct uio_pruss_shmem_dev { + struct uio_info *info; + void __iomem *prussio_vaddr; +}; + +static void pruss_shmem_cleanup(struct device *dev, struct uio_pruss_shmem_dev *gdev) +{ + struct uio_info *p = gdev->info; + + uio_unregister_device(p); + iounmap(gdev->prussio_vaddr); + kfree(gdev->info); + kfree(gdev); +} + +static int pruss_shmem_probe(struct platform_device *pdev) +{ + struct uio_pruss_shmem_dev *gdev; + struct resource *regs_prussio; + struct resource res; + struct device *dev = &pdev->dev; + int ret, len; + struct uio_info *p; + + dev_info(dev, "Allocating gdev\n"); + gdev = kzalloc(sizeof(struct uio_pruss_shmem_dev), GFP_KERNEL); + if (!gdev) + return -ENOMEM; + + dev_info(dev, "Allocating info\n"); + gdev->info = kzalloc(sizeof(struct uio_info), GFP_KERNEL); + if (!gdev->info) { + ret = -ENOMEM; + goto err_free_gdev; + } + + dev_info(dev, "Requesting resource\n"); + if (dev->of_node) { + ret = of_address_to_resource(dev->of_node, 0, &res); + if (IS_ERR_VALUE(ret)) { + dev_err(dev, "Failed to parse DT reg\n"); + goto err_free_info; + } + regs_prussio = &res; + } else { + regs_prussio = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!regs_prussio) { + dev_err(dev, "No PRUSS I/O resource specified\n"); + goto err_free_info; + } + } + if (!regs_prussio) { + dev_err(dev, "No PRUSS I/O resource specified\n"); + ret = -EIO; + goto err_free_info; + } + if (!regs_prussio->start) { + dev_err(dev, "Invalid memory resource\n"); + ret = -EIO; + goto err_free_info; + } + dev_info(dev, "Mapping resource\n"); + len = resource_size(regs_prussio); + gdev->prussio_vaddr = ioremap(regs_prussio->start, len); + if (!gdev->prussio_vaddr) { + dev_err(dev, "Can't remap PRUSS I/O address range\n"); + ret = -ENOMEM; + goto err_free_info; + } + + p = gdev->info; + p->mem[0].addr = regs_prussio->start; + p->mem[0].size = resource_size(regs_prussio); + p->mem[0].memtype = UIO_MEM_PHYS; + + p->name = "pruss_shmem"; + p->version = DRV_VERSION; + + dev_info(dev, "Registering with uio driver\n"); + ret = uio_register_device(dev, p); + if (ret < 0) { + goto err_free_register; + } + + dev_info(dev, "Saving platform data\n"); + platform_set_drvdata(pdev, gdev); + return 0; + +err_free_register: + uio_unregister_device(p); + iounmap(gdev->prussio_vaddr); +err_free_info: + kfree(gdev->info); +err_free_gdev: + kfree(gdev); + + return ret; +} + +static int pruss_shmem_remove(struct platform_device *dev) +{ + struct uio_pruss_shmem_dev *gdev = platform_get_drvdata(dev); + + pruss_shmem_cleanup(&dev->dev, gdev); + return 0; +} + +static const struct of_device_id pruss_shmem_dt_ids[] = { + { .compatible = "ti,pruss-shmem", .data = NULL, }, + {}, +}; +MODULE_DEVICE_TABLE(of, pruss_shmem_dt_ids); + +static struct platform_driver pruss_shmem_driver = { + .probe = pruss_shmem_probe, + .remove = pruss_shmem_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = pruss_shmem_dt_ids, + }, +}; + +module_platform_driver(pruss_shmem_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_VERSION(DRV_VERSION); +MODULE_AUTHOR("Amit Chatterjee "); +MODULE_AUTHOR("Pratheesh Gangadhar "); +MODULE_AUTHOR("Jason Kridner "); From 502ae6518ac3ede2c03db99ec9481c25f844cc68 Mon Sep 17 00:00:00 2001 From: Matthijs van Duin Date: Sat, 9 Feb 2019 17:44:46 +0100 Subject: [PATCH 193/213] uio-pruss: cleanups and pruss v2 (pru-icss) support --- drivers/uio/uio_pruss.c | 124 +++++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 34 deletions(-) diff --git a/drivers/uio/uio_pruss.c b/drivers/uio/uio_pruss.c index 1cc175d3c25c5..dabc37ed64594 100644 --- a/drivers/uio/uio_pruss.c +++ b/drivers/uio/uio_pruss.c @@ -26,14 +26,22 @@ #include #include #include +#include +#ifdef CONFIG_ARCH_DAVINCI_DA850 #include +#else +#include +#endif #define DRV_NAME "pruss_uio" #define DRV_VERSION "1.0" +#ifdef CONFIG_ARCH_DAVINCI_DA850 static int sram_pool_sz = SZ_16K; module_param(sram_pool_sz, int, 0); -MODULE_PARM_DESC(sram_pool_sz, "sram pool size to allocate "); +MODULE_PARM_DESC(sram_pool_sz, "sram pool size to allocate"); +#endif + static int extram_pool_sz = SZ_256K; module_param(extram_pool_sz, int, 0); @@ -61,22 +69,23 @@ MODULE_PARM_DESC(extram_pool_sz, "external ram pool size to allocate"); #define PINTC_HIER 0x1500 struct uio_pruss_dev { - struct uio_info *info; - struct clk *pruss_clk; - dma_addr_t sram_paddr; + struct uio_info info[MAX_PRUSS_EVT]; dma_addr_t ddr_paddr; void __iomem *prussio_vaddr; - unsigned long sram_vaddr; void *ddr_vaddr; - unsigned int hostirq_start; unsigned int pintc_base; +#ifdef CONFIG_ARCH_DAVINCI_DA850 + struct clk *pruss_clk; struct gen_pool *sram_pool; + dma_addr_t sram_paddr; + unsigned long sram_vaddr; +#endif }; static irqreturn_t pruss_handler(int irq, struct uio_info *info) { struct uio_pruss_dev *gdev = info->priv; - int intr_bit = (irq - gdev->hostirq_start + 2); + int intr_bit = 2 + (info - gdev->info); int val, intr_mask = (1 << intr_bit); void __iomem *base = gdev->prussio_vaddr + gdev->pintc_base; void __iomem *intren_reg = base + PINTC_HIER; @@ -94,53 +103,50 @@ static irqreturn_t pruss_handler(int irq, struct uio_info *info) static void pruss_cleanup(struct device *dev, struct uio_pruss_dev *gdev) { - int cnt; - struct uio_info *p = gdev->info; + int i; - for (cnt = 0; cnt < MAX_PRUSS_EVT; cnt++, p++) { - uio_unregister_device(p); - kfree(p->name); + for (i = 0; i < MAX_PRUSS_EVT; i++) { + uio_unregister_device(&gdev->info[i]); + kfree(gdev->info[i].name); } iounmap(gdev->prussio_vaddr); if (gdev->ddr_vaddr) { dma_free_coherent(dev, extram_pool_sz, gdev->ddr_vaddr, gdev->ddr_paddr); } +#ifdef CONFIG_ARCH_DAVINCI_DA850 if (gdev->sram_vaddr) gen_pool_free(gdev->sram_pool, gdev->sram_vaddr, sram_pool_sz); - kfree(gdev->info); clk_disable(gdev->pruss_clk); clk_put(gdev->pruss_clk); +#else + pm_runtime_put(dev); + pm_runtime_disable(dev); +#endif kfree(gdev); } static int pruss_probe(struct platform_device *pdev) { - struct uio_info *p; struct uio_pruss_dev *gdev; struct resource *regs_prussio; struct device *dev = &pdev->dev; - int ret, cnt, i, len; + int ret, i, len; struct uio_pruss_pdata *pdata = dev_get_platdata(dev); gdev = kzalloc(sizeof(struct uio_pruss_dev), GFP_KERNEL); if (!gdev) return -ENOMEM; - gdev->info = kcalloc(MAX_PRUSS_EVT, sizeof(*p), GFP_KERNEL); - if (!gdev->info) { - ret = -ENOMEM; - goto err_free_gdev; - } - +#ifdef CONFIG_ARCH_DAVINCI_DA850 /* Power on PRU in case its not done as part of boot-loader */ gdev->pruss_clk = clk_get(dev, "pruss"); if (IS_ERR(gdev->pruss_clk)) { dev_err(dev, "Failed to get clock\n"); ret = PTR_ERR(gdev->pruss_clk); - goto err_free_info; + goto err_free_gdev; } ret = clk_enable(gdev->pruss_clk); @@ -148,6 +154,15 @@ static int pruss_probe(struct platform_device *pdev) dev_err(dev, "Failed to enable clock\n"); goto err_clk_put; } +#else + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "pm_runtime_get_sync() failed\n"); + pm_runtime_disable(dev); + goto err_free_gdev; + } +#endif regs_prussio = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!regs_prussio) { @@ -162,6 +177,18 @@ static int pruss_probe(struct platform_device *pdev) goto err_clk_disable; } + if (dev->of_node) { + ret = of_property_read_u32(dev->of_node, + "ti,pintc-offset", + &gdev->pintc_base); + if (ret < 0) { + dev_err(dev, "Can't parse ti,pintc-offset property\n"); + goto err_clk_disable; + } + } else + gdev->pintc_base = pdata->pintc_base; + +#ifdef CONFIG_ARCH_DAVINCI_DA850 if (pdata->sram_pool) { gdev->sram_pool = pdata->sram_pool; gdev->sram_vaddr = @@ -173,9 +200,10 @@ static int pruss_probe(struct platform_device *pdev) goto err_clk_disable; } } +#endif gdev->ddr_vaddr = dma_alloc_coherent(dev, extram_pool_sz, - &(gdev->ddr_paddr), GFP_KERNEL | GFP_DMA); + &gdev->ddr_paddr, GFP_KERNEL | GFP_DMA); if (!gdev->ddr_vaddr) { dev_err(dev, "Could not allocate external memory\n"); ret = -ENOMEM; @@ -185,32 +213,48 @@ static int pruss_probe(struct platform_device *pdev) len = resource_size(regs_prussio); gdev->prussio_vaddr = ioremap(regs_prussio->start, len); if (!gdev->prussio_vaddr) { - dev_err(dev, "Can't remap PRUSS I/O address range\n"); + dev_err(dev, "Can't remap PRUSS I/O address range\n"); ret = -ENOMEM; goto err_free_ddr_vaddr; } - gdev->pintc_base = pdata->pintc_base; - gdev->hostirq_start = platform_get_irq(pdev, 0); + for (i = 0; i < MAX_PRUSS_EVT; i++) { + struct uio_info *p = &gdev->info[i]; - for (cnt = 0, p = gdev->info; cnt < MAX_PRUSS_EVT; cnt++, p++) { + p->mem[0].name = "pruss"; p->mem[0].addr = regs_prussio->start; p->mem[0].size = resource_size(regs_prussio); p->mem[0].memtype = UIO_MEM_PHYS; + /* note: some userspace code uses hardcoded mem indices... */ +#ifdef CONFIG_ARCH_DAVINCI_DA850 + p->mem[1].name = "sram"; p->mem[1].addr = gdev->sram_paddr; p->mem[1].size = sram_pool_sz; p->mem[1].memtype = UIO_MEM_PHYS; + p->mem[2].name = "ddr"; p->mem[2].addr = gdev->ddr_paddr; p->mem[2].size = extram_pool_sz; p->mem[2].memtype = UIO_MEM_PHYS; +#else + p->mem[1].name = "ddr"; + p->mem[1].addr = gdev->ddr_paddr; + p->mem[1].size = extram_pool_sz; + p->mem[1].memtype = UIO_MEM_PHYS; +#endif - p->name = kasprintf(GFP_KERNEL, "pruss_evt%d", cnt); + ret = platform_get_irq(pdev, i); + if (ret < 0) { + dev_err(dev, "Failed to obtain irq %d (%d)\n", i, ret); + goto err_unloop; + } + + p->name = kasprintf(GFP_KERNEL, "pruss_evt%d", i); p->version = DRV_VERSION; /* Register PRUSS IRQ lines */ - p->irq = gdev->hostirq_start + cnt; + p->irq = ret; p->handler = pruss_handler; p->priv = gdev; @@ -225,23 +269,27 @@ static int pruss_probe(struct platform_device *pdev) return 0; err_unloop: - for (i = 0, p = gdev->info; i < cnt; i++, p++) { - uio_unregister_device(p); - kfree(p->name); + while( --i >= 0 ) { + uio_unregister_device(&gdev->info[i]); + kfree(gdev->info[i].name); } iounmap(gdev->prussio_vaddr); err_free_ddr_vaddr: dma_free_coherent(dev, extram_pool_sz, gdev->ddr_vaddr, gdev->ddr_paddr); err_free_sram: +#ifdef CONFIG_ARCH_DAVINCI_DA850 if (pdata->sram_pool) gen_pool_free(gdev->sram_pool, gdev->sram_vaddr, sram_pool_sz); err_clk_disable: clk_disable(gdev->pruss_clk); err_clk_put: clk_put(gdev->pruss_clk); -err_free_info: - kfree(gdev->info); +#else +err_clk_disable: + pm_runtime_put(dev); + pm_runtime_disable(dev); +#endif err_free_gdev: kfree(gdev); @@ -256,11 +304,19 @@ static int pruss_remove(struct platform_device *dev) return 0; } +static const struct of_device_id pruss_dt_ids[] = { + { .compatible = "ti,pruss-v1" }, + { .compatible = "ti,pruss-v2" }, + {}, +}; +MODULE_DEVICE_TABLE(of, pruss_dt_ids); + static struct platform_driver pruss_driver = { .probe = pruss_probe, .remove = pruss_remove, .driver = { .name = DRV_NAME, + .of_match_table = pruss_dt_ids, }, }; From 36552c0131b165860778e479a4b8072ddb9e5dcf Mon Sep 17 00:00:00 2001 From: Matthijs van Duin Date: Wed, 13 Feb 2019 08:00:11 +0100 Subject: [PATCH 194/213] ARM: omap2: support deasserting reset from dts --- .../devicetree/bindings/arm/omap/omap.txt | 2 ++ arch/arm/mach-omap2/omap_device.c | 24 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/arm/omap/omap.txt b/Documentation/devicetree/bindings/arm/omap/omap.txt index 2ecc712bf7075..e026391599fbe 100644 --- a/Documentation/devicetree/bindings/arm/omap/omap.txt +++ b/Documentation/devicetree/bindings/arm/omap/omap.txt @@ -24,6 +24,8 @@ Optional properties: - ti,no-reset-on-init: When present, the module should not be reset at init - ti,no-idle-on-init: When present, the module should not be idled at init - ti,no-idle: When present, the module is never allowed to idle. +- ti,deassert-hard-reset: list of hwmod and hardware reset line name pairs + (ascii strings) to be deasserted upon device instantiation. Example: diff --git a/arch/arm/mach-omap2/omap_device.c b/arch/arm/mach-omap2/omap_device.c index 4933e44a7e7e9..2839e77258c59 100644 --- a/arch/arm/mach-omap2/omap_device.c +++ b/arch/arm/mach-omap2/omap_device.c @@ -141,8 +141,8 @@ static int omap_device_build_from_dt(struct platform_device *pdev) struct omap_hwmod *oh; struct device_node *node = pdev->dev.of_node; struct resource res; - const char *oh_name; - int oh_cnt, i, ret = 0; + const char *oh_name, *rst_name; + int oh_cnt, dstr_cnt, i, ret = 0; bool device_active = false, skip_pm_domain = false; oh_cnt = of_property_count_strings(node, "ti,hwmods"); @@ -189,6 +189,26 @@ static int omap_device_build_from_dt(struct platform_device *pdev) ret = PTR_ERR(od); goto odbfd_exit1; } + dstr_cnt = + of_property_count_strings(node, "ti,deassert-hard-reset"); + if (dstr_cnt > 0) { + for (i = 0; i < dstr_cnt; i += 2) { + of_property_read_string_index( + node, "ti,deassert-hard-reset", i, + &oh_name); + of_property_read_string_index( + node, "ti,deassert-hard-reset", i+1, + &rst_name); + oh = omap_hwmod_lookup(oh_name); + if (!oh) { + dev_warn(&pdev->dev, + "Cannot parse deassert property for '%s'\n", + oh_name); + break; + } + omap_hwmod_deassert_hardreset(oh, rst_name); + } + } /* Fix up missing resource names */ for (i = 0; i < pdev->num_resources; i++) { From 87b2ac295229d7bc662d0a33690d993e1b2d22d5 Mon Sep 17 00:00:00 2001 From: Dimitar Dimitrov Date: Wed, 31 Aug 2016 10:39:07 -0600 Subject: [PATCH 195/213] Fix remoteproc to work with the PRU GNU Binutils port PRU IRAM addresses need to be masked before being handled to remoteproc. This is due to PRU Binutils' lack of separate address spaces for IRAM and DRAM. Signed-off-by: Dimitar Dimitrov --- drivers/remoteproc/pru_rproc.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/remoteproc/pru_rproc.c b/drivers/remoteproc/pru_rproc.c index 9e08243758195..2df1ccf4506c9 100644 --- a/drivers/remoteproc/pru_rproc.c +++ b/drivers/remoteproc/pru_rproc.c @@ -955,6 +955,14 @@ static void *pru_i_da_to_va(struct pru_rproc *pru, u32 da, int len) u32 offset; void *va = NULL; + /* GNU binutils do not support multiple address spaces. The + * default linker script from the official GNU pru-ld places + * IRAM at an arbitrary high offset, in order to differentiate it + * from DRAM. Hence we need to strip the artificial offset + * from the IRAM address. + */ + da &= ~0xf0000000u; + if (len <= 0) return NULL; From d3b6d82557af186f1c87148102fdda7add5fe543 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Wed, 18 Mar 2020 13:10:20 -0500 Subject: [PATCH 196/213] bootup hacks: move mmc early Signed-off-by: Robert Nelson --- drivers/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/Makefile b/drivers/Makefile index 91189b7c8ffe8..ed7270f3657b0 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -55,6 +55,9 @@ obj-$(CONFIG_RESET_CONTROLLER) += reset/ obj-y += tty/ obj-y += char/ +# put mmc early as many morden devices use emm/sd card as rootfs storage +obj-y += mmc/ + # iommu/ comes before gpu as gpu are using iommu controllers obj-$(CONFIG_IOMMU_SUPPORT) += iommu/ @@ -128,7 +131,6 @@ obj-$(CONFIG_EISA) += eisa/ obj-$(CONFIG_PM_OPP) += opp/ obj-$(CONFIG_CPU_FREQ) += cpufreq/ obj-$(CONFIG_CPU_IDLE) += cpuidle/ -obj-y += mmc/ obj-$(CONFIG_MEMSTICK) += memstick/ obj-$(CONFIG_NEW_LEDS) += leds/ obj-$(CONFIG_INFINIBAND) += infiniband/ From 2e7c948a98ca6b5f5ceb1b406ecb6e473add5df6 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 19 Mar 2020 14:00:05 -0500 Subject: [PATCH 197/213] bootup-hacks: xor select neon or arm4regs already default on BBB/X15, just calcuated everybootup Signed-off-by: Robert Nelson --- arch/arm/include/asm/xor.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h index 4ffb26d4cad8d..cea3e0c6469bb 100644 --- a/arch/arm/include/asm/xor.h +++ b/arch/arm/include/asm/xor.h @@ -212,3 +212,9 @@ static struct xor_block_template xor_block_neon = { #else #define NEON_TEMPLATES #endif + +#ifdef CONFIG_KERNEL_MODE_NEON +#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_neon) +#else +#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_arm4regs) +#endif From 0ebaf58295642a2cbbd5caff141568e3718df6ee Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Mon, 8 Jun 2020 11:02:13 -0500 Subject: [PATCH 198/213] backports: bindeb-pkg: from: linux.git Reference: v5.2.21 Signed-off-by: Robert Nelson --- scripts/package/Makefile | 12 +++++------ scripts/package/builddeb | 44 ++++++++++++++++++++++----------------- scripts/package/buildtar | 4 ++-- scripts/package/mkdebian | 45 +++++++++++++++++++++++----------------- scripts/package/mkspec | 11 +++++----- 5 files changed, 65 insertions(+), 51 deletions(-) diff --git a/scripts/package/Makefile b/scripts/package/Makefile index 73503ebce6326..ca7f46b562a4f 100644 --- a/scripts/package/Makefile +++ b/scripts/package/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only # Makefile for the different targets used to generate full packages of a kernel # It uses the generic clean infrastructure of kbuild @@ -33,7 +34,6 @@ MKSPEC := $(srctree)/scripts/package/mkspec quiet_cmd_src_tar = TAR $(2).tar.gz cmd_src_tar = \ -set -e; \ if test "$(objtree)" != "$(srctree)"; then \ echo >&2; \ echo >&2 " ERROR:"; \ @@ -60,7 +60,7 @@ rpm-pkg: FORCE # binrpm-pkg # --------------------------------------------------------------------------- binrpm-pkg: FORCE - $(MAKE) KBUILD_SRC= + $(MAKE) -f $(srctree)/Makefile $(CONFIG_SHELL) $(MKSPEC) prebuilt > $(objtree)/binkernel.spec +rpmbuild $(RPMOPTS) --define "_builddir $(objtree)" --target \ $(UTS_MACHINE) -bb $(objtree)/binkernel.spec @@ -73,11 +73,11 @@ deb-pkg: FORCE $(call cmd,src_tar,$(KDEB_SOURCENAME)) origversion=$$(dpkg-parsechangelog -SVersion |sed 's/-[^-]*$$//');\ mv $(KDEB_SOURCENAME).tar.gz ../$(KDEB_SOURCENAME)_$${origversion}.orig.tar.gz - +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) -i.git -us -uc + +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) -i.git -us -uc bindeb-pkg: FORCE $(CONFIG_SHELL) $(srctree)/scripts/package/mkdebian - +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) -b -nc -uc + +dpkg-buildpackage -r$(KBUILD_PKG_ROOTCMD) -a$$(cat debian/arch) $(DPKG_FLAGS) -b -nc -uc intdeb-pkg: FORCE +$(CONFIG_SHELL) $(srctree)/scripts/package/builddeb @@ -103,8 +103,8 @@ clean-dirs += $(objtree)/snap/ # tarball targets # --------------------------------------------------------------------------- tar%pkg: FORCE - $(MAKE) KBUILD_SRC= - $(CONFIG_SHELL) $(srctree)/scripts/package/buildtar $@ + $(MAKE) -f $(srctree)/Makefile + +$(CONFIG_SHELL) $(srctree)/scripts/package/buildtar $@ clean-dirs += $(objtree)/tar-install/ diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 0b31f4f1f92cf..b03dd56a4782b 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -12,6 +12,18 @@ set -e +is_enabled() { + grep -q "^$1=y" include/config/auto.conf +} + +if_enabled_echo() { + if is_enabled "$1"; then + echo -n "$2" + elif [ $# -ge 3 ]; then + echo -n "$3" + fi +} + create_package() { local pname="$1" pdir="$2" @@ -62,7 +74,7 @@ parisc|mips|powerpc) installed_image_path="boot/vmlinuz-$version" esac -BUILD_DEBUG="$(grep -s '^CONFIG_DEBUG_INFO=y' $KCONFIG_CONFIG || true)" +BUILD_DEBUG=$(if_enabled_echo CONFIG_DEBUG_INFO Yes) # Setup the directory structure rm -rf "$tmpdir" "$kernel_headers_dir" "$libc_headers_dir" "$dbg_dir" $objtree/debian/files @@ -83,15 +95,15 @@ else fi cp "$($MAKE -s -f $srctree/Makefile image_name)" "$tmpdir/$installed_image_path" -if grep -q "^CONFIG_OF=y" $KCONFIG_CONFIG ; then +if is_enabled CONFIG_OF_EARLY_FLATTREE; then # Only some architectures with OF support have this target - if grep -q dtbs_install "${srctree}/arch/$SRCARCH/Makefile"; then - $MAKE KBUILD_SRC= INSTALL_DTBS_PATH="$tmpdir/usr/lib/$packagename" dtbs_install + if [ -d "${srctree}/arch/$SRCARCH/boot/dts" ]; then + $MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/usr/lib/$packagename" dtbs_install fi fi -if grep -q '^CONFIG_MODULES=y' $KCONFIG_CONFIG ; then - INSTALL_MOD_PATH="$tmpdir" $MAKE KBUILD_SRC= modules_install +if is_enabled CONFIG_MODULES; then + INSTALL_MOD_PATH="$tmpdir" $MAKE -f $srctree/Makefile modules_install rm -f "$tmpdir/lib/modules/$version/build" rm -f "$tmpdir/lib/modules/$version/source" if [ "$ARCH" = "um" ] ; then @@ -111,16 +123,15 @@ if grep -q '^CONFIG_MODULES=y' $KCONFIG_CONFIG ; then done # resign stripped modules - MODULE_SIG_ALL="$(grep -s '^CONFIG_MODULE_SIG_ALL=y' $KCONFIG_CONFIG || true)" - if [ -n "$MODULE_SIG_ALL" ]; then - INSTALL_MOD_PATH="$tmpdir" $MAKE KBUILD_SRC= modules_sign + if is_enabled CONFIG_MODULE_SIG_ALL; then + INSTALL_MOD_PATH="$tmpdir" $MAKE -f $srctree/Makefile modules_sign fi fi fi if [ "$ARCH" != "um" ]; then - $MAKE headers_check KBUILD_SRC= - $MAKE headers_install KBUILD_SRC= INSTALL_HDR_PATH="$libc_headers_dir/usr" + $MAKE -f $srctree/Makefile headers_check + $MAKE -f $srctree/Makefile headers_install INSTALL_HDR_PATH="$libc_headers_dir/usr" fi # Install the maintainer scripts @@ -129,11 +140,6 @@ fi # make-kpkg sets $INITRD to indicate whether an initramfs is wanted, and # so do we; recent versions of dracut and initramfs-tools will obey this. debhookdir=${KDEB_HOOKDIR:-/etc/kernel} -if grep -q '^CONFIG_BLK_DEV_INITRD=y' $KCONFIG_CONFIG; then - want_initrd=Yes -else - want_initrd=No -fi for script in postinst postrm preinst prerm ; do mkdir -p "$tmpdir$debhookdir/$script.d" cat < "$tmpdir/DEBIAN/$script" @@ -145,7 +151,7 @@ set -e export DEB_MAINT_PARAMS="\$*" # Tell initramfs builder whether it's wanted -export INITRD=$want_initrd +export INITRD=$(if_enabled_echo CONFIG_BLK_DEV_INITRD Yes No) test -d $debhookdir/$script.d && run-parts --arg="$version" --arg="/$installed_image_path" $debhookdir/$script.d exit 0 @@ -158,11 +164,11 @@ done (cd $srctree; find arch/*/include include scripts -type f -o -type l) >> "$objtree/debian/hdrsrcfiles" (cd $srctree; find arch/$SRCARCH -name module.lds -o -name Kbuild.platforms -o -name Platform) >> "$objtree/debian/hdrsrcfiles" (cd $srctree; find $(find arch/$SRCARCH -name include -o -name scripts -type d) -type f) >> "$objtree/debian/hdrsrcfiles" -if grep -q '^CONFIG_STACK_VALIDATION=y' $KCONFIG_CONFIG ; then +if is_enabled CONFIG_STACK_VALIDATION; then (cd $objtree; find tools/objtool -type f -executable) >> "$objtree/debian/hdrobjfiles" fi (cd $objtree; find arch/$SRCARCH/include Module.symvers include scripts -type f) >> "$objtree/debian/hdrobjfiles" -if grep -q '^CONFIG_GCC_PLUGINS=y' $KCONFIG_CONFIG ; then +if is_enabled CONFIG_GCC_PLUGINS; then (cd $objtree; find scripts/gcc-plugins -name \*.so -o -name gcc-common.h) >> "$objtree/debian/hdrobjfiles" fi destdir=$kernel_headers_dir/usr/src/linux-headers-$version diff --git a/scripts/package/buildtar b/scripts/package/buildtar index d624a07a4e77a..2f66c81e4021b 100755 --- a/scripts/package/buildtar +++ b/scripts/package/buildtar @@ -56,8 +56,8 @@ dirs=boot # # Try to install modules # -if grep -q '^CONFIG_MODULES=y' "${KCONFIG_CONFIG}"; then - make ARCH="${ARCH}" O="${objtree}" KBUILD_SRC= INSTALL_MOD_PATH="${tmpdir}" modules_install +if grep -q '^CONFIG_MODULES=y' include/config/auto.conf; then + make ARCH="${ARCH}" -f ${srctree}/Makefile INSTALL_MOD_PATH="${tmpdir}" modules_install dirs="$dirs lib" fi diff --git a/scripts/package/mkdebian b/scripts/package/mkdebian index edcad61fe3cda..8351584cb24e6 100755 --- a/scripts/package/mkdebian +++ b/scripts/package/mkdebian @@ -7,7 +7,7 @@ set -e is_enabled() { - grep -q "^CONFIG_$1=y" $KCONFIG_CONFIG + grep -q "^$1=y" include/config/auto.conf } if_enabled_echo() { @@ -31,23 +31,23 @@ set_debarch() { x86_64) debarch=amd64 ;; sparc*) - debarch=sparc$(if_enabled_echo 64BIT 64) ;; + debarch=sparc$(if_enabled_echo CONFIG_64BIT 64) ;; s390*) debarch=s390x ;; ppc*) - if is_enabled 64BIT; then - debarch=ppc64$(if_enabled_echo CPU_LITTLE_ENDIAN el) + if is_enabled CONFIG_64BIT; then + debarch=ppc64$(if_enabled_echo CONFIG_CPU_LITTLE_ENDIAN el) else - debarch=powerpc$(if_enabled_echo SPE spe) + debarch=powerpc$(if_enabled_echo CONFIG_SPE spe) fi ;; parisc*) debarch=hppa ;; mips*) - if is_enabled CPU_LITTLE_ENDIAN; then - debarch=mips$(if_enabled_echo 64BIT 64)$(if_enabled_echo CPU_MIPSR6 r6)el - elif is_enabled CPU_MIPSR6; then - debarch=mips$(if_enabled_echo 64BIT 64)r6 + if is_enabled CONFIG_CPU_LITTLE_ENDIAN; then + debarch=mips$(if_enabled_echo CONFIG_64BIT 64)$(if_enabled_echo CONFIG_CPU_MIPSR6 r6)el + elif is_enabled CONFIG_CPU_MIPSR6; then + debarch=mips$(if_enabled_echo CONFIG_64BIT 64)r6 else debarch=mips fi @@ -55,8 +55,8 @@ set_debarch() { aarch64|arm64) debarch=arm64 ;; arm*) - if is_enabled AEABI; then - debarch=arm$(if_enabled_echo VFP hf el) + if is_enabled CONFIG_AEABI; then + debarch=arm$(if_enabled_echo CONFIG_VFP hf el) else debarch=arm fi @@ -64,10 +64,10 @@ set_debarch() { openrisc) debarch=or1k ;; sh) - if is_enabled CPU_SH3; then - debarch=sh3$(if_enabled_echo CPU_BIG_ENDIAN eb) - elif is_enabled CPU_SH4; then - debarch=sh4$(if_enabled_echo CPU_BIG_ENDIAN eb) + if is_enabled CONFIG_CPU_SH3; then + debarch=sh3$(if_enabled_echo CONFIG_CPU_BIG_ENDIAN eb) + elif is_enabled CONFIG_CPU_SH4; then + debarch=sh4$(if_enabled_echo CONFIG_CPU_BIG_ENDIAN eb) fi ;; esac @@ -132,8 +132,12 @@ else echo >&2 "Install lsb-release or set \$KDEB_CHANGELOG_DIST explicitly" fi -mkdir -p debian/ +mkdir -p debian/source/ +echo "1.0" > debian/source/format + echo $debarch > debian/arch +extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev)" +extra_build_depends="$extra_build_depends, $(if_enabled_echo CONFIG_SYSTEM_TRUSTED_KEYRING libssl-dev:native)" # Generate a simple changelog template cat < debian/changelog @@ -170,7 +174,7 @@ Source: $sourcename Section: kernel Priority: optional Maintainer: $maintainer -Build-Depends: bc, kmod, cpio +Build-Depends: bc, kmod, cpio, bison, flex | flex:native $extra_build_depends Homepage: http://www.kernel.org/ Package: $packagename @@ -205,13 +209,15 @@ EOF cat < debian/rules #!$(command -v $MAKE) -f +srctree ?= . + build: \$(MAKE) KERNELRELEASE=${version} ARCH=${ARCH} \ - KBUILD_BUILD_VERSION=${revision} KBUILD_SRC= + KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile binary-arch: \$(MAKE) KERNELRELEASE=${version} ARCH=${ARCH} \ - KBUILD_BUILD_VERSION=${revision} KBUILD_SRC= intdeb-pkg + KBUILD_BUILD_VERSION=${revision} -f \$(srctree)/Makefile intdeb-pkg clean: rm -rf debian/*tmp debian/files @@ -219,5 +225,6 @@ clean: binary: binary-arch EOF +chmod +x debian/rules exit 0 diff --git a/scripts/package/mkspec b/scripts/package/mkspec index e05646dc24dcf..009147d4718ee 100755 --- a/scripts/package/mkspec +++ b/scripts/package/mkspec @@ -12,6 +12,7 @@ # how we were called determines which rpms we build and how we build them if [ "$1" = prebuilt ]; then S=DEL + MAKE="$MAKE -f $srctree/Makefile" else S= fi @@ -78,19 +79,19 @@ $S %prep $S %setup -q $S $S %build -$S make %{?_smp_mflags} KBUILD_BUILD_VERSION=%{release} +$S $MAKE %{?_smp_mflags} KBUILD_BUILD_VERSION=%{release} $S %install mkdir -p %{buildroot}/boot %ifarch ia64 mkdir -p %{buildroot}/boot/efi - cp \$(make image_name) %{buildroot}/boot/efi/vmlinuz-$KERNELRELEASE + cp \$($MAKE image_name) %{buildroot}/boot/efi/vmlinuz-$KERNELRELEASE ln -s efi/vmlinuz-$KERNELRELEASE %{buildroot}/boot/ %else - cp \$(make image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE + cp \$($MAKE image_name) %{buildroot}/boot/vmlinuz-$KERNELRELEASE %endif -$M make %{?_smp_mflags} INSTALL_MOD_PATH=%{buildroot} KBUILD_SRC= modules_install - make %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr KBUILD_SRC= headers_install +$M $MAKE %{?_smp_mflags} INSTALL_MOD_PATH=%{buildroot} modules_install + $MAKE %{?_smp_mflags} INSTALL_HDR_PATH=%{buildroot}/usr headers_install cp System.map %{buildroot}/boot/System.map-$KERNELRELEASE cp .config %{buildroot}/boot/config-$KERNELRELEASE bzip2 -9 --keep vmlinux From 86940d34728f53ac2728d587bc82861252813359 Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Fri, 6 Mar 2020 16:53:00 -0600 Subject: [PATCH 199/213] builddeb: Install our dtbs under /boot/dtbs/$version Signed-off-by: Robert Nelson --- scripts/package/builddeb | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/package/builddeb b/scripts/package/builddeb index b03dd56a4782b..96321fe491322 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -99,6 +99,7 @@ if is_enabled CONFIG_OF_EARLY_FLATTREE; then # Only some architectures with OF support have this target if [ -d "${srctree}/arch/$SRCARCH/boot/dts" ]; then $MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/usr/lib/$packagename" dtbs_install + $MAKE -f $srctree/Makefile INSTALL_DTBS_PATH="$tmpdir/boot/dtbs/$version" dtbs_install fi fi From 78ae3ba617e80f8a68e092677dba5c306ed63ced Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 13:45:32 -0500 Subject: [PATCH 200/213] enable: Jenkins: http://gfnd.rcn-ee.org:8080 Signed-off-by: Robert Nelson --- .github/ISSUE_TEMPLATE/bug_report.md | 20 +++++++++ Jenkinsfile | 11 +++++ README.md | 7 ++++ jenkins_build.sh | 62 ++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 Jenkinsfile create mode 100644 README.md create mode 100755 jenkins_build.sh diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000000..94be17e338b34 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,20 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Describe how to reproduce the bug** +List all the steps needed to reproduce the bug + +**REQUIRED INFORMATION** +Run this command and paste the output here: +``` +sudo /opt/scripts/tools/version.sh +``` diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000000000..1e73bc717e366 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,11 @@ +pipeline { + agent any + + stages { + stage('Build') { + steps { + sh '/bin/bash ./jenkins_build.sh' + } + } + } +} diff --git a/README.md b/README.md new file mode 100644 index 0000000000000..a35e1f064aa79 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +| kernel | normal | rt | +|:---------:|:--------:|:-------:| +|4.4 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.4)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.4/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.4-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.4-rt/) | +|4.9 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.9)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.9/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.9-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.9-rt/) | +|4.14 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.14)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.14/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.14-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.14-rt/) | +|4.19 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.19)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.19/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/4.19-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/4.19-rt/) | +|5.4 | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/5.4)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/5.4/) | [![Build Status](http://gfnd.rcn-ee.org:8080/buildStatus/icon?job=beagleboard_kernel_builder/5.4-rt)](http://gfnd.rcn-ee.org:8080/job/beagleboard_kernel_builder/job/5.4-rt/) | diff --git a/jenkins_build.sh b/jenkins_build.sh new file mode 100755 index 0000000000000..8fba37bce2880 --- /dev/null +++ b/jenkins_build.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +#gcc: + +# +#https://releases.linaro.org/components/toolchain/binaries/7.1-2017.05/arm-linux-gnueabihf/gcc-linaro-7.1.1-2017.05-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.1-2017.08/arm-linux-gnueabihf/gcc-linaro-7.1.1-2017.08-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.2-2017.11/arm-linux-gnueabihf/gcc-linaro-7.2.1-2017.11-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.3-2018.05/arm-linux-gnueabihf/gcc-linaro-7.3.1-2018.05-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.4-2019.02/arm-linux-gnueabihf/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz +#https://releases.linaro.org/components/toolchain/binaries/7.5-2019.12/arm-linux-gnueabihf/gcc-linaro-7.5.0-2019.12-x86_64_arm-linux-gnueabihf.tar.xz +# + +gcc_html_path="https://releases.linaro.org/components/toolchain/binaries/7.5-2019.12/arm-linux-gnueabihf/" +gcc_filename_prefix="gcc-linaro-7.5.0-2019.12-x86_64_arm-linux-gnueabihf" +gcc_banner="arm-linux-gnueabihf-gcc (Linaro GCC 7.5-2019.12) 7.5.0" +gcc_copyright="2017" +datestamp="2019.12-gcc-arm-linux-gnueabihf" + +# + +if [ ! -d ${gcc_filename_prefix}/ ] ; then + rm -rf ./gcc-* || true + #wget -c ${site}/${version}/${filename} + wget -c http://gfnd.rcn-ee.org/farm/jenkins-dl/${gcc_filename_prefix}.tar.xz + tar xf ${gcc_filename_prefix}.tar.xz +fi + +export CC=`pwd`/${gcc_filename_prefix}/bin/arm-linux-gnueabihf- + +make ARCH=arm clean +make ARCH=arm bb.org_defconfig + +echo "[make ARCH=arm -j4 CROSS_COMPILE=\"${binary}\" zImage]" +make ARCH=arm -j4 CROSS_COMPILE="ccache ${CC}" zImage +if [ ! -f arch/arm/boot/zImage ] ; then + echo "failed: [arch/arm/boot/zImage]" + exit 1 +fi + +echo "[make ARCH=arm -j4 CROSS_COMPILE=\"${binary}\" modules]" +make ARCH=arm -j4 CROSS_COMPILE="ccache ${CC}" modules +if [ ! -f drivers/spi/spidev.ko ] ; then + echo "failed: [drivers/spi/spidev.ko]" + exit 1 +fi + +echo "[make ARCH=arm CROSS_COMPILE=\"${binary}\" dtbs]" +make ARCH=arm CROSS_COMPILE="ccache ${CC}" dtbs +if [ ! -f arch/arm/boot/dts/am335x-boneblack.dtb ] ; then + echo "failed: [arch/arm/boot/dts/am335x-boneblack.dtb]" + exit 1 +else + if [ -f arch/arm/boot/dts/am335x-pocketbeagle.dts ] ; then + if [ ! -f arch/arm/boot/dts/am335x-pocketbeagle.dtb ] ; then + echo "failed: [arch/arm/boot/dts/am335x-pocketbeagle.dtb]" + exit 1 + fi + fi +fi + +make ARCH=arm clean From 905439288ac7e37aa94f390657e33183071017ec Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 14:03:31 -0500 Subject: [PATCH 201/213] 4.19.94-ti-r45 patchset From 38b462b53e1ce18b0727a6852c8c87dd96f56f6f Mon Sep 17 00:00:00 2001 From: Robert Nelson Date: Thu, 18 Jun 2020 14:03:53 -0500 Subject: [PATCH 202/213] 4.19.94-ti-r45 4.19.94-ti-r45 bb.org_defconfig AUFS: https://github.com/sfjro/aufs4-standalone/commit/ccb15898ed1eb7fe2e914439aca993e903e8ed51 BBDTBS: https://github.com/beagleboard/BeagleBoard-DeviceTrees/commit/af07ef77cc6f8f94568a4c238cc6d41fb8c81931 CAN-ISOTP: https://github.com/hartkopp/can-isotp/commit/ced84cab120a1ddde2bd4561a18514707c8a6184 TI_AMX3_CM3: http://git.ti.com/gitweb/?p=processor-firmware/ti-amx3-cm3-pm-firmware.git;a=commit;h=6a849767df85ce9399494f53fb5c753665396653 WIREGUARD: https://git.zx2c4.com/WireGuard/commit/edad0d6e99e5133b1e8e865d727a25fff6399cb4 Signed-off-by: Robert Nelson --- arch/arm/configs/bb.org_defconfig | 2518 +++++++++++++++++++++++++++++ 1 file changed, 2518 insertions(+) create mode 100644 arch/arm/configs/bb.org_defconfig diff --git a/arch/arm/configs/bb.org_defconfig b/arch/arm/configs/bb.org_defconfig new file mode 100644 index 0000000000000..f471935f97590 --- /dev/null +++ b/arch/arm/configs/bb.org_defconfig @@ -0,0 +1,2518 @@ +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_KERNEL_LZO=y +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_NO_HZ_IDLE=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_PREEMPT=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_BSD_PROCESS_ACCT_V3=y +CONFIG_TASKSTATS=y +CONFIG_TASK_DELAY_ACCT=y +CONFIG_TASK_XACCT=y +CONFIG_TASK_IO_ACCOUNTING=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_MEMCG=y +CONFIG_MEMCG_SWAP=y +CONFIG_BLK_CGROUP=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_RT_GROUP_SCHED=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_PERF=y +CONFIG_CGROUP_BPF=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_SCHED_AUTOGROUP=y +CONFIG_BLK_DEV_INITRD=y +# CONFIG_SYSFS_SYSCALL is not set +CONFIG_KALLSYMS_ALL=y +CONFIG_BPF_SYSCALL=y +CONFIG_USERFAULTFD=y +CONFIG_EMBEDDED=y +# CONFIG_COMPAT_BRK is not set +CONFIG_SLAB_FREELIST_RANDOM=y +CONFIG_SLAB_FREELIST_HARDENED=y +CONFIG_PROFILING=y +CONFIG_OMAP_RESET_CLOCKS=y +CONFIG_SOC_OMAP5=y +CONFIG_SOC_AM33XX=y +CONFIG_SOC_AM43XX=y +CONFIG_SOC_DRA7XX=y +CONFIG_SOC_HAS_OMAP2_SDRC=y +CONFIG_ARM_THUMBEE=y +CONFIG_PL310_ERRATA_588369=y +CONFIG_PL310_ERRATA_727915=y +CONFIG_PL310_ERRATA_753970=y +CONFIG_ARM_ERRATA_430973=y +CONFIG_ARM_ERRATA_773022=y +CONFIG_SMP=y +CONFIG_NR_CPUS=2 +CONFIG_ARM_PSCI=y +CONFIG_HZ_250=y +CONFIG_SECCOMP=y +# CONFIG_ATAGS is not set +CONFIG_ZBOOT_ROM_TEXT=0x0 +CONFIG_ZBOOT_ROM_BSS=0x0 +CONFIG_EFI=y +# CONFIG_DMI is not set +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_GOV_POWERSAVE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_ONDEMAND=y +CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y +CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y +CONFIG_CPUFREQ_DT=y +# CONFIG_ARM_OMAP2PLUS_CPUFREQ is not set +CONFIG_ARM_TI_CPUFREQ=y +CONFIG_HIBERNATION=y +CONFIG_PM_AUTOSLEEP=y +CONFIG_PM_WAKELOCKS=y +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y +CONFIG_APM_EMULATION=y +CONFIG_ARM_CRYPTO=y +CONFIG_CRYPTO_SHA1_ARM_NEON=m +CONFIG_CRYPTO_SHA256_ARM=m +CONFIG_CRYPTO_SHA512_ARM=m +CONFIG_CRYPTO_AES_ARM=m +CONFIG_CRYPTO_AES_ARM_BS=m +CONFIG_CRYPTO_CHACHA20_NEON=m +CONFIG_OPROFILE=m +CONFIG_KPROBES=y +CONFIG_JUMP_LABEL=y +CONFIG_MODULES=y +CONFIG_MODULE_FORCE_LOAD=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_MODULE_COMPRESS=y +CONFIG_MODULE_COMPRESS_XZ=y +CONFIG_BLK_DEV_THROTTLING=y +CONFIG_BLK_WBT=y +CONFIG_BLK_SED_OPAL=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_KARMA_PARTITION=y +CONFIG_CFQ_GROUP_IOSCHED=y +CONFIG_MQ_IOSCHED_KYBER=m +CONFIG_IOSCHED_BFQ=m +CONFIG_BFQ_GROUP_IOSCHED=y +CONFIG_BINFMT_MISC=m +CONFIG_KSM=y +CONFIG_FRONTSWAP=y +CONFIG_ZSWAP=y +CONFIG_ZBUD=y +CONFIG_ZSMALLOC=m +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=m +CONFIG_UNIX=y +CONFIG_UNIX_DIAG=m +CONFIG_XFRM_USER=m +CONFIG_XFRM_SUB_POLICY=y +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_FIB_TRIE_STATS=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_IP_MROUTE=y +CONFIG_IP_MROUTE_MULTIPLE_TABLES=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_NET_IPVTI=m +CONFIG_NET_FOU_IP_TUNNELS=y +CONFIG_INET_AH=m +CONFIG_INET_ESP=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_XFRM_MODE_TRANSPORT=m +CONFIG_INET_XFRM_MODE_TUNNEL=m +CONFIG_INET_XFRM_MODE_BEET=m +CONFIG_INET_DIAG=m +CONFIG_INET_UDP_DIAG=m +CONFIG_INET_DIAG_DESTROY=y +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_HSTCP=m +CONFIG_TCP_CONG_HYBLA=m +CONFIG_TCP_CONG_NV=m +CONFIG_TCP_CONG_SCALABLE=m +CONFIG_TCP_CONG_LP=m +CONFIG_TCP_CONG_VENO=m +CONFIG_TCP_CONG_YEAH=m +CONFIG_TCP_CONG_ILLINOIS=m +CONFIG_TCP_CONG_DCTCP=m +CONFIG_TCP_CONG_CDG=m +CONFIG_TCP_CONG_BBR=m +CONFIG_TCP_MD5SIG=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_MIP6=y +CONFIG_IPV6_ILA=m +CONFIG_INET6_XFRM_MODE_TRANSPORT=m +CONFIG_INET6_XFRM_MODE_TUNNEL=m +CONFIG_INET6_XFRM_MODE_BEET=m +CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m +CONFIG_IPV6_VTI=m +CONFIG_IPV6_SIT=m +CONFIG_IPV6_SIT_6RD=y +CONFIG_IPV6_GRE=m +CONFIG_IPV6_SUBTREES=y +CONFIG_IPV6_MROUTE=y +CONFIG_IPV6_MROUTE_MULTIPLE_TABLES=y +CONFIG_IPV6_PIMSM_V2=y +CONFIG_IPV6_SEG6_LWTUNNEL=y +CONFIG_IPV6_SEG6_HMAC=y +CONFIG_NETLABEL=y +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_LOG_NETDEV=m +CONFIG_NF_CONNTRACK_SECMARK=y +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CONNTRACK_EVENTS=y +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_SET=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_CT=y +CONFIG_NFT_COUNTER=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_NAT=y +CONFIG_NFT_OBJREF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_HASH=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NFT_FIB_NETDEV=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_LOG=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETFILTER_XT_TARGET_SECMARK=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_IPVS=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_IP_VS=m +CONFIG_IP_VS_IPV6=y +CONFIG_IP_VS_PROTO_TCP=y +CONFIG_IP_VS_PROTO_UDP=y +CONFIG_IP_VS_PROTO_ESP=y +CONFIG_IP_VS_PROTO_AH=y +CONFIG_IP_VS_PROTO_SCTP=y +CONFIG_IP_VS_RR=m +CONFIG_IP_VS_WRR=m +CONFIG_IP_VS_LC=m +CONFIG_IP_VS_WLC=m +CONFIG_IP_VS_FO=m +CONFIG_IP_VS_OVF=m +CONFIG_IP_VS_LBLC=m +CONFIG_IP_VS_LBLCR=m +CONFIG_IP_VS_DH=m +CONFIG_IP_VS_SH=m +CONFIG_IP_VS_SED=m +CONFIG_IP_VS_NQ=m +CONFIG_IP_VS_FTP=m +CONFIG_IP_VS_PE_SIP=m +CONFIG_NFT_CHAIN_ROUTE_IPV4=m +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_LOG_ARP=m +CONFIG_NFT_CHAIN_NAT_IPV4=m +CONFIG_NFT_MASQ_IPV4=m +CONFIG_NFT_REDIR_IPV4=m +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_SYNPROXY=m +CONFIG_IP_NF_NAT=y +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_TARGET_CLUSTERIP=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_SECURITY=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_NFT_CHAIN_ROUTE_IPV6=m +CONFIG_NFT_CHAIN_NAT_IPV6=m +CONFIG_NFT_MASQ_IPV6=m +CONFIG_NFT_REDIR_IPV6=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IP6_NF_TARGET_SYNPROXY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_SECURITY=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_NF_TABLES_BRIDGE=y +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NF_LOG_BRIDGE=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_IP_DCCP=m +CONFIG_SCTP_COOKIE_HMAC_SHA1=y +CONFIG_RDS=m +CONFIG_RDS_TCP=m +CONFIG_TIPC=m +CONFIG_ATM=m +CONFIG_ATM_CLIP=m +CONFIG_ATM_LANE=m +CONFIG_ATM_MPOA=m +CONFIG_ATM_BR2684=m +CONFIG_L2TP=m +CONFIG_L2TP_DEBUGFS=m +CONFIG_L2TP_V3=y +CONFIG_L2TP_IP=m +CONFIG_L2TP_ETH=m +CONFIG_BRIDGE=m +CONFIG_BRIDGE_VLAN_FILTERING=y +CONFIG_VLAN_8021Q=y +CONFIG_VLAN_8021Q_GVRP=y +CONFIG_VLAN_8021Q_MVRP=y +CONFIG_LLC2=m +CONFIG_ATALK=m +CONFIG_DEV_APPLETALK=m +CONFIG_IPDDP=m +CONFIG_IPDDP_ENCAP=y +CONFIG_PHONET=m +CONFIG_6LOWPAN=m +CONFIG_6LOWPAN_GHC_EXT_HDR_HOP=m +CONFIG_6LOWPAN_GHC_UDP=m +CONFIG_6LOWPAN_GHC_ICMPV6=m +CONFIG_6LOWPAN_GHC_EXT_HDR_DEST=m +CONFIG_6LOWPAN_GHC_EXT_HDR_FRAG=m +CONFIG_6LOWPAN_GHC_EXT_HDR_ROUTE=m +CONFIG_IEEE802154=m +CONFIG_IEEE802154_6LOWPAN=m +CONFIG_MAC802154=m +CONFIG_NET_SCHED=y +CONFIG_NET_SCH_CBQ=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_ATM=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_DSMARK=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_MQPRIO=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_INGRESS=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_CLS_BASIC=m +CONFIG_NET_CLS_TCINDEX=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_U32=m +CONFIG_CLS_U32_PERF=y +CONFIG_CLS_U32_MARK=y +CONFIG_NET_CLS_RSVP=m +CONFIG_NET_CLS_RSVP6=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_CGROUP=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOWER=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_CANID=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_CLS_ACT=y +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_GACT=m +CONFIG_GACT_PROB=y +CONFIG_NET_ACT_MIRRED=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_IPT=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_IFE=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_IFE_SKBMARK=m +CONFIG_NET_IFE_SKBPRIO=m +CONFIG_NET_IFE_SKBTCINDEX=m +CONFIG_NET_CLS_IND=y +CONFIG_DCB=y +CONFIG_BATMAN_ADV=m +# CONFIG_BATMAN_ADV_BATMAN_V is not set +CONFIG_BATMAN_ADV_NC=y +CONFIG_BATMAN_ADV_MCAST=y +CONFIG_BATMAN_ADV_DEBUGFS=y +CONFIG_OPENVSWITCH=m +CONFIG_VSOCKETS=m +CONFIG_VIRTIO_VSOCKETS=m +CONFIG_NETLINK_DIAG=m +CONFIG_NET_MPLS_GSO=y +CONFIG_MPLS_ROUTING=m +CONFIG_MPLS_IPTUNNEL=m +CONFIG_NET_SWITCHDEV=y +CONFIG_CGROUP_NET_PRIO=y +CONFIG_BPF_JIT=y +CONFIG_NET_PKTGEN=m +CONFIG_NET_DROP_MONITOR=m +CONFIG_HAMRADIO=y +CONFIG_AX25=m +CONFIG_NETROM=m +CONFIG_ROSE=m +CONFIG_MKISS=m +CONFIG_6PACK=m +CONFIG_BPQETHER=m +CONFIG_BAYCOM_SER_FDX=m +CONFIG_BAYCOM_SER_HDX=m +CONFIG_YAM=m +CONFIG_CAN=y +CONFIG_CAN_RAW=m +CONFIG_CAN_BCM=m +CONFIG_CAN_GW=m +CONFIG_CAN_ISOTP=m +CONFIG_CAN_VCAN=m +CONFIG_CAN_SLCAN=m +CONFIG_CAN_C_CAN=y +CONFIG_CAN_C_CAN_PLATFORM=y +CONFIG_CAN_HI311X=m +CONFIG_CAN_MCP251X=m +CONFIG_CAN_8DEV_USB=m +CONFIG_CAN_EMS_USB=m +CONFIG_CAN_ESD_USB2=m +CONFIG_CAN_GS_USB=m +CONFIG_CAN_KVASER_USB=m +CONFIG_CAN_PEAK_USB=m +CONFIG_BT=m +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y +CONFIG_BT_HIDP=m +CONFIG_BT_6LOWPAN=m +CONFIG_BT_LEDS=y +CONFIG_BT_HCIBTUSB=m +CONFIG_BT_HCIBTSDIO=m +CONFIG_BT_HCIUART=m +CONFIG_BT_HCIUART_ATH3K=y +CONFIG_BT_HCIUART_LL=y +CONFIG_BT_HCIUART_3WIRE=y +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MRVL=m +CONFIG_BT_MRVL_SDIO=m +CONFIG_BT_ATH3K=m +CONFIG_BT_WILINK=m +CONFIG_AF_RXRPC_IPV6=y +CONFIG_RXKAD=y +CONFIG_CFG80211=m +# CONFIG_CFG80211_DEFAULT_PS is not set +CONFIG_CFG80211_WEXT=y +CONFIG_MAC80211=m +CONFIG_MAC80211_RC_MINSTREL_VHT=y +CONFIG_MAC80211_MESH=y +CONFIG_WIMAX=m +CONFIG_RFKILL=y +CONFIG_RFKILL_INPUT=y +CONFIG_NET_9P=m +CONFIG_NET_9P_VIRTIO=m +CONFIG_NFC=m +CONFIG_NFC_DIGITAL=m +CONFIG_NFC_NCI=m +CONFIG_NFC_NCI_SPI=m +CONFIG_NFC_NCI_UART=m +CONFIG_NFC_HCI=m +CONFIG_NFC_SHDLC=y +CONFIG_NFC_TRF7970A=m +CONFIG_NFC_SIM=m +CONFIG_NFC_PORT100=m +CONFIG_NFC_PN544_I2C=m +CONFIG_NFC_PN533_USB=m +CONFIG_NFC_PN533_I2C=m +CONFIG_NFC_MICROREAD_I2C=m +CONFIG_NFC_ST21NFCA_I2C=m +CONFIG_NFC_ST_NCI_I2C=m +CONFIG_NFC_ST_NCI_SPI=m +CONFIG_NFC_NXP_NCI=m +CONFIG_NFC_NXP_NCI_I2C=m +CONFIG_NFC_ST95HF=m +CONFIG_NET_DEVLINK=m +# CONFIG_UEVENT_HELPER is not set +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_EXTRA_FIRMWARE="am335x-pm-firmware.elf am335x-bone-scale-data.bin am335x-evm-scale-data.bin am43x-evm-scale-data.bin" +CONFIG_EXTRA_FIRMWARE_DIR="firmware" +CONFIG_CMA_SIZE_MBYTES=48 +CONFIG_OMAP_OCP2SCP=y +CONFIG_SIMPLE_PM_BUS=y +CONFIG_CONNECTOR=y +CONFIG_GNSS=y +CONFIG_GNSS_SIRF_SERIAL=y +CONFIG_GNSS_UBX_SERIAL=y +CONFIG_MTD=y +CONFIG_MTD_OF_PARTS=m +CONFIG_MTD_AR7_PARTS=m +CONFIG_MTD_BLOCK=m +CONFIG_MTD_BLOCK_RO=m +CONFIG_RFD_FTL=m +CONFIG_SSFDC=m +CONFIG_MTD_OOPS=m +CONFIG_MTD_SWAP=m +CONFIG_MTD_PHYSMAP=m +CONFIG_MTD_PLATRAM=m +CONFIG_MTD_DATAFLASH=m +CONFIG_MTD_M25P80=m +CONFIG_MTD_SST25L=m +CONFIG_MTD_ONENAND=y +CONFIG_MTD_ONENAND_VERIFY_WRITE=y +CONFIG_MTD_ONENAND_2X_PROGRAM=y +CONFIG_MTD_NAND=y +CONFIG_MTD_NAND_ECC_BCH=y +CONFIG_MTD_NAND_OMAP2=m +CONFIG_MTD_NAND_NANDSIM=m +CONFIG_MTD_LPDDR=m +CONFIG_MTD_SPI_NOR=m +CONFIG_MTD_UBI=y +CONFIG_MTD_UBI_BLOCK=y +CONFIG_OF_OVERLAY=y +CONFIG_BLK_DEV_NULL_BLK=m +CONFIG_ZRAM=m +CONFIG_BLK_DEV_LOOP=m +CONFIG_BLK_DEV_DRBD=m +CONFIG_BLK_DEV_NBD=m +CONFIG_BLK_DEV_RAM=m +CONFIG_BLK_DEV_RAM_SIZE=16384 +CONFIG_ATA_OVER_ETH=m +CONFIG_VIRTIO_BLK=m +CONFIG_BLK_DEV_RBD=m +CONFIG_AD525X_DPOT=m +CONFIG_AD525X_DPOT_I2C=m +CONFIG_AD525X_DPOT_SPI=m +CONFIG_ICS932S401=m +CONFIG_APDS9802ALS=m +CONFIG_ISL29003=m +CONFIG_ISL29020=m +CONFIG_SENSORS_TSL2550=m +CONFIG_SENSORS_BH1770=m +CONFIG_SENSORS_APDS990X=m +CONFIG_HMC6352=m +CONFIG_DS1682=m +CONFIG_SRAM=y +CONFIG_DMA_BUF_PHYS=y +CONFIG_TIEQEP=y +CONFIG_EEPROM_AT24=y +CONFIG_EEPROM_AT25=y +CONFIG_EEPROM_LEGACY=m +CONFIG_EEPROM_MAX6875=m +CONFIG_EEPROM_93XX46=m +CONFIG_TI_ST=m +CONFIG_BEAGLEBONE_PINMUX_HELPER=y +CONFIG_RAID_ATTRS=m +# CONFIG_SCSI_PROC_FS is not set +CONFIG_BLK_DEV_SD=y +CONFIG_CHR_DEV_ST=m +CONFIG_CHR_DEV_OSST=m +CONFIG_BLK_DEV_SR=m +CONFIG_BLK_DEV_SR_VENDOR=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_SCH=m +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SCAN_ASYNC=y +CONFIG_SCSI_SPI_ATTRS=m +CONFIG_SCSI_FC_ATTRS=m +CONFIG_SCSI_SAS_LIBSAS=m +CONFIG_SCSI_SAS_ATA=y +CONFIG_SCSI_SRP_ATTRS=m +CONFIG_ISCSI_TCP=m +CONFIG_ISCSI_BOOT_SYSFS=m +CONFIG_SCSI_UFSHCD=m +CONFIG_LIBFC=m +CONFIG_LIBFCOE=m +CONFIG_SCSI_VIRTIO=m +CONFIG_SCSI_DH=y +CONFIG_SCSI_DH_RDAC=m +CONFIG_SCSI_DH_HP_SW=m +CONFIG_SCSI_DH_EMC=m +CONFIG_SCSI_DH_ALUA=m +CONFIG_SCSI_OSD_INITIATOR=m +CONFIG_SCSI_OSD_ULD=m +CONFIG_ATA=y +CONFIG_SATA_AHCI_PLATFORM=y +CONFIG_MD=y +CONFIG_BLK_DEV_MD=m +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_MULTIPATH=m +CONFIG_MD_FAULTY=m +CONFIG_BCACHE=m +CONFIG_BLK_DEV_DM=m +CONFIG_DM_MQ_DEFAULT=y +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_CACHE=m +CONFIG_DM_ERA=m +CONFIG_DM_MIRROR=m +CONFIG_DM_LOG_USERSPACE=m +CONFIG_DM_ZERO=m +CONFIG_DM_MULTIPATH=m +CONFIG_DM_MULTIPATH_QL=m +CONFIG_DM_MULTIPATH_ST=m +CONFIG_DM_DELAY=m +CONFIG_DM_UEVENT=y +CONFIG_DM_FLAKEY=m +CONFIG_DM_VERITY=m +CONFIG_DM_SWITCH=m +CONFIG_DM_LOG_WRITES=m +CONFIG_DM_INTEGRITY=m +CONFIG_TARGET_CORE=m +CONFIG_TCM_IBLOCK=m +CONFIG_TCM_FILEIO=m +CONFIG_TCM_PSCSI=m +CONFIG_TCM_USER2=m +CONFIG_LOOPBACK_TARGET=m +CONFIG_TCM_FC=m +CONFIG_ISCSI_TARGET=m +CONFIG_NETDEVICES=y +CONFIG_BONDING=m +CONFIG_DUMMY=m +CONFIG_EQUALIZER=m +CONFIG_IFB=m +CONFIG_NET_TEAM=m +CONFIG_NET_TEAM_MODE_BROADCAST=m +CONFIG_NET_TEAM_MODE_ROUNDROBIN=m +CONFIG_NET_TEAM_MODE_RANDOM=m +CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m +CONFIG_NET_TEAM_MODE_LOADBALANCE=m +CONFIG_MACVLAN=m +CONFIG_MACVTAP=m +CONFIG_IPVLAN=m +CONFIG_IPVTAP=m +CONFIG_VXLAN=m +CONFIG_GENEVE=m +CONFIG_GTP=m +CONFIG_MACSEC=m +CONFIG_NETCONSOLE=m +CONFIG_NETCONSOLE_DYNAMIC=y +CONFIG_TUN=m +CONFIG_VETH=m +CONFIG_VIRTIO_NET=m +CONFIG_NLMON=m +CONFIG_NET_VRF=m +CONFIG_ATM_DUMMY=m +# CONFIG_NET_VENDOR_ALACRITECH is not set +# CONFIG_NET_VENDOR_AMAZON is not set +# CONFIG_NET_VENDOR_AQUANTIA is not set +# CONFIG_NET_VENDOR_ARC is not set +# CONFIG_NET_VENDOR_AURORA is not set +# CONFIG_NET_VENDOR_BROADCOM is not set +# CONFIG_NET_VENDOR_CADENCE is not set +# CONFIG_NET_VENDOR_CAVIUM is not set +# CONFIG_NET_VENDOR_CIRRUS is not set +# CONFIG_NET_VENDOR_CORTINA is not set +# CONFIG_NET_VENDOR_EZCHIP is not set +# CONFIG_NET_VENDOR_FARADAY is not set +# CONFIG_NET_VENDOR_HISILICON is not set +# CONFIG_NET_VENDOR_HUAWEI is not set +# CONFIG_NET_VENDOR_INTEL is not set +# CONFIG_NET_VENDOR_MARVELL is not set +# CONFIG_NET_VENDOR_MELLANOX is not set +CONFIG_KS8851=m +CONFIG_ENC28J60=y +CONFIG_ENCX24J600=y +# CONFIG_NET_VENDOR_NATSEMI is not set +# CONFIG_NET_VENDOR_NETRONOME is not set +# CONFIG_NET_VENDOR_NI is not set +# CONFIG_NET_VENDOR_QUALCOMM is not set +# CONFIG_NET_VENDOR_RENESAS is not set +# CONFIG_NET_VENDOR_ROCKER is not set +# CONFIG_NET_VENDOR_SAMSUNG is not set +# CONFIG_NET_VENDOR_SEEQ is not set +# CONFIG_NET_VENDOR_SOLARFLARE is not set +CONFIG_SMC91X=m +CONFIG_SMC911X=m +CONFIG_SMSC911X=m +# CONFIG_NET_VENDOR_SOCIONEXT is not set +# CONFIG_NET_VENDOR_STMICRO is not set +# CONFIG_NET_VENDOR_SYNOPSYS is not set +CONFIG_TI_CPSW=y +CONFIG_TI_CPTS=y +# CONFIG_NET_VENDOR_VIA is not set +CONFIG_WIZNET_W5100=y +CONFIG_WIZNET_W5100_SPI=y +CONFIG_AT803X_PHY=y +CONFIG_DP83848_PHY=y +CONFIG_DP83867_PHY=y +CONFIG_MICREL_PHY=y +CONFIG_MICROSEMI_PHY=y +CONFIG_SMSC_PHY=y +CONFIG_VITESSE_PHY=y +CONFIG_PPP=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPPOATM=m +CONFIG_PPPOE=m +CONFIG_PPTP=m +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_SYNC_TTY=m +CONFIG_SLIP=m +CONFIG_SLIP_COMPRESSED=y +CONFIG_SLIP_SMART=y +CONFIG_SLIP_MODE_SLIP6=y +CONFIG_USB_CATC=m +CONFIG_USB_KAWETH=m +CONFIG_USB_PEGASUS=m +CONFIG_USB_RTL8150=m +CONFIG_USB_RTL8152=m +CONFIG_USB_LAN78XX=m +CONFIG_USB_USBNET=y +CONFIG_USB_NET_AX8817X=m +CONFIG_USB_NET_AX88179_178A=m +CONFIG_USB_NET_CDCETHER=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_NCM=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_DM9601=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_NET_SR9800=m +CONFIG_USB_NET_SMSC75XX=m +CONFIG_USB_NET_SMSC95XX=y +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_NET1080=m +CONFIG_USB_NET_PLUSB=m +CONFIG_USB_NET_MCS7830=m +CONFIG_USB_NET_CDC_SUBSET=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_EPSON2888=y +CONFIG_USB_KC2190=y +CONFIG_USB_NET_ZAURUS=m +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_HSO=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_CDC_PHONET=m +CONFIG_USB_IPHETH=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_VL600=m +CONFIG_USB_NET_CH9200=m +CONFIG_ATH9K=m +CONFIG_ATH9K_HTC=m +CONFIG_ATH9K_HWRNG=y +CONFIG_CARL9170=m +CONFIG_ATH6KL=m +CONFIG_ATH6KL_SDIO=m +CONFIG_ATH6KL_USB=m +CONFIG_AR5523=m +CONFIG_ATH10K=m +CONFIG_ATH10K_USB=m +CONFIG_AT76C50X_USB=m +CONFIG_BRCMFMAC=m +CONFIG_HOSTAP=m +CONFIG_HOSTAP_FIRMWARE=y +CONFIG_P54_COMMON=m +CONFIG_P54_USB=m +CONFIG_LIBERTAS=m +CONFIG_LIBERTAS_USB=m +CONFIG_LIBERTAS_SDIO=m +CONFIG_LIBERTAS_MESH=y +CONFIG_LIBERTAS_THINFIRM=m +CONFIG_LIBERTAS_THINFIRM_USB=m +CONFIG_MT7601U=m +CONFIG_RT2X00=m +CONFIG_RT2500USB=m +CONFIG_RT73USB=m +CONFIG_RT2800USB=m +CONFIG_RT2800USB_RT3573=y +CONFIG_RT2800USB_RT53XX=y +CONFIG_RT2800USB_RT55XX=y +CONFIG_RTL8187=m +CONFIG_RTL8192CU=m +# CONFIG_RTLWIFI_DEBUG is not set +CONFIG_RTL8XXXU=m +CONFIG_RSI_91X=m +# CONFIG_RSI_SDIO is not set +CONFIG_WL1251=m +CONFIG_WL1251_SPI=m +CONFIG_WL1251_SDIO=m +CONFIG_WL12XX=m +CONFIG_WL18XX=m +CONFIG_WLCORE_SPI=m +CONFIG_WLCORE_SDIO=m +CONFIG_ZD1211RW=m +CONFIG_MAC80211_HWSIM=m +CONFIG_USB_NET_RNDIS_WLAN=m +CONFIG_WIMAX_I2400M_USB=m +CONFIG_IEEE802154_FAKELB=m +CONFIG_IEEE802154_AT86RF230=m +CONFIG_IEEE802154_MRF24J40=m +CONFIG_IEEE802154_CC2520=m +CONFIG_IEEE802154_ATUSB=m +CONFIG_IEEE802154_ADF7242=m +CONFIG_INPUT_SPARSEKMAP=m +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_ADP5588=m +# CONFIG_KEYBOARD_ATKBD is not set +CONFIG_KEYBOARD_QT2160=m +CONFIG_KEYBOARD_GPIO=y +CONFIG_KEYBOARD_LM8323=m +CONFIG_KEYBOARD_MAX7359=m +CONFIG_KEYBOARD_OPENCORES=m +CONFIG_KEYBOARD_STOWAWAY=m +# CONFIG_MOUSE_PS2 is not set +CONFIG_MOUSE_APPLETOUCH=m +CONFIG_MOUSE_ELAN_I2C=m +CONFIG_MOUSE_SYNAPTICS_I2C=m +CONFIG_MOUSE_SYNAPTICS_USB=m +CONFIG_INPUT_JOYSTICK=y +CONFIG_JOYSTICK_IFORCE=m +CONFIG_JOYSTICK_IFORCE_USB=y +CONFIG_JOYSTICK_IFORCE_232=y +CONFIG_JOYSTICK_WARRIOR=m +CONFIG_JOYSTICK_MAGELLAN=m +CONFIG_JOYSTICK_SPACEORB=m +CONFIG_JOYSTICK_SPACEBALL=m +CONFIG_JOYSTICK_STINGER=m +CONFIG_JOYSTICK_TWIDJOY=m +CONFIG_JOYSTICK_ZHENHUA=m +CONFIG_JOYSTICK_AS5011=m +CONFIG_JOYSTICK_XPAD=m +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_JOYSTICK_PSXPAD_SPI=y +CONFIG_JOYSTICK_PSXPAD_SPI_FF=y +CONFIG_JOYSTICK_PXRC=m +CONFIG_INPUT_TABLET=y +CONFIG_TABLET_USB_ACECAD=m +CONFIG_TABLET_USB_AIPTEK=m +CONFIG_TABLET_USB_GTCO=m +CONFIG_TABLET_USB_HANWANG=m +CONFIG_TABLET_USB_KBTAB=m +CONFIG_TABLET_USB_PEGASUS=m +CONFIG_TABLET_SERIAL_WACOM4=m +CONFIG_INPUT_TOUCHSCREEN=y +CONFIG_TOUCHSCREEN_ADS7846=m +CONFIG_TOUCHSCREEN_AD7877=m +CONFIG_TOUCHSCREEN_AD7879=m +CONFIG_TOUCHSCREEN_AD7879_I2C=m +CONFIG_TOUCHSCREEN_AR1021_I2C=y +CONFIG_TOUCHSCREEN_ATMEL_MXT=m +CONFIG_TOUCHSCREEN_DYNAPRO=m +CONFIG_TOUCHSCREEN_HAMPSHIRE=m +CONFIG_TOUCHSCREEN_FUJITSU=m +CONFIG_TOUCHSCREEN_GOODIX=m +CONFIG_TOUCHSCREEN_GUNZE=m +CONFIG_TOUCHSCREEN_ELO=m +CONFIG_TOUCHSCREEN_WACOM_W8001=m +CONFIG_TOUCHSCREEN_MCS5000=m +CONFIG_TOUCHSCREEN_MTOUCH=m +CONFIG_TOUCHSCREEN_INEXIO=m +CONFIG_TOUCHSCREEN_MK712=m +CONFIG_TOUCHSCREEN_PENMOUNT=m +CONFIG_TOUCHSCREEN_EDT_FT5X06=y +CONFIG_TOUCHSCREEN_TOUCHRIGHT=m +CONFIG_TOUCHSCREEN_TOUCHWIN=m +CONFIG_TOUCHSCREEN_TI_AM335X_TSC=y +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_TOUCHSCREEN_TSC2005=m +CONFIG_TOUCHSCREEN_TSC2007=m +CONFIG_TOUCHSCREEN_SILEAD=y +CONFIG_TOUCHSCREEN_STMPE=y +CONFIG_TOUCHSCREEN_TPS6507X=m +CONFIG_INPUT_MISC=y +CONFIG_INPUT_MMA8450=m +CONFIG_INPUT_ATI_REMOTE2=m +CONFIG_INPUT_KEYSPAN_REMOTE=m +CONFIG_INPUT_POWERMATE=m +CONFIG_INPUT_YEALINK=m +CONFIG_INPUT_CM109=m +CONFIG_INPUT_TPS65218_PWRBUTTON=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_PALMAS_PWRBUTTON=y +# CONFIG_LEGACY_PTYS is not set +CONFIG_N_GSM=m +CONFIG_SERIAL_8250=y +# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set +CONFIG_SERIAL_8250_CONSOLE=y +# CONFIG_SERIAL_8250_DMA is not set +CONFIG_SERIAL_8250_NR_UARTS=6 +CONFIG_SERIAL_8250_RUNTIME_UARTS=6 +CONFIG_SERIAL_8250_OMAP=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_MAX3100=m +CONFIG_SERIAL_MAX310X=m +CONFIG_SERIAL_SC16IS7XX=m +CONFIG_SERIAL_SC16IS7XX_SPI=y +CONFIG_SERIAL_DEV_BUS=y +# CONFIG_SERIAL_DEV_CTRL_TTYPORT is not set +CONFIG_TTY_PRINTK=m +CONFIG_VIRTIO_CONSOLE=m +CONFIG_HW_RANDOM=y +CONFIG_HW_RANDOM_VIRTIO=m +CONFIG_TCG_TPM=y +CONFIG_TCG_TIS_I2C_ATMEL=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_ARB_GPIO_CHALLENGE=m +CONFIG_I2C_MUX_GPIO=y +CONFIG_I2C_MUX_PCA954x=m +CONFIG_I2C_MUX_PINCTRL=y +CONFIG_I2C_GPIO=y +CONFIG_I2C_DIOLAN_U2C=m +CONFIG_I2C_ROBOTFUZZ_OSIF=m +CONFIG_I2C_TAOS_EVM=m +CONFIG_I2C_TINY_USB=m +CONFIG_SPI=y +CONFIG_SPI_GPIO=m +CONFIG_SPI_OMAP24XX=y +CONFIG_SPI_TI_QSPI=y +CONFIG_SPI_SPIDEV=m +CONFIG_SPI_SLAVE=y +CONFIG_SPI_SLAVE_TIME=m +CONFIG_SPI_SLAVE_SYSTEM_CONTROL=m +CONFIG_PPS_CLIENT_KTIMER=m +CONFIG_PPS_CLIENT_LDISC=m +CONFIG_PPS_CLIENT_GPIO=m +CONFIG_PINCTRL_MCP23S08=m +CONFIG_PINCTRL_SINGLE=y +CONFIG_PINCTRL_PALMAS=y +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_OF_HELPER=y +CONFIG_GPIO_GENERIC_PLATFORM=y +CONFIG_GPIO_SYSCON=y +CONFIG_GPIO_ADP5588=m +CONFIG_GPIO_ADNP=m +CONFIG_GPIO_MAX7300=m +CONFIG_GPIO_MAX732X=m +CONFIG_GPIO_PCA953X=y +CONFIG_GPIO_PCA953X_IRQ=y +CONFIG_GPIO_PCF857X=m +CONFIG_GPIO_TPIC2810=m +CONFIG_GPIO_PALMAS=y +CONFIG_GPIO_STMPE=y +CONFIG_GPIO_TPS65218=y +CONFIG_GPIO_74X164=m +CONFIG_GPIO_MAX3191X=m +CONFIG_GPIO_MAX7301=m +CONFIG_GPIO_MC33880=m +CONFIG_GPIO_PISOSR=m +CONFIG_GPIO_XRA1403=m +CONFIG_W1=m +CONFIG_W1_MASTER_DS2490=m +CONFIG_W1_MASTER_DS2482=m +CONFIG_W1_MASTER_GPIO=m +CONFIG_W1_SLAVE_THERM=m +CONFIG_W1_SLAVE_SMEM=m +CONFIG_W1_SLAVE_DS2405=m +CONFIG_W1_SLAVE_DS2408=m +CONFIG_W1_SLAVE_DS2413=m +CONFIG_W1_SLAVE_DS2406=m +CONFIG_W1_SLAVE_DS2423=m +CONFIG_W1_SLAVE_DS2805=m +CONFIG_W1_SLAVE_DS2431=m +CONFIG_W1_SLAVE_DS2433=m +CONFIG_W1_SLAVE_DS2433_CRC=y +CONFIG_W1_SLAVE_DS2438=m +CONFIG_W1_SLAVE_DS2780=m +CONFIG_W1_SLAVE_DS2781=m +CONFIG_W1_SLAVE_DS28E04=m +CONFIG_W1_SLAVE_DS28E17=m +CONFIG_POWER_AVS=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_GPIO=y +CONFIG_POWER_RESET_GPIO_RESTART=y +CONFIG_POWER_RESET_RESTART=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_POWER_RESET_SYSCON_POWEROFF=y +CONFIG_GENERIC_ADC_BATTERY=m +CONFIG_CHARGER_GPIO=m +CONFIG_SENSORS_AD7314=m +CONFIG_SENSORS_AD7414=m +CONFIG_SENSORS_AD7418=m +CONFIG_SENSORS_ADM1021=m +CONFIG_SENSORS_ADM1025=m +CONFIG_SENSORS_ADM1026=m +CONFIG_SENSORS_ADM1029=m +CONFIG_SENSORS_ADM1031=m +CONFIG_SENSORS_ADM9240=m +CONFIG_SENSORS_ADT7310=m +CONFIG_SENSORS_ADT7410=m +CONFIG_SENSORS_ADT7411=m +CONFIG_SENSORS_ADT7462=m +CONFIG_SENSORS_ADT7470=m +CONFIG_SENSORS_ADT7475=m +CONFIG_SENSORS_ASC7621=m +CONFIG_SENSORS_ASPEED=m +CONFIG_SENSORS_ATXP1=m +CONFIG_SENSORS_DS620=m +CONFIG_SENSORS_DS1621=m +CONFIG_SENSORS_F71805F=m +CONFIG_SENSORS_F71882FG=m +CONFIG_SENSORS_F75375S=m +CONFIG_SENSORS_FTSTEUTATES=m +CONFIG_SENSORS_GL518SM=m +CONFIG_SENSORS_GL520SM=m +CONFIG_SENSORS_G760A=m +CONFIG_SENSORS_G762=m +CONFIG_SENSORS_GPIO_FAN=y +CONFIG_SENSORS_HIH6130=m +CONFIG_SENSORS_IIO_HWMON=m +CONFIG_SENSORS_IT87=m +CONFIG_SENSORS_JC42=m +CONFIG_SENSORS_POWR1220=m +CONFIG_SENSORS_LINEAGE=m +CONFIG_SENSORS_LTC2945=m +CONFIG_SENSORS_LTC2990=m +CONFIG_SENSORS_LTC4151=m +CONFIG_SENSORS_LTC4215=m +CONFIG_SENSORS_LTC4222=m +CONFIG_SENSORS_LTC4245=m +CONFIG_SENSORS_LTC4260=m +CONFIG_SENSORS_LTC4261=m +CONFIG_SENSORS_MAX1111=m +CONFIG_SENSORS_MAX16065=m +CONFIG_SENSORS_MAX1619=m +CONFIG_SENSORS_MAX1668=m +CONFIG_SENSORS_MAX197=m +CONFIG_SENSORS_MAX31722=m +CONFIG_SENSORS_MAX6621=m +CONFIG_SENSORS_MAX6639=m +CONFIG_SENSORS_MAX6642=m +CONFIG_SENSORS_MAX6650=m +CONFIG_SENSORS_MAX6697=m +CONFIG_SENSORS_MAX31790=m +CONFIG_SENSORS_MCP3021=m +CONFIG_SENSORS_TC654=m +CONFIG_SENSORS_ADCXX=m +CONFIG_SENSORS_LM63=m +CONFIG_SENSORS_LM70=m +CONFIG_SENSORS_LM73=m +CONFIG_SENSORS_LM75=m +CONFIG_SENSORS_LM77=m +CONFIG_SENSORS_LM78=m +CONFIG_SENSORS_LM80=m +CONFIG_SENSORS_LM83=m +CONFIG_SENSORS_LM85=m +CONFIG_SENSORS_LM87=m +CONFIG_SENSORS_LM90=m +CONFIG_SENSORS_LM92=m +CONFIG_SENSORS_LM93=m +CONFIG_SENSORS_LM95234=m +CONFIG_SENSORS_LM95241=m +CONFIG_SENSORS_LM95245=m +CONFIG_SENSORS_PC87360=m +CONFIG_SENSORS_PC87427=m +CONFIG_SENSORS_NTC_THERMISTOR=m +CONFIG_SENSORS_NCT6683=m +CONFIG_SENSORS_NCT6775=m +CONFIG_SENSORS_NCT7802=m +CONFIG_SENSORS_NCT7904=m +CONFIG_SENSORS_NPCM7XX=m +CONFIG_SENSORS_PCF8591=m +CONFIG_PMBUS=m +CONFIG_SENSORS_ADM1275=m +CONFIG_SENSORS_IBM_CFFPS=m +CONFIG_SENSORS_IR35221=m +CONFIG_SENSORS_LM25066=m +CONFIG_SENSORS_LTC2978=m +CONFIG_SENSORS_LTC2978_REGULATOR=y +CONFIG_SENSORS_LTC3815=m +CONFIG_SENSORS_MAX16064=m +CONFIG_SENSORS_MAX20751=m +CONFIG_SENSORS_MAX31785=m +CONFIG_SENSORS_MAX34440=m +CONFIG_SENSORS_MAX8688=m +CONFIG_SENSORS_TPS40422=m +CONFIG_SENSORS_TPS53679=m +CONFIG_SENSORS_UCD9000=m +CONFIG_SENSORS_UCD9200=m +CONFIG_SENSORS_ZL6100=m +CONFIG_SENSORS_PWM_FAN=m +CONFIG_SENSORS_SHT15=m +CONFIG_SENSORS_SHT21=m +CONFIG_SENSORS_SHTC1=m +CONFIG_SENSORS_DME1737=m +CONFIG_SENSORS_EMC1403=m +CONFIG_SENSORS_EMC2103=m +CONFIG_SENSORS_EMC6W201=m +CONFIG_SENSORS_SMSC47M1=m +CONFIG_SENSORS_SMSC47M192=m +CONFIG_SENSORS_SMSC47B397=m +CONFIG_SENSORS_SCH5627=m +CONFIG_SENSORS_SCH5636=m +CONFIG_SENSORS_STTS751=m +CONFIG_SENSORS_SMM665=m +CONFIG_SENSORS_ADC128D818=m +CONFIG_SENSORS_ADS1015=m +CONFIG_SENSORS_ADS7828=m +CONFIG_SENSORS_ADS7871=m +CONFIG_SENSORS_AMC6821=m +CONFIG_SENSORS_INA209=m +CONFIG_SENSORS_INA2XX=m +CONFIG_SENSORS_INA3221=m +CONFIG_SENSORS_TC74=m +CONFIG_SENSORS_THMC50=m +CONFIG_SENSORS_TMP102=y +CONFIG_SENSORS_TMP103=m +CONFIG_SENSORS_TMP108=m +CONFIG_SENSORS_TMP401=m +CONFIG_SENSORS_TMP421=m +CONFIG_SENSORS_VT1211=m +CONFIG_SENSORS_W83773G=m +CONFIG_SENSORS_W83781D=m +CONFIG_SENSORS_W83791D=m +CONFIG_SENSORS_W83792D=m +CONFIG_SENSORS_W83793=m +CONFIG_SENSORS_W83795=m +CONFIG_SENSORS_W83L785TS=m +CONFIG_SENSORS_W83L786NG=m +CONFIG_SENSORS_W83627HF=m +CONFIG_SENSORS_W83627EHF=m +CONFIG_THERMAL_GOV_FAIR_SHARE=y +CONFIG_THERMAL_GOV_BANG_BANG=y +CONFIG_CPU_THERMAL=y +CONFIG_CLOCK_THERMAL=y +CONFIG_DEVFREQ_THERMAL=y +CONFIG_TI_THERMAL=y +CONFIG_OMAP5_THERMAL=y +CONFIG_DRA752_THERMAL=y +CONFIG_GENERIC_ADC_THERMAL=m +CONFIG_WATCHDOG=y +CONFIG_WATCHDOG_SYSFS=y +CONFIG_SOFT_WATCHDOG=y +CONFIG_OMAP_WATCHDOG=y +CONFIG_MFD_STMPE=y +CONFIG_MFD_TI_AM335X_TSCADC=y +CONFIG_MFD_PALMAS=y +CONFIG_MFD_TPS65217=y +CONFIG_MFD_TPS65218=y +CONFIG_MFD_WL1273_CORE=m +CONFIG_REGULATOR_USERSPACE_CONSUMER=y +CONFIG_REGULATOR_GPIO=y +CONFIG_REGULATOR_PALMAS=y +CONFIG_REGULATOR_PBIAS=y +CONFIG_REGULATOR_PWM=y +CONFIG_REGULATOR_TI_ABB=y +CONFIG_REGULATOR_TPS65217=y +CONFIG_REGULATOR_TPS65218=y +CONFIG_RC_CORE=m +CONFIG_LIRC=y +CONFIG_RC_DECODERS=y +CONFIG_IR_NEC_DECODER=m +CONFIG_IR_RC5_DECODER=m +CONFIG_IR_RC6_DECODER=m +CONFIG_IR_JVC_DECODER=m +CONFIG_IR_SONY_DECODER=m +CONFIG_IR_SANYO_DECODER=m +CONFIG_IR_SHARP_DECODER=m +CONFIG_IR_MCE_KBD_DECODER=m +CONFIG_IR_XMP_DECODER=m +CONFIG_RC_DEVICES=y +CONFIG_RC_ATI_REMOTE=m +CONFIG_IR_IMON=m +CONFIG_IR_MCEUSB=m +CONFIG_IR_REDRAT3=m +CONFIG_IR_STREAMZAP=m +CONFIG_IR_IGORPLUGUSB=m +CONFIG_IR_IGUANA=m +CONFIG_IR_TTUSBIR=m +CONFIG_RC_LOOPBACK=m +CONFIG_IR_GPIO_CIR=m +CONFIG_MEDIA_SUPPORT=m +CONFIG_MEDIA_CAMERA_SUPPORT=y +CONFIG_MEDIA_ANALOG_TV_SUPPORT=y +CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y +CONFIG_MEDIA_RADIO_SUPPORT=y +CONFIG_MEDIA_SDR_SUPPORT=y +CONFIG_MEDIA_CEC_SUPPORT=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_V4L2_SUBDEV_API=y +CONFIG_DVB_DYNAMIC_MINORS=y +CONFIG_MEDIA_USB_SUPPORT=y +CONFIG_USB_VIDEO_CLASS=m +CONFIG_USB_M5602=m +CONFIG_USB_STV06XX=m +CONFIG_USB_GL860=m +CONFIG_USB_GSPCA_BENQ=m +CONFIG_USB_GSPCA_CONEX=m +CONFIG_USB_GSPCA_CPIA1=m +CONFIG_USB_GSPCA_DTCS033=m +CONFIG_USB_GSPCA_ETOMS=m +CONFIG_USB_GSPCA_FINEPIX=m +CONFIG_USB_GSPCA_JEILINJ=m +CONFIG_USB_GSPCA_JL2005BCD=m +CONFIG_USB_GSPCA_KINECT=m +CONFIG_USB_GSPCA_KONICA=m +CONFIG_USB_GSPCA_MARS=m +CONFIG_USB_GSPCA_MR97310A=m +CONFIG_USB_GSPCA_NW80X=m +CONFIG_USB_GSPCA_OV519=m +CONFIG_USB_GSPCA_OV534=m +CONFIG_USB_GSPCA_OV534_9=m +CONFIG_USB_GSPCA_PAC207=m +CONFIG_USB_GSPCA_PAC7302=m +CONFIG_USB_GSPCA_PAC7311=m +CONFIG_USB_GSPCA_SE401=m +CONFIG_USB_GSPCA_SN9C2028=m +CONFIG_USB_GSPCA_SN9C20X=m +CONFIG_USB_GSPCA_SONIXB=m +CONFIG_USB_GSPCA_SONIXJ=m +CONFIG_USB_GSPCA_SPCA500=m +CONFIG_USB_GSPCA_SPCA501=m +CONFIG_USB_GSPCA_SPCA505=m +CONFIG_USB_GSPCA_SPCA506=m +CONFIG_USB_GSPCA_SPCA508=m +CONFIG_USB_GSPCA_SPCA561=m +CONFIG_USB_GSPCA_SPCA1528=m +CONFIG_USB_GSPCA_SQ905=m +CONFIG_USB_GSPCA_SQ905C=m +CONFIG_USB_GSPCA_SQ930X=m +CONFIG_USB_GSPCA_STK014=m +CONFIG_USB_GSPCA_STK1135=m +CONFIG_USB_GSPCA_STV0680=m +CONFIG_USB_GSPCA_SUNPLUS=m +CONFIG_USB_GSPCA_T613=m +CONFIG_USB_GSPCA_TOPRO=m +CONFIG_USB_GSPCA_TOUPTEK=m +CONFIG_USB_GSPCA_TV8532=m +CONFIG_USB_GSPCA_VC032X=m +CONFIG_USB_GSPCA_VICAM=m +CONFIG_USB_GSPCA_XIRLINK_CIT=m +CONFIG_USB_GSPCA_ZC3XX=m +CONFIG_USB_PWC=m +CONFIG_VIDEO_CPIA2=m +CONFIG_USB_ZR364XX=m +CONFIG_USB_STKWEBCAM=m +CONFIG_USB_S2255=m +CONFIG_VIDEO_USBTV=m +CONFIG_VIDEO_PVRUSB2=m +CONFIG_VIDEO_HDPVR=m +CONFIG_VIDEO_USBVISION=m +CONFIG_VIDEO_STK1160_COMMON=m +CONFIG_VIDEO_GO7007=m +CONFIG_VIDEO_GO7007_USB=m +CONFIG_VIDEO_GO7007_USB_S2250_BOARD=m +CONFIG_VIDEO_AU0828=m +CONFIG_VIDEO_AU0828_RC=y +CONFIG_VIDEO_CX231XX=m +CONFIG_VIDEO_CX231XX_ALSA=m +CONFIG_VIDEO_CX231XX_DVB=m +CONFIG_DVB_USB=m +CONFIG_DVB_USB_A800=m +CONFIG_DVB_USB_DIBUSB_MB=m +CONFIG_DVB_USB_DIBUSB_MB_FAULTY=y +CONFIG_DVB_USB_DIBUSB_MC=m +CONFIG_DVB_USB_DIB0700=m +CONFIG_DVB_USB_UMT_010=m +CONFIG_DVB_USB_CXUSB=m +CONFIG_DVB_USB_M920X=m +CONFIG_DVB_USB_DIGITV=m +CONFIG_DVB_USB_VP7045=m +CONFIG_DVB_USB_VP702X=m +CONFIG_DVB_USB_GP8PSK=m +CONFIG_DVB_USB_NOVA_T_USB2=m +CONFIG_DVB_USB_TTUSB2=m +CONFIG_DVB_USB_DTT200U=m +CONFIG_DVB_USB_OPERA1=m +CONFIG_DVB_USB_AF9005=m +CONFIG_DVB_USB_AF9005_REMOTE=m +CONFIG_DVB_USB_PCTV452E=m +CONFIG_DVB_USB_DW2102=m +CONFIG_DVB_USB_CINERGY_T2=m +CONFIG_DVB_USB_DTV5100=m +CONFIG_DVB_USB_AZ6027=m +CONFIG_DVB_USB_TECHNISAT_USB2=m +CONFIG_DVB_USB_V2=m +CONFIG_DVB_USB_AF9015=m +CONFIG_DVB_USB_AF9035=m +CONFIG_DVB_USB_ANYSEE=m +CONFIG_DVB_USB_AU6610=m +CONFIG_DVB_USB_AZ6007=m +CONFIG_DVB_USB_CE6230=m +CONFIG_DVB_USB_EC168=m +CONFIG_DVB_USB_GL861=m +CONFIG_DVB_USB_LME2510=m +CONFIG_DVB_USB_MXL111SF=m +CONFIG_DVB_USB_RTL28XXU=m +CONFIG_DVB_USB_DVBSKY=m +CONFIG_SMS_USB_DRV=m +CONFIG_DVB_B2C2_FLEXCOP_USB=m +CONFIG_DVB_AS102=m +CONFIG_VIDEO_EM28XX=m +CONFIG_VIDEO_EM28XX_V4L2=m +CONFIG_VIDEO_EM28XX_ALSA=m +CONFIG_VIDEO_EM28XX_DVB=m +CONFIG_USB_AIRSPY=m +CONFIG_USB_HACKRF=m +CONFIG_USB_MSI2500=m +CONFIG_USB_PULSE8_CEC=m +CONFIG_USB_RAINSHADOW_CEC=m +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_TI_CAL=m +CONFIG_VIDEO_TI_VIP=m +CONFIG_V4L_MEM2MEM_DRIVERS=y +CONFIG_VIDEO_TI_VPE=m +CONFIG_SMS_SDIO_DRV=m +CONFIG_RADIO_SI470X=m +CONFIG_USB_SI470X=m +CONFIG_RADIO_SI4713=m +CONFIG_I2C_SI4713=m +CONFIG_USB_MR800=m +CONFIG_RADIO_SHARK=m +CONFIG_RADIO_SHARK2=m +CONFIG_USB_KEENE=m +CONFIG_USB_RAREMONO=m +CONFIG_USB_MA901=m +CONFIG_RADIO_WL128X=m +CONFIG_DRM=y +CONFIG_DRM_DP_AUX_CHARDEV=y +CONFIG_DRM_LOAD_EDID_FIRMWARE=y +CONFIG_DRM_I2C_NXP_TDA998X=y +CONFIG_DRM_VGEM=m +CONFIG_DRM_UDL=m +CONFIG_DRM_OMAP=y +# CONFIG_OMAP2_DSS_VENC is not set +CONFIG_OMAP5_DSS_HDMI=y +CONFIG_DRM_OMAP_ENCODER_TPD12S015=y +CONFIG_DRM_OMAP_DRA7EVM_ENCODER_TPD12S015=y +CONFIG_DRM_OMAP_CONNECTOR_HDMI=y +CONFIG_DRM_TILCDC=y +CONFIG_DRM_PANEL_SIMPLE=y +CONFIG_DRM_PANEL_OSD_OSD101T2587_53TS=y +CONFIG_DRM_SII902X=y +CONFIG_DRM_TI_TFP410=y +CONFIG_DRM_I2C_ADV7511=y +CONFIG_DRM_I2C_ADV7511_AUDIO=y +CONFIG_DRM_ETNAVIV=y +CONFIG_DRM_TINYDRM=m +CONFIG_TINYDRM_ILI9225=m +CONFIG_TINYDRM_ILI9341=m +CONFIG_TINYDRM_MI0283QT=m +CONFIG_TINYDRM_REPAPER=m +CONFIG_TINYDRM_ST7586=m +CONFIG_TINYDRM_ST7735R=m +CONFIG_DRM_LEGACY=y +CONFIG_FIRMWARE_EDID=y +CONFIG_FB_TILEBLITTING=y +CONFIG_FB_SMSCUFX=m +CONFIG_FB_UDL=m +CONFIG_FB_SIMPLE=y +CONFIG_FB_SSD1307=y +# CONFIG_LCD_CLASS_DEVICE is not set +CONFIG_BACKLIGHT_PWM=y +CONFIG_BACKLIGHT_GPIO=y +CONFIG_BACKLIGHT_LED=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_ROTATION=y +CONFIG_LOGO=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_OSSEMUL=y +CONFIG_SND_MIXER_OSS=m +CONFIG_SND_PCM_OSS=m +CONFIG_SND_HRTIMER=m +CONFIG_SND_DYNAMIC_MINORS=y +CONFIG_SND_SEQUENCER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_DUMMY=m +CONFIG_SND_ALOOP=m +CONFIG_SND_VIRMIDI=m +CONFIG_SND_MTPAV=m +CONFIG_SND_SERIAL_U16550=m +CONFIG_SND_MPU401=m +CONFIG_SND_HDA_PREALLOC_SIZE=2048 +CONFIG_SND_USB_AUDIO=m +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_BCD2000=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SND_SOC=y +CONFIG_SND_SOC_DAVINCI_MCASP=y +CONFIG_SND_SOC_OMAP_DMIC=y +CONFIG_SND_SOC_OMAP_MCBSP=y +CONFIG_SND_SOC_OMAP_MCPDM=y +CONFIG_SND_SOC_OMAP_HDMI=y +CONFIG_SND_SOC_ADAU1701=m +CONFIG_SND_SOC_ADAU7002=m +CONFIG_SND_SOC_AK4554=m +CONFIG_SND_SOC_CS4265=m +CONFIG_SND_SOC_CS4271_I2C=m +CONFIG_SND_SOC_PCM3168A_I2C=m +CONFIG_SND_SOC_PCM512x_I2C=m +CONFIG_SND_SOC_SPDIF=m +CONFIG_SND_SOC_TLV320AIC23_I2C=y +CONFIG_SND_SOC_TLV320AIC31XX=y +CONFIG_SND_SOC_TLV320AIC3X=y +CONFIG_SND_SOC_TS3A227E=m +CONFIG_SND_SOC_WM8753=m +CONFIG_SND_SOC_WM8804_I2C=m +CONFIG_SND_SOC_WM8903=m +CONFIG_SND_SOC_TPA6130A2=m +CONFIG_SND_SIMPLE_CARD=y +CONFIG_SND_SIMPLE_SCU_CARD=m +CONFIG_SND_AUDIO_GRAPH_CARD=m +CONFIG_SND_AUDIO_GRAPH_SCU_CARD=m +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HIDRAW=y +CONFIG_UHID=y +CONFIG_HID_A4TECH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_APPLE=m +CONFIG_HID_APPLEIR=m +CONFIG_HID_ASUS=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CP2112=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_DRAGONRISE_FF=y +CONFIG_HID_EMS_FF=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EZKEY=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GFRM=m +CONFIG_HID_HOLTEK=m +CONFIG_HOLTEK_FF=y +CONFIG_HID_GT683R=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_WALTOP=m +CONFIG_HID_GYRATION=m +CONFIG_HID_ICADE=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LOGITECH=y +CONFIG_HID_LOGITECH_DJ=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGIG940_FF=y +CONFIG_HID_MAGICMOUSE=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_MULTITOUCH=m +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_PANTHERLORD_FF=y +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PICOLCD_FB=y +CONFIG_HID_PICOLCD_BACKLIGHT=y +CONFIG_HID_PICOLCD_LEDS=y +CONFIG_HID_PICOLCD_CIR=y +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_RETRODE=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SONY=m +CONFIG_SONY_FF=y +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEAM=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_RMI=m +CONFIG_HID_GREENASIA=m +CONFIG_GREENASIA_FF=y +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_HID_TIVO=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_THRUSTMASTER_FF=y +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_WACOM=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_ZEROPLUS_FF=y +CONFIG_HID_ZYDACRON=m +CONFIG_HID_SENSOR_CUSTOM_SENSOR=m +CONFIG_HID_ALPS=m +CONFIG_USB_HID=m +CONFIG_HID_PID=y +CONFIG_USB_HIDDEV=y +CONFIG_I2C_HID=m +CONFIG_USB=y +CONFIG_USB_ANNOUNCE_NEW_DEVICES=y +CONFIG_USB_DYNAMIC_MINORS=y +CONFIG_USB_OTG=y +CONFIG_USB_LEDS_TRIGGER_USBPORT=m +CONFIG_USB_MON=m +CONFIG_USB_WUSB_CBAF=m +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_EHCI_ROOT_HUB_TT=y +CONFIG_USB_EHCI_HCD_PLATFORM=y +CONFIG_USB_PRINTER=m +CONFIG_USB_TMC=m +CONFIG_USB_STORAGE=y +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_STORAGE_DATAFAB=m +CONFIG_USB_STORAGE_FREECOM=m +CONFIG_USB_STORAGE_ISD200=m +CONFIG_USB_STORAGE_USBAT=m +CONFIG_USB_STORAGE_SDDR09=m +CONFIG_USB_STORAGE_SDDR55=m +CONFIG_USB_STORAGE_JUMPSHOT=m +CONFIG_USB_STORAGE_ALAUDA=m +CONFIG_USB_STORAGE_ONETOUCH=m +CONFIG_USB_STORAGE_KARMA=m +CONFIG_USB_STORAGE_CYPRESS_ATACB=m +CONFIG_USB_STORAGE_ENE_UB6250=m +CONFIG_USB_UAS=m +CONFIG_USB_MDC800=m +CONFIG_USB_MICROTEK=m +CONFIG_USBIP_CORE=m +CONFIG_USBIP_VHCI_HCD=m +CONFIG_USBIP_VHCI_HC_PORTS=15 +CONFIG_USBIP_VHCI_NR_HCS=8 +CONFIG_USBIP_HOST=m +CONFIG_USBIP_VUDC=m +CONFIG_USB_MUSB_HDRC=y +CONFIG_USB_MUSB_DSPS=y +CONFIG_MUSB_PIO_ONLY=y +CONFIG_USB_DWC3=y +CONFIG_USB_SERIAL=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_CP210X=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_FTDI_SIO=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_METRO=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_MXUPORT=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_XIRCOM=m +CONFIG_USB_SERIAL_OPTION=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SERIAL_WISHBONE=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_DEBUG=m +CONFIG_USB_EMI62=m +CONFIG_USB_EMI26=m +CONFIG_USB_ADUTUX=m +CONFIG_USB_SEVSEG=m +CONFIG_USB_LEGOTOWER=m +CONFIG_USB_LCD=m +CONFIG_USB_CYPRESS_CY7C63=m +CONFIG_USB_CYTHERM=m +CONFIG_USB_IDMOUSE=m +CONFIG_USB_FTDI_ELAN=m +CONFIG_USB_APPLEDISPLAY=m +CONFIG_USB_SISUSBVGA=m +CONFIG_USB_SISUSBVGA_CON=y +CONFIG_USB_LD=m +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_USB_IOWARRIOR=m +CONFIG_USB_TEST=m +CONFIG_USB_EHSET_TEST_FIXTURE=m +CONFIG_USB_ISIGHTFW=m +CONFIG_USB_YUREX=m +CONFIG_USB_HSIC_USB3503=m +CONFIG_USB_CHAOSKEY=m +CONFIG_NOP_USB_XCEIV=y +CONFIG_AM335X_PHY_USB=y +CONFIG_USB_GPIO_VBUS=y +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_VBUS_DRAW=500 +CONFIG_USB_DUMMY_HCD=m +CONFIG_USB_CONFIGFS=m +CONFIG_USB_CONFIGFS_SERIAL=y +CONFIG_USB_CONFIGFS_ACM=y +CONFIG_USB_CONFIGFS_OBEX=y +CONFIG_USB_CONFIGFS_NCM=y +CONFIG_USB_CONFIGFS_ECM=y +CONFIG_USB_CONFIGFS_ECM_SUBSET=y +CONFIG_USB_CONFIGFS_RNDIS=y +CONFIG_USB_CONFIGFS_EEM=y +CONFIG_USB_CONFIGFS_PHONET=y +CONFIG_USB_CONFIGFS_MASS_STORAGE=y +CONFIG_USB_CONFIGFS_F_LB_SS=y +CONFIG_USB_CONFIGFS_F_FS=y +CONFIG_USB_CONFIGFS_F_UAC1=y +CONFIG_USB_CONFIGFS_F_UAC2=y +CONFIG_USB_CONFIGFS_F_MIDI=y +CONFIG_USB_CONFIGFS_F_HID=y +CONFIG_USB_CONFIGFS_F_UVC=y +CONFIG_USB_CONFIGFS_F_PRINTER=y +CONFIG_USB_ZERO=m +CONFIG_USB_AUDIO=m +CONFIG_USB_ETH=m +CONFIG_USB_G_NCM=m +CONFIG_USB_GADGETFS=m +CONFIG_USB_FUNCTIONFS=m +CONFIG_USB_FUNCTIONFS_ETH=y +CONFIG_USB_FUNCTIONFS_RNDIS=y +CONFIG_USB_FUNCTIONFS_GENERIC=y +CONFIG_USB_MASS_STORAGE=m +CONFIG_USB_G_SERIAL=m +CONFIG_USB_MIDI_GADGET=m +CONFIG_USB_G_PRINTER=m +CONFIG_USB_CDC_COMPOSITE=m +CONFIG_USB_G_NOKIA=m +CONFIG_USB_G_ACM_MS=m +CONFIG_USB_G_MULTI=m +CONFIG_USB_G_HID=m +CONFIG_USB_G_DBGP=m +CONFIG_TYPEC=y +CONFIG_TYPEC_TCPM=y +CONFIG_TYPEC_UCSI=y +CONFIG_USB_LED_TRIG=y +CONFIG_MMC=y +CONFIG_SDIO_UART=m +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MMC_OMAP_HS=y +CONFIG_MMC_SPI=y +CONFIG_MMC_VUB300=m +CONFIG_MMC_USHC=m +CONFIG_MMC_SDHCI_OMAP=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_BRIGHTNESS_HW_CHANGED=y +CONFIG_LEDS_LM3530=m +CONFIG_LEDS_LM3642=m +CONFIG_LEDS_PCA9532=m +CONFIG_LEDS_PCA9532_GPIO=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_LP3944=m +CONFIG_LEDS_LP5521=m +CONFIG_LEDS_LP5523=m +CONFIG_LEDS_LP5562=m +CONFIG_LEDS_LP8501=m +CONFIG_LEDS_LP8860=m +CONFIG_LEDS_PCA955X=m +CONFIG_LEDS_PCA963X=m +CONFIG_LEDS_DAC124S085=m +CONFIG_LEDS_PWM=m +CONFIG_LEDS_REGULATOR=m +CONFIG_LEDS_BD2802=m +CONFIG_LEDS_LT3593=m +CONFIG_LEDS_TCA6507=m +CONFIG_LEDS_TLC591XX=m +CONFIG_LEDS_LM355x=m +CONFIG_LEDS_IS31FL319X=m +CONFIG_LEDS_IS31FL32XX=m +CONFIG_LEDS_TRIGGER_TIMER=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_DISK=y +CONFIG_LEDS_TRIGGER_MTD=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=y +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_ACTIVITY=y +CONFIG_LEDS_TRIGGER_GPIO=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_LEDS_TRIGGER_TRANSIENT=m +CONFIG_LEDS_TRIGGER_CAMERA=m +CONFIG_LEDS_TRIGGER_PANIC=y +CONFIG_LEDS_TRIGGER_NETDEV=y +CONFIG_ACCESSIBILITY=y +CONFIG_A11Y_BRAILLE_CONSOLE=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_ABB5ZES3=y +CONFIG_RTC_DRV_ABX80X=y +CONFIG_RTC_DRV_DS1307=y +CONFIG_RTC_DRV_DS1374=y +CONFIG_RTC_DRV_DS1374_WDT=y +CONFIG_RTC_DRV_DS1672=y +CONFIG_RTC_DRV_HYM8563=y +CONFIG_RTC_DRV_MAX6900=y +CONFIG_RTC_DRV_RS5C372=y +CONFIG_RTC_DRV_ISL1208=y +CONFIG_RTC_DRV_ISL12022=y +CONFIG_RTC_DRV_ISL12026=y +CONFIG_RTC_DRV_X1205=y +CONFIG_RTC_DRV_PCF8523=y +CONFIG_RTC_DRV_PCF85063=y +CONFIG_RTC_DRV_PCF85363=y +CONFIG_RTC_DRV_PCF8563=y +CONFIG_RTC_DRV_PCF8583=y +CONFIG_RTC_DRV_M41T80=y +CONFIG_RTC_DRV_M41T80_WDT=y +CONFIG_RTC_DRV_BQ32K=y +CONFIG_RTC_DRV_PALMAS=y +CONFIG_RTC_DRV_S35390A=y +CONFIG_RTC_DRV_FM3130=y +CONFIG_RTC_DRV_RX8010=y +CONFIG_RTC_DRV_RX8581=y +CONFIG_RTC_DRV_RX8025=y +CONFIG_RTC_DRV_EM3027=y +CONFIG_RTC_DRV_RV8803=y +CONFIG_RTC_DRV_M41T93=y +CONFIG_RTC_DRV_M41T94=y +CONFIG_RTC_DRV_DS1302=y +CONFIG_RTC_DRV_DS1305=y +CONFIG_RTC_DRV_DS1343=y +CONFIG_RTC_DRV_DS1347=y +CONFIG_RTC_DRV_DS1390=y +CONFIG_RTC_DRV_MAX6916=y +CONFIG_RTC_DRV_R9701=y +CONFIG_RTC_DRV_RX4581=y +CONFIG_RTC_DRV_RX6110=y +CONFIG_RTC_DRV_RS5C348=y +CONFIG_RTC_DRV_MAX6902=y +CONFIG_RTC_DRV_PCF2123=y +CONFIG_RTC_DRV_MCP795=y +CONFIG_RTC_DRV_DS3232=y +CONFIG_RTC_DRV_PCF2127=y +CONFIG_RTC_DRV_RV3029C2=y +CONFIG_RTC_DRV_CMOS=m +CONFIG_RTC_DRV_DS1286=m +CONFIG_RTC_DRV_DS1511=m +CONFIG_RTC_DRV_DS1553=m +CONFIG_RTC_DRV_DS1685_FAMILY=m +CONFIG_RTC_DRV_DS1742=m +CONFIG_RTC_DRV_DS2404=m +CONFIG_RTC_DRV_STK17TA8=m +CONFIG_RTC_DRV_M48T86=m +CONFIG_RTC_DRV_M48T35=m +CONFIG_RTC_DRV_M48T59=m +CONFIG_RTC_DRV_MSM6242=m +CONFIG_RTC_DRV_BQ4802=m +CONFIG_RTC_DRV_RP5C01=m +CONFIG_RTC_DRV_V3020=m +CONFIG_RTC_DRV_OMAP=y +CONFIG_RTC_DRV_HID_SENSOR_TIME=m +CONFIG_DMADEVICES=y +CONFIG_TI_CPPI41=y +CONFIG_ASYNC_TX_DMA=y +CONFIG_AUXDISPLAY=y +CONFIG_HD44780=m +CONFIG_IMG_ASCII_LCD=m +CONFIG_HT16K33=m +CONFIG_UIO=m +CONFIG_UIO_PDRV_GENIRQ=m +CONFIG_UIO_DMEM_GENIRQ=m +CONFIG_UIO_PRUSS=m +CONFIG_UIO_PRUSS_SHMEM=m +CONFIG_VIRT_DRIVERS=y +CONFIG_VIRTIO_BALLOON=m +CONFIG_VIRTIO_INPUT=m +CONFIG_VIRTIO_MMIO=m +CONFIG_STAGING=y +CONFIG_RTLLIB=m +CONFIG_RTL8723BS=m +CONFIG_R8712U=m +CONFIG_R8188EU=m +CONFIG_ADIS16203=m +CONFIG_ADIS16240=m +CONFIG_AD7606=m +CONFIG_AD7606_IFACE_SPI=m +CONFIG_AD7780=m +CONFIG_AD7816=m +CONFIG_AD7192=m +CONFIG_AD7280=m +CONFIG_ADT7316=m +CONFIG_ADT7316_I2C=m +CONFIG_AD7150=m +CONFIG_AD7152=m +CONFIG_AD7746=m +CONFIG_AD9832=m +CONFIG_AD9834=m +CONFIG_AD5933=m +CONFIG_ADE7854=m +CONFIG_AD2S90=m +CONFIG_AD2S1210=m +CONFIG_ION=y +CONFIG_ION_SYSTEM_HEAP=y +CONFIG_ION_CARVEOUT_HEAP=y +CONFIG_ION_CHUNK_HEAP=y +CONFIG_ION_TI=y +CONFIG_STAGING_BOARD=y +CONFIG_FB_TFT=m +CONFIG_FB_TFT_AGM1264K_FL=m +CONFIG_FB_TFT_BD663474=m +CONFIG_FB_TFT_HX8340BN=m +CONFIG_FB_TFT_HX8347D=m +CONFIG_FB_TFT_HX8353D=m +CONFIG_FB_TFT_HX8357D=m +CONFIG_FB_TFT_ILI9163=m +CONFIG_FB_TFT_ILI9320=m +CONFIG_FB_TFT_ILI9325=m +CONFIG_FB_TFT_ILI9340=m +CONFIG_FB_TFT_ILI9341=m +CONFIG_FB_TFT_ILI9481=m +CONFIG_FB_TFT_ILI9486=m +CONFIG_FB_TFT_PCD8544=m +CONFIG_FB_TFT_RA8875=m +CONFIG_FB_TFT_S6D02A1=m +CONFIG_FB_TFT_S6D1121=m +CONFIG_FB_TFT_SH1106=m +CONFIG_FB_TFT_SSD1289=m +CONFIG_FB_TFT_SSD1305=m +CONFIG_FB_TFT_SSD1306=m +CONFIG_FB_TFT_SSD1331=m +CONFIG_FB_TFT_SSD1351=m +CONFIG_FB_TFT_ST7735R=m +CONFIG_FB_TFT_ST7789V=m +CONFIG_FB_TFT_TINYLCD=m +CONFIG_FB_TFT_TLS8204=m +CONFIG_FB_TFT_UC1611=m +CONFIG_FB_TFT_UC1701=m +CONFIG_FB_TFT_UPD161704=m +CONFIG_FB_TFT_WATTEROTT=m +CONFIG_FB_FLEX=m +CONFIG_FB_TFT_FBTFT_DEVICE=m +CONFIG_GREYBUS=m +CONFIG_GREYBUS_ES2=m +CONFIG_GREYBUS_AUDIO=m +CONFIG_GREYBUS_BOOTROM=m +CONFIG_GREYBUS_FIRMWARE=m +CONFIG_GREYBUS_HID=m +CONFIG_GREYBUS_LIGHT=m +CONFIG_GREYBUS_LOG=m +CONFIG_GREYBUS_LOOPBACK=m +CONFIG_GREYBUS_POWER=m +CONFIG_GREYBUS_RAW=m +CONFIG_GREYBUS_VIBRATOR=m +CONFIG_GREYBUS_BRIDGED_PHY=m +CONFIG_GREYBUS_GPIO=m +CONFIG_GREYBUS_I2C=m +CONFIG_GREYBUS_PWM=m +CONFIG_GREYBUS_SDIO=m +CONFIG_GREYBUS_SPI=m +CONFIG_GREYBUS_UART=m +CONFIG_GREYBUS_USB=m +CONFIG_COMMON_CLK_PALMAS=y +CONFIG_COMMON_CLK_TI_ADPLL=y +CONFIG_HWSPINLOCK=y +CONFIG_HWSPINLOCK_OMAP=y +CONFIG_OMAP2PLUS_MBOX=y +CONFIG_OMAP_IOMMU=y +CONFIG_REMOTEPROC=m +CONFIG_OMAP_REMOTEPROC=m +CONFIG_OMAP_REMOTEPROC_WATCHDOG=y +CONFIG_WKUP_M3_RPROC=m +CONFIG_RPMSG_VIRTIO=m +CONFIG_RPMSG_RPC=m +CONFIG_RPMSG_PRU=m +CONFIG_RPMSG_KDRV_DISPLAY=y +CONFIG_SOC_TI=y +CONFIG_AMX3_PM=m +CONFIG_WKUP_M3_IPC=m +CONFIG_TI_PRUSS=m +CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND=y +CONFIG_DEVFREQ_GOV_PERFORMANCE=y +CONFIG_DEVFREQ_GOV_POWERSAVE=y +CONFIG_DEVFREQ_GOV_USERSPACE=y +CONFIG_DEVFREQ_GOV_PASSIVE=y +CONFIG_PM_DEVFREQ_EVENT=y +CONFIG_EXTCON_GPIO=y +CONFIG_EXTCON_PALMAS=y +CONFIG_EXTCON_USB_GPIO=y +CONFIG_TI_EMIF=y +CONFIG_TI_EMIF_SRAM=y +CONFIG_IIO=y +CONFIG_IIO_SW_DEVICE=m +CONFIG_IIO_SW_TRIGGER=m +CONFIG_ADIS16201=m +CONFIG_ADIS16209=m +CONFIG_ADXL345_I2C=m +CONFIG_ADXL345_SPI=m +CONFIG_BMA180=m +CONFIG_BMA220=m +CONFIG_BMC150_ACCEL=m +CONFIG_DA280=m +CONFIG_DA311=m +CONFIG_DMARD06=m +CONFIG_DMARD09=m +CONFIG_DMARD10=m +CONFIG_HID_SENSOR_ACCEL_3D=m +CONFIG_IIO_ST_ACCEL_3AXIS=m +CONFIG_KXSD9=m +CONFIG_KXCJK1013=m +CONFIG_MC3230=m +CONFIG_MMA7455_I2C=m +CONFIG_MMA7455_SPI=m +CONFIG_MMA7660=m +CONFIG_MMA8452=m +CONFIG_MMA9551=m +CONFIG_MMA9553=m +CONFIG_MXC4005=m +CONFIG_MXC6255=m +CONFIG_SCA3000=m +CONFIG_STK8312=m +CONFIG_STK8BA50=m +CONFIG_AD7266=m +CONFIG_AD7291=m +CONFIG_AD7298=m +CONFIG_AD7476=m +CONFIG_AD7766=m +CONFIG_AD7791=m +CONFIG_AD7793=m +CONFIG_AD7887=m +CONFIG_AD7923=m +CONFIG_AD799X=m +CONFIG_CC10001_ADC=m +CONFIG_ENVELOPE_DETECTOR=m +CONFIG_HI8435=m +CONFIG_HX711=m +CONFIG_INA2XX_ADC=m +CONFIG_LTC2471=m +CONFIG_LTC2485=m +CONFIG_LTC2497=m +CONFIG_MAX1027=m +CONFIG_MAX11100=m +CONFIG_MAX1118=m +CONFIG_MAX1363=m +CONFIG_MAX9611=m +CONFIG_MCP320X=m +CONFIG_MCP3422=m +CONFIG_NAU7802=m +CONFIG_PALMAS_GPADC=m +CONFIG_STMPE_ADC=m +CONFIG_TI_ADC081C=m +CONFIG_TI_ADC0832=m +CONFIG_TI_ADC084S021=m +CONFIG_TI_ADC12138=m +CONFIG_TI_ADC108S102=m +CONFIG_TI_ADC128S052=m +CONFIG_TI_ADC161S626=m +CONFIG_TI_ADS1015=m +CONFIG_TI_ADS7950=m +CONFIG_TI_ADS8688=m +CONFIG_TI_AM335X_ADC=y +CONFIG_TI_TLC4541=m +CONFIG_IIO_RESCALE=m +CONFIG_AD8366=m +CONFIG_ATLAS_PH_SENSOR=m +CONFIG_BME680=m +CONFIG_CCS811=m +CONFIG_IAQCORE=m +CONFIG_VZ89X=m +CONFIG_AD5064=m +CONFIG_AD5360=m +CONFIG_AD5380=m +CONFIG_AD5421=m +CONFIG_AD5446=m +CONFIG_AD5449=m +CONFIG_AD5592R=m +CONFIG_AD5593R=m +CONFIG_AD5504=m +CONFIG_AD5624R_SPI=m +CONFIG_LTC2632=m +CONFIG_AD5686_SPI=m +CONFIG_AD5696_I2C=m +CONFIG_AD5755=m +CONFIG_AD5758=m +CONFIG_AD5761=m +CONFIG_AD5764=m +CONFIG_AD5791=m +CONFIG_AD7303=m +CONFIG_AD8801=m +CONFIG_DPOT_DAC=m +CONFIG_DS4424=m +CONFIG_M62332=m +CONFIG_MAX517=m +CONFIG_MAX5821=m +CONFIG_MCP4725=m +CONFIG_MCP4922=m +CONFIG_TI_DAC082S085=m +CONFIG_TI_DAC5571=m +CONFIG_AD9523=m +CONFIG_ADF4350=m +CONFIG_ADIS16080=m +CONFIG_ADIS16130=m +CONFIG_ADIS16136=m +CONFIG_ADIS16260=m +CONFIG_ADXRS450=m +CONFIG_BMG160=m +CONFIG_HID_SENSOR_GYRO_3D=m +CONFIG_MPU3050_I2C=m +CONFIG_IIO_ST_GYRO_3AXIS=m +CONFIG_ITG3200=m +CONFIG_AFE4403=m +CONFIG_AFE4404=m +CONFIG_MAX30100=m +CONFIG_MAX30102=m +CONFIG_AM2315=m +CONFIG_DHT11=m +CONFIG_HDC100X=m +CONFIG_HID_SENSOR_HUMIDITY=m +CONFIG_HTS221=m +CONFIG_HTU21=m +CONFIG_SI7005=m +CONFIG_SI7020=m +CONFIG_ADIS16400=m +CONFIG_ADIS16480=m +CONFIG_BMI160_I2C=m +CONFIG_BMI160_SPI=m +CONFIG_KMX61=m +CONFIG_INV_MPU6050_I2C=m +CONFIG_INV_MPU6050_SPI=m +CONFIG_IIO_ST_LSM6DSX=m +CONFIG_ADJD_S311=m +CONFIG_AL3320A=m +CONFIG_APDS9300=m +CONFIG_APDS9960=m +CONFIG_BH1750=m +CONFIG_BH1780=m +CONFIG_CM32181=m +CONFIG_CM3232=m +CONFIG_CM3323=m +CONFIG_CM3605=m +CONFIG_CM36651=m +CONFIG_GP2AP020A00F=m +CONFIG_SENSORS_ISL29018=m +CONFIG_SENSORS_ISL29028=m +CONFIG_ISL29125=m +CONFIG_HID_SENSOR_ALS=m +CONFIG_HID_SENSOR_PROX=m +CONFIG_JSA1212=m +CONFIG_RPR0521=m +CONFIG_LTR501=m +CONFIG_LV0104CS=m +CONFIG_MAX44000=m +CONFIG_OPT3001=m +CONFIG_PA12203001=m +CONFIG_SI1133=m +CONFIG_SI1145=m +CONFIG_STK3310=m +CONFIG_ST_UVIS25=m +CONFIG_TCS3414=m +CONFIG_TCS3472=m +CONFIG_SENSORS_TSL2563=m +CONFIG_TSL2583=m +CONFIG_TSL2772=m +CONFIG_TSL4531=m +CONFIG_US5182D=m +CONFIG_VCNL4000=m +CONFIG_VEML6070=m +CONFIG_VL6180=m +CONFIG_ZOPT2201=m +CONFIG_AK8974=m +CONFIG_AK09911=m +CONFIG_BMC150_MAGN_I2C=m +CONFIG_BMC150_MAGN_SPI=m +CONFIG_MAG3110=m +CONFIG_HID_SENSOR_MAGNETOMETER_3D=m +CONFIG_MMC35240=m +CONFIG_IIO_ST_MAGN_3AXIS=m +CONFIG_SENSORS_HMC5843_I2C=m +CONFIG_SENSORS_HMC5843_SPI=m +CONFIG_HID_SENSOR_INCLINOMETER_3D=m +CONFIG_HID_SENSOR_DEVICE_ROTATION=m +CONFIG_IIO_HRTIMER_TRIGGER=m +CONFIG_IIO_INTERRUPT_TRIGGER=m +CONFIG_IIO_TIGHTLOOP_TRIGGER=m +CONFIG_IIO_SYSFS_TRIGGER=m +CONFIG_AD5272=m +CONFIG_DS1803=m +CONFIG_MAX5481=m +CONFIG_MAX5487=m +CONFIG_MCP4018=m +CONFIG_MCP4131=m +CONFIG_MCP4531=m +CONFIG_TPL0102=m +CONFIG_LMP91000=m +CONFIG_ABP060MG=m +CONFIG_BMP280=m +CONFIG_HID_SENSOR_PRESS=m +CONFIG_HP03=m +CONFIG_MPL115_I2C=m +CONFIG_MPL115_SPI=m +CONFIG_MPL3115=m +CONFIG_MS5611=m +CONFIG_MS5611_I2C=m +CONFIG_MS5611_SPI=m +CONFIG_MS5637=m +CONFIG_IIO_ST_PRESS=m +CONFIG_T5403=m +CONFIG_HP206C=m +CONFIG_ZPA2326=m +CONFIG_AS3935=m +CONFIG_ISL29501=m +CONFIG_LIDAR_LITE_V2=m +CONFIG_RFD77402=m +CONFIG_SRF04=m +CONFIG_SX9500=m +CONFIG_SRF08=m +CONFIG_VL53L0X_I2C=m +CONFIG_AD2S1200=m +CONFIG_MAXIM_THERMOCOUPLE=m +CONFIG_HID_SENSOR_TEMP=m +CONFIG_MLX90614=m +CONFIG_MLX90632=m +CONFIG_TMP006=m +CONFIG_TMP007=m +CONFIG_TSYS01=m +CONFIG_TSYS02D=m +CONFIG_PWM_OMAP_DMTIMER=y +CONFIG_PWM_PCA9685=y +CONFIG_PWM_STMPE=y +CONFIG_PWM_TIECAP=y +CONFIG_PWM_TIEHRPWM=y +CONFIG_RESET_CONTROLLER=y +CONFIG_RESET_TI_SYSCON=y +CONFIG_OMAP_USB2=y +CONFIG_TI_PIPE3=y +CONFIG_RAS=y +CONFIG_ANDROID=y +CONFIG_FPGA=m +CONFIG_ALTERA_PR_IP_CORE=m +CONFIG_ALTERA_PR_IP_CORE_PLAT=m +CONFIG_FPGA_MGR_ALTERA_PS_SPI=m +CONFIG_FPGA_MGR_XILINX_SPI=m +CONFIG_FPGA_MGR_ICE40_SPI=m +CONFIG_FPGA_MGR_MACHXO2_SPI=m +CONFIG_FPGA_BRIDGE=m +CONFIG_XILINX_PR_DECOUPLER=m +CONFIG_FPGA_REGION=m +CONFIG_OF_FPGA_REGION=m +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_ENCRYPTION=y +CONFIG_JFS_FS=m +CONFIG_JFS_POSIX_ACL=y +CONFIG_JFS_SECURITY=y +CONFIG_GFS2_FS=m +CONFIG_GFS2_FS_LOCKING_DLM=y +CONFIG_BTRFS_FS=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_F2FS_FS=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_F2FS_FS_ENCRYPTION=y +CONFIG_FANOTIFY=y +CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_QUOTA=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +CONFIG_QFMT_V2=m +CONFIG_AUTOFS4_FS=y +CONFIG_FUSE_FS=y +CONFIG_CUSE=m +CONFIG_OVERLAY_FS=y +CONFIG_FSCACHE=m +CONFIG_FSCACHE_STATS=y +CONFIG_CACHEFILES=m +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_ZISOFS=y +CONFIG_UDF_FS=m +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_DEFAULT_IOCHARSET="ascii" +CONFIG_FAT_DEFAULT_UTF8=y +CONFIG_NTFS_FS=m +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_CONFIGFS_FS=y +CONFIG_ECRYPT_FS=m +CONFIG_ECRYPT_FS_MESSAGING=y +CONFIG_JFFS2_FS=m +CONFIG_JFFS2_SUMMARY=y +CONFIG_JFFS2_FS_XATTR=y +CONFIG_JFFS2_COMPRESSION_OPTIONS=y +CONFIG_JFFS2_LZO=y +CONFIG_UBIFS_FS=y +CONFIG_UBIFS_FS_ADVANCED_COMPR=y +CONFIG_SQUASHFS=m +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_ROMFS_FS=m +CONFIG_ROMFS_BACKED_BY_BOTH=y +CONFIG_AUFS_FS=m +CONFIG_AUFS_EXPORT=y +CONFIG_AUFS_XATTR=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_NFS_V4=y +CONFIG_NFS_SWAP=y +CONFIG_NFS_V4_1=y +CONFIG_NFS_V4_2=y +CONFIG_ROOT_NFS=y +CONFIG_NFSD=m +CONFIG_NFSD_V3_ACL=y +CONFIG_NFSD_V4=y +CONFIG_NFSD_BLOCKLAYOUT=y +CONFIG_NFSD_V4_SECURITY_LABEL=y +CONFIG_RPCSEC_GSS_KRB5=m +CONFIG_SUNRPC_DEBUG=y +CONFIG_CEPH_FS=m +CONFIG_CEPH_FSCACHE=y +CONFIG_CEPH_FS_POSIX_ACL=y +CONFIG_CIFS=m +CONFIG_CIFS_WEAK_PW_HASH=y +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_CIFS_POSIX=y +CONFIG_CIFS_ACL=y +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_FSCACHE=y +CONFIG_CODA_FS=m +CONFIG_AFS_FS=m +CONFIG_AFS_FSCACHE=y +CONFIG_9P_FS=m +CONFIG_9P_FSCACHE=y +CONFIG_9P_FS_POSIX_ACL=y +CONFIG_9P_FS_SECURITY=y +CONFIG_NLS_DEFAULT="utf8" +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_ASCII=y +CONFIG_NLS_ISO8859_1=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=y +CONFIG_DLM=m +CONFIG_DLM_DEBUG=y +CONFIG_SECURITY=y +CONFIG_SECURITY_NETWORK_XFRM=y +CONFIG_SECURITY_SELINUX=y +CONFIG_SECURITY_TOMOYO=y +CONFIG_SECURITY_APPARMOR=y +CONFIG_SECURITY_YAMA=y +CONFIG_INTEGRITY_SIGNATURE=y +CONFIG_INTEGRITY_ASYMMETRIC_KEYS=y +CONFIG_DEFAULT_SECURITY_APPARMOR=y +CONFIG_CRYPTO_USER=m +CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_CCM=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_CMAC=y +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_VMAC=m +CONFIG_CRYPTO_RMD128=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_RMD256=m +CONFIG_CRYPTO_RMD320=m +CONFIG_CRYPTO_SHA3=m +CONFIG_CRYPTO_TGR192=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_ANUBIS=m +CONFIG_CRYPTO_ARC4=y +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_KHAZAD=m +CONFIG_CRYPTO_SALSA20=m +CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TEA=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_LZ4=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_ANSI_CPRNG=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_USER_API_RNG=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_DEV_OMAP=y +CONFIG_CRYPTO_DEV_OMAP_SHAM=y +CONFIG_CRYPTO_DEV_OMAP_AES=y +CONFIG_CRYPTO_DEV_OMAP_DES=y +# CONFIG_RAID6_PQ_BENCHMARK is not set +# CONFIG_XZ_DEC_X86 is not set +# CONFIG_XZ_DEC_POWERPC is not set +# CONFIG_XZ_DEC_IA64 is not set +# CONFIG_XZ_DEC_SPARC is not set +CONFIG_CORDIC=m +CONFIG_IRQ_POLL=y +CONFIG_PRINTK_TIME=y +CONFIG_BOOT_PRINTK_DELAY=y +CONFIG_DYNAMIC_DEBUG=y +CONFIG_STRIP_ASM_SYMS=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_MAGIC_SYSRQ_DEFAULT_ENABLE=0x01b6 +CONFIG_DETECT_HUNG_TASK=y +CONFIG_SCHEDSTATS=y +CONFIG_SCHED_STACK_END_CHECK=y +CONFIG_NOTIFIER_ERROR_INJECTION=m +CONFIG_FTRACE_SYSCALLS=y +CONFIG_TRACER_SNAPSHOT=y +CONFIG_STACK_TRACER=y +CONFIG_BLK_DEV_IO_TRACE=y +# CONFIG_RUNTIME_TESTING_MENU is not set +CONFIG_BUG_ON_DATA_CORRUPTION=y From 2996485244842915558cfa50f50bc1c2a309b537 Mon Sep 17 00:00:00 2001 From: Biju Das Date: Wed, 4 Sep 2019 09:15:40 +0100 Subject: [PATCH 203/213] usb: typec: driver for TI HD3SS3220 USB Type-C DRP port controller Driver for TI HD3SS3220 USB Type-C DRP port controller. The driver currently registers the port and supports data role swapping. Signed-off-by: Biju Das Acked-by: Heikki Krogerus Link: https://lore.kernel.org/r/1567584941-13690-3-git-send-email-biju.das@bp.renesas.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/Kconfig | 10 ++ drivers/usb/typec/Makefile | 1 + drivers/usb/typec/hd3ss3220.c | 259 ++++++++++++++++++++++++++++++++++ 3 files changed, 270 insertions(+) create mode 100644 drivers/usb/typec/hd3ss3220.c diff --git a/drivers/usb/typec/Kconfig b/drivers/usb/typec/Kconfig index 00878c386dd09..fde51c59c6996 100644 --- a/drivers/usb/typec/Kconfig +++ b/drivers/usb/typec/Kconfig @@ -92,6 +92,16 @@ endif # TYPEC_TCPM source "drivers/usb/typec/ucsi/Kconfig" +config TYPEC_HD3SS3220 + tristate "TI HD3SS3220 Type-C DRP Port controller driver" + depends on I2C + help + Say Y or M here if your system has TI HD3SS3220 Type-C DRP Port + controller driver. + + If you choose to build this driver as a dynamically linked module, the + module will be called hd3ss3220.ko. + config TYPEC_TPS6598X tristate "TI TPS6598x USB Power Delivery controller driver" depends on I2C diff --git a/drivers/usb/typec/Makefile b/drivers/usb/typec/Makefile index 45b0aef428a82..ff58cea5407dc 100644 --- a/drivers/usb/typec/Makefile +++ b/drivers/usb/typec/Makefile @@ -6,6 +6,7 @@ obj-$(CONFIG_TYPEC_TCPM) += tcpm.o obj-y += fusb302/ obj-$(CONFIG_TYPEC_WCOVE) += typec_wcove.o obj-$(CONFIG_TYPEC_UCSI) += ucsi/ +obj-$(CONFIG_TYPEC_HD3SS3220) += hd3ss3220.o obj-$(CONFIG_TYPEC_TPS6598X) += tps6598x.o obj-$(CONFIG_TYPEC) += mux/ obj-$(CONFIG_TYPEC_TCPCI) += tcpci.o diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c new file mode 100644 index 0000000000000..b8f247e792b87 --- /dev/null +++ b/drivers/usb/typec/hd3ss3220.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * TI HD3SS3220 Type-C DRP Port Controller Driver + * + * Copyright (C) 2019 Renesas Electronics Corp. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HD3SS3220_REG_CN_STAT_CTRL 0x09 +#define HD3SS3220_REG_GEN_CTRL 0x0A +#define HD3SS3220_REG_DEV_REV 0xA0 + +/* Register HD3SS3220_REG_CN_STAT_CTRL*/ +#define HD3SS3220_REG_CN_STAT_CTRL_ATTACHED_STATE_MASK (BIT(7) | BIT(6)) +#define HD3SS3220_REG_CN_STAT_CTRL_AS_DFP BIT(6) +#define HD3SS3220_REG_CN_STAT_CTRL_AS_UFP BIT(7) +#define HD3SS3220_REG_CN_STAT_CTRL_TO_ACCESSORY (BIT(7) | BIT(6)) +#define HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS BIT(4) + +/* Register HD3SS3220_REG_GEN_CTRL*/ +#define HD3SS3220_REG_GEN_CTRL_SRC_PREF_MASK (BIT(2) | BIT(1)) +#define HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_DEFAULT 0x00 +#define HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_TRY_SNK BIT(1) +#define HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_TRY_SRC (BIT(2) | BIT(1)) + +struct hd3ss3220 { + struct device *dev; + struct regmap *regmap; + struct usb_role_switch *role_sw; + struct typec_port *port; + struct typec_capability typec_cap; +}; + +static int hd3ss3220_set_source_pref(struct hd3ss3220 *hd3ss3220, int src_pref) +{ + return regmap_update_bits(hd3ss3220->regmap, HD3SS3220_REG_GEN_CTRL, + HD3SS3220_REG_GEN_CTRL_SRC_PREF_MASK, + src_pref); +} + +static enum usb_role hd3ss3220_get_attached_state(struct hd3ss3220 *hd3ss3220) +{ + unsigned int reg_val; + enum usb_role attached_state; + int ret; + + ret = regmap_read(hd3ss3220->regmap, HD3SS3220_REG_CN_STAT_CTRL, + ®_val); + if (ret < 0) + return ret; + + switch (reg_val & HD3SS3220_REG_CN_STAT_CTRL_ATTACHED_STATE_MASK) { + case HD3SS3220_REG_CN_STAT_CTRL_AS_DFP: + attached_state = USB_ROLE_HOST; + break; + case HD3SS3220_REG_CN_STAT_CTRL_AS_UFP: + attached_state = USB_ROLE_DEVICE; + break; + default: + attached_state = USB_ROLE_NONE; + break; + } + + return attached_state; +} + +static int hd3ss3220_dr_set(const struct typec_capability *cap, + enum typec_data_role role) +{ + struct hd3ss3220 *hd3ss3220 = container_of(cap, struct hd3ss3220, + typec_cap); + enum usb_role role_val; + int pref, ret = 0; + + if (role == TYPEC_HOST) { + role_val = USB_ROLE_HOST; + pref = HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_TRY_SRC; + } else { + role_val = USB_ROLE_DEVICE; + pref = HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_TRY_SNK; + } + + ret = hd3ss3220_set_source_pref(hd3ss3220, pref); + usleep_range(10, 100); + + usb_role_switch_set_role(hd3ss3220->role_sw, role_val); + typec_set_data_role(hd3ss3220->port, role); + + return ret; +} + +static void hd3ss3220_set_role(struct hd3ss3220 *hd3ss3220) +{ + enum usb_role role_state = hd3ss3220_get_attached_state(hd3ss3220); + + usb_role_switch_set_role(hd3ss3220->role_sw, role_state); + if (role_state == USB_ROLE_NONE) + hd3ss3220_set_source_pref(hd3ss3220, + HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_DEFAULT); + + switch (role_state) { + case USB_ROLE_HOST: + typec_set_data_role(hd3ss3220->port, TYPEC_HOST); + break; + case USB_ROLE_DEVICE: + typec_set_data_role(hd3ss3220->port, TYPEC_DEVICE); + break; + default: + break; + } +} + +irqreturn_t hd3ss3220_irq(struct hd3ss3220 *hd3ss3220) +{ + int err; + + hd3ss3220_set_role(hd3ss3220); + err = regmap_update_bits_base(hd3ss3220->regmap, + HD3SS3220_REG_CN_STAT_CTRL, + HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS, + HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS, + NULL, false, true); + if (err < 0) + return IRQ_NONE; + + return IRQ_HANDLED; +} + +static irqreturn_t hd3ss3220_irq_handler(int irq, void *data) +{ + struct i2c_client *client = to_i2c_client(data); + struct hd3ss3220 *hd3ss3220 = i2c_get_clientdata(client); + + return hd3ss3220_irq(hd3ss3220); +} + +static const struct regmap_config config = { + .reg_bits = 8, + .val_bits = 8, + .max_register = 0x0A, +}; + +static int hd3ss3220_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct hd3ss3220 *hd3ss3220; + struct fwnode_handle *connector; + int ret; + unsigned int data; + + hd3ss3220 = devm_kzalloc(&client->dev, sizeof(struct hd3ss3220), + GFP_KERNEL); + if (!hd3ss3220) + return -ENOMEM; + + i2c_set_clientdata(client, hd3ss3220); + + hd3ss3220->dev = &client->dev; + hd3ss3220->regmap = devm_regmap_init_i2c(client, &config); + if (IS_ERR(hd3ss3220->regmap)) + return PTR_ERR(hd3ss3220->regmap); + + hd3ss3220_set_source_pref(hd3ss3220, + HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_DEFAULT); + connector = device_get_named_child_node(hd3ss3220->dev, "connector"); + if (IS_ERR(connector)) + return PTR_ERR(connector); + + hd3ss3220->role_sw = fwnode_usb_role_switch_get(connector); + fwnode_handle_put(connector); + if (IS_ERR_OR_NULL(hd3ss3220->role_sw)) + return PTR_ERR(hd3ss3220->role_sw); + + hd3ss3220->typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; + hd3ss3220->typec_cap.dr_set = hd3ss3220_dr_set; + hd3ss3220->typec_cap.type = TYPEC_PORT_DRP; + hd3ss3220->typec_cap.data = TYPEC_PORT_DRD; + + hd3ss3220->port = typec_register_port(&client->dev, + &hd3ss3220->typec_cap); + if (IS_ERR(hd3ss3220->port)) + return PTR_ERR(hd3ss3220->port); + + hd3ss3220_set_role(hd3ss3220); + ret = regmap_read(hd3ss3220->regmap, HD3SS3220_REG_CN_STAT_CTRL, &data); + if (ret < 0) + goto error; + + if (data & HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS) { + ret = regmap_write(hd3ss3220->regmap, + HD3SS3220_REG_CN_STAT_CTRL, + data | HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS); + if (ret < 0) + goto error; + } + + if (client->irq > 0) { + ret = devm_request_threaded_irq(&client->dev, client->irq, NULL, + hd3ss3220_irq_handler, + IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + "hd3ss3220", &client->dev); + if (ret) + goto error; + } + + ret = i2c_smbus_read_byte_data(client, HD3SS3220_REG_DEV_REV); + if (ret < 0) + goto error; + + dev_info(&client->dev, "probed revision=0x%x\n", ret); + + return 0; +error: + typec_unregister_port(hd3ss3220->port); + usb_role_switch_put(hd3ss3220->role_sw); + + return ret; +} + +static int hd3ss3220_remove(struct i2c_client *client) +{ + struct hd3ss3220 *hd3ss3220 = i2c_get_clientdata(client); + + typec_unregister_port(hd3ss3220->port); + usb_role_switch_put(hd3ss3220->role_sw); + + return 0; +} + +static const struct of_device_id dev_ids[] = { + { .compatible = "ti,hd3ss3220"}, + {} +}; +MODULE_DEVICE_TABLE(of, dev_ids); + +static struct i2c_driver hd3ss3220_driver = { + .driver = { + .name = "hd3ss3220", + .of_match_table = of_match_ptr(dev_ids), + }, + .probe = hd3ss3220_probe, + .remove = hd3ss3220_remove, +}; + +module_i2c_driver(hd3ss3220_driver); + +MODULE_AUTHOR("Biju Das "); +MODULE_DESCRIPTION("TI HD3SS3220 DRP Port Controller Driver"); +MODULE_LICENSE("GPL"); From 9e3a4ea60254212004f8dd146a107a07e4f2c676 Mon Sep 17 00:00:00 2001 From: kbuild test robot Date: Sun, 6 Oct 2019 05:57:27 +0800 Subject: [PATCH 204/213] usb: typec: hd3ss3220_irq() can be static Fixes: 1c48c759ef4b ("usb: typec: driver for TI HD3SS3220 USB Type-C DRP port controller") Signed-off-by: kbuild test robot Link: https://lore.kernel.org/r/20191005215727.qfypxoswkiyu45ak@332d0cec05f4 Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/hd3ss3220.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index b8f247e792b87..1900910c637e3 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -120,7 +120,7 @@ static void hd3ss3220_set_role(struct hd3ss3220 *hd3ss3220) } } -irqreturn_t hd3ss3220_irq(struct hd3ss3220 *hd3ss3220) +static irqreturn_t hd3ss3220_irq(struct hd3ss3220 *hd3ss3220) { int err; From a6cda709c655307a51962e06b6edac634f0fbf5c Mon Sep 17 00:00:00 2001 From: Biju Das Date: Mon, 7 Oct 2019 16:38:49 +0100 Subject: [PATCH 205/213] usb: typec: hd3ss3220: hd3ss3220_probe() warn: passing zero to 'PTR_ERR' This patch fixes the warning passing zero to 'PTR_ERR' by changing the check from 'IS_ERR_OR_NULL' to 'IS_ERR'. Also improved the error handling on probe function. Fixes: 1c48c759ef4b ("usb: typec: driver for TI HD3SS3220 USB Type-C DRP port controller") Reported-by: kbuild test robot Reported-by: Dan Carpenter Signed-off-by: Biju Das Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/1570462729-25722-1-git-send-email-biju.das@bp.renesas.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/hd3ss3220.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index 1900910c637e3..7e5f3f73f6f4d 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -178,7 +178,7 @@ static int hd3ss3220_probe(struct i2c_client *client, hd3ss3220->role_sw = fwnode_usb_role_switch_get(connector); fwnode_handle_put(connector); - if (IS_ERR_OR_NULL(hd3ss3220->role_sw)) + if (IS_ERR(hd3ss3220->role_sw)) return PTR_ERR(hd3ss3220->role_sw); hd3ss3220->typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; @@ -188,20 +188,22 @@ static int hd3ss3220_probe(struct i2c_client *client, hd3ss3220->port = typec_register_port(&client->dev, &hd3ss3220->typec_cap); - if (IS_ERR(hd3ss3220->port)) - return PTR_ERR(hd3ss3220->port); + if (IS_ERR(hd3ss3220->port)) { + ret = PTR_ERR(hd3ss3220->port); + goto err_put_role; + } hd3ss3220_set_role(hd3ss3220); ret = regmap_read(hd3ss3220->regmap, HD3SS3220_REG_CN_STAT_CTRL, &data); if (ret < 0) - goto error; + goto err_unreg_port; if (data & HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS) { ret = regmap_write(hd3ss3220->regmap, HD3SS3220_REG_CN_STAT_CTRL, data | HD3SS3220_REG_CN_STAT_CTRL_INT_STATUS); if (ret < 0) - goto error; + goto err_unreg_port; } if (client->irq > 0) { @@ -210,18 +212,19 @@ static int hd3ss3220_probe(struct i2c_client *client, IRQF_TRIGGER_FALLING | IRQF_ONESHOT, "hd3ss3220", &client->dev); if (ret) - goto error; + goto err_unreg_port; } ret = i2c_smbus_read_byte_data(client, HD3SS3220_REG_DEV_REV); if (ret < 0) - goto error; + goto err_unreg_port; dev_info(&client->dev, "probed revision=0x%x\n", ret); return 0; -error: +err_unreg_port: typec_unregister_port(hd3ss3220->port); +err_put_role: usb_role_switch_put(hd3ss3220->role_sw); return ret; From ed365616ff23153320aa8ee82c3dcde2c774189d Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Wed, 9 Oct 2019 12:04:49 +0000 Subject: [PATCH 206/213] usb: typec: remove duplicated include from hd3ss3220.c Remove duplicated include. Signed-off-by: YueHaibing Link: https://lore.kernel.org/r/20191009120449.44899-1-yuehaibing@huawei.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/hd3ss3220.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index 7e5f3f73f6f4d..9715600aeb04f 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include From a9af09809db39b14936a82766c5fc21f369841b2 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 11 Oct 2019 21:50:55 +0300 Subject: [PATCH 207/213] usb: typec: fix an IS_ERR() vs NULL bug in hd3ss3220_probe() The device_get_named_child_node() function doesn't return error pointers, it returns NULL on error. Fixes: 1c48c759ef4b ("usb: typec: driver for TI HD3SS3220 USB Type-C DRP port controller") Signed-off-by: Dan Carpenter Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20191011185055.GA20972@mwanda Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/hd3ss3220.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index 9715600aeb04f..8afaf5768a17d 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -172,8 +172,8 @@ static int hd3ss3220_probe(struct i2c_client *client, hd3ss3220_set_source_pref(hd3ss3220, HD3SS3220_REG_GEN_CTRL_SRC_PREF_DRP_DEFAULT); connector = device_get_named_child_node(hd3ss3220->dev, "connector"); - if (IS_ERR(connector)) - return PTR_ERR(connector); + if (!connector) + return -ENODEV; hd3ss3220->role_sw = fwnode_usb_role_switch_get(connector); fwnode_handle_put(connector); From 48ee8ac4d65e747e48e3eca89c88c923d40d7dda Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Mon, 4 Nov 2019 17:24:24 +0300 Subject: [PATCH 208/213] usb: typec: hd3ss3220: Start using struct typec_operations Supplying the operation callbacks as part of a struct typec_operations instead of as part of struct typec_capability during port registration. After this there is not need to keep the capabilities stored anywhere in the driver. Signed-off-by: Heikki Krogerus Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20191104142435.29960-8-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/hd3ss3220.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index 8afaf5768a17d..db09fa0d85f2c 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -37,7 +37,6 @@ struct hd3ss3220 { struct regmap *regmap; struct usb_role_switch *role_sw; struct typec_port *port; - struct typec_capability typec_cap; }; static int hd3ss3220_set_source_pref(struct hd3ss3220 *hd3ss3220, int src_pref) @@ -73,11 +72,9 @@ static enum usb_role hd3ss3220_get_attached_state(struct hd3ss3220 *hd3ss3220) return attached_state; } -static int hd3ss3220_dr_set(const struct typec_capability *cap, - enum typec_data_role role) +static int hd3ss3220_dr_set(struct typec_port *port, enum typec_data_role role) { - struct hd3ss3220 *hd3ss3220 = container_of(cap, struct hd3ss3220, - typec_cap); + struct hd3ss3220 *hd3ss3220 = typec_get_drvdata(port); enum usb_role role_val; int pref, ret = 0; @@ -98,6 +95,10 @@ static int hd3ss3220_dr_set(const struct typec_capability *cap, return ret; } +static const struct typec_operations hd3ss3220_ops = { + .dr_set = hd3ss3220_dr_set +}; + static void hd3ss3220_set_role(struct hd3ss3220 *hd3ss3220) { enum usb_role role_state = hd3ss3220_get_attached_state(hd3ss3220); @@ -152,6 +153,7 @@ static const struct regmap_config config = { static int hd3ss3220_probe(struct i2c_client *client, const struct i2c_device_id *id) { + struct typec_capability typec_cap = { }; struct hd3ss3220 *hd3ss3220; struct fwnode_handle *connector; int ret; @@ -180,13 +182,13 @@ static int hd3ss3220_probe(struct i2c_client *client, if (IS_ERR(hd3ss3220->role_sw)) return PTR_ERR(hd3ss3220->role_sw); - hd3ss3220->typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; - hd3ss3220->typec_cap.dr_set = hd3ss3220_dr_set; - hd3ss3220->typec_cap.type = TYPEC_PORT_DRP; - hd3ss3220->typec_cap.data = TYPEC_PORT_DRD; + typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; + typec_cap.driver_data = hd3ss3220; + typec_cap.type = TYPEC_PORT_DRP; + typec_cap.data = TYPEC_PORT_DRD; + typec_cap.ops = &hd3ss3220_ops; - hd3ss3220->port = typec_register_port(&client->dev, - &hd3ss3220->typec_cap); + hd3ss3220->port = typec_register_port(&client->dev, &typec_cap); if (IS_ERR(hd3ss3220->port)) { ret = PTR_ERR(hd3ss3220->port); goto err_put_role; From ef05517e99ccdf7ff693e6c6129119dcf7bb1b6f Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Mon, 4 Nov 2019 17:24:27 +0300 Subject: [PATCH 209/213] usb: typec: hd3ss3220: Give the connector fwnode to the port device The driver already finds the node in order to get reference to the USB role switch. Signed-off-by: Heikki Krogerus Tested-by: Biju Das Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20191104142435.29960-11-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/hd3ss3220.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c index db09fa0d85f2c..323dfa8160ab0 100644 --- a/drivers/usb/typec/hd3ss3220.c +++ b/drivers/usb/typec/hd3ss3220.c @@ -178,15 +178,17 @@ static int hd3ss3220_probe(struct i2c_client *client, return -ENODEV; hd3ss3220->role_sw = fwnode_usb_role_switch_get(connector); - fwnode_handle_put(connector); - if (IS_ERR(hd3ss3220->role_sw)) - return PTR_ERR(hd3ss3220->role_sw); + if (IS_ERR(hd3ss3220->role_sw)) { + ret = PTR_ERR(hd3ss3220->role_sw); + goto err_put_fwnode; + } typec_cap.prefer_role = TYPEC_NO_PREFERRED_ROLE; typec_cap.driver_data = hd3ss3220; typec_cap.type = TYPEC_PORT_DRP; typec_cap.data = TYPEC_PORT_DRD; typec_cap.ops = &hd3ss3220_ops; + typec_cap.fwnode = connector; hd3ss3220->port = typec_register_port(&client->dev, &typec_cap); if (IS_ERR(hd3ss3220->port)) { @@ -220,6 +222,8 @@ static int hd3ss3220_probe(struct i2c_client *client, if (ret < 0) goto err_unreg_port; + fwnode_handle_put(connector); + dev_info(&client->dev, "probed revision=0x%x\n", ret); return 0; @@ -227,6 +231,8 @@ static int hd3ss3220_probe(struct i2c_client *client, typec_unregister_port(hd3ss3220->port); err_put_role: usb_role_switch_put(hd3ss3220->role_sw); +err_put_fwnode: + fwnode_handle_put(connector); return ret; } From 7c6ebfc838337d7ec01593551ea9c7b9ecd86925 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Tue, 20 Mar 2018 15:57:04 +0300 Subject: [PATCH 210/213] usb: common: Small class for USB role switches USB role switch is a device that can be used to choose the data role for USB connector. With dual-role capable USB controllers, the controller itself will be the switch, but on some platforms the USB host and device controllers are separate IPs and there is a mux between them and the connector. On those platforms the mux driver will need to register the switch. With USB Type-C connectors, the host-to-device relationship is negotiated over the Configuration Channel (CC). That means the USB Type-C drivers need to be in control of the role switch. The class provides a simple API for the USB Type-C drivers for the control. For other types of USB connectors (mainly microAB) the class provides user space control via sysfs attribute file that can be used to request role swapping from the switch. Reviewed-by: Hans de Goede Reviewed-by: Andy Shevchenko Signed-off-by: Heikki Krogerus Signed-off-by: Greg Kroah-Hartman --- drivers/usb/Kconfig | 3 + drivers/usb/common/Makefile | 1 + drivers/usb/common/roles.c | 305 ++++++++++++++++++++++++++++++++++++ 3 files changed, 309 insertions(+) create mode 100644 drivers/usb/common/roles.c diff --git a/drivers/usb/Kconfig b/drivers/usb/Kconfig index e52e55d05db33..c4e9c5048d3b8 100644 --- a/drivers/usb/Kconfig +++ b/drivers/usb/Kconfig @@ -207,4 +207,7 @@ config USB_ULPI_BUS To compile this driver as a module, choose M here: the module will be called ulpi. +config USB_ROLE_SWITCH + tristate + endif # USB_SUPPORT diff --git a/drivers/usb/common/Makefile b/drivers/usb/common/Makefile index 76f3e80623db8..72d4272a4b591 100644 --- a/drivers/usb/common/Makefile +++ b/drivers/usb/common/Makefile @@ -10,3 +10,4 @@ usb-common-$(CONFIG_USB_LED_TRIG) += led.o obj-$(CONFIG_USB_OTG_FSM) += usb-otg-fsm.o obj-$(CONFIG_USB_ULPI_BUS) += ulpi.o +obj-$(CONFIG_USB_ROLE_SWITCH) += roles.o diff --git a/drivers/usb/common/roles.c b/drivers/usb/common/roles.c new file mode 100644 index 0000000000000..15cc76e22123e --- /dev/null +++ b/drivers/usb/common/roles.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * USB Role Switch Support + * + * Copyright (C) 2018 Intel Corporation + * Author: Heikki Krogerus + * Hans de Goede + */ + +#include +#include +#include +#include +#include + +static struct class *role_class; + +struct usb_role_switch { + struct device dev; + struct mutex lock; /* device lock*/ + enum usb_role role; + + /* From descriptor */ + struct device *usb2_port; + struct device *usb3_port; + struct device *udc; + usb_role_switch_set_t set; + usb_role_switch_get_t get; + bool allow_userspace_control; +}; + +#define to_role_switch(d) container_of(d, struct usb_role_switch, dev) + +/** + * usb_role_switch_set_role - Set USB role for a switch + * @sw: USB role switch + * @role: USB role to be switched to + * + * Set USB role @role for @sw. + */ +int usb_role_switch_set_role(struct usb_role_switch *sw, enum usb_role role) +{ + int ret; + + if (IS_ERR_OR_NULL(sw)) + return 0; + + mutex_lock(&sw->lock); + + ret = sw->set(sw->dev.parent, role); + if (!ret) + sw->role = role; + + mutex_unlock(&sw->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(usb_role_switch_set_role); + +/** + * usb_role_switch_get_role - Get the USB role for a switch + * @sw: USB role switch + * + * Depending on the role-switch-driver this function returns either a cached + * value of the last set role, or reads back the actual value from the hardware. + */ +enum usb_role usb_role_switch_get_role(struct usb_role_switch *sw) +{ + enum usb_role role; + + if (IS_ERR_OR_NULL(sw)) + return USB_ROLE_NONE; + + mutex_lock(&sw->lock); + + if (sw->get) + role = sw->get(sw->dev.parent); + else + role = sw->role; + + mutex_unlock(&sw->lock); + + return role; +} +EXPORT_SYMBOL_GPL(usb_role_switch_get_role); + +static int __switch_match(struct device *dev, const void *name) +{ + return !strcmp((const char *)name, dev_name(dev)); +} + +static void *usb_role_switch_match(struct device_connection *con, int ep, + void *data) +{ + struct device *dev; + + dev = class_find_device(role_class, NULL, con->endpoint[ep], + __switch_match); + + return dev ? to_role_switch(dev) : ERR_PTR(-EPROBE_DEFER); +} + +/** + * usb_role_switch_get - Find USB role switch linked with the caller + * @dev: The caller device + * + * Finds and returns role switch linked with @dev. The reference count for the + * found switch is incremented. + */ +struct usb_role_switch *usb_role_switch_get(struct device *dev) +{ + return device_connection_find_match(dev, "usb-role-switch", NULL, + usb_role_switch_match); +} +EXPORT_SYMBOL_GPL(usb_role_switch_get); + +/** + * usb_role_switch_put - Release handle to a switch + * @sw: USB Role Switch + * + * Decrement reference count for @sw. + */ +void usb_role_switch_put(struct usb_role_switch *sw) +{ + if (!IS_ERR_OR_NULL(sw)) + put_device(&sw->dev); +} +EXPORT_SYMBOL_GPL(usb_role_switch_put); + +static umode_t +usb_role_switch_is_visible(struct kobject *kobj, struct attribute *attr, int n) +{ + struct device *dev = container_of(kobj, typeof(*dev), kobj); + struct usb_role_switch *sw = to_role_switch(dev); + + if (sw->allow_userspace_control) + return attr->mode; + + return 0; +} + +static const char * const usb_roles[] = { + [USB_ROLE_NONE] = "none", + [USB_ROLE_HOST] = "host", + [USB_ROLE_DEVICE] = "device", +}; + +static ssize_t +role_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct usb_role_switch *sw = to_role_switch(dev); + enum usb_role role = usb_role_switch_get_role(sw); + + return sprintf(buf, "%s\n", usb_roles[role]); +} + +static ssize_t role_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t size) +{ + struct usb_role_switch *sw = to_role_switch(dev); + int ret; + + ret = sysfs_match_string(usb_roles, buf); + if (ret < 0) { + bool res; + + /* Extra check if the user wants to disable the switch */ + ret = kstrtobool(buf, &res); + if (ret || res) + return -EINVAL; + } + + ret = usb_role_switch_set_role(sw, ret); + if (ret) + return ret; + + return size; +} +static DEVICE_ATTR_RW(role); + +static struct attribute *usb_role_switch_attrs[] = { + &dev_attr_role.attr, + NULL, +}; + +static const struct attribute_group usb_role_switch_group = { + .is_visible = usb_role_switch_is_visible, + .attrs = usb_role_switch_attrs, +}; + +static const struct attribute_group *usb_role_switch_groups[] = { + &usb_role_switch_group, + NULL, +}; + +static int +usb_role_switch_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + int ret; + + ret = add_uevent_var(env, "USB_ROLE_SWITCH=%s", dev_name(dev)); + if (ret) + dev_err(dev, "failed to add uevent USB_ROLE_SWITCH\n"); + + return ret; +} + +static void usb_role_switch_release(struct device *dev) +{ + struct usb_role_switch *sw = to_role_switch(dev); + + kfree(sw); +} + +static const struct device_type usb_role_dev_type = { + .name = "usb_role_switch", + .groups = usb_role_switch_groups, + .uevent = usb_role_switch_uevent, + .release = usb_role_switch_release, +}; + +/** + * usb_role_switch_register - Register USB Role Switch + * @parent: Parent device for the switch + * @desc: Description of the switch + * + * USB Role Switch is a device capable or choosing the role for USB connector. + * On platforms where the USB controller is dual-role capable, the controller + * driver will need to register the switch. On platforms where the USB host and + * USB device controllers behind the connector are separate, there will be a + * mux, and the driver for that mux will need to register the switch. + * + * Returns handle to a new role switch or ERR_PTR. The content of @desc is + * copied. + */ +struct usb_role_switch * +usb_role_switch_register(struct device *parent, + const struct usb_role_switch_desc *desc) +{ + struct usb_role_switch *sw; + int ret; + + if (!desc || !desc->set) + return ERR_PTR(-EINVAL); + + sw = kzalloc(sizeof(*sw), GFP_KERNEL); + if (!sw) + return ERR_PTR(-ENOMEM); + + mutex_init(&sw->lock); + + sw->allow_userspace_control = desc->allow_userspace_control; + sw->usb2_port = desc->usb2_port; + sw->usb3_port = desc->usb3_port; + sw->udc = desc->udc; + sw->set = desc->set; + sw->get = desc->get; + + sw->dev.parent = parent; + sw->dev.class = role_class; + sw->dev.type = &usb_role_dev_type; + dev_set_name(&sw->dev, "%s-role-switch", dev_name(parent)); + + ret = device_register(&sw->dev); + if (ret) { + put_device(&sw->dev); + return ERR_PTR(ret); + } + + /* TODO: Symlinks for the host port and the device controller. */ + + return sw; +} +EXPORT_SYMBOL_GPL(usb_role_switch_register); + +/** + * usb_role_switch_unregister - Unregsiter USB Role Switch + * @sw: USB Role Switch + * + * Unregister switch that was registered with usb_role_switch_register(). + */ +void usb_role_switch_unregister(struct usb_role_switch *sw) +{ + if (!IS_ERR_OR_NULL(sw)) + device_unregister(&sw->dev); +} +EXPORT_SYMBOL_GPL(usb_role_switch_unregister); + +static int __init usb_roles_init(void) +{ + role_class = class_create(THIS_MODULE, "usb_role"); + return PTR_ERR_OR_ZERO(role_class); +} +subsys_initcall(usb_roles_init); + +static void __exit usb_roles_exit(void) +{ + class_destroy(role_class); +} +module_exit(usb_roles_exit); + +MODULE_AUTHOR("Heikki Krogerus "); +MODULE_AUTHOR("Hans de Goede "); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("USB Role Class"); From dbe42e6b872b72805960dad54574bbb195a4cc3f Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Wed, 13 Feb 2019 10:45:54 +0300 Subject: [PATCH 211/213] usb: roles: Find the muxes by also matching against the device node When the connections are defined in firmware, struct device_connection will have the fwnode member pointing to the device node (struct fwnode_handle) of the requested device, and the endpoint will not be used at all in that case. Acked-by: Hans de Goede Reviewed-by: Andy Shevchenko Reviewed-by: Jun Li Tested-by: Jun Li Signed-off-by: Heikki Krogerus Signed-off-by: Greg Kroah-Hartman --- drivers/usb/roles/class.c | 21 ++++++++++++++++++--- include/linux/usb/role.h | 2 ++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/drivers/usb/roles/class.c b/drivers/usb/roles/class.c index 1dd492e89719e..44a4af292f3cf 100644 --- a/drivers/usb/roles/class.c +++ b/drivers/usb/roles/class.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -84,7 +85,12 @@ enum usb_role usb_role_switch_get_role(struct usb_role_switch *sw) } EXPORT_SYMBOL_GPL(usb_role_switch_get_role); -static int __switch_match(struct device *dev, const void *name) +static int switch_fwnode_match(struct device *dev, const void *fwnode) +{ + return dev_fwnode(dev) == fwnode; +} + +static int switch_name_match(struct device *dev, const void *name) { return !strcmp((const char *)name, dev_name(dev)); } @@ -94,8 +100,16 @@ static void *usb_role_switch_match(struct device_connection *con, int ep, { struct device *dev; - dev = class_find_device(role_class, NULL, con->endpoint[ep], - __switch_match); + if (con->fwnode) { + if (!fwnode_property_present(con->fwnode, con->id)) + return NULL; + + dev = class_find_device(role_class, NULL, con->fwnode, + switch_fwnode_match); + } else { + dev = class_find_device(role_class, NULL, con->endpoint[ep], + switch_name_match); + } return dev ? to_role_switch(dev) : ERR_PTR(-EPROBE_DEFER); } @@ -266,6 +280,7 @@ usb_role_switch_register(struct device *parent, sw->get = desc->get; sw->dev.parent = parent; + sw->dev.fwnode = desc->fwnode; sw->dev.class = role_class; sw->dev.type = &usb_role_dev_type; dev_set_name(&sw->dev, "%s-role-switch", dev_name(parent)); diff --git a/include/linux/usb/role.h b/include/linux/usb/role.h index edc51be4a77c6..c05ffa6abda9f 100644 --- a/include/linux/usb/role.h +++ b/include/linux/usb/role.h @@ -18,6 +18,7 @@ typedef enum usb_role (*usb_role_switch_get_t)(struct device *dev); /** * struct usb_role_switch_desc - USB Role Switch Descriptor + * @fwnode: The device node to be associated with the role switch * @usb2_port: Optional reference to the host controller port device (USB2) * @usb3_port: Optional reference to the host controller port device (USB3) * @udc: Optional reference to the peripheral controller device @@ -32,6 +33,7 @@ typedef enum usb_role (*usb_role_switch_get_t)(struct device *dev); * usb_role_switch_register() before registering the switch. */ struct usb_role_switch_desc { + struct fwnode_handle *fwnode; struct device *usb2_port; struct device *usb3_port; struct device *udc; From ad9dd945d7e751a80ca609370bbf5c8f7b0429ac Mon Sep 17 00:00:00 2001 From: Yu Chen Date: Thu, 29 Aug 2019 17:22:32 +0800 Subject: [PATCH 212/213] usb: roles: Introduce stubs for the exiting functions in role.h This patch adds stubs for the exiting functions while CONFIG_USB_ROLE_SWITCH does not enabled. Cc: Greg Kroah-Hartman Cc: Heikki Krogerus Cc: Hans de Goede Cc: Andy Shevchenko Cc: John Stultz Reviewed-by: Heikki Krogerus Signed-off-by: Yu Chen Signed-off-by: Chunfeng Yun Link: https://lore.kernel.org/r/1567070558-29417-6-git-send-email-chunfeng.yun@mediatek.com Signed-off-by: Greg Kroah-Hartman --- include/linux/usb/role.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/include/linux/usb/role.h b/include/linux/usb/role.h index c05ffa6abda9f..da2b9641b877c 100644 --- a/include/linux/usb/role.h +++ b/include/linux/usb/role.h @@ -42,6 +42,8 @@ struct usb_role_switch_desc { bool allow_userspace_control; }; + +#if IS_ENABLED(CONFIG_USB_ROLE_SWITCH) int usb_role_switch_set_role(struct usb_role_switch *sw, enum usb_role role); enum usb_role usb_role_switch_get_role(struct usb_role_switch *sw); struct usb_role_switch *usb_role_switch_get(struct device *dev); @@ -51,5 +53,33 @@ struct usb_role_switch * usb_role_switch_register(struct device *parent, const struct usb_role_switch_desc *desc); void usb_role_switch_unregister(struct usb_role_switch *sw); +#else +static inline int usb_role_switch_set_role(struct usb_role_switch *sw, + enum usb_role role) +{ + return 0; +} + +static inline enum usb_role usb_role_switch_get_role(struct usb_role_switch *sw) +{ + return USB_ROLE_NONE; +} + +static inline struct usb_role_switch *usb_role_switch_get(struct device *dev) +{ + return ERR_PTR(-ENODEV); +} + +static inline void usb_role_switch_put(struct usb_role_switch *sw) { } + +static inline struct usb_role_switch * +usb_role_switch_register(struct device *parent, + const struct usb_role_switch_desc *desc) +{ + return ERR_PTR(-ENODEV); +} + +static inline void usb_role_switch_unregister(struct usb_role_switch *sw) { } +#endif #endif /* __LINUX_USB_ROLE_H */ From 767d7c3371c2bca6895f1d55a5328925ac5f91c5 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Thu, 29 Aug 2019 17:22:34 +0800 Subject: [PATCH 213/213] usb: roles: Add fwnode_usb_role_switch_get() function The fwnode_usb_role_switch_get() function is exactly the same as usb_role_switch_get(), except that it takes struct fwnode_handle as parameter instead of struct device. Signed-off-by: Heikki Krogerus Signed-off-by: Chunfeng Yun Tested-by: Biju Das Link: https://lore.kernel.org/r/1567070558-29417-8-git-send-email-chunfeng.yun@mediatek.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/roles/class.c | 20 ++++++++++++++++++++ include/linux/usb/role.h | 7 +++++++ 2 files changed, 27 insertions(+) diff --git a/drivers/usb/roles/class.c b/drivers/usb/roles/class.c index 44a4af292f3cf..9b81d9a0a9388 100644 --- a/drivers/usb/roles/class.c +++ b/drivers/usb/roles/class.c @@ -135,6 +135,26 @@ struct usb_role_switch *usb_role_switch_get(struct device *dev) } EXPORT_SYMBOL_GPL(usb_role_switch_get); +/** + * fwnode_usb_role_switch_get - Find USB role switch linked with the caller + * @fwnode: The caller device node + * + * This is similar to the usb_role_switch_get() function above, but it searches + * the switch using fwnode instead of device entry. + */ +struct usb_role_switch *fwnode_usb_role_switch_get(struct fwnode_handle *fwnode) +{ + struct usb_role_switch *sw; + + sw = fwnode_connection_find_match(fwnode, "usb-role-switch", NULL, + usb_role_switch_match); + if (!IS_ERR_OR_NULL(sw)) + WARN_ON(!try_module_get(sw->dev.parent->driver->owner)); + + return sw; +} +EXPORT_SYMBOL_GPL(fwnode_usb_role_switch_get); + /** * usb_role_switch_put - Release handle to a switch * @sw: USB Role Switch diff --git a/include/linux/usb/role.h b/include/linux/usb/role.h index da2b9641b877c..2d77f97df72d6 100644 --- a/include/linux/usb/role.h +++ b/include/linux/usb/role.h @@ -47,6 +47,7 @@ struct usb_role_switch_desc { int usb_role_switch_set_role(struct usb_role_switch *sw, enum usb_role role); enum usb_role usb_role_switch_get_role(struct usb_role_switch *sw); struct usb_role_switch *usb_role_switch_get(struct device *dev); +struct usb_role_switch *fwnode_usb_role_switch_get(struct fwnode_handle *node); void usb_role_switch_put(struct usb_role_switch *sw); struct usb_role_switch * @@ -70,6 +71,12 @@ static inline struct usb_role_switch *usb_role_switch_get(struct device *dev) return ERR_PTR(-ENODEV); } +static inline struct usb_role_switch * +fwnode_usb_role_switch_get(struct fwnode_handle *node) +{ + return ERR_PTR(-ENODEV); +} + static inline void usb_role_switch_put(struct usb_role_switch *sw) { } static inline struct usb_role_switch *