Merge branch 'release/1.7.0'

README.md

@@ -28,3 +28,6 @@ Licensed under the [Apache 2.0](LICENSE.md) license.
 
 [github-actions-creator 文档](https://wiki.yoqi.me/github-actions-creator/#/)
 
+## Reference
+
+- [docs](https://wiki.yoqi.me/github-actions-creator/#/)

ai/.devcontainer2/Dockerfile

@@ -0,0 +1,11 @@
+FROM pytorch/pytorch:2.2.0-cuda12.1-cudnn8-runtime
+
+RUN <<EOF
+apt-get update
+apt-get install -y --no-install-recommends git
+rm -rf /var/lib/apt/lists/*
+EOF
+
+COPY requirements.txt requirements.txt
+
+RUN pip install --no-cache-dir -r requirements.txt

ai/.devcontainer2/devcontainer.json

@@ -0,0 +1,21 @@
+{
+	"name": "LLMs From Scratch",
+	"build": {
+		"context": "..",
+		"dockerfile": "Dockerfile"
+	},
+	"runArgs": [
+		"--runtime=nvidia",
+		"--gpus=all"
+	],
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"ms-azuretools.vscode-docker",
+				"ms-toolsai.jupyter",
+				"esbenp.prettier-vscode"
+			]
+		}
+	}
+}

ai/requirements.txt

@@ -1,11 +1,17 @@
 transformers
-torch
 click==7.1.2
 tensorflow==2.9.1
 beautifulsoup4==4.11.1
 wikipedia==1.4.0
-pylint==2.15.0
 ipython==8.4.0
 gradio
-numpy==1.23.2
-sentencepiece
+sentencepiece
+
+numpy >= 1.24.3
+scipy >= 1.10.1
+pandas >= 2.0.2
+matplotlib >= 3.7.1
+jupyterlab >= 4.0
+watermark >= 2.4.2
+torch >= 2.0.1
+tiktoken >= 0.5.1

android/android-dev-eclipse/.dockerignore

@@ -0,0 +1,13 @@
+node_modules
+npm-debug.log
+Dockerfile*
+docker-compose*
+.dockerignore
+.git
+.gitignore
+.env
+*/bin
+*/obj
+README.md
+LICENSE
+.vscode

android/android-dev-eclipse/Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:18.10
+
+ENV ANDROID_COMPILE_SDK 25
+ENV VERSION_SDK_TOOLS 3859397
+ENV ANDROID_HOME "/sdk"
+ENV PATH "${PATH}:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools"
+
+RUN sed -i "s|http://archive.ubuntu.com|http://mirrors.aliyun.com|g" /etc/apt/sources.list && apt-get -y update
+
+RUN apt-get install -qqy --no-install-recommends \
+    curl unzip lib32stdc++6 lib32z1 lib32ncurses5 lib32gcc1 lib32stdc++6 libc6-i386 html2text openjdk-8-jdk \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# download android-sdk
+RUN curl -s -o sdk-tools.zip https://dl.google.com/android/repository/sdk-tools-linux-${VERSION_SDK_TOOLS}.zip && \
+    unzip /sdk-tools.zip -d /sdk && rm -v /sdk-tools.zip
+
+RUN mkdir -p $ANDROID_HOME/licenses/ && \
+    echo "8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e" > $ANDROID_HOME/licenses/android-sdk-license && \
+    echo "84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
+
+# Upadte sdkmanager
+RUN mkdir -p /root/.android && \
+    touch /root/.android/repositories.cfg && \
+    sdkmanager --update
+
+CMD ["/bin/bash" ,"-c" ,"while true;do echo hello docker;sleep 1;done"]

android/android-dev-eclipse/README.md

@@ -0,0 +1,28 @@
+## docker-android
+
+
+| images     | desc | other |
+| -------- | ---- | ---- |
+| android-dev |   基于Ubuntu的Android编译环境。   |      |
+| android-source     |  基于Ubuntu的安卓系统源码编译。    |      |
+|          |      |      |
+
+## usage
+
+1. 启动docker环境
+
+```
+docker-compose up -d
+```
+
+2. 进入容器编译项目
+
+```
+aapt package -f \
+-M AndroidManifest.xml \ 
+-I "$ANDROID_HOME/platforms/android-N/android.jar" \
+-S res/ \
+-J gen/ \
+-m \
+
+```

android/android-dev-eclipse/docker-compose.debug.yml

@@ -0,0 +1,8 @@
+version: '2.1'
+
+services:
+  docker-android-sdk:
+    image: docker-android-sdk
+    build:
+      context: .
+      dockerfile: Dockerfile

android/android-dev-eclipse/docker-compose.yml

@@ -0,0 +1,5 @@
+version: '2.1'
+
+services:
+  docker-android-sdk:
+    image: docker-android-sdk

android/android-dev-eclipse/tools/build_project.sh

@@ -0,0 +1,9 @@
+
+
+aapt package -f \
+-M AndroidManifest.xml \ 
+-I "$ANDROID_HOME/platforms/android-N/android.jar" \
+-S res/ \
+-J gen/ \
+-m \
+

android/android-dev-studio/.dockerignore

@@ -0,0 +1,13 @@
+node_modules
+npm-debug.log
+Dockerfile*
+docker-compose*
+.dockerignore
+.git
+.gitignore
+.env
+*/bin
+*/obj
+README.md
+LICENSE
+.vscode

android/android-dev-studio/Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:18.10
+
+ENV ANDROID_COMPILE_SDK 25
+ENV VERSION_SDK_TOOLS 3859397
+ENV ANDROID_HOME "/sdk"
+ENV PATH "${PATH}:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools"
+
+RUN sed -i "s|http://archive.ubuntu.com|http://mirrors.aliyun.com|g" /etc/apt/sources.list && apt-get -y update
+
+RUN apt-get install -qqy --no-install-recommends \
+    curl unzip lib32stdc++6 lib32z1 lib32ncurses5 lib32gcc1 lib32stdc++6 libc6-i386 html2text openjdk-8-jdk \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# download android-sdk
+RUN curl -s -o sdk-tools.zip https://dl.google.com/android/repository/sdk-tools-linux-${VERSION_SDK_TOOLS}.zip && \
+    unzip /sdk-tools.zip -d /sdk && rm -v /sdk-tools.zip
+
+RUN mkdir -p $ANDROID_HOME/licenses/ && \
+    echo "8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e" > $ANDROID_HOME/licenses/android-sdk-license && \
+    echo "84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
+
+# Upadte sdkmanager
+RUN mkdir -p /root/.android && \
+    touch /root/.android/repositories.cfg && \
+    sdkmanager --update
+
+CMD ["/bin/bash" ,"-c" ,"while true;do echo hello docker;sleep 1;done"]

android/android-dev-studio/README.md

@@ -0,0 +1,28 @@
+## docker-android
+
+
+| images     | desc | other |
+| -------- | ---- | ---- |
+| android-dev |   基于Ubuntu的Android编译环境。   |      |
+| android-source     |  基于Ubuntu的安卓系统源码编译。    |      |
+|          |      |      |
+
+## usage
+
+1. 启动docker环境
+
+```
+docker-compose up -d
+```
+
+2. 进入容器编译项目
+
+```
+aapt package -f \
+-M AndroidManifest.xml \ 
+-I "$ANDROID_HOME/platforms/android-N/android.jar" \
+-S res/ \
+-J gen/ \
+-m \
+
+```

android/android-dev-studio/docker-compose.debug.yml

@@ -0,0 +1,8 @@
+version: '2.1'
+
+services:
+  docker-android-sdk:
+    image: docker-android-sdk
+    build:
+      context: .
+      dockerfile: Dockerfile

android/android-dev-studio/docker-compose.yml

@@ -0,0 +1,5 @@
+version: '2.1'
+
+services:
+  docker-android-sdk:
+    image: docker-android-sdk

android/android-dev-studio/tools/build_project.sh

@@ -0,0 +1,9 @@
+
+
+aapt package -f \
+-M AndroidManifest.xml \ 
+-I "$ANDROID_HOME/platforms/android-N/android.jar" \
+-S res/ \
+-J gen/ \
+-m \
+

android/android-source/.dockerignore

@@ -0,0 +1,13 @@
+node_modules
+npm-debug.log
+Dockerfile*
+docker-compose*
+.dockerignore
+.git
+.gitignore
+.env
+*/bin
+*/obj
+README.md
+LICENSE
+.vscode

android/android-source/Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:18.10
+
+WORKDIR /root/aosp
+
+RUN sed -i "s|http://archive.ubuntu.com|http://mirrors.aliyun.com|g" /etc/apt/sources.list && apt-get -y update
+
+RUN apt-get install -y git-core gnupg flex bison gperf build-essential zip curl zlib1g-dev \ 
+gcc-multilib g++-multilib libc6-dev-i386 lib32ncurses5-dev x11proto-core-dev libx11-dev \ 
+lib32z-dev libgl1-mesa-dev libxml2-utils xsltproc unzip openjdk-7-jdk python	
+
+VOLUME [ "/root/aosp" ]
+
+CMD ["/bin/bash" ,"-c" ,"while true;do echo hello docker;sleep 1;done"]

android/android-source/README.md

@@ -0,0 +1,49 @@
+## android-source
+
+android 系统源码编译，android源码特别大，所以不要放到镜像中打包,所以本质上本镜像就是一个 Ubuntu附带gcc等编译工具而已。
+
+**系统要求**
+
+* docker中Ubuntu18
+* 内存16G
+* 磁盘200G SSD
+
+## 使用
+
+1、启动编译系统（略），下面通过docker-compose启动。
+
+```
+docker run --name aosp6 -it android-source /bin/bash
+
+```
+
+2、下载源码，启动编译系统
+
+下载源码可能耗时一两天，大概50G，解压80G，在aosp还原则会120G左右。
+
+```
+sh tools/download_android_repo.sh
+```
+
+这个可以直接插上已经下载好Android系统源码的磁盘，然后把指定目录挂载到docker精选即可。
+
+```
+cd xx/aosp
+docker-composer up -d
+```
+
+3、编译源码
+
+```
+sh tools/build_src.sh
+
+```
+
+4、测试
+
+编译后，在 build会有对象的系统镜像文件，最好真机中刷机体验。或者虚拟机中更加镜像创建，这个时候就需要可视化环境，也就不能在docker中跑了，windows系统中安装好emulator，不是某某模拟器，某某模拟器自带镜像：
+
+```
+emulator -kernel ./kernel/goldfish-android-3.4/arch/x86/boot/zImage　&
+```
+

android/android-source/docker-compose.debug.yml

@@ -0,0 +1,8 @@
+version: '2.1'
+
+services:
+  docker-android-sdk:
+    image: docker-android-sdk
+    build:
+      context: .
+      dockerfile: Dockerfile

android/android-source/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '2.1'
+
+services:
+  android-source:
+    image: android-source
+    volumes:
+      - .:/root/aosp

android/android-source/tools/build_src.sh

@@ -0,0 +1,5 @@
+
+source build/envsetup.sh
+export USER=$(whoami)
+lunch aosp_hammerhead-eng
+make -j16

android/android-source/tools/download_android_repo.sh

@@ -0,0 +1,16 @@
+#!bin/sh
+
+mkdir ~/bin
+PATH=~/bin:$PATH
+
+curl https://storage.googleapis.com/git-repo-downloads/repo > ~/bin/repo
+chmod a+x ~/bin/repo
+
+# 创建一个空目录用来保存android源代码
+mkdir ~/aosp
+cd ~/aosp
+
+# 指定从清华的镜像，只下6.0.1版本Android 
+repo init -u https://aosp.tuna.tsinghua.edu.cn/platform/manifest -b android-6.0.1_r46
+#同步源码树，并发设置为8，不要太大
+repo sync -j8

build.sh

@@ -10,8 +10,8 @@ export JAVA_HOME=/usr/local/openjdk-11
 # export JAVA_HOME=/root/.sdkman/candidates/java/21.0.1-ms/
 # source "$HOME/.sdkman/bin/sdkman-init.sh"
 # sdk list
-# sdk install 8.0.392-librca
-# sdk default 8.0.392-librca
+# sdk install java 8.0.392-librca
+# sdk default java 8.0.392-librca
 
 # flutter
 export PUB_HOSTED_URL=https://pub.flutter-io.cn

docker/.github/workflows/vue_deploy.yml

@@ -10,6 +10,11 @@ on:
   pull_request:
     branches: [master]
 
+# 并发组
+concurrency:
+  group: vue-deploy
+  cancel-in-progress: true
+
 jobs:
   build_vue:
     runs-on: ubuntu-latest
@@ -66,6 +71,14 @@ jobs:
       docker login -u ${{secrets.DOCKER_HUB_NAME}} -p ${{secrets.DOCKER_HUB_PASSWORD}}
       docker-compose -f docker-compose.yml up -d
 
+# 登录GitHub Container Registry，推送镜像到仓库
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          
 # 登录远程服务器，拉取代码，部署
   deploy2:
     runs-on: ubuntu-latest

docs/ai.md

@@ -14,6 +14,21 @@ Repo that allows me to build AI tools on top of Hugging Face
 * run `python utils/verify_cuda_pytorch.py`
 * run `nvidia-smi` should show a GPU
 
+```
+conda config --set solver libmamba
+
+conda create -n LLMs python=3.10
+
+conda activate dl-workshop
+
+conda install jupyterlab watermark
+
+pip install torch==2.0.1
+
+pip install -r requirements.txt
+
+```
+
 
 ## Dev Container
 

docs/dev/android.md

@@ -0,0 +1,17 @@
+# Android 
+
+| images     | desc | other |
+| -------- | ---- | ---- |
+| android-dev-eclipse |   基于 Ubuntu 的 Android 项目开发编译环境。   |   eclipse项目打包apk比较复杂   |
+| android-dev-studio |   基于 Ubuntu 的 Android 项目开发编译环境。   |    studio打包，需要管理依赖  |
+| android-source     |  基于 Ubuntu 的 Android 系统源码编译。    |      |
+|          |      |      |
+
+## usage
+
+1. 启动 docker 环境
+
+```
+docker-compose up -d
+```
+

docs/dev/others.md

@@ -12,92 +12,3 @@ mkdir -p php-demo
 chmod 777 php-demo && cd php-demo
 docker run --rm -it -w /var/www/html -v $(pwd):/var/www/html -p 8000:8000 groovy:jdk17-alpine /bin/bash
 cd /var/www/html
-
-
-
-## 详情
-
-
-**runs-on** **可选：**
-
-| **名称**                                       | [已安装](https://github.com/actions/virtual-environments/blob/master/images/linux/Ubuntu1804-README.md) |
-| ---------------------------------------------- | ------------------------------------------------------------ |
-| windows-2019                                   | docker/vs   2019/**android**/ruby/git/go/php/.net/minGW/miniconda/Alibaba Cloud CLI |
-| **ubuntu-latest** or   ubuntu-18.04   ubuntu-16.04 | docker/kubect/php/python/java/**android**/zip/.net/powershell/mysql/ms   sql/nodejs/webpack/yarn/gradle/maven/ant/ |
-| macos-latest or   macos-10.15                  | php/python/go/java/yarn/git/**xcode**/android                |
-
-### **azure 部分**
-
-关于 azure 云平台配合 github 相关 CI/CD配置。
-
-| **位置**                                       | 描述 |
-| ---------------------------------------------- | ------------------------------------------------------------ |
-| azure\.github\workflow\deploy-app-to-azure.yml | 部署resources app到azure  |
-| azure\.github\workflow\set-up-azure-resources.yml                  |  设置resources资源                |
-
-### **Vue 部分**
-
-使用 github action 自动构建/部署 Vue 项目。
-
-| **位置**                                       | 描述 |
-| ---------------------------------------------- | ------------------------------------------------------------ |
-|  |   |
-
-
-**Python 部分**
-
-| **位置**                                       | 描述 |
-| ---------------------------------------------- | ------------------------------------------------------------ |
-|  |   |
-
-
-
-**Go 部分**
-
-使用 github action 编译/发布 go 项目，以及使用 go web 功能。
-
-| **位置**                                       | 描述 |
-| ---------------------------------------------- | ------------------------------------------------------------ |
-|  |   |
-
-
-
-**PHP 部分**
-
-| **位置**                                       | 描述 |
-| ---------------------------------------------- | ------------------------------------------------------------ |
-|  |   |
-
-
-**Java 部分**
-
-| **位置**                                       | 描述 |
-| ---------------------------------------------- | ------------------------------------------------------------ |
-|  |   |
-
-
-**Android 部分**
-
-本项目刚启动时候，很多 android 使用 github action 自动构建的很少，本项目将归纳基于 eclipse 和 android studio 等开发的 android 项目使用 github action 自动构建方案。
-
-注意： 使用 eclipse IDE 开发的 andorid 项目，其实采用的是 ant 构建。而采用 android studio IDE 开发的项目采用的是 gradle 构建。
-
-| **位置**                                       | 描述 |
-| ---------------------------------------------- | ------------------------------------------------------------ |
-| android-studio\.github\workflows\main.yml |  默认 android studio 开发的项目采用这个 action |
-| android-eclipse\.github\workflows\main.yml |  采用 eclipse 开发的采用这个 action |
-
-
-20 年 vscode 开源的容器 demo 项目，修改为国内镜像。如今，使用通用镜像即可。
-```
-git clone https://github.com/Microsoft/vscode-remote-try-node
-git clone https://github.com/Microsoft/vscode-remote-try-python
-git clone https://github.com/Microsoft/vscode-remote-try-go
-git clone https://github.com/Microsoft/vscode-remote-try-java
-git clone https://github.com/Microsoft/vscode-remote-try-dotnetcore
-git clone https://github.com/Microsoft/vscode-remote-try-php
-git clone https://github.com/Microsoft/vscode-remote-try-rust
-git clone https://github.com/Microsoft/vscode-remote-try-cpp
-
-```
-

docs/dev/php.md

@@ -109,4 +109,7 @@ composer create-project --prefer-dist yiisoft/yii2-app-basic demo
 
 ```
 
-```
+```
+
+## Rust
+

nodejs/Dockerfile

@@ -24,10 +24,4 @@ RUN npm install
 COPY . .
 RUN chown -R appuser:appgroup /app
 EXPOSE 8080
-
-FROM base as builder
-
-
-FROM base as runner
-
 CMD [ "npm", "start" ]

rust/Dockerfile

@@ -0,0 +1,62 @@
+FROM ubuntu:22.04
+SHELL ["/bin/bash", "-c"]
+
+# 安装必要依赖与开发工具
+RUN sed -i 's/archive.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list && \
+    apt-get update && apt-get install -y \
+    gcc-riscv64-unknown-elf gdb-multiarch dosfstools cmake \
+    git wget python3 vim file curl \
+    autoconf automake autotools-dev  libmpc-dev libmpfr-dev libgmp-dev \
+    gawk build-essential bison flex texinfo gperf libtool patchutils bc \
+    zlib1g-dev libexpat-dev \
+    ninja-build pkg-config libglib2.0-dev libpixman-1-dev libsdl2-dev \ 
+    && rm -rf /var/lib/apt/lists/*
+
+# 安装 QEMU
+ARG QEMU_VERSION=7.0.0
+RUN cd /tmp && \
+    wget https://download.qemu.org/qemu-${QEMU_VERSION}.tar.xz && \
+    tar xf qemu-${QEMU_VERSION}.tar.xz && \
+    cd qemu-${QEMU_VERSION} && \
+    ./configure --target-list=riscv64-softmmu,riscv64-linux-user && \
+    make -j && \
+    make install && \
+    cd .. && \
+    rm -rf qemu-${QEMU_VERSION} qemu-${QEMU_VERSION}.tar.xz
+
+# 安装 code-server 和扩展
+ARG CODE_VERSION=4.10.1
+RUN cd /usr/local/ && \
+    wget https://github.com/coder/code-server/releases/download/v${CODE_VERSION}/code-server-${CODE_VERSION}-linux-amd64.tar.gz && \
+    tar xf code-server-${CODE_VERSION}-linux-amd64.tar.gz && \
+    ln -s  /usr/local/code-server-${CODE_VERSION}-linux-amd64/bin/code-server /usr/bin/code && \
+    rm code-server-${CODE_VERSION}-linux-amd64.tar.gz && \
+    wget https://openvsxorg.blob.core.windows.net/resources/rust-lang/rust-analyzer/linux-x64/0.3.1435/rust-lang.rust-analyzer-0.3.1435@linux-x64.vsix && \
+    code --install-extension rust-lang.rust-analyzer-0.3.1435@linux-x64.vsix && \
+    rm rust-lang.rust-analyzer-0.3.1435@linux-x64.vsix && \
+    wget https://open-vsx.org/api/MS-CEINTL/vscode-language-pack-zh-hans/1.75.0/file/MS-CEINTL.vscode-language-pack-zh-hans-1.75.0.vsix && \
+    code --install-extension MS-CEINTL.vscode-language-pack-zh-hans-1.75.0.vsix && \
+    rm MS-CEINTL.vscode-language-pack-zh-hans-1.75.0.vsix
+
+WORKDIR /root
+# 安装 rust
+ARG RUST_VERSION=nightly
+ENV RUSTUP_DIST_SERVER=https://mirrors.ustc.edu.cn/rust-static
+ENV RUSTUP_UPDATE_ROOT=https://mirrors.ustc.edu.cn/rust-static/rustup
+RUN mkdir .cargo && \
+    echo '[source.crates-io]' >> .cargo/config && \
+    echo 'registry = "https://github.com/rust-lang/crates.io-index"' >> .cargo/config && \
+    echo 'replace-with = "ustc"' >> .cargo/config && \
+    echo '[source.ustc]' >> .cargo/config && \
+    echo 'registry = "git://mirrors.ustc.edu.cn/crates.io-index"' >> .cargo/config && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init && \
+    chmod +x rustup-init && \
+    ./rustup-init -y --default-toolchain ${RUST_VERSION} --target riscv64imac-unknown-none-elf && \
+    rm rustup-init && \
+    source $HOME/.cargo/env && \
+    cargo install cargo-binutils && \
+    rustup component add llvm-tools-preview && \
+    rustup component add rust-src
+
+EXPOSE 8080/tcp
+CMD ["code", "--auth", "none", "--bind-addr", "0.0.0.0:8080"]