ci: build and upload releases on Travis

Author: igrr

.travis.yml

@@ -1,48 +1,29 @@
 sudo: false
 language: bash
-os:
-  - linux
-
+os: linux
 dist: trusty
 
+env:
+  - BUILD_TYPE=build
+
 install:
   - pip install --user -r doc/requirements.txt
 
 script:
-  - set -e
-  - echo -e "travis_fold:start:docs"
-  - pushd $TRAVIS_BUILD_DIR/doc
-  - SPHINXOPTS="-W" make html
-  - popd
-  - echo -e "travis_fold:end:docs"
-  - echo -e "travis_fold:start:host_tests"
-  - pushd $TRAVIS_BUILD_DIR/tests/host
-  - make
-  - make clean-objects
-  - popd
-  - echo -e "travis_fold:end:host_tests"
-  - echo -e "travis_fold:start:sketch_test_env_prepare"
-  - wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
-  - tar xf arduino.tar.xz
-  - mv arduino-nightly $HOME/arduino_ide
-  - cd $HOME/arduino_ide/hardware
-  - mkdir esp8266com
-  - cd esp8266com
-  - ln -s $TRAVIS_BUILD_DIR esp8266
-  - cd esp8266/tools
-  - python get.py
-  - export PATH="$HOME/arduino_ide:$TRAVIS_BUILD_DIR/tools/xtensa-lx106-elf/bin:$PATH"
-  - which arduino
-  - cd $TRAVIS_BUILD_DIR
-  - source tests/common.sh
-  - install_libraries
-  - echo -e "travis_fold:end:sketch_test_env_prepare"
-  - echo -e "travis_fold:start:sketch_test"
-  - build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries"
-  - echo -e "travis_fold:end:sketch_test"
-  - echo -e "travis_fold:start:size_report"
-  - cat size.log
-  - echo -e "travis_fold:end:size_report"
+  - $TRAVIS_BUILD_DIR/tests/common.sh
+
+deploy:
+  provider: releases
+  prerelease: true
+  api_key:
+    secure: A4FBmqyhlzy33oPeZVolg2Q/A3ZcJ3WnRQqQJ3NAPy+qGM5xcboOYtwcLL9vKaHZGfUB7lUP9QVZFGou1Wrmo9DnPvAoe3+XvCaDRGzVMxeIpu7UStbBD4Knbh98tlbMvZCXYRlT4VcusI9bMLK6UWw4sMdPislBh2FEfglTiag=
+  file_glob: true
+  file:
+    - package/versions/$TRAVIS_TAG/esp8266-$TRAVIS_TAG.zip
+    - package/versions/$TRAVIS_TAG/package_esp8266com_index.json
+  on:
+    repo: esp8266/Arduino
+    tags: true
 
 notifications:
   email:

README.md

@@ -38,11 +38,6 @@ Boards manager link: `http://arduino.esp8266.com/stable/package_esp8266com_index
 
 Documentation: [http://esp8266.github.io/Arduino/versions/2.3.0/](http://esp8266.github.io/Arduino/versions/2.3.0/)
 
-##### Staging version ![](http://arduino.esp8266.com/staging/badge.svg)
-Boards manager link: `http://arduino.esp8266.com/staging/package_esp8266com_index.json`
-
-Documentation: [http://esp8266.github.io/Arduino/versions/2.3.0-rc2/](http://esp8266.github.io/Arduino/versions/2.3.0-rc2/)
-
 ### Using git version
 [![Linux build status](https://travis-ci.org/esp8266/Arduino.svg)](https://travis-ci.org/esp8266/Arduino) [![codecov.io](https://codecov.io/github/esp8266/Arduino/coverage.svg?branch=master)](https://codecov.io/github/esp8266/Arduino?branch=master)
 

package/build_boards_manager_package.sh

@@ -1,8 +1,21 @@
 #!/bin/bash
 #
 
+# Extract next version from platform.txt
+next=`sed -n -E 's/version=([0-9.]+)/\1/p' ../platform.txt`
+
 # Figure out how will the package be called
-ver=`git describe --tags --always`
+ver=`git describe --exact-match`
+if [ $? -ne 0 ]; then
+    # not tagged version; generate nightly package
+    date_str=`date +"%Y%m%d"`
+    is_nightly=1
+    plain_ver="${next}-nightly"
+    ver="${plain_ver}+${date_str}"
+else
+    plain_ver=$ver
+fi
+
 package_name=esp8266-$ver
 echo "Version: $ver"
 echo "Package name: $package_name"
@@ -16,14 +29,14 @@ fi
 echo "Remote: $REMOTE_URL"
 
 if [ -z "$PKG_URL" ]; then
-    PKG_URL="$REMOTE_URL/versions/$ver/$package_name.zip"
+    if [ -z "$PKG_URL_PREFIX" ]; then
+        PKG_URL_PREFIX="$REMOTE_URL/versions/$ver"
+    fi
+    PKG_URL="$PKG_URL_PREFIX/$package_name.zip"
 fi
 echo "Package: $PKG_URL"
-
-if [ -z "$DOC_URL" ]; then
-    DOC_URL="$REMOTE_URL/versions/$ver/doc/reference.html"
-fi
 echo "Docs: $DOC_URL"
+
 pushd ..
 # Create directory for the package
 outdir=package/versions/$ver/$package_name
@@ -37,6 +50,7 @@ cat << EOF > exclude.txt
 .gitignore
 .travis.yml
 package
+doc
 EOF
 # Also include all files which are ignored by git
 git ls-files --other --directory >> exclude.txt
@@ -47,7 +61,7 @@ rm exclude.txt
 # Get additional libraries (TODO: add them as git submodule or subtree?)
 
 # SoftwareSerial library
-wget -q -O SoftwareSerial.zip https://github.com/plerup/espsoftwareserial/archive/3.1.0.zip
+curl -L -o SoftwareSerial.zip https://github.com/plerup/espsoftwareserial/archive/3.1.0.zip
 unzip -q SoftwareSerial.zip
 rm -rf SoftwareSerial.zip
 mv espsoftwareserial-* SoftwareSerial
@@ -71,7 +85,7 @@ $SED 's/recipe.hooks.core.prebuild.1.pattern.*//g' \
  > $outdir/platform.txt
 
 # Put core version and short hash of git version into core_version.h
-ver_define=`echo $ver | tr "[:lower:].-" "[:upper:]_"`
+ver_define=`echo $plain_ver | tr "[:lower:].\055" "[:upper:]_"`
 echo Ver define: $ver_define
 echo \#define ARDUINO_ESP8266_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null` >$outdir/cores/esp8266/core_version.h
 echo \#define ARDUINO_ESP8266_RELEASE_$ver_define >>$outdir/cores/esp8266/core_version.h
@@ -90,14 +104,49 @@ echo Size: $size
 echo SHA-256: $sha
 
 echo "Making package_esp8266com_index.json"
-cat $srcdir/package/package_esp8266com_index.template.json | \
-jq ".packages[0].platforms[0].version = \"$ver\" | \
+
+jq_arg=".packages[0].platforms[0].version = \"$ver\" | \
     .packages[0].platforms[0].url = \"$PKG_URL\" |\
-    .packages[0].platforms[0].archiveFileName = \"$package_name.zip\" |\
-    .packages[0].platforms[0].checksum = \"SHA-256:$sha\" |\
-    .packages[0].platforms[0].size = \"$size\" |\
-    .packages[0].platforms[0].help.online = \"$DOC_URL\"" \
-    > package_esp8266com_index.json
+    .packages[0].platforms[0].archiveFileName = \"$package_name.zip\""
+
+if [ -z "$is_nightly" ]; then
+    jq_arg="$jq_arg |\
+        .packages[0].platforms[0].size = \"$size\" |\
+        .packages[0].platforms[0].checksum = \"SHA-256:$sha\""
+fi
+
+if [ ! -z "$DOC_URL" ]; then
+    jq_arg="$jq_arg |\
+        .packages[0].platforms[0].help.online = \"$DOC_URL\""
+fi
+
+cat $srcdir/package/package_esp8266com_index.template.json | \
+    jq "$jq_arg" > package_esp8266com_index.json
+
+# Get previous release name
+curl --silent https://api.github.com/repos/esp8266/Arduino/releases > releases.json
+# Previous final release (prerelase == false)
+prev_release=$(jq -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' releases.json)
+# Previous release (possibly a pre-release)
+prev_any_release=$(jq -r '. | map(select(.draft == false)) | sort_by(.created_at | - fromdateiso8601)  | .[0].tag_name' releases.json)
+# Previous pre-release
+prev_pre_release=$(jq -r '. | map(select(.draft == false and .prerelease == true)) | sort_by(.created_at | - fromdateiso8601)  | .[0].tag_name' releases.json)
+
+echo "Previous release: $prev_release"
+echo "Previous (pre-?)release: $prev_any_release"
+echo "Previous pre-release: $prev_pre_release"
+
+# Make all released versions available in one package (i.e. don't separate stable/staging versions)
+base_ver=$prev_any_release
+
+# Download previous release
+echo "Downloading base package: $base_ver"
+old_json=package_esp8266com_index_stable.json
+curl -L -o $old_json "https://github.com/esp8266/Arduino/releases/download/${base_ver}/package_esp8266com_index.json"
+new_json=package_esp8266com_index.json
+
+set +e
+python ../../merge_packages.py $new_json $old_json >tmp && mv tmp $new_json && rm $old_json
 
 popd
 popd

package/package_esp8266com_index.template.json

@@ -16,8 +16,6 @@
           "category": "ESP8266",
           "url": "",
           "archiveFileName": "",
-          "checksum": "",
-          "size": "",
           "help": {
             "online": ""
           },

platform.txt

@@ -6,7 +6,7 @@
 # https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5---3rd-party-Hardware-specification
 
 name=ESP8266 Modules
-version=2.3.0
+version=2.4.0
 
 runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}/tools/xtensa-lx106-elf
 runtime.tools.esptool.path={runtime.platform.path}/tools/esptool

tests/common.sh

@@ -84,3 +84,91 @@ function install_libraries()
 
     popd
 }
+
+function install_ide()
+{
+    local ide_path=$1
+    local core_path=$2
+    wget -O arduino.tar.xz https://www.arduino.cc/download.php?f=/arduino-nightly-linux64.tar.xz
+    tar xf arduino.tar.xz
+    mv arduino-nightly $ide_path
+    cd $ide_path/hardware
+    mkdir esp8266com
+    cd esp8266com
+    ln -s $core_path esp8266
+    cd esp8266/tools
+    python get.py
+    export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH"
+}
+
+function build_docs()
+{
+    SPHINXOPTS="-W" make html
+}
+
+function run_host_tests()
+{
+    pushd host
+    make
+    make clean-objects
+    popd
+}
+
+function build_package()
+{
+    export PKG_URL=https://github.com/esp8266/Arduino/releases/download/$TRAVIS_TAG/esp8266-$TRAVIS_TAG.zip
+    export DOC_URL=https://arduino-esp8266.readthedocs.io/en/$TRAVIS_TAG/
+    ./build_boards_manager_package.sh
+}
+
+function run_travis_ci_build()
+{
+    # Build documentation using Sphinx
+    echo -e "travis_fold:start:docs"
+    cd $TRAVIS_BUILD_DIR/doc
+    build_docs
+    echo -e "travis_fold:end:docs"
+
+    # Build release package
+    echo -e "travis_fold:start:build_package"
+    cd $TRAVIS_BUILD_DIR/package
+    build_package
+    echo -e "travis_fold:end:build_package"
+
+    if [ "$TRAVIS_TAG" != "" ]; then
+        echo "Skipping tests for tagged build"
+        return 0;
+    fi
+
+    # Run host side tests
+    echo -e "travis_fold:start:host_tests"
+    cd $TRAVIS_BUILD_DIR/tests
+    run_host_tests
+    echo -e "travis_fold:end:host_tests"
+
+    # Install Arduino IDE and required libraries
+    echo -e "travis_fold:start:sketch_test_env_prepare"
+    cd $TRAVIS_BUILD_DIR
+    install_ide $HOME/arduino_ide $TRAVIS_BUILD_DIR
+    which arduino
+    cd $TRAVIS_BUILD_DIR
+    install_libraries
+    echo -e "travis_fold:end:sketch_test_env_prepare"
+
+    # Compile sketches
+    echo -e "travis_fold:start:sketch_test"
+    build_sketches $HOME/arduino_ide $TRAVIS_BUILD_DIR/libraries "-l $HOME/Arduino/libraries"
+    echo -e "travis_fold:end:sketch_test"
+
+    # Generate size report
+    echo -e "travis_fold:start:size_report"
+    cat size.log
+    echo -e "travis_fold:end:size_report"
+}
+
+set -e
+
+if [ "$BUILD_TYPE" = "build" ]; then
+    run_travis_ci_build
+fi
+