From 3e359351826a12c5324079b167f5a6962c55cb76 Mon Sep 17 00:00:00 2001 From: heff Date: Tue, 9 Jun 2015 18:57:20 -0700 Subject: [PATCH] Sped up tests by relying on watchify and splitting up tests - Temporarily killed API tests. Combining them with other tests. - Using browserify:watch for build/temp/video.js (sandbox testing) - Using karma:watch for automated tests Using individual watch tasks allows watchify to use smart caching in both instances. - Switched to grunt-concurrent for watch tasks - Switched to travis containers, sudo: false - Added caching of npm modules in Travis - Consolidated travis testing - Cleaned up grunt file - Fixed travis.yml spacing - Added the watchAll task for trying it out - Moved travis test script logic to package.json - Moved coverage reporting to Travis only closes #2254 --- .travis.yml | 27 +++--- CHANGELOG.md | 1 + build/tasks/test-travis.js | 28 ------ grunt.js | 169 +++++++++++++++++---------------- package.json | 7 +- test/index.html | 3 - test/karma.conf.js | 189 ++++++++++++++++++++++--------------- 7 files changed, 216 insertions(+), 208 deletions(-) delete mode 100644 build/tasks/test-travis.js diff --git a/.travis.yml b/.travis.yml index 3e31a08e9..e6b5397b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,24 +1,27 @@ language: node_js node_js: -- iojs + - iojs before_script: -- if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then curl https://gist.githubusercontent.com/santiycr/5139565/raw/sauce_connect_setup.sh - | bash; fi + - if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then curl https://gist.githubusercontent.com/santiycr/5139565/raw/sauce_connect_setup.sh | bash; fi before_install: -- export CHROME_BIN=chromium-browser -- export DISPLAY=:99.0 -- sh -e /etc/init.d/xvfb start + - export CHROME_BIN=chromium-browser + - export DISPLAY=:99.0 + - sh -e /etc/init.d/xvfb start notifications: irc: channels: - - chat.freenode.net#videojs + - chat.freenode.net#videojs use_notice: true on_success: never webhooks: - - http://pam.videojs.com/savage/travis + - http://pam.videojs.com/savage/travis env: global: - - secure: K6JpKwMkfNaJix3Bb0tLjVMzHMJgtBXdd/dvfw1BMb9DCBpd81PqXbDs7yXCddUxnUPTBPxZCrQgWsw71Wn+qEoIG5MU3uOT5A2rBbx/yZonVAGv5ed/9w0xk0OzO383CmPMFqwqtp9YmdmqGjQBkYXVXJjTvNTOAExFSdhO+3U= - - secure: GIbhjUJapvC70nIZVlhVyK+3KAD2TVKpiY/q412OO7V2izbvcM1tvU3LBoMZbROzrt5TT84tCoJDvHnrpL0OvxPwrzL5CUU7h4UTxhTOyQkEinbYAnWlW9wdrvtdczsEvANkFPqBZ53B3hVHZHMLOG8QRWaTBicF68vSHEJFqb4= - - secure: gglh7xDnURKfXp9T543DD7NG1pQ8HeWh1XtRspBAwr0H7RqJBVDqqODSYSPRFhfld7M6sYmvQIXgil7XlyefnKNTXqCarvaoTg3lbip8kSltXMiNw2V6AVpsQGuja7+XbaM0do70ETTKjW4Kw6wnxEHb78BvGN/hXIeqizUAjanlDAjd7fouaxpTBIbMESe2rI+WRHPis1cmnv8v70Mrh/8Un/NO4gkebGyvA47LTDNIaVqIVjonsndr8WjMv1/PNxQ8LyCO6D64MufrobS7Sec+VuN30apwEsBw8v82MK/MZ3qXu0lUp4+ERTbuc/rymh2wDFTQeG20Kf/NTauSaH6f414KNzIRFj0/xyLAzVZKIscXM2DKXMuskkZuvHLZvaspnZWcPYTjPZl0P88N0RBqnoLdR80dR5bDljNwU2QnSBeol/q1wXNEr6I1VTRFOB+qsHrD1blVMB1I5W3I0ti1aQ7XtgMOGi1kcPb4oFcJdl+3dLFDnyRyaNfdMOnOZYBBHdQCo19Mj/L+nqPGWeeYiEAM6JsuhNjHn5Za5nGf1ztXTimVPOQjyATin0x9kST3soLWSVmdW2dBHUGDVSMhvoLLR+nKSdNQ0KfpqtgrzeLxoVnRYHVBlih41tapM9IG/6BMYnDMaRcc0i54YeUP4oxlxGSyASIenkAgC6w= - - secure: WtIEOSnqDkCZuTlBsxwlVwaRpVTbz7ol8+XSJIZb0aFo1lLisF9cz6s9WrAfX36MaxIcDN9LFZkpXzMvNrNkZWQa1kacGWH1rbx0SiiQ8LMweAcKdnZ5uXlSplBxbJ8bZfXKB1sIHsOsYw/vWhHKkcsDUkAEzQrIiMOhuoUV3s0uKM0knKXIAfNIF0EbDzLIojm+nm+F0n5vM60LRdKesaSt/o2p2LKxdZVoFGrg48D7bdA9VEfMWWRL/evDxJmnX4p+AjBc7mklqZ5F2pYsY6XXQuuS+2Sy+lnxz01kLg+RC4Cpv5dyYfK3h0j8KeyK8IuixycVONWVe9rANq8UaIsMrRN+6uDSC8zXiH4P+h6UDMm3jetc2ZyAfhBA8OyIs5QEShae2Rd7Y3WFJxBp6UVgyj6SkXGxrEdb1ZJgTTl4dyqiP0bYrLePNP2qSJ6OTfNdG791HF077uzXI96ABdMG54Wv9N9T/hmxKwV2Lajx/GZJMmHuwT9tkHKhkcxWea1HYam9QYSFUyJ5THfNk2A9u/r8DkL62MZ85zIQBisrlFjbPAGRejq6qyirBJPAy+FCjhM+oO/i2f2bGkkAfHGT0Og1BcrWVXs54yWdO7UZgie2F+Rmdwinb/GxebZJ+21ZQ4OkVr2t1Skr/PRni9+U7q/6xCLwUJgx45XJ0FE= + - secure: K6JpKwMkfNaJix3Bb0tLjVMzHMJgtBXdd/dvfw1BMb9DCBpd81PqXbDs7yXCddUxnUPTBPxZCrQgWsw71Wn+qEoIG5MU3uOT5A2rBbx/yZonVAGv5ed/9w0xk0OzO383CmPMFqwqtp9YmdmqGjQBkYXVXJjTvNTOAExFSdhO+3U= + - secure: GIbhjUJapvC70nIZVlhVyK+3KAD2TVKpiY/q412OO7V2izbvcM1tvU3LBoMZbROzrt5TT84tCoJDvHnrpL0OvxPwrzL5CUU7h4UTxhTOyQkEinbYAnWlW9wdrvtdczsEvANkFPqBZ53B3hVHZHMLOG8QRWaTBicF68vSHEJFqb4= + - secure: gglh7xDnURKfXp9T543DD7NG1pQ8HeWh1XtRspBAwr0H7RqJBVDqqODSYSPRFhfld7M6sYmvQIXgil7XlyefnKNTXqCarvaoTg3lbip8kSltXMiNw2V6AVpsQGuja7+XbaM0do70ETTKjW4Kw6wnxEHb78BvGN/hXIeqizUAjanlDAjd7fouaxpTBIbMESe2rI+WRHPis1cmnv8v70Mrh/8Un/NO4gkebGyvA47LTDNIaVqIVjonsndr8WjMv1/PNxQ8LyCO6D64MufrobS7Sec+VuN30apwEsBw8v82MK/MZ3qXu0lUp4+ERTbuc/rymh2wDFTQeG20Kf/NTauSaH6f414KNzIRFj0/xyLAzVZKIscXM2DKXMuskkZuvHLZvaspnZWcPYTjPZl0P88N0RBqnoLdR80dR5bDljNwU2QnSBeol/q1wXNEr6I1VTRFOB+qsHrD1blVMB1I5W3I0ti1aQ7XtgMOGi1kcPb4oFcJdl+3dLFDnyRyaNfdMOnOZYBBHdQCo19Mj/L+nqPGWeeYiEAM6JsuhNjHn5Za5nGf1ztXTimVPOQjyATin0x9kST3soLWSVmdW2dBHUGDVSMhvoLLR+nKSdNQ0KfpqtgrzeLxoVnRYHVBlih41tapM9IG/6BMYnDMaRcc0i54YeUP4oxlxGSyASIenkAgC6w= + - secure: WtIEOSnqDkCZuTlBsxwlVwaRpVTbz7ol8+XSJIZb0aFo1lLisF9cz6s9WrAfX36MaxIcDN9LFZkpXzMvNrNkZWQa1kacGWH1rbx0SiiQ8LMweAcKdnZ5uXlSplBxbJ8bZfXKB1sIHsOsYw/vWhHKkcsDUkAEzQrIiMOhuoUV3s0uKM0knKXIAfNIF0EbDzLIojm+nm+F0n5vM60LRdKesaSt/o2p2LKxdZVoFGrg48D7bdA9VEfMWWRL/evDxJmnX4p+AjBc7mklqZ5F2pYsY6XXQuuS+2Sy+lnxz01kLg+RC4Cpv5dyYfK3h0j8KeyK8IuixycVONWVe9rANq8UaIsMrRN+6uDSC8zXiH4P+h6UDMm3jetc2ZyAfhBA8OyIs5QEShae2Rd7Y3WFJxBp6UVgyj6SkXGxrEdb1ZJgTTl4dyqiP0bYrLePNP2qSJ6OTfNdG791HF077uzXI96ABdMG54Wv9N9T/hmxKwV2Lajx/GZJMmHuwT9tkHKhkcxWea1HYam9QYSFUyJ5THfNk2A9u/r8DkL62MZ85zIQBisrlFjbPAGRejq6qyirBJPAy+FCjhM+oO/i2f2bGkkAfHGT0Og1BcrWVXs54yWdO7UZgie2F+Rmdwinb/GxebZJ+21ZQ4OkVr2t1Skr/PRni9+U7q/6xCLwUJgx45XJ0FE= +sudo: false +cache: + directories: + - node_modules diff --git a/CHANGELOG.md b/CHANGELOG.md index 8733b1fd9..bfa25b255 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ CHANGELOG * @gkatsev updated vttjs to fix a trailing comma JSON error ([view](https://github.com/videojs/video.js/pull/2331)) * @gkatsev exported the videojs.bind() function ([view](https://github.com/videojs/video.js/pull/2332)) * Insert cloned el back into DOM. Fixes #2214 ([view](https://github.com/videojs/video.js/pull/2334)) +* @heff sped up testing ([view](https://github.com/videojs/video.js/pull/2254)) -------------------- diff --git a/build/tasks/test-travis.js b/build/tasks/test-travis.js deleted file mode 100644 index 5c6a72513..000000000 --- a/build/tasks/test-travis.js +++ /dev/null @@ -1,28 +0,0 @@ -module.exports = function(grunt) { - // The test task will run `karma:saucelabs` when running in travis, - // when running via a PR from a fork, it'll run qunit tests in phantom using - // karma otherwise, it'll run the tests in chrome via karma - // You can specify which browsers to build with by using grunt-style arguments - // or separating them with a comma: - // grunt test:chrome:firefox # grunt-style - // grunt test:chrome,firefox # comma-separated - grunt.registerTask('test-travis', function() { - let tasks = this.args; - let tasksMinified; - - // I believe this was done originally because of security implications around running - // Saucelabs automatically on PRs. - if (!process.env.SAUCE_ACCESS_KEY) { - grunt.task.run(['karma:firefox']); - } else { - grunt.task.run([ - 'karma:chrome_sl', - 'karma:firefox_sl', - 'karma:safari_sl', - 'karma:ipad_sl', - 'karma:android_sl', - 'karma:ie_sl' - ]); - } - }); -}; diff --git a/grunt.js b/grunt.js index 68dc4e438..c7085dede 100644 --- a/grunt.js +++ b/grunt.js @@ -55,9 +55,9 @@ module.exports = function(grunt) { }, dist: {}, watch: { - default: { - files: [ 'src/**/*', 'test/unit/**/*.js', 'Gruntfile.js' ], - tasks: 'dev' + jshint: { + files: ['src/**/*', 'test/unit/**/*.js', 'Gruntfile.js'], + tasks: 'jshint' }, skin: { files: ['src/css/**/*'], @@ -65,12 +65,6 @@ module.exports = function(grunt) { } }, connect: { - preview: { - options: { - port: 9999, - keepalive: true - } - }, dev: { options: { port: 9999, @@ -106,7 +100,7 @@ module.exports = function(grunt) { } }, sass: { - dist: { + build: { files: { 'build/temp/video-js.css': 'src/css/video-js.scss' } @@ -118,10 +112,15 @@ module.exports = function(grunt) { configFile: 'test/karma.conf.js' }, - // these are run locally on local browsers - dev: { - browsers: ['Chrome', 'Firefox', 'Safari'] + defaults: {}, + + watch: { + autoWatch: true, + singleRun: false }, + + // these are run locally on local browsers + dev: { browsers: ['Chrome', 'Firefox', 'Safari'] }, chromecanary: { browsers: ['ChromeCanary'] }, chrome: { browsers: ['Chrome'] }, firefox: { browsers: ['Firefox'] }, @@ -219,72 +218,59 @@ module.exports = function(grunt) { } }, browserify: { + options: { + browserifyOptions: { + debug: true, + standalone: 'videojs' + }, + banner: license, + plugin: [ + ['browserify-derequire'] + ], + transform: [ + require('babelify').configure({ + sourceMapRelative: './', + loose: ['all'] + }), + ['browserify-versionify', { + placeholder: '__VERSION__', + version: pkg.version + }], + ['browserify-versionify', { + placeholder: '__VERSION_NO_PATCH__', + version: version.majorMinor + }] + ] + }, build: { files: { 'build/temp/video.js': ['src/js/video.js'] - }, - options: { - browserifyOptions: { - debug: true, - standalone: 'videojs' - }, - banner: license, - plugin: [ - [ 'browserify-derequire' ] - ], - transform: [ - require('babelify').configure({ - sourceMapRelative: './src/js', - loose: 'all' - }), - ['browserify-versionify', { - placeholder: '__VERSION__', - version: pkg.version - }], - ['browserify-versionify', { - placeholder: '__VERSION_NO_PATCH__', - version: version.majorMinor - }] - ] - } - }, - test: { - files: { - 'build/temp/tests.js': [ - 'test/globals-shim.js', - 'test/unit/**/*.js' - ] - }, - options: { - browserifyOptions: { - debug: true, - standalone: 'videojs' - }, - transform: [ - require('babelify').configure(), - 'browserify-istanbul' - ] } }, watch: { + options: { + watch: true, + keepAlive: true + }, + files: { + 'build/temp/video.js': ['src/js/video.js'] + } + }, + tests: { + options: { + browserifyOptions: { + debug: true, + standalone: false + }, + banner: false, + watch: true, + keepAlive: true + }, files: { - 'build/temp/video.js': ['src/js/video.js'], 'build/temp/tests.js': [ 'test/globals-shim.js', 'test/unit/**/*.js' ] - }, - options: { - watch: true, - keepAlive: true, - browserifyOptions: { - standalone: 'videojs' - }, - banner: license, - transform: ['babelify'], - plugin: [ - [ 'browserify-derequire' ] - ] } } }, @@ -314,6 +300,23 @@ module.exports = function(grunt) { src: ['build/temp/video.js', 'node_modules/vtt.js/dist/vtt.js'], dest: 'build/temp/video.js', }, + }, + concurrent: { + options: { + logConcurrentOutput: true + }, + // Run multiple watch tasks in parallel + // Needed so watchify can cache intelligently + watchAll: [ + 'watch', + 'browserify:watch', + 'browserify:tests', + 'karma:watch' + ], + watchSandbox: [ + 'watch', + 'browserify:watch' + ] } }); @@ -324,16 +327,18 @@ module.exports = function(grunt) { grunt.registerTask('build', [ 'clean:build', + 'jshint', 'browserify:build', - 'browserify:test', + 'exorcise:build', 'copy:novtt', 'concat:vtt', - 'exorcise', 'uglify', + 'sass', 'version:css', 'cssmin', + 'copy:fonts', 'copy:swf', 'copy:ie8', @@ -348,25 +353,19 @@ module.exports = function(grunt) { 'zip:dist' ]); - // Remove this and add to the test task once mmcc's coverall changes are merged - grunt.registerTask('newtest', ['build', 'karma:chrome']); + // Default task - build and test + grunt.registerTask('default', ['test']); - // Default task. - grunt.registerTask('default', ['build', 'test-local']); + grunt.registerTask('test', ['build', 'karma:defaults']); - // Development watch task. Doing the minimum required. - grunt.registerTask('dev', ['jshint', 'sass', 'browserify:build', 'karma:chrome']); + // Run while developing + grunt.registerTask('dev', ['build', 'connect:dev', 'concurrent:watchSandbox']); - // Skin development watch task. - grunt.registerTask('skin-dev', ['connect:dev', 'watch:skin']); + grunt.registerTask('watchAll', ['build', 'connect:dev', 'concurrent:watchAll']); - // Tests. - // We want to run things a little differently if it's coming from Travis vs local - if (process.env.TRAVIS) { - grunt.registerTask('test', ['build', 'test-travis', 'coveralls']); - } else { - grunt.registerTask('test', ['build', 'test-local']); - } + // Pick your testing, or run both in different terminals + grunt.registerTask('test-ui', ['browserify:tests']); + grunt.registerTask('test-cli', ['karma:watch']); // Load all the tasks in the tasks directory grunt.loadTasks('build/tasks'); diff --git a/package.json b/package.json index 58101f72e..4cd1d9a57 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "homepage": "http://videojs.com", "author": "Steve Heffernan", "scripts": { - "test": "grunt test" + "test": "grunt test && if [ '$TRAVIS' ]; then grunt coveralls; fi;" }, "repository": { "type": "git", @@ -47,6 +47,7 @@ "grunt-banner": "^0.3.1", "grunt-browserify": "3.5.1", "grunt-cli": "~0.1.0", + "grunt-concurrent": "^1.0.0", "grunt-contrib-clean": "~0.4.0a", "grunt-contrib-concat": "^0.5.1", "grunt-contrib-connect": "~0.7.1", @@ -65,10 +66,10 @@ "grunt-version": "~0.3.0", "grunt-videojs-languages": "0.0.4", "grunt-zip": "0.10.2", - "karma": "^0.12.14", + "karma": "^0.12.36", "karma-browserify": "^4.0.0", "karma-chrome-launcher": "^0.1.3", - "karma-coverage": "^0.2.7", + "karma-coverage": "^0.4.0", "karma-firefox-launcher": "^0.1.3", "karma-ie-launcher": "^0.1.5", "karma-opera-launcher": "~0.1.0", diff --git a/test/index.html b/test/index.html index 0e450bcfc..a1edc36ac 100644 --- a/test/index.html +++ b/test/index.html @@ -13,8 +13,5 @@ - - - diff --git a/test/karma.conf.js b/test/karma.conf.js index 830468bae..f0b481e30 100644 --- a/test/karma.conf.js +++ b/test/karma.conf.js @@ -1,5 +1,115 @@ module.exports = function(config) { - var customLaunchers = { + // Creating settings object first so we can modify based on travis + var settings = { + basePath: '', + + frameworks: ['browserify', 'qunit'], + autoWatch: false, + singleRun: true, + browsers: ['Chrome'], + + // Compling tests here + files: [ + '../build/temp/video-js.css', + '../test/globals-shim.js', + '../test/unit/**/*.js', + { pattern: '../src/**/*.js', watched: true, included: false, served: false } + ], + + // Using precompiled tests + // files: [ + // '../build/temp/video-js.css', + // '../build/temp/tests.js' + // ], + + preprocessors: { + '../test/**/*.js': [ 'browserify' ] + }, + + browserify: { + debug: true, + transform: [ + require('babelify').configure({ + sourceMapRelative: './', + loose: ['all'] + }) + ] + }, + + plugins: [ + 'karma-qunit', + 'karma-chrome-launcher', + 'karma-firefox-launcher', + 'karma-ie-launcher', + 'karma-opera-launcher', + 'karma-phantomjs-launcher', + 'karma-safari-launcher', + 'karma-sauce-launcher', + 'karma-browserify', + 'karma-coverage' + ], + + reporters: ['dots', 'saucelabs'], + + // web server port + port: 9876, + + // cli runner port + runnerPort: 9100, + colors: true, + logLevel: config.LOG_INFO, + captureTimeout: 60000, + browserNoActivityTimeout: 60000, + + sauceLabs: { + startConnect: true, + tunnelIdentifier: process.env.TRAVIS_JOB_NUMBER, + build: process.env.TRAVIS_BUILD_NUMBER, + testName: process.env.TRAVIS_BUILD_NUMBER + process.env.TRAVIS_BRANCH, + recordScreenshots: false + }, + customLaunchers: getCustomLaunchers(), + + // The HTML reporter seems to be busted right now, so we're just using text in the meantime + // along with the summary after the test run. + coverageReporter: { + reporters: [ + { + type: 'text', + dir: 'coverage/', + file: 'coverage.txt' + }, + { + type: 'lcovonly', + dir: 'coverage/', + subdir: '.' + }, + { type: 'text-summary' } + ] + } + }; + + if (process.env.TRAVIS) { + settings.browserify.transform.push('browserify-istanbul'); + settings.reporters.push('coverage'); + + if (process.env.SAUCE_ACCESS_KEY) { + settings.browsers = [ + 'chrome_sl', + 'firefox_sl', + 'safari_sl', + 'ie_sl' + ]; + } else { + settings.browsers = ['Firefox']; + } + } + + config.set(settings); +}; + +function getCustomLaunchers(){ + return { chrome_sl: { singleRun: true, base: 'SauceLabs', @@ -46,79 +156,4 @@ module.exports = function(config) { version: '11' } }; - - config.set({ - basePath: '', - - frameworks: ['qunit'], - - autoWatch: false, - - singleRun: true, - - files: [ - // include and execute the standalone test bundle first - '../build/temp/tests.js', - - // then include video.js through globals to run the API tests - '../build/temp/video-js.min.css', - '../build/temp/video.min.js', - 'api/api.js' - ], - - plugins: [ - 'karma-qunit', - 'karma-chrome-launcher', - 'karma-firefox-launcher', - 'karma-ie-launcher', - 'karma-opera-launcher', - 'karma-phantomjs-launcher', - 'karma-safari-launcher', - 'karma-sauce-launcher', - 'karma-coverage' - ], - - reporters: ['dots', 'saucelabs', 'coverage'], - - // web server port - port: 9876, - - // cli runner port - runnerPort: 9100, - - colors: true, - - logLevel: config.LOG_INFO, - - captureTimeout: 60000, - - browserNoActivityTimeout: 60000, - - sauceLabs: { - startConnect: true, - tunnelIdentifier: process.env.TRAVIS_JOB_NUMBER, - build: process.env.TRAVIS_BUILD_NUMBER, - testName: process.env.TRAVIS_BUILD_NUMBER + process.env.TRAVIS_BRANCH, - recordScreenshots: false - }, - customLaunchers: customLaunchers, - - // The HTML reporter seems to be busted right now, so we're just using text in the meantime - // along with the summary after the test run. - coverageReporter: { - reporters: [ - { - type: 'text', - dir: 'coverage/', - file: 'coverage.txt' - }, - { - type: 'lcovonly', - dir: 'coverage/', - subdir: '.' - }, - { type: 'text-summary' } - ] - } - }); -}; +}