Corruption of image files during migration or optimization

[arturvader]

When the script is triggered, the file from the app/images/brand_logo.jpg folder is copied to dist/images/brand_logo.jpg. There should be no changes to the file but it changes and becomes unreadable.

In another project, when using "gulp-tinify": "^1.0.2" the same problem.

I checked with gulp version 4.0.2 installed and everything works correctly.

Node version:
20.13.0
npm version:
10.5.2
gulp --version
CLI version: 3.0.0
Local version: 5.0.0

package.json:
{
"name": "test",
"version": "1.0.0",
"main": "gulpfile.js",
"private": true,
"type": "module",
"browserslist": [
"last 2 version",
"not dead",
"not ie <= 11",
"iOS >= 12"
],
"scripts": {
"test": "echo "Error: no test specified" && exit 1"
},
"author": "",
"license": "ISC",
"devDependencies": {
"gulp": "^5.0.0",
"gulp-imagemin": "^9.1.0",
"gulp-newer": "^1.4.0"
}
}

gulpfile.js:
import gulp from 'gulp';
import imagemin from 'gulp-imagemin';
import newer from 'gulp-newer';

const app = 'app',
dist = 'dist';

const config = {
app : {
svg : app + '/images//*.svg',
img : app + '/images//.{jpg,jpeg,png,gif,webp}',
},
dist : {
svg : dist + '/images/**/.svg',
img : dist + '/images//*',
all : dist + '//*',
},
}

// Images

export const images = gulp.series(
function(cb){
return gulp.src(config.app.svg, { base: 'app' })
.pipe(newer(config.dist.svg))
.pipe(imagemin())
.pipe(gulp.dest(dist))
cb();
},
);
export const imgtinify = () => {
return gulp.src(config.app.img, { base: 'app', encoding: false, buffer: false })
.pipe(newer(config.dist.img))
.pipe(gulp.dest(dist))
};

// Watch

export const startwatch = () => {
gulp.watch(config.app.img, gulp.series(
imgtinify,
));
gulp.watch(config.app.svg, gulp.series(images, ));
};

// Default

export default gulp.series(
gulp.parallel(
images,
imgtinify,
),
gulp.parallel(
startwatch,
),
);

[yocontra]

Add {encoding: false} to any gulp.src call where you are using binary files - see now docs and release blog for v5 for info.

[arturvader]

Add {encoding: false} to any gulp.src call where you are using binary files - see now docs and release blog for v5 for info.

Pay attention to the code that I provided as a sample. What you wrote is indicated in the code ( gulp.src(config.app.img, { base: 'app', encoding: false, buffer: false }) ) and it does not help.

[yocontra]

Sorry, the code wasn't formatted in the issue very well and I was looking at the SVG logic. With encoding: false it shouldn't make any changes to the file at all.

Just going to repost your code simplified and formatted for reference:

import gulp from 'gulp'
import newer from 'gulp-newer'

const app = 'app'
const dist = 'dist'

const config = {
  app: {
    img: app + '/images//.{jpg,jpeg,png,gif,webp}'
  },
  dist: {
    img: dist + '/images//*'
  }
}
export const imgtinify = () =>
  gulp
    .src(config.app.img, {
      base: 'app',
      encoding: false,
      buffer: false
    })
    .pipe(newer(config.dist.img))
    .pipe(gulp.dest(dist))

export default gulp.series(imgtinify)

Are you able to reproduce this with the above code, and does removing gulp-newer also reproduce it? Could you provide an example of a before and after file so I can look at the binary?

[arturvader]

Sorry, the code wasn't formatted in the issue very well and I was looking at the SVG logic. With encoding: false it shouldn't make any changes to the file at all.

Just going to repost your code simplified and formatted for reference:

import gulp from 'gulp'
import newer from 'gulp-newer'

const app = 'app'
const dist = 'dist'

const config = {
  app: {
    img: app + '/images//.{jpg,jpeg,png,gif,webp}'
  },
  dist: {
    img: dist + '/images//*'
  }
}
export const imgtinify = () =>
  gulp
    .src(config.app.img, {
      base: 'app',
      encoding: false,
      buffer: false
    })
    .pipe(newer(config.dist.img))
    .pipe(gulp.dest(dist))

export default gulp.series(imgtinify)

Are you able to reproduce this with the above code, and does removing gulp-newer also reproduce it? Could you provide an example of a before and after file so I can look at the binary?

Original image:

Resulting file:

[arturvader]

Sorry, the code wasn't formatted in the issue very well and I was looking at the SVG logic. With encoding: false it shouldn't make any changes to the file at all.

Just going to repost your code simplified and formatted for reference:

import gulp from 'gulp'
import newer from 'gulp-newer'

const app = 'app'
const dist = 'dist'

const config = {
  app: {
    img: app + '/images//.{jpg,jpeg,png,gif,webp}'
  },
  dist: {
    img: dist + '/images//*'
  }
}
export const imgtinify = () =>
  gulp
    .src(config.app.img, {
      base: 'app',
      encoding: false,
      buffer: false
    })
    .pipe(newer(config.dist.img))
    .pipe(gulp.dest(dist))

export default gulp.series(imgtinify)

Are you able to reproduce this with the above code, and does removing gulp-newer also reproduce it? Could you provide an example of a before and after file so I can look at the binary?

I checked the test assembly with the code you wrote and got the same result.

[phated]

I don't have much input right now, but I'm noticing that the globs look very broken. Perhaps this isn't even picking up the files at all and the output is an older generated file

[arturvader]

when I copied the code here some characters are not published (double *, /images/**/ )

[yocontra]

To simplify further and fix the globs in the code, does this reproduce?

import gulp from 'gulp'

export const imgtinify = () =>
  gulp
    .src('app/images/**/*.{jpg,jpeg,png,gif,webp}', {
      base: 'app',
      encoding: false,
      buffer: false
    })
    .pipe(gulp.dest('dist'))

export default gulp.series(imgtinify)

Is it possible that you ran it without encoding: false, it produced broken images, then you added encoding: false but the new images didn't get written because gulp-newer is just looking at the mtime of the original file? Try clearing the dist folder out before reproducing again.

[arturvader]

To simplify further and fix the globs in the code, does this reproduce?

import gulp from 'gulp'

export const imgtinify = () =>
  gulp
    .src('app/images/**/*.{jpg,jpeg,png,gif,webp}', {
      base: 'app',
      encoding: false,
      buffer: false
    })
    .pipe(gulp.dest('dist'))

export default gulp.series(imgtinify)

Is it possible that you ran it without encoding: false, it produced broken images, then you added encoding: false but the new images didn't get written because gulp-newer is just looking at the mtime of the original file? Try clearing the dist folder out before reproducing again.

The source file was made in Photoshop, I deleted all the files in dist before launching gulp. This is a test build, the problem was discovered in 2 new projects with a lot of image files and gulp version 5 was installed for the first time. In projects where the old version (previous) of gulp remains, everything works correctly.

[arturvader]

Please try to create a test project with the settings that I specified. Maybe I made a mistake somewhere. I tried it on my MacBook Pro and on Linux (Raspberry PI 5). Sorry for my English, it's not my native language.

[Nezarik-Intmax]

Try to add removeBOM: false to src options

[mjot]

I had the same problem with this simple task for copying font files:

async function fontsBuild(site) {
    let path = setPathsForSite(site);
    let fontsConfig = {
        dir: path.src + 'fonts/',
        src: path.src + 'fonts/*.{woff,woff2,eot,ttf,svg}',
        build: path.dst + 'assets/fonts/'
    };

    const directoryExists = await checkDirectory(fontsConfig.dir);
    if (directoryExists) {
        return gulp.src(fontsConfig.src)
            .pipe(newer(fontsConfig.build))
            .pipe(gulp.dest(fontsConfig.build));
    }
}

By adding encoding: false to gulp.src like mentioned from @yocontra its works again:

return gulp.src(fontsConfig.src, { encoding: false })

[arturvader]

Try to add removeBOM: false to src options

I tried what you wrote, it didn't help.

[vkorytska]

This is probably related to gulpjs/vinyl-fs#351

[ernilambar]

I was having same issue of corrupted image. Adding following arguments in src() fixed the issue.

{
  buffer: true,
  removeBOM: false
}

Example:

return gulp.src( [ 'src/images/**/*.*' ], {
  buffer: true,
  removeBOM: false
  } )
  .pipe( gulp.dest( 'assets/images' ) );
} );

[arturvader]

true

I confirm that after setting the buffer value to true (buffer: true), the file was copied without changes and working.

[BlueAbdul]

true

I confirm that after setting the buffer value to true (buffer: true), the file was copied without changes and working.

Yeah it fixes the problem, I don't know why my comment got minimized as I'm only trying to help...

[woody-li]

{ buffer: true, removeBOM: false }

It still don't working for font sources, such as gulp-iconfont.

[PRR24]

According to the doc, buffer is true by default. It changing it affects the outcome, there is possibly another smaller defect.

[PRR24]

I think I found the cause of the problem:
line 34 in write-stream.js is currently reading ... encoding.enc !== .... It should be ... codec.enc !== ... instead.
After making the change, all my scripts started to work correctly.

While looking around in the code, two additional things caught my eye:

• optResolver used in write-stream is not in sync with the resolver in gulp.src, so the encoding used in stream writing differs from stream reading. As write-stream is calling stream reading functions, it feels bit scary.
• Resolver.resolve is called with second argument (file), but the function has only a single argument.

Possibly these are there for a reason, but just wanted to point them out.

[KarelWintersky]

Same problem.

Yeah, buffer: true solves problem for binary files, but this solution is absolutely not obvious.

The need for this line breaks many packages that don't know about it. Such as:

• https://www.npmjs.com/package/@hh.ru/gulp-minify-images

Is it possible to specify this parameter somewhere globally for all tasks?

[PRR24]

@KarelWintersky You may try, if an in-place patch of
node_modules/vinyl-fs/lib/dest/write-contents/write-stream.js:34
as per above solves the problem for you.
If yes, this is a global fix.

[yocontra]

I think I found the cause of the problem: line 34 in write-stream.js is currently reading ... encoding.enc !== .... It should be ... codec.enc !== ... instead. After making the change, all my scripts started to work correctly.

While looking around in the code, two additional things caught my eye:

* optResolver used in write-stream is not in sync with the resolver in _gulp.src_, so the encoding used in stream writing differs from stream reading. As write-stream is calling stream reading functions, it feels bit scary.

* Resolver.resolve is called with second argument (file), but the function has only a single argument.

Possibly these are there for a reason, but just wanted to point them out.

@PRR24 Can you submit a PR or open an issue on vinyl-fs? Thank you for looking deeper into this.

[keitetran]

Add {encoding: false} to any gulp.src call where you are using binary files - see now docs and release blog for v5 for info.

I was add this for copy excel file.

[alex-lenk]

encoding: false

{
  buffer: true,
  removeBOM: false
}

Praise be to you, kind person. You have solved my problem. I was already considering switching to version 4.

[twolfson]

In gulp.spritesmith we had a comparable problem but the fix had to also include add a {encoding: false} on the .dest() as well

twolfson/gulp.spritesmith#159

Example of fixed Gulpfile.js:

var gulp = require('gulp');
var spritesmith = require('gulp.spritesmith');

gulp.task('sprite', function () {
  var spriteData = gulp.src('images/*.png', {encoding: false}).pipe(spritesmith({
    imgName: 'sprite.png',
    cssName: 'sprite.css'
  }));
  return spriteData.pipe(gulp.dest('path/to/output/', {encoding: false}));
});

EDIT: I've also opened a complaint about encoding here, gulpjs/vinyl-fs#355

[GeorgeFlorian]

The following code produces broken images, after decompressing:

import gulp from "gulp";
import gzip from "gulp-gzip";
import gulpIf from "gulp-if";
import path from "path";

// Define file type checks
const isTextFile = (file) => /\.(html|css|js)$/.test(path.extname(file.path));
const isBinaryFile = (file) =>
  /\.(png|jpg|jpeg|gif|svg|ico)$/.test(path.extname(file.path));

export function compress() {
  return (
    gulp
      .src("dist/**/*")
      // Gzip text files with default encoding
      .pipe(
        gulpIf(
          isTextFile,
          gzip({
            gzipOptions: { level: 9 },
            append: true,
          })
        )
      )
      // Gzip binary files with { encoding: false }
      .pipe(
        gulpIf(
          isBinaryFile,
          gzip({
            gzipOptions: { level: 9 },
            append: true,
            encoding: false,
            buffer: true,
            removeBOM: false,
          })
        )
      )
      .pipe(gulp.dest("data"))
  ); // Output to LittleFS data directory
}

gulp.task("compress", compress);